xref: /petsc/src/dm/impls/plex/plexfem.c (revision 48a46eb9bd028bec07ec0f396b1a3abb43f14558)
1af0996ceSBarry Smith #include <petsc/private/dmpleximpl.h> /*I      "petscdmplex.h"   I*/
2da97024aSMatthew G. Knepley #include <petscsf.h>
3cb1e1211SMatthew G Knepley 
48e3a2eefSMatthew G. Knepley #include <petscblaslapack.h>
5e8f14785SLisandro Dalcin #include <petsc/private/hashsetij.h>
6af0996ceSBarry Smith #include <petsc/private/petscfeimpl.h>
7af0996ceSBarry Smith #include <petsc/private/petscfvimpl.h>
8a0845e3aSMatthew G. Knepley 
95f0b18bfSMatthew G. Knepley PetscBool  Clementcite       = PETSC_FALSE;
105f0b18bfSMatthew G. Knepley const char ClementCitation[] = "@article{clement1975approximation,\n"
115f0b18bfSMatthew G. Knepley                                "  title   = {Approximation by finite element functions using local regularization},\n"
125f0b18bfSMatthew G. Knepley                                "  author  = {Philippe Cl{\\'e}ment},\n"
135f0b18bfSMatthew G. Knepley                                "  journal = {Revue fran{\\c{c}}aise d'automatique, informatique, recherche op{\\'e}rationnelle. Analyse num{\\'e}rique},\n"
145f0b18bfSMatthew G. Knepley                                "  volume  = {9},\n"
155f0b18bfSMatthew G. Knepley                                "  number  = {R2},\n"
165f0b18bfSMatthew G. Knepley                                "  pages   = {77--84},\n"
175f0b18bfSMatthew G. Knepley                                "  year    = {1975}\n}\n";
185f0b18bfSMatthew G. Knepley 
199371c9d4SSatish Balay static PetscErrorCode DMPlexConvertPlex(DM dm, DM *plex, PetscBool copy) {
202f856554SMatthew G. Knepley   PetscBool isPlex;
212f856554SMatthew G. Knepley 
222f856554SMatthew G. Knepley   PetscFunctionBegin;
239566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex));
242f856554SMatthew G. Knepley   if (isPlex) {
252f856554SMatthew G. Knepley     *plex = dm;
269566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)dm));
272f856554SMatthew G. Knepley   } else {
289566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex));
292f856554SMatthew G. Knepley     if (!*plex) {
309566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm, DMPLEX, plex));
319566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex));
322f856554SMatthew G. Knepley       if (copy) {
332f856554SMatthew G. Knepley         DMSubDomainHookLink link;
349a2a23afSMatthew G. Knepley 
359566063dSJacob Faibussowitsch         PetscCall(DMCopyAuxiliaryVec(dm, *plex));
369a2a23afSMatthew G. Knepley         /* Run the subdomain hook (this will copy the DMSNES/DMTS) */
372f856554SMatthew G. Knepley         for (link = dm->subdomainhook; link; link = link->next) {
389566063dSJacob Faibussowitsch           if (link->ddhook) PetscCall((*link->ddhook)(dm, *plex, link->ctx));
392f856554SMatthew G. Knepley         }
402f856554SMatthew G. Knepley       }
412f856554SMatthew G. Knepley     } else {
429566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)*plex));
432f856554SMatthew G. Knepley     }
442f856554SMatthew G. Knepley   }
452f856554SMatthew G. Knepley   PetscFunctionReturn(0);
462f856554SMatthew G. Knepley }
472f856554SMatthew G. Knepley 
489371c9d4SSatish Balay static PetscErrorCode PetscContainerUserDestroy_PetscFEGeom(void *ctx) {
499b6f715bSMatthew G. Knepley   PetscFEGeom *geom = (PetscFEGeom *)ctx;
509b6f715bSMatthew G. Knepley 
519b6f715bSMatthew G. Knepley   PetscFunctionBegin;
529566063dSJacob Faibussowitsch   PetscCall(PetscFEGeomDestroy(&geom));
539b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
549b6f715bSMatthew G. Knepley }
559b6f715bSMatthew G. Knepley 
569371c9d4SSatish Balay static PetscErrorCode DMPlexGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom) {
579b6f715bSMatthew G. Knepley   char           composeStr[33] = {0};
589b6f715bSMatthew G. Knepley   PetscObjectId  id;
599b6f715bSMatthew G. Knepley   PetscContainer container;
609b6f715bSMatthew G. Knepley 
619b6f715bSMatthew G. Knepley   PetscFunctionBegin;
629566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad, &id));
6363a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMPlexGetFEGeom_%" PetscInt64_FMT "\n", id));
649566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container));
659b6f715bSMatthew G. Knepley   if (container) {
669566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **)geom));
679b6f715bSMatthew G. Knepley   } else {
689566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom));
699566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container));
709566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *)*geom));
719566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom));
729566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container));
739566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
749b6f715bSMatthew G. Knepley   }
759b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
769b6f715bSMatthew G. Knepley }
779b6f715bSMatthew G. Knepley 
789371c9d4SSatish Balay static PetscErrorCode DMPlexRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom) {
799b6f715bSMatthew G. Knepley   PetscFunctionBegin;
809b6f715bSMatthew G. Knepley   *geom = NULL;
819b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
829b6f715bSMatthew G. Knepley }
839b6f715bSMatthew G. Knepley 
8446fa42a0SMatthew G. Knepley /*@
8546fa42a0SMatthew G. Knepley   DMPlexGetScale - Get the scale for the specified fundamental unit
8646fa42a0SMatthew G. Knepley 
8746fa42a0SMatthew G. Knepley   Not collective
8846fa42a0SMatthew G. Knepley 
894165533cSJose E. Roman   Input Parameters:
9046fa42a0SMatthew G. Knepley + dm   - the DM
9146fa42a0SMatthew G. Knepley - unit - The SI unit
9246fa42a0SMatthew G. Knepley 
934165533cSJose E. Roman   Output Parameter:
9446fa42a0SMatthew G. Knepley . scale - The value used to scale all quantities with this unit
9546fa42a0SMatthew G. Knepley 
9646fa42a0SMatthew G. Knepley   Level: advanced
9746fa42a0SMatthew G. Knepley 
98db781477SPatrick Sanan .seealso: `DMPlexSetScale()`, `PetscUnit`
9946fa42a0SMatthew G. Knepley @*/
1009371c9d4SSatish Balay PetscErrorCode DMPlexGetScale(DM dm, PetscUnit unit, PetscReal *scale) {
101cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
102cb1e1211SMatthew G Knepley 
103cb1e1211SMatthew G Knepley   PetscFunctionBegin;
104cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
105dadcf809SJacob Faibussowitsch   PetscValidRealPointer(scale, 3);
106cb1e1211SMatthew G Knepley   *scale = mesh->scale[unit];
107cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
108cb1e1211SMatthew G Knepley }
109cb1e1211SMatthew G Knepley 
11046fa42a0SMatthew G. Knepley /*@
11146fa42a0SMatthew G. Knepley   DMPlexSetScale - Set the scale for the specified fundamental unit
11246fa42a0SMatthew G. Knepley 
11346fa42a0SMatthew G. Knepley   Not collective
11446fa42a0SMatthew G. Knepley 
1154165533cSJose E. Roman   Input Parameters:
11646fa42a0SMatthew G. Knepley + dm   - the DM
11746fa42a0SMatthew G. Knepley . unit - The SI unit
11846fa42a0SMatthew G. Knepley - scale - The value used to scale all quantities with this unit
11946fa42a0SMatthew G. Knepley 
12046fa42a0SMatthew G. Knepley   Level: advanced
12146fa42a0SMatthew G. Knepley 
122db781477SPatrick Sanan .seealso: `DMPlexGetScale()`, `PetscUnit`
12346fa42a0SMatthew G. Knepley @*/
1249371c9d4SSatish Balay PetscErrorCode DMPlexSetScale(DM dm, PetscUnit unit, PetscReal scale) {
125cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex *)dm->data;
126cb1e1211SMatthew G Knepley 
127cb1e1211SMatthew G Knepley   PetscFunctionBegin;
128cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
129cb1e1211SMatthew G Knepley   mesh->scale[unit] = scale;
130cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
131cb1e1211SMatthew G Knepley }
132cb1e1211SMatthew G Knepley 
1339371c9d4SSatish Balay static PetscErrorCode DMPlexProjectRigidBody_Private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nc, PetscScalar *mode, void *ctx) {
1349371c9d4SSatish Balay   const PetscInt eps[3][3][3] = {
1359371c9d4SSatish Balay     {{0, 0, 0},  {0, 0, 1},  {0, -1, 0}},
1369371c9d4SSatish Balay     {{0, 0, -1}, {0, 0, 0},  {1, 0, 0} },
1379371c9d4SSatish Balay     {{0, 1, 0},  {-1, 0, 0}, {0, 0, 0} }
1389371c9d4SSatish Balay   };
139026175e5SToby Isaac   PetscInt *ctxInt = (PetscInt *)ctx;
140ad917190SMatthew G. Knepley   PetscInt  dim2   = ctxInt[0];
141026175e5SToby Isaac   PetscInt  d      = ctxInt[1];
142026175e5SToby Isaac   PetscInt  i, j, k = dim > 2 ? d - dim : d;
143026175e5SToby Isaac 
144ad917190SMatthew G. Knepley   PetscFunctionBegin;
14563a3b9bcSJacob Faibussowitsch   PetscCheck(dim == dim2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Input dimension %" PetscInt_FMT " does not match context dimension %" PetscInt_FMT, dim, dim2);
146026175e5SToby Isaac   for (i = 0; i < dim; i++) mode[i] = 0.;
147026175e5SToby Isaac   if (d < dim) {
14812adca46SMatthew G. Knepley     mode[d] = 1.; /* Translation along axis d */
149ad917190SMatthew G. Knepley   } else {
150026175e5SToby Isaac     for (i = 0; i < dim; i++) {
1519371c9d4SSatish Balay       for (j = 0; j < dim; j++) { mode[j] += eps[i][j][k] * X[i]; /* Rotation about axis d */ }
152026175e5SToby Isaac     }
153026175e5SToby Isaac   }
154ad917190SMatthew G. Knepley   PetscFunctionReturn(0);
155026175e5SToby Isaac }
156026175e5SToby Isaac 
157cc4e42d9SMartin Diehl /*@
15812adca46SMatthew G. Knepley   DMPlexCreateRigidBody - For the default global section, create rigid body modes by function space interpolation
159cb1e1211SMatthew G Knepley 
160d083f849SBarry Smith   Collective on dm
161cb1e1211SMatthew G Knepley 
1624165533cSJose E. Roman   Input Parameters:
16356cf3b9cSMatthew G. Knepley + dm - the DM
16456cf3b9cSMatthew G. Knepley - field - The field number for the rigid body space, or 0 for the default
165cb1e1211SMatthew G Knepley 
1664165533cSJose E. Roman   Output Parameter:
167cb1e1211SMatthew G Knepley . sp - the null space
168cb1e1211SMatthew G Knepley 
16912adca46SMatthew G. Knepley   Note: This is necessary to provide a suitable coarse space for algebraic multigrid
170cb1e1211SMatthew G Knepley 
171cb1e1211SMatthew G Knepley   Level: advanced
172cb1e1211SMatthew G Knepley 
173db781477SPatrick Sanan .seealso: `MatNullSpaceCreate()`, `PCGAMG`
174cb1e1211SMatthew G Knepley @*/
1759371c9d4SSatish Balay PetscErrorCode DMPlexCreateRigidBody(DM dm, PetscInt field, MatNullSpace *sp) {
17656cf3b9cSMatthew G. Knepley   PetscErrorCode (**func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *);
177cb1e1211SMatthew G Knepley   MPI_Comm     comm;
178026175e5SToby Isaac   Vec          mode[6];
179026175e5SToby Isaac   PetscSection section, globalSection;
18056cf3b9cSMatthew G. Knepley   PetscInt     dim, dimEmbed, Nf, n, m, mmin, d, i, j;
181cb1e1211SMatthew G Knepley 
182cb1e1211SMatthew G Knepley   PetscFunctionBegin;
1839566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
1849566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
1859566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
1869566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
18763a3b9bcSJacob Faibussowitsch   PetscCheck(!Nf || !(field < 0 || field >= Nf), comm, PETSC_ERR_ARG_OUTOFRANGE, "Field %" PetscInt_FMT " is not in [0, %" PetscInt_FMT ")", field, Nf);
18856cf3b9cSMatthew G. Knepley   if (dim == 1 && Nf < 2) {
1899566063dSJacob Faibussowitsch     PetscCall(MatNullSpaceCreate(comm, PETSC_TRUE, 0, NULL, sp));
190cb1e1211SMatthew G Knepley     PetscFunctionReturn(0);
191cb1e1211SMatthew G Knepley   }
1929566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
1939566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
1949566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
1959566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(Nf, &func));
196b247467aSMatthew G. Knepley   m = (dim * (dim + 1)) / 2;
1979566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
1989566063dSJacob Faibussowitsch   PetscCall(VecSetType(mode[0], dm->vectype));
1999566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
2009566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
2019566063dSJacob Faibussowitsch   PetscCall(VecGetSize(mode[0], &n));
202b247467aSMatthew G. Knepley   mmin        = PetscMin(m, n);
20356cf3b9cSMatthew G. Knepley   func[field] = DMPlexProjectRigidBody_Private;
2049566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
205026175e5SToby Isaac   for (d = 0; d < m; d++) {
206330485fdSToby Isaac     PetscInt ctx[2];
207026175e5SToby Isaac     void    *voidctx = (void *)(&ctx[0]);
208cb1e1211SMatthew G Knepley 
2099d8fbdeaSMatthew G. Knepley     ctx[0] = dimEmbed;
210330485fdSToby Isaac     ctx[1] = d;
2119566063dSJacob Faibussowitsch     PetscCall(DMProjectFunction(dm, 0.0, func, &voidctx, INSERT_VALUES, mode[d]));
212cb1e1211SMatthew G Knepley   }
2133b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
214b50a2c0aSJacob Faibussowitsch   for (i = 0; i < mmin; ++i) {
215b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
216b50a2c0aSJacob Faibussowitsch 
2179566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
2189566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], mmin - i - 1, mode + i + 1, dots + i + 1));
219b50a2c0aSJacob Faibussowitsch     for (j = i + 1; j < mmin; ++j) {
220b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
2219566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
222b50a2c0aSJacob Faibussowitsch     }
223cb1e1211SMatthew G Knepley   }
2249566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, mmin, mode, sp));
2259566063dSJacob Faibussowitsch   for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i]));
2269566063dSJacob Faibussowitsch   PetscCall(PetscFree(func));
227cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
228cb1e1211SMatthew G Knepley }
229cb1e1211SMatthew G Knepley 
230cc4e42d9SMartin Diehl /*@
23112adca46SMatthew G. Knepley   DMPlexCreateRigidBodies - For the default global section, create rigid body modes by function space interpolation
23212adca46SMatthew G. Knepley 
233d083f849SBarry Smith   Collective on dm
23412adca46SMatthew G. Knepley 
2354165533cSJose E. Roman   Input Parameters:
23612adca46SMatthew G. Knepley + dm    - the DM
23712adca46SMatthew G. Knepley . nb    - The number of bodies
23812adca46SMatthew G. Knepley . label - The DMLabel marking each domain
23912adca46SMatthew G. Knepley . nids  - The number of ids per body
24012adca46SMatthew G. Knepley - ids   - An array of the label ids in sequence for each domain
24112adca46SMatthew G. Knepley 
2424165533cSJose E. Roman   Output Parameter:
24312adca46SMatthew G. Knepley . sp - the null space
24412adca46SMatthew G. Knepley 
24512adca46SMatthew G. Knepley   Note: This is necessary to provide a suitable coarse space for algebraic multigrid
24612adca46SMatthew G. Knepley 
24712adca46SMatthew G. Knepley   Level: advanced
24812adca46SMatthew G. Knepley 
249db781477SPatrick Sanan .seealso: `MatNullSpaceCreate()`
25012adca46SMatthew G. Knepley @*/
2519371c9d4SSatish Balay PetscErrorCode DMPlexCreateRigidBodies(DM dm, PetscInt nb, DMLabel label, const PetscInt nids[], const PetscInt ids[], MatNullSpace *sp) {
25212adca46SMatthew G. Knepley   MPI_Comm     comm;
25312adca46SMatthew G. Knepley   PetscSection section, globalSection;
25412adca46SMatthew G. Knepley   Vec         *mode;
25512adca46SMatthew G. Knepley   PetscScalar *dots;
25612adca46SMatthew G. Knepley   PetscInt     dim, dimEmbed, n, m, b, d, i, j, off;
25712adca46SMatthew G. Knepley 
25812adca46SMatthew G. Knepley   PetscFunctionBegin;
2599566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
2609566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
2619566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
2629566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
2639566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
2649566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
26512adca46SMatthew G. Knepley   m = nb * (dim * (dim + 1)) / 2;
2669566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(m, &mode, m, &dots));
2679566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
2689566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
2699566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
2709566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
27112adca46SMatthew G. Knepley   for (b = 0, off = 0; b < nb; ++b) {
27212adca46SMatthew G. Knepley     for (d = 0; d < m / nb; ++d) {
27312adca46SMatthew G. Knepley       PetscInt ctx[2];
27412adca46SMatthew G. Knepley       PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private;
27512adca46SMatthew G. Knepley       void *voidctx                                                                                   = (void *)(&ctx[0]);
27612adca46SMatthew G. Knepley 
27712adca46SMatthew G. Knepley       ctx[0] = dimEmbed;
27812adca46SMatthew G. Knepley       ctx[1] = d;
2799566063dSJacob Faibussowitsch       PetscCall(DMProjectFunctionLabel(dm, 0.0, label, nids[b], &ids[off], 0, NULL, &func, &voidctx, INSERT_VALUES, mode[d]));
28012adca46SMatthew G. Knepley       off += nids[b];
28112adca46SMatthew G. Knepley     }
28212adca46SMatthew G. Knepley   }
2833b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
284606c1a1cSJacob Faibussowitsch   for (i = 0; i < m; ++i) {
285b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
2865a0e29b9SJacob Faibussowitsch 
2879566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
2889566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], m - i - 1, mode + i + 1, dots + i + 1));
2895a0e29b9SJacob Faibussowitsch     for (j = i + 1; j < m; ++j) {
290b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
2919566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
2925a0e29b9SJacob Faibussowitsch     }
29312adca46SMatthew G. Knepley   }
2949566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, m, mode, sp));
2959566063dSJacob Faibussowitsch   for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i]));
2969566063dSJacob Faibussowitsch   PetscCall(PetscFree2(mode, dots));
29712adca46SMatthew G. Knepley   PetscFunctionReturn(0);
29812adca46SMatthew G. Knepley }
29912adca46SMatthew G. Knepley 
300b29cfa1cSToby Isaac /*@
301b29cfa1cSToby Isaac   DMPlexSetMaxProjectionHeight - In DMPlexProjectXXXLocal() functions, the projected values of a basis function's dofs
302b29cfa1cSToby Isaac   are computed by associating the basis function with one of the mesh points in its transitively-closed support, and
303b29cfa1cSToby Isaac   evaluating the dual space basis of that point.  A basis function is associated with the point in its
304b29cfa1cSToby Isaac   transitively-closed support whose mesh height is highest (w.r.t. DAG height), but not greater than the maximum
305b29cfa1cSToby Isaac   projection height, which is set with this function.  By default, the maximum projection height is zero, which means
306b29cfa1cSToby Isaac   that only mesh cells are used to project basis functions.  A height of one, for example, evaluates a cell-interior
307b29cfa1cSToby Isaac   basis functions using its cells dual space basis, but all other basis functions with the dual space basis of a face.
308b29cfa1cSToby Isaac 
309b29cfa1cSToby Isaac   Input Parameters:
310b29cfa1cSToby Isaac + dm - the DMPlex object
311b29cfa1cSToby Isaac - height - the maximum projection height >= 0
312b29cfa1cSToby Isaac 
313b29cfa1cSToby Isaac   Level: advanced
314b29cfa1cSToby Isaac 
315db781477SPatrick Sanan .seealso: `DMPlexGetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()`
316b29cfa1cSToby Isaac @*/
3179371c9d4SSatish Balay PetscErrorCode DMPlexSetMaxProjectionHeight(DM dm, PetscInt height) {
318b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *)dm->data;
319b29cfa1cSToby Isaac 
320b29cfa1cSToby Isaac   PetscFunctionBegin;
321b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
322b29cfa1cSToby Isaac   plex->maxProjectionHeight = height;
323b29cfa1cSToby Isaac   PetscFunctionReturn(0);
324b29cfa1cSToby Isaac }
325b29cfa1cSToby Isaac 
326b29cfa1cSToby Isaac /*@
327b29cfa1cSToby Isaac   DMPlexGetMaxProjectionHeight - Get the maximum height (w.r.t. DAG) of mesh points used to evaluate dual bases in
328b29cfa1cSToby Isaac   DMPlexProjectXXXLocal() functions.
329b29cfa1cSToby Isaac 
330b29cfa1cSToby Isaac   Input Parameters:
331b29cfa1cSToby Isaac . dm - the DMPlex object
332b29cfa1cSToby Isaac 
333b29cfa1cSToby Isaac   Output Parameters:
334b29cfa1cSToby Isaac . height - the maximum projection height
335b29cfa1cSToby Isaac 
336b29cfa1cSToby Isaac   Level: intermediate
337b29cfa1cSToby Isaac 
338db781477SPatrick Sanan .seealso: `DMPlexSetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()`
339b29cfa1cSToby Isaac @*/
3409371c9d4SSatish Balay PetscErrorCode DMPlexGetMaxProjectionHeight(DM dm, PetscInt *height) {
341b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *)dm->data;
342b29cfa1cSToby Isaac 
343b29cfa1cSToby Isaac   PetscFunctionBegin;
344b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
345b29cfa1cSToby Isaac   *height = plex->maxProjectionHeight;
346b29cfa1cSToby Isaac   PetscFunctionReturn(0);
347b29cfa1cSToby Isaac }
348b29cfa1cSToby Isaac 
349ca3d3a14SMatthew G. Knepley typedef struct {
350ca3d3a14SMatthew G. Knepley   PetscReal    alpha; /* The first Euler angle, and in 2D the only one */
351ca3d3a14SMatthew G. Knepley   PetscReal    beta;  /* The second Euler angle */
352ca3d3a14SMatthew G. Knepley   PetscReal    gamma; /* The third Euler angle */
353ca3d3a14SMatthew G. Knepley   PetscInt     dim;   /* The dimension of R */
354ca3d3a14SMatthew G. Knepley   PetscScalar *R;     /* The rotation matrix, transforming a vector in the local basis to the global basis */
355ca3d3a14SMatthew G. Knepley   PetscScalar *RT;    /* The transposed rotation matrix, transforming a vector in the global basis to the local basis */
356ca3d3a14SMatthew G. Knepley } RotCtx;
357ca3d3a14SMatthew G. Knepley 
358ca3d3a14SMatthew G. Knepley /*
359ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
360ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
361ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
362ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
363ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
364ca3d3a14SMatthew G. Knepley */
3659371c9d4SSatish Balay static PetscErrorCode DMPlexBasisTransformSetUp_Rotation_Internal(DM dm, void *ctx) {
366ca3d3a14SMatthew G. Knepley   RotCtx   *rc  = (RotCtx *)ctx;
367ca3d3a14SMatthew G. Knepley   PetscInt  dim = rc->dim;
368ca3d3a14SMatthew G. Knepley   PetscReal c1, s1, c2, s2, c3, s3;
369ca3d3a14SMatthew G. Knepley 
370ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
3719566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(PetscSqr(dim), &rc->R, PetscSqr(dim), &rc->RT));
372ca3d3a14SMatthew G. Knepley   switch (dim) {
373ca3d3a14SMatthew G. Knepley   case 2:
3749371c9d4SSatish Balay     c1       = PetscCosReal(rc->alpha);
3759371c9d4SSatish Balay     s1       = PetscSinReal(rc->alpha);
3769371c9d4SSatish Balay     rc->R[0] = c1;
3779371c9d4SSatish Balay     rc->R[1] = s1;
3789371c9d4SSatish Balay     rc->R[2] = -s1;
3799371c9d4SSatish Balay     rc->R[3] = c1;
3809566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
381b458e8f1SJose E. Roman     DMPlex_Transpose2D_Internal(rc->RT);
382ca3d3a14SMatthew G. Knepley     break;
383ca3d3a14SMatthew G. Knepley   case 3:
3849371c9d4SSatish Balay     c1       = PetscCosReal(rc->alpha);
3859371c9d4SSatish Balay     s1       = PetscSinReal(rc->alpha);
3869371c9d4SSatish Balay     c2       = PetscCosReal(rc->beta);
3879371c9d4SSatish Balay     s2       = PetscSinReal(rc->beta);
3889371c9d4SSatish Balay     c3       = PetscCosReal(rc->gamma);
3899371c9d4SSatish Balay     s3       = PetscSinReal(rc->gamma);
3909371c9d4SSatish Balay     rc->R[0] = c1 * c3 - c2 * s1 * s3;
3919371c9d4SSatish Balay     rc->R[1] = c3 * s1 + c1 * c2 * s3;
3929371c9d4SSatish Balay     rc->R[2] = s2 * s3;
3939371c9d4SSatish Balay     rc->R[3] = -c1 * s3 - c2 * c3 * s1;
3949371c9d4SSatish Balay     rc->R[4] = c1 * c2 * c3 - s1 * s3;
3959371c9d4SSatish Balay     rc->R[5] = c3 * s2;
3969371c9d4SSatish Balay     rc->R[6] = s1 * s2;
3979371c9d4SSatish Balay     rc->R[7] = -c1 * s2;
3989371c9d4SSatish Balay     rc->R[8] = c2;
3999566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
400b458e8f1SJose E. Roman     DMPlex_Transpose3D_Internal(rc->RT);
401ca3d3a14SMatthew G. Knepley     break;
40263a3b9bcSJacob Faibussowitsch   default: SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " not supported", dim);
403ca3d3a14SMatthew G. Knepley   }
404ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
405ca3d3a14SMatthew G. Knepley }
406ca3d3a14SMatthew G. Knepley 
4079371c9d4SSatish Balay static PetscErrorCode DMPlexBasisTransformDestroy_Rotation_Internal(DM dm, void *ctx) {
408ca3d3a14SMatthew G. Knepley   RotCtx *rc = (RotCtx *)ctx;
409ca3d3a14SMatthew G. Knepley 
410ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
4119566063dSJacob Faibussowitsch   PetscCall(PetscFree2(rc->R, rc->RT));
4129566063dSJacob Faibussowitsch   PetscCall(PetscFree(rc));
413ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
414ca3d3a14SMatthew G. Knepley }
415ca3d3a14SMatthew G. Knepley 
4169371c9d4SSatish Balay static PetscErrorCode DMPlexBasisTransformGetMatrix_Rotation_Internal(DM dm, const PetscReal x[], PetscBool l2g, const PetscScalar **A, void *ctx) {
417ca3d3a14SMatthew G. Knepley   RotCtx *rc = (RotCtx *)ctx;
418ca3d3a14SMatthew G. Knepley 
419ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
420ca3d3a14SMatthew G. Knepley   PetscValidPointer(ctx, 5);
4219371c9d4SSatish Balay   if (l2g) {
4229371c9d4SSatish Balay     *A = rc->R;
4239371c9d4SSatish Balay   } else {
4249371c9d4SSatish Balay     *A = rc->RT;
4259371c9d4SSatish Balay   }
426ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
427ca3d3a14SMatthew G. Knepley }
428ca3d3a14SMatthew G. Knepley 
4299371c9d4SSatish Balay PetscErrorCode DMPlexBasisTransformApplyReal_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscReal *y, PetscReal *z, void *ctx) {
430ec277c0fSMatthew G. Knepley   PetscFunctionBegin;
431ab6a9622SMatthew G. Knepley #if defined(PETSC_USE_COMPLEX)
432ab6a9622SMatthew G. Knepley   switch (dim) {
4339371c9d4SSatish Balay   case 2: {
43427104ee2SJacob Faibussowitsch     PetscScalar yt[2] = {y[0], y[1]}, zt[2] = {0.0, 0.0};
435ab6a9622SMatthew G. Knepley 
4369566063dSJacob Faibussowitsch     PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
4379371c9d4SSatish Balay     z[0] = PetscRealPart(zt[0]);
4389371c9d4SSatish Balay     z[1] = PetscRealPart(zt[1]);
4399371c9d4SSatish Balay   } break;
4409371c9d4SSatish Balay   case 3: {
44127104ee2SJacob Faibussowitsch     PetscScalar yt[3] = {y[0], y[1], y[2]}, zt[3] = {0.0, 0.0, 0.0};
442ab6a9622SMatthew G. Knepley 
4439566063dSJacob Faibussowitsch     PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
4449371c9d4SSatish Balay     z[0] = PetscRealPart(zt[0]);
4459371c9d4SSatish Balay     z[1] = PetscRealPart(zt[1]);
4469371c9d4SSatish Balay     z[2] = PetscRealPart(zt[2]);
4479371c9d4SSatish Balay   } break;
448ab6a9622SMatthew G. Knepley   }
449ab6a9622SMatthew G. Knepley #else
4509566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, y, z, ctx));
451ab6a9622SMatthew G. Knepley #endif
452ec277c0fSMatthew G. Knepley   PetscFunctionReturn(0);
453ab6a9622SMatthew G. Knepley }
454ab6a9622SMatthew G. Knepley 
4559371c9d4SSatish Balay PetscErrorCode DMPlexBasisTransformApply_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscScalar *y, PetscScalar *z, void *ctx) {
456ca3d3a14SMatthew G. Knepley   const PetscScalar *A;
457ca3d3a14SMatthew G. Knepley 
458ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
4599566063dSJacob Faibussowitsch   PetscCall((*dm->transformGetMatrix)(dm, x, l2g, &A, ctx));
460ca3d3a14SMatthew G. Knepley   switch (dim) {
4611fba962aSMatthew G. Knepley   case 2: DMPlex_Mult2D_Internal(A, 1, y, z); break;
4621fba962aSMatthew G. Knepley   case 3: DMPlex_Mult3D_Internal(A, 1, y, z); break;
463ca3d3a14SMatthew G. Knepley   }
464ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
465ca3d3a14SMatthew G. Knepley }
466ca3d3a14SMatthew G. Knepley 
4679371c9d4SSatish Balay static PetscErrorCode DMPlexBasisTransformField_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscInt f, PetscBool l2g, PetscScalar *a) {
468ca3d3a14SMatthew G. Knepley   PetscSection       ts;
469ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tva;
470ca3d3a14SMatthew G. Knepley   PetscInt           dof;
471ca3d3a14SMatthew G. Knepley 
472ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
4739566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
4749566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, p, f, &dof));
4759566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
4769566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, p, f, ta, &tva));
477ca3d3a14SMatthew G. Knepley   if (l2g) {
478ca3d3a14SMatthew G. Knepley     switch (dof) {
4791fba962aSMatthew G. Knepley     case 4: DMPlex_Mult2D_Internal(tva, 1, a, a); break;
4801fba962aSMatthew G. Knepley     case 9: DMPlex_Mult3D_Internal(tva, 1, a, a); break;
481ca3d3a14SMatthew G. Knepley     }
482ca3d3a14SMatthew G. Knepley   } else {
483ca3d3a14SMatthew G. Knepley     switch (dof) {
4841fba962aSMatthew G. Knepley     case 4: DMPlex_MultTranspose2D_Internal(tva, 1, a, a); break;
4851fba962aSMatthew G. Knepley     case 9: DMPlex_MultTranspose3D_Internal(tva, 1, a, a); break;
486ca3d3a14SMatthew G. Knepley     }
487ca3d3a14SMatthew G. Knepley   }
4889566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
489ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
490ca3d3a14SMatthew G. Knepley }
491ca3d3a14SMatthew G. Knepley 
4929371c9d4SSatish Balay static PetscErrorCode DMPlexBasisTransformFieldTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt pf, PetscInt f, PetscInt pg, PetscInt g, PetscBool l2g, PetscInt lda, PetscScalar *a) {
493ca3d3a14SMatthew G. Knepley   PetscSection       s, ts;
494ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tvaf, *tvag;
495ca3d3a14SMatthew G. Knepley   PetscInt           fdof, gdof, fpdof, gpdof;
496ca3d3a14SMatthew G. Knepley 
497ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
4989566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
4999566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
5009566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pf, f, &fpdof));
5019566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pg, g, &gpdof));
5029566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pf, f, &fdof));
5039566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pg, g, &gdof));
5049566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
5059566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pf, f, ta, &tvaf));
5069566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pg, g, ta, &tvag));
507ca3d3a14SMatthew G. Knepley   if (l2g) {
508ca3d3a14SMatthew G. Knepley     switch (fdof) {
509ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMult2D_Internal(tvaf, gpdof, lda, a, a); break;
510ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMult3D_Internal(tvaf, gpdof, lda, a, a); break;
511ca3d3a14SMatthew G. Knepley     }
512ca3d3a14SMatthew G. Knepley     switch (gdof) {
513ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultTransposeLeft2D_Internal(tvag, fpdof, lda, a, a); break;
514ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultTransposeLeft3D_Internal(tvag, fpdof, lda, a, a); break;
515ca3d3a14SMatthew G. Knepley     }
516ca3d3a14SMatthew G. Knepley   } else {
517ca3d3a14SMatthew G. Knepley     switch (fdof) {
518ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultTranspose2D_Internal(tvaf, gpdof, lda, a, a); break;
519ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultTranspose3D_Internal(tvaf, gpdof, lda, a, a); break;
520ca3d3a14SMatthew G. Knepley     }
521ca3d3a14SMatthew G. Knepley     switch (gdof) {
522ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultLeft2D_Internal(tvag, fpdof, lda, a, a); break;
523ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultLeft3D_Internal(tvag, fpdof, lda, a, a); break;
524ca3d3a14SMatthew G. Knepley     }
525ca3d3a14SMatthew G. Knepley   }
5269566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
527ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
528ca3d3a14SMatthew G. Knepley }
529ca3d3a14SMatthew G. Knepley 
5309371c9d4SSatish Balay PetscErrorCode DMPlexBasisTransformPoint_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool fieldActive[], PetscBool l2g, PetscScalar *a) {
531ca3d3a14SMatthew G. Knepley   PetscSection    s;
532ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
533ca3d3a14SMatthew G. Knepley   IS              clPoints;
534ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
535ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
536ca3d3a14SMatthew G. Knepley   PetscInt        Nf, f, Np, cp, dof, d = 0;
537ca3d3a14SMatthew G. Knepley 
538ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5399566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
5409566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
5419566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
542ca3d3a14SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
543ca3d3a14SMatthew G. Knepley     for (cp = 0; cp < Np * 2; cp += 2) {
5449566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cp], f, &dof));
545ca3d3a14SMatthew G. Knepley       if (!dof) continue;
5469566063dSJacob Faibussowitsch       if (fieldActive[f]) PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, points[cp], f, l2g, &a[d]));
547ca3d3a14SMatthew G. Knepley       d += dof;
548ca3d3a14SMatthew G. Knepley     }
549ca3d3a14SMatthew G. Knepley   }
5509566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
551ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
552ca3d3a14SMatthew G. Knepley }
553ca3d3a14SMatthew G. Knepley 
5549371c9d4SSatish Balay PetscErrorCode DMPlexBasisTransformPointTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool l2g, PetscInt lda, PetscScalar *a) {
555ca3d3a14SMatthew G. Knepley   PetscSection    s;
556ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
557ca3d3a14SMatthew G. Knepley   IS              clPoints;
558ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
559ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
5608bdb3c71SMatthew G. Knepley   PetscInt        Nf, f, g, Np, cpf, cpg, fdof, gdof, r, c = 0;
561ca3d3a14SMatthew G. Knepley 
562ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5639566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
5649566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
5659566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
566ca3d3a14SMatthew G. Knepley   for (f = 0, r = 0; f < Nf; ++f) {
567ca3d3a14SMatthew G. Knepley     for (cpf = 0; cpf < Np * 2; cpf += 2) {
5689566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cpf], f, &fdof));
569ca3d3a14SMatthew G. Knepley       for (g = 0, c = 0; g < Nf; ++g) {
570ca3d3a14SMatthew G. Knepley         for (cpg = 0; cpg < Np * 2; cpg += 2) {
5719566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldDof(s, points[cpg], g, &gdof));
5729566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformFieldTensor_Internal(dm, tdm, tv, points[cpf], f, points[cpg], g, l2g, lda, &a[r * lda + c]));
573ca3d3a14SMatthew G. Knepley           c += gdof;
574ca3d3a14SMatthew G. Knepley         }
575ca3d3a14SMatthew G. Knepley       }
57663a3b9bcSJacob Faibussowitsch       PetscCheck(c == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of columns %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
577ca3d3a14SMatthew G. Knepley       r += fdof;
578ca3d3a14SMatthew G. Knepley     }
579ca3d3a14SMatthew G. Knepley   }
58063a3b9bcSJacob Faibussowitsch   PetscCheck(r == lda, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of rows %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
5819566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
582ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
583ca3d3a14SMatthew G. Knepley }
584ca3d3a14SMatthew G. Knepley 
5859371c9d4SSatish Balay static PetscErrorCode DMPlexBasisTransform_Internal(DM dm, Vec lv, PetscBool l2g) {
586ca3d3a14SMatthew G. Knepley   DM                 tdm;
587ca3d3a14SMatthew G. Knepley   Vec                tv;
588ca3d3a14SMatthew G. Knepley   PetscSection       ts, s;
589ca3d3a14SMatthew G. Knepley   const PetscScalar *ta;
590ca3d3a14SMatthew G. Knepley   PetscScalar       *a, *va;
591ca3d3a14SMatthew G. Knepley   PetscInt           pStart, pEnd, p, Nf, f;
592ca3d3a14SMatthew G. Knepley 
593ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5949566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
5959566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
5969566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
5979566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
5989566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetChart(s, &pStart, &pEnd));
5999566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
6009566063dSJacob Faibussowitsch   PetscCall(VecGetArray(lv, &a));
6019566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
602ca3d3a14SMatthew G. Knepley   for (p = pStart; p < pEnd; ++p) {
603ca3d3a14SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
6049566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalFieldRef(dm, p, f, a, &va));
6059566063dSJacob Faibussowitsch       PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, p, f, l2g, va));
606ca3d3a14SMatthew G. Knepley     }
607ca3d3a14SMatthew G. Knepley   }
6089566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(lv, &a));
6099566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
610ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
611ca3d3a14SMatthew G. Knepley }
612ca3d3a14SMatthew G. Knepley 
613ca3d3a14SMatthew G. Knepley /*@
614ca3d3a14SMatthew G. Knepley   DMPlexGlobalToLocalBasis - Transform the values in the given local vector from the global basis to the local basis
615ca3d3a14SMatthew G. Knepley 
616ca3d3a14SMatthew G. Knepley   Input Parameters:
617ca3d3a14SMatthew G. Knepley + dm - The DM
618ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the global basis
619ca3d3a14SMatthew G. Knepley 
620ca3d3a14SMatthew G. Knepley   Output Parameters:
621ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the local basis
622ca3d3a14SMatthew G. Knepley 
623c0f8e1fdSMatthew G. Knepley   Note: This method is only intended to be called inside DMGlobalToLocal(). It is unlikely that a user will have a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
624c0f8e1fdSMatthew G. Knepley 
625ca3d3a14SMatthew G. Knepley   Level: developer
626ca3d3a14SMatthew G. Knepley 
627db781477SPatrick Sanan .seealso: `DMPlexLocalToGlobalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()`
628ca3d3a14SMatthew G. Knepley @*/
6299371c9d4SSatish Balay PetscErrorCode DMPlexGlobalToLocalBasis(DM dm, Vec lv) {
630ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
631ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
632ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
6339566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_FALSE));
634ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
635ca3d3a14SMatthew G. Knepley }
636ca3d3a14SMatthew G. Knepley 
637ca3d3a14SMatthew G. Knepley /*@
638ca3d3a14SMatthew G. Knepley   DMPlexLocalToGlobalBasis - Transform the values in the given local vector from the local basis to the global basis
639ca3d3a14SMatthew G. Knepley 
640ca3d3a14SMatthew G. Knepley   Input Parameters:
641ca3d3a14SMatthew G. Knepley + dm - The DM
642ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the local basis
643ca3d3a14SMatthew G. Knepley 
644ca3d3a14SMatthew G. Knepley   Output Parameters:
645ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the global basis
646ca3d3a14SMatthew G. Knepley 
647c0f8e1fdSMatthew G. Knepley   Note: This method is only intended to be called inside DMGlobalToLocal(). It is unlikely that a user would want a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
648c0f8e1fdSMatthew G. Knepley 
649ca3d3a14SMatthew G. Knepley   Level: developer
650ca3d3a14SMatthew G. Knepley 
651db781477SPatrick Sanan .seealso: `DMPlexGlobalToLocalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()`
652ca3d3a14SMatthew G. Knepley @*/
6539371c9d4SSatish Balay PetscErrorCode DMPlexLocalToGlobalBasis(DM dm, Vec lv) {
654ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
655ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
656ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
6579566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_TRUE));
658ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
659ca3d3a14SMatthew G. Knepley }
660ca3d3a14SMatthew G. Knepley 
661ca3d3a14SMatthew G. Knepley /*@
662ca3d3a14SMatthew G. Knepley   DMPlexCreateBasisRotation - Create an internal transformation from the global basis, used to specify boundary conditions
663ca3d3a14SMatthew G. Knepley     and global solutions, to a local basis, appropriate for discretization integrals and assembly.
664ca3d3a14SMatthew G. Knepley 
665ca3d3a14SMatthew G. Knepley   Input Parameters:
666ca3d3a14SMatthew G. Knepley + dm    - The DM
667ca3d3a14SMatthew G. Knepley . alpha - The first Euler angle, and in 2D the only one
668ca3d3a14SMatthew G. Knepley . beta  - The second Euler angle
669f0fc11ceSJed Brown - gamma - The third Euler angle
670ca3d3a14SMatthew G. Knepley 
671ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
672ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
673ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
674ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
675ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
676ca3d3a14SMatthew G. Knepley 
677ca3d3a14SMatthew G. Knepley   Level: developer
678ca3d3a14SMatthew G. Knepley 
679db781477SPatrick Sanan .seealso: `DMPlexGlobalToLocalBasis()`, `DMPlexLocalToGlobalBasis()`
680ca3d3a14SMatthew G. Knepley @*/
6819371c9d4SSatish Balay PetscErrorCode DMPlexCreateBasisRotation(DM dm, PetscReal alpha, PetscReal beta, PetscReal gamma) {
682ca3d3a14SMatthew G. Knepley   RotCtx  *rc;
683ca3d3a14SMatthew G. Knepley   PetscInt cdim;
684ca3d3a14SMatthew G. Knepley 
685362febeeSStefano Zampini   PetscFunctionBegin;
6869566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
6879566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(1, &rc));
688ca3d3a14SMatthew G. Knepley   dm->transformCtx       = rc;
689ca3d3a14SMatthew G. Knepley   dm->transformSetUp     = DMPlexBasisTransformSetUp_Rotation_Internal;
690ca3d3a14SMatthew G. Knepley   dm->transformDestroy   = DMPlexBasisTransformDestroy_Rotation_Internal;
691ca3d3a14SMatthew G. Knepley   dm->transformGetMatrix = DMPlexBasisTransformGetMatrix_Rotation_Internal;
692ca3d3a14SMatthew G. Knepley   rc->dim                = cdim;
693ca3d3a14SMatthew G. Knepley   rc->alpha              = alpha;
694ca3d3a14SMatthew G. Knepley   rc->beta               = beta;
695ca3d3a14SMatthew G. Knepley   rc->gamma              = gamma;
6969566063dSJacob Faibussowitsch   PetscCall((*dm->transformSetUp)(dm, dm->transformCtx));
6979566063dSJacob Faibussowitsch   PetscCall(DMConstructBasisTransform_Internal(dm));
698ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
699ca3d3a14SMatthew G. Knepley }
700ca3d3a14SMatthew G. Knepley 
701b278463cSMatthew G. Knepley /*@C
702ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssential - Insert boundary values into a local vector using a function of the coordinates
703b278463cSMatthew G. Knepley 
704b278463cSMatthew G. Knepley   Input Parameters:
705b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
706b278463cSMatthew G. Knepley . time   - The time
707b278463cSMatthew G. Knepley . field  - The field to constrain
7081c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
7091c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
710b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
711b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
712b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
713b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
714b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
715b278463cSMatthew G. Knepley 
716b278463cSMatthew G. Knepley   Output Parameter:
717b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
718b278463cSMatthew G. Knepley 
719b278463cSMatthew G. Knepley   Level: developer
720b278463cSMatthew G. Knepley 
721db781477SPatrick Sanan .seealso: `DMPlexInsertBoundaryValuesEssentialField()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()`
722b278463cSMatthew G. Knepley @*/
7239371c9d4SSatish Balay PetscErrorCode DMPlexInsertBoundaryValuesEssential(DM dm, PetscReal time, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void *ctx, Vec locX) {
7240163fd50SMatthew G. Knepley   PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx);
72555f2e967SMatthew G. Knepley   void   **ctxs;
726d7ddef95SMatthew G. Knepley   PetscInt numFields;
727d7ddef95SMatthew G. Knepley 
728d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
7299566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
7309566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs));
731d7ddef95SMatthew G. Knepley   funcs[field] = func;
732d7ddef95SMatthew G. Knepley   ctxs[field]  = ctx;
7339566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_BC_VALUES, locX));
7349566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs, ctxs));
735d7ddef95SMatthew G. Knepley   PetscFunctionReturn(0);
736d7ddef95SMatthew G. Knepley }
737d7ddef95SMatthew G. Knepley 
738b278463cSMatthew G. Knepley /*@C
739ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialField - Insert boundary values into a local vector using a function of the coordinates and field data
740b278463cSMatthew G. Knepley 
741b278463cSMatthew G. Knepley   Input Parameters:
742b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
743b278463cSMatthew G. Knepley . time   - The time
744b278463cSMatthew G. Knepley . locU   - A local vector with the input solution values
745b278463cSMatthew G. Knepley . field  - The field to constrain
7461c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
7471c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
748b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
749b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
750b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
751b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
752b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
753b278463cSMatthew G. Knepley 
754b278463cSMatthew G. Knepley   Output Parameter:
755b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
756b278463cSMatthew G. Knepley 
757b278463cSMatthew G. Knepley   Level: developer
758b278463cSMatthew G. Knepley 
759db781477SPatrick Sanan .seealso: `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()`
760b278463cSMatthew G. Knepley @*/
7619371c9d4SSatish Balay PetscErrorCode DMPlexInsertBoundaryValuesEssentialField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX) {
7629371c9d4SSatish Balay   void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
763c60e475cSMatthew G. Knepley   void   **ctxs;
764c60e475cSMatthew G. Knepley   PetscInt numFields;
765c60e475cSMatthew G. Knepley 
766c60e475cSMatthew G. Knepley   PetscFunctionBegin;
7679566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
7689566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs));
769c60e475cSMatthew G. Knepley   funcs[field] = func;
770c60e475cSMatthew G. Knepley   ctxs[field]  = ctx;
7719566063dSJacob Faibussowitsch   PetscCall(DMProjectFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
7729566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs, ctxs));
773c60e475cSMatthew G. Knepley   PetscFunctionReturn(0);
774c60e475cSMatthew G. Knepley }
775c60e475cSMatthew G. Knepley 
776b278463cSMatthew G. Knepley /*@C
777ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialBdField - Insert boundary values into a local vector using a function of the coodinates and boundary field data
778ece3a9fcSMatthew G. Knepley 
779ece3a9fcSMatthew G. Knepley   Collective on dm
780ece3a9fcSMatthew G. Knepley 
781ece3a9fcSMatthew G. Knepley   Input Parameters:
782ece3a9fcSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
783ece3a9fcSMatthew G. Knepley . time   - The time
784ece3a9fcSMatthew G. Knepley . locU   - A local vector with the input solution values
785ece3a9fcSMatthew G. Knepley . field  - The field to constrain
786ece3a9fcSMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
787ece3a9fcSMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
788ece3a9fcSMatthew G. Knepley . label  - The DMLabel defining constrained points
789ece3a9fcSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
790ece3a9fcSMatthew G. Knepley . ids    - An array of ids for constrained points
791ece3a9fcSMatthew G. Knepley . func   - A pointwise function giving boundary values, the calling sequence is given in DMProjectBdFieldLabelLocal()
792ece3a9fcSMatthew G. Knepley - ctx    - An optional user context for bcFunc
793ece3a9fcSMatthew G. Knepley 
794ece3a9fcSMatthew G. Knepley   Output Parameter:
795ece3a9fcSMatthew G. Knepley . locX   - A local vector to receive the boundary values
796ece3a9fcSMatthew G. Knepley 
797ece3a9fcSMatthew G. Knepley   Level: developer
798ece3a9fcSMatthew G. Knepley 
799db781477SPatrick Sanan .seealso: `DMProjectBdFieldLabelLocal()`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()`
800ece3a9fcSMatthew G. Knepley @*/
8019371c9d4SSatish Balay PetscErrorCode DMPlexInsertBoundaryValuesEssentialBdField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), void *ctx, Vec locX) {
8029371c9d4SSatish Balay   void (**funcs)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
803ece3a9fcSMatthew G. Knepley   void   **ctxs;
804ece3a9fcSMatthew G. Knepley   PetscInt numFields;
805ece3a9fcSMatthew G. Knepley 
806ece3a9fcSMatthew G. Knepley   PetscFunctionBegin;
8079566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
8089566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields, &funcs, numFields, &ctxs));
809ece3a9fcSMatthew G. Knepley   funcs[field] = func;
810ece3a9fcSMatthew G. Knepley   ctxs[field]  = ctx;
8119566063dSJacob Faibussowitsch   PetscCall(DMProjectBdFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
8129566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs, ctxs));
813ece3a9fcSMatthew G. Knepley   PetscFunctionReturn(0);
814ece3a9fcSMatthew G. Knepley }
815ece3a9fcSMatthew G. Knepley 
816ece3a9fcSMatthew G. Knepley /*@C
817b278463cSMatthew G. Knepley   DMPlexInsertBoundaryValuesRiemann - Insert boundary values into a local vector
818b278463cSMatthew G. Knepley 
819b278463cSMatthew G. Knepley   Input Parameters:
820b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
821b278463cSMatthew G. Knepley . time   - The time
822b278463cSMatthew G. Knepley . faceGeometry - A vector with the FVM face geometry information
823b278463cSMatthew G. Knepley . cellGeometry - A vector with the FVM cell geometry information
824b278463cSMatthew G. Knepley . Grad         - A vector with the FVM cell gradient information
825b278463cSMatthew G. Knepley . field  - The field to constrain
8261c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
8271c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
828b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
829b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
830b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
831b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
832b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
833b278463cSMatthew G. Knepley 
834b278463cSMatthew G. Knepley   Output Parameter:
835b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
836b278463cSMatthew G. Knepley 
837b278463cSMatthew G. Knepley   Note: This implementation currently ignores the numcomps/comps argument from DMAddBoundary()
838b278463cSMatthew G. Knepley 
839b278463cSMatthew G. Knepley   Level: developer
840b278463cSMatthew G. Knepley 
841db781477SPatrick Sanan .seealso: `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()`
842b278463cSMatthew G. Knepley @*/
8439371c9d4SSatish Balay PetscErrorCode DMPlexInsertBoundaryValuesRiemann(DM dm, PetscReal time, Vec faceGeometry, Vec cellGeometry, Vec Grad, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *), void *ctx, Vec locX) {
84461f58d28SMatthew G. Knepley   PetscDS            prob;
845da97024aSMatthew G. Knepley   PetscSF            sf;
846d7ddef95SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad;
84720369375SToby Isaac   const PetscScalar *facegeom, *cellgeom = NULL, *grad;
848da97024aSMatthew G. Knepley   const PetscInt    *leaves;
849d7ddef95SMatthew G. Knepley   PetscScalar       *x, *fx;
850520b3818SMatthew G. Knepley   PetscInt           dim, nleaves, loc, fStart, fEnd, pdim, i;
8515f80ce2aSJacob Faibussowitsch   PetscErrorCode     ierru = 0;
852d7ddef95SMatthew G. Knepley 
853d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
8549566063dSJacob Faibussowitsch   PetscCall(DMGetPointSF(dm, &sf));
8559566063dSJacob Faibussowitsch   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
856da97024aSMatthew G. Knepley   nleaves = PetscMax(0, nleaves);
8579566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
8589566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
8599566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
8609566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
8619566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
86220369375SToby Isaac   if (cellGeometry) {
8639566063dSJacob Faibussowitsch     PetscCall(VecGetDM(cellGeometry, &dmCell));
8649566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
86520369375SToby Isaac   }
866d7ddef95SMatthew G. Knepley   if (Grad) {
867c0a6632aSMatthew G. Knepley     PetscFV fv;
868c0a6632aSMatthew G. Knepley 
8699566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fv));
8709566063dSJacob Faibussowitsch     PetscCall(VecGetDM(Grad, &dmGrad));
8719566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(Grad, &grad));
8729566063dSJacob Faibussowitsch     PetscCall(PetscFVGetNumComponents(fv, &pdim));
8739566063dSJacob Faibussowitsch     PetscCall(DMGetWorkArray(dm, pdim, MPIU_SCALAR, &fx));
874d7ddef95SMatthew G. Knepley   }
8759566063dSJacob Faibussowitsch   PetscCall(VecGetArray(locX, &x));
876d7ddef95SMatthew G. Knepley   for (i = 0; i < numids; ++i) {
877d7ddef95SMatthew G. Knepley     IS              faceIS;
878d7ddef95SMatthew G. Knepley     const PetscInt *faces;
879d7ddef95SMatthew G. Knepley     PetscInt        numFaces, f;
880d7ddef95SMatthew G. Knepley 
8819566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, ids[i], &faceIS));
882d7ddef95SMatthew G. Knepley     if (!faceIS) continue; /* No points with that id on this process */
8839566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(faceIS, &numFaces));
8849566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(faceIS, &faces));
885d7ddef95SMatthew G. Knepley     for (f = 0; f < numFaces; ++f) {
886d7ddef95SMatthew G. Knepley       const PetscInt   face = faces[f], *cells;
887640bce14SSatish Balay       PetscFVFaceGeom *fg;
888d7ddef95SMatthew G. Knepley 
889d7ddef95SMatthew G. Knepley       if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */
8909566063dSJacob Faibussowitsch       PetscCall(PetscFindInt(face, nleaves, (PetscInt *)leaves, &loc));
891da97024aSMatthew G. Knepley       if (loc >= 0) continue;
8929566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
8939566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, face, &cells));
894d7ddef95SMatthew G. Knepley       if (Grad) {
895640bce14SSatish Balay         PetscFVCellGeom *cg;
896640bce14SSatish Balay         PetscScalar     *cx, *cgrad;
897d7ddef95SMatthew G. Knepley         PetscScalar     *xG;
898d7ddef95SMatthew G. Knepley         PetscReal        dx[3];
899d7ddef95SMatthew G. Knepley         PetscInt         d;
900d7ddef95SMatthew G. Knepley 
9019566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg));
9029566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &cx));
9039566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad));
9049566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
905d7ddef95SMatthew G. Knepley         DMPlex_WaxpyD_Internal(dim, -1, cg->centroid, fg->centroid, dx);
906d7ddef95SMatthew G. Knepley         for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DMPlex_DotD_Internal(dim, &cgrad[d * dim], dx);
9079566063dSJacob Faibussowitsch         PetscCall((*func)(time, fg->centroid, fg->normal, fx, xG, ctx));
908d7ddef95SMatthew G. Knepley       } else {
909640bce14SSatish Balay         PetscScalar *xI;
910d7ddef95SMatthew G. Knepley         PetscScalar *xG;
911d7ddef95SMatthew G. Knepley 
9129566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &xI));
9139566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
914e735a8a9SMatthew G. Knepley         ierru = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx);
915e735a8a9SMatthew G. Knepley         if (ierru) {
9169566063dSJacob Faibussowitsch           PetscCall(ISRestoreIndices(faceIS, &faces));
9179566063dSJacob Faibussowitsch           PetscCall(ISDestroy(&faceIS));
918e735a8a9SMatthew G. Knepley           goto cleanup;
919e735a8a9SMatthew G. Knepley         }
920d7ddef95SMatthew G. Knepley       }
921d7ddef95SMatthew G. Knepley     }
9229566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(faceIS, &faces));
9239566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&faceIS));
924d7ddef95SMatthew G. Knepley   }
925e735a8a9SMatthew G. Knepley cleanup:
9269566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(locX, &x));
927d7ddef95SMatthew G. Knepley   if (Grad) {
9289566063dSJacob Faibussowitsch     PetscCall(DMRestoreWorkArray(dm, pdim, MPIU_SCALAR, &fx));
9299566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(Grad, &grad));
930d7ddef95SMatthew G. Knepley   }
9319566063dSJacob Faibussowitsch   if (cellGeometry) PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
9329566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
9339566063dSJacob Faibussowitsch   PetscCall(ierru);
934d7ddef95SMatthew G. Knepley   PetscFunctionReturn(0);
935d7ddef95SMatthew G. Knepley }
936d7ddef95SMatthew G. Knepley 
9379371c9d4SSatish Balay static PetscErrorCode zero(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nc, PetscScalar *u, void *ctx) {
9380c364540SMatthew G. Knepley   PetscInt c;
9390c364540SMatthew G. Knepley   for (c = 0; c < Nc; ++c) u[c] = 0.0;
9400c364540SMatthew G. Knepley   return 0;
9410c364540SMatthew G. Knepley }
9420c364540SMatthew G. Knepley 
9439371c9d4SSatish Balay PetscErrorCode DMPlexInsertBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) {
9440c364540SMatthew G. Knepley   PetscObject isZero;
945e5e52638SMatthew G. Knepley   PetscDS     prob;
946d7ddef95SMatthew G. Knepley   PetscInt    numBd, b;
94755f2e967SMatthew G. Knepley 
94855f2e967SMatthew G. Knepley   PetscFunctionBegin;
9499566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
9509566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
9519566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero));
95255f2e967SMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
95345480ffeSMatthew G. Knepley     PetscWeakForm           wf;
954f971fd6bSMatthew G. Knepley     DMBoundaryConditionType type;
95545480ffeSMatthew G. Knepley     const char             *name;
956d7ddef95SMatthew G. Knepley     DMLabel                 label;
9571c531cf8SMatthew G. Knepley     PetscInt                field, Nc;
9581c531cf8SMatthew G. Knepley     const PetscInt         *comps;
959d7ddef95SMatthew G. Knepley     PetscObject             obj;
960d7ddef95SMatthew G. Knepley     PetscClassId            id;
96145480ffeSMatthew G. Knepley     void (*bvfunc)(void);
962d7ddef95SMatthew G. Knepley     PetscInt        numids;
963d7ddef95SMatthew G. Knepley     const PetscInt *ids;
96455f2e967SMatthew G. Knepley     void           *ctx;
96555f2e967SMatthew G. Knepley 
9669566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, &bvfunc, NULL, &ctx));
967f971fd6bSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
9689566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
9699566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
970d7ddef95SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
971c60e475cSMatthew G. Knepley       switch (type) {
972c60e475cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
9739371c9d4SSatish Balay       case DM_BC_ESSENTIAL: {
97445480ffeSMatthew G. Knepley         PetscSimplePointFunc func = (PetscSimplePointFunc)bvfunc;
97545480ffeSMatthew G. Knepley 
97645480ffeSMatthew G. Knepley         if (isZero) func = zero;
9779566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
9789566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func, ctx, locX));
9799566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
9809371c9d4SSatish Balay       } break;
9819371c9d4SSatish Balay       case DM_BC_ESSENTIAL_FIELD: {
98245480ffeSMatthew G. Knepley         PetscPointFunc func = (PetscPointFunc)bvfunc;
98345480ffeSMatthew G. Knepley 
9849566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
9859566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func, ctx, locX));
9869566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
9879371c9d4SSatish Balay       } break;
988c60e475cSMatthew G. Knepley       default: break;
989c60e475cSMatthew G. Knepley       }
990d7ddef95SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
99145480ffeSMatthew G. Knepley       {
99245480ffeSMatthew G. Knepley         PetscErrorCode (*func)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *) = (PetscErrorCode(*)(PetscReal, const PetscReal *, const PetscReal *, const PetscScalar *, PetscScalar *, void *))bvfunc;
99345480ffeSMatthew G. Knepley 
99443ea7facSMatthew G. Knepley         if (!faceGeomFVM) continue;
9959566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesRiemann(dm, time, faceGeomFVM, cellGeomFVM, gradFVM, field, Nc, comps, label, numids, ids, func, ctx, locX));
99645480ffeSMatthew G. Knepley       }
99763a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
99855f2e967SMatthew G. Knepley   }
99955f2e967SMatthew G. Knepley   PetscFunctionReturn(0);
100055f2e967SMatthew G. Knepley }
100155f2e967SMatthew G. Knepley 
10029371c9d4SSatish Balay PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) {
100356cf3b9cSMatthew G. Knepley   PetscObject isZero;
100456cf3b9cSMatthew G. Knepley   PetscDS     prob;
100556cf3b9cSMatthew G. Knepley   PetscInt    numBd, b;
100656cf3b9cSMatthew G. Knepley 
100756cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
100856cf3b9cSMatthew G. Knepley   if (!locX) PetscFunctionReturn(0);
10099566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
10109566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
10119566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)locX, "__Vec_bc_zero__", &isZero));
101256cf3b9cSMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
101345480ffeSMatthew G. Knepley     PetscWeakForm           wf;
101456cf3b9cSMatthew G. Knepley     DMBoundaryConditionType type;
101545480ffeSMatthew G. Knepley     const char             *name;
101656cf3b9cSMatthew G. Knepley     DMLabel                 label;
101756cf3b9cSMatthew G. Knepley     PetscInt                field, Nc;
101856cf3b9cSMatthew G. Knepley     const PetscInt         *comps;
101956cf3b9cSMatthew G. Knepley     PetscObject             obj;
102056cf3b9cSMatthew G. Knepley     PetscClassId            id;
102156cf3b9cSMatthew G. Knepley     PetscInt                numids;
102256cf3b9cSMatthew G. Knepley     const PetscInt         *ids;
102345480ffeSMatthew G. Knepley     void (*bvfunc)(void);
102456cf3b9cSMatthew G. Knepley     void *ctx;
102556cf3b9cSMatthew G. Knepley 
10269566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, NULL, &bvfunc, &ctx));
102756cf3b9cSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
10289566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
10299566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
103056cf3b9cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
103156cf3b9cSMatthew G. Knepley       switch (type) {
103256cf3b9cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
10339371c9d4SSatish Balay       case DM_BC_ESSENTIAL: {
103445480ffeSMatthew G. Knepley         PetscSimplePointFunc func_t = (PetscSimplePointFunc)bvfunc;
103545480ffeSMatthew G. Knepley 
103645480ffeSMatthew G. Knepley         if (isZero) func_t = zero;
10379566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
10389566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
10399566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
10409371c9d4SSatish Balay       } break;
10419371c9d4SSatish Balay       case DM_BC_ESSENTIAL_FIELD: {
104245480ffeSMatthew G. Knepley         PetscPointFunc func_t = (PetscPointFunc)bvfunc;
104345480ffeSMatthew G. Knepley 
10449566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelAddCells(dm, label));
10459566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
10469566063dSJacob Faibussowitsch         PetscCall(DMPlexLabelClearCells(dm, label));
10479371c9d4SSatish Balay       } break;
104856cf3b9cSMatthew G. Knepley       default: break;
104956cf3b9cSMatthew G. Knepley       }
105063a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
105156cf3b9cSMatthew G. Knepley   }
105256cf3b9cSMatthew G. Knepley   PetscFunctionReturn(0);
105356cf3b9cSMatthew G. Knepley }
105456cf3b9cSMatthew G. Knepley 
1055f1d73a7aSMatthew G. Knepley /*@
1056f1d73a7aSMatthew G. Knepley   DMPlexInsertBoundaryValues - Puts coefficients which represent boundary values into the local solution vector
1057f1d73a7aSMatthew G. Knepley 
1058ed808b8fSJed Brown   Not Collective
1059ed808b8fSJed Brown 
1060f1d73a7aSMatthew G. Knepley   Input Parameters:
1061f1d73a7aSMatthew G. Knepley + dm - The DM
1062f1d73a7aSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
1063f1d73a7aSMatthew G. Knepley . time - The time
1064f1d73a7aSMatthew G. Knepley . faceGeomFVM - Face geometry data for FV discretizations
1065f1d73a7aSMatthew G. Knepley . cellGeomFVM - Cell geometry data for FV discretizations
1066f1d73a7aSMatthew G. Knepley - gradFVM - Gradient reconstruction data for FV discretizations
1067f1d73a7aSMatthew G. Knepley 
1068f1d73a7aSMatthew G. Knepley   Output Parameters:
1069f1d73a7aSMatthew G. Knepley . locX - Solution updated with boundary values
1070f1d73a7aSMatthew G. Knepley 
1071ed808b8fSJed Brown   Level: intermediate
1072f1d73a7aSMatthew G. Knepley 
1073db781477SPatrick Sanan .seealso: `DMProjectFunctionLabelLocal()`, `DMAddBoundary()`
1074f1d73a7aSMatthew G. Knepley @*/
10759371c9d4SSatish Balay PetscErrorCode DMPlexInsertBoundaryValues(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) {
1076f1d73a7aSMatthew G. Knepley   PetscFunctionBegin;
1077f1d73a7aSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1078064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
1079064a246eSJacob Faibussowitsch   if (faceGeomFVM) { PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5); }
1080064a246eSJacob Faibussowitsch   if (cellGeomFVM) { PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6); }
1081064a246eSJacob Faibussowitsch   if (gradFVM) { PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7); }
1082cac4c232SBarry Smith   PetscTryMethod(dm, "DMPlexInsertBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX, time, faceGeomFVM, cellGeomFVM, gradFVM));
1083f1d73a7aSMatthew G. Knepley   PetscFunctionReturn(0);
1084f1d73a7aSMatthew G. Knepley }
1085f1d73a7aSMatthew G. Knepley 
108656cf3b9cSMatthew G. Knepley /*@
1087a5b23f4aSJose E. Roman   DMPlexInsertTimeDerivativeBoundaryValues - Puts coefficients which represent boundary values of the time derivative into the local solution vector
108856cf3b9cSMatthew G. Knepley 
108956cf3b9cSMatthew G. Knepley   Input Parameters:
109056cf3b9cSMatthew G. Knepley + dm - The DM
109156cf3b9cSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
109256cf3b9cSMatthew G. Knepley . time - The time
109356cf3b9cSMatthew G. Knepley . faceGeomFVM - Face geometry data for FV discretizations
109456cf3b9cSMatthew G. Knepley . cellGeomFVM - Cell geometry data for FV discretizations
109556cf3b9cSMatthew G. Knepley - gradFVM - Gradient reconstruction data for FV discretizations
109656cf3b9cSMatthew G. Knepley 
109756cf3b9cSMatthew G. Knepley   Output Parameters:
109856cf3b9cSMatthew G. Knepley . locX_t - Solution updated with boundary values
109956cf3b9cSMatthew G. Knepley 
110056cf3b9cSMatthew G. Knepley   Level: developer
110156cf3b9cSMatthew G. Knepley 
1102db781477SPatrick Sanan .seealso: `DMProjectFunctionLabelLocal()`
110356cf3b9cSMatthew G. Knepley @*/
11049371c9d4SSatish Balay PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues(DM dm, PetscBool insertEssential, Vec locX_t, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM) {
110556cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
110656cf3b9cSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1107064a246eSJacob Faibussowitsch   if (locX_t) { PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 3); }
1108064a246eSJacob Faibussowitsch   if (faceGeomFVM) { PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5); }
1109064a246eSJacob Faibussowitsch   if (cellGeomFVM) { PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6); }
1110064a246eSJacob Faibussowitsch   if (gradFVM) { PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7); }
1111cac4c232SBarry Smith   PetscTryMethod(dm, "DMPlexInsertTimeDerviativeBoundaryValues_C", (DM, PetscBool, Vec, PetscReal, Vec, Vec, Vec), (dm, insertEssential, locX_t, time, faceGeomFVM, cellGeomFVM, gradFVM));
111256cf3b9cSMatthew G. Knepley   PetscFunctionReturn(0);
111356cf3b9cSMatthew G. Knepley }
111456cf3b9cSMatthew G. Knepley 
11159371c9d4SSatish Balay PetscErrorCode DMComputeL2Diff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff) {
1116574a98acSMatthew G. Knepley   Vec localX;
1117574a98acSMatthew G. Knepley 
1118574a98acSMatthew G. Knepley   PetscFunctionBegin;
11199566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
11209566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, localX, time, NULL, NULL, NULL));
11219566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
11229566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
11239566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeL2DiffLocal(dm, time, funcs, ctxs, localX, diff));
11249566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
1125574a98acSMatthew G. Knepley   PetscFunctionReturn(0);
1126574a98acSMatthew G. Knepley }
1127574a98acSMatthew G. Knepley 
1128574a98acSMatthew G. Knepley /*@C
1129ca3d3a14SMatthew G. Knepley   DMComputeL2DiffLocal - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h.
1130574a98acSMatthew G. Knepley 
1131d083f849SBarry Smith   Collective on dm
1132c0f8e1fdSMatthew G. Knepley 
1133574a98acSMatthew G. Knepley   Input Parameters:
1134574a98acSMatthew G. Knepley + dm     - The DM
1135574a98acSMatthew G. Knepley . time   - The time
1136574a98acSMatthew G. Knepley . funcs  - The functions to evaluate for each field component
1137574a98acSMatthew G. Knepley . ctxs   - Optional array of contexts to pass to each function, or NULL.
1138574a98acSMatthew G. Knepley - localX - The coefficient vector u_h, a local vector
1139574a98acSMatthew G. Knepley 
1140574a98acSMatthew G. Knepley   Output Parameter:
1141574a98acSMatthew G. Knepley . diff - The diff ||u - u_h||_2
1142574a98acSMatthew G. Knepley 
1143574a98acSMatthew G. Knepley   Level: developer
1144574a98acSMatthew G. Knepley 
1145db781477SPatrick Sanan .seealso: `DMProjectFunction()`, `DMComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
1146574a98acSMatthew G. Knepley @*/
11479371c9d4SSatish Balay PetscErrorCode DMPlexComputeL2DiffLocal(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec localX, PetscReal *diff) {
11480f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex *)dm->data)->printL2;
1149ca3d3a14SMatthew G. Knepley   DM               tdm;
1150ca3d3a14SMatthew G. Knepley   Vec              tv;
1151cb1e1211SMatthew G Knepley   PetscSection     section;
1152c5bbbd5bSMatthew G. Knepley   PetscQuadrature  quad;
11534bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
115415496722SMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
11554bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
1156cb1e1211SMatthew G Knepley   PetscReal        localDiff = 0.0;
11577318780aSToby Isaac   const PetscReal *quadWeights;
1158412e9a14SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cellHeight, cStart, cEnd, c, field, fieldOffset;
1159ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1160cb1e1211SMatthew G Knepley 
1161cb1e1211SMatthew G Knepley   PetscFunctionBegin;
11629566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
11639566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
11642a4e142eSMatthew G. Knepley   fegeom.dimEmbed = coordDim;
11659566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
11669566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
11679566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
11689566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
11699566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1170cb1e1211SMatthew G Knepley   for (field = 0; field < numFields; ++field) {
117115496722SMatthew G. Knepley     PetscObject  obj;
117215496722SMatthew G. Knepley     PetscClassId id;
1173c5bbbd5bSMatthew G. Knepley     PetscInt     Nc;
1174c5bbbd5bSMatthew G. Knepley 
11759566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
11769566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
117715496722SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
117815496722SMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
117915496722SMatthew G. Knepley 
11809566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
11819566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
118215496722SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
118315496722SMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
118415496722SMatthew G. Knepley 
11859566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
11869566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
118763a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1188c5bbbd5bSMatthew G. Knepley     numComponents += Nc;
1189cb1e1211SMatthew G Knepley   }
11909566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
119163a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
11929566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ));
11939566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
11949566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
1195cb1e1211SMatthew G Knepley   for (c = cStart; c < cEnd; ++c) {
1196a1e44745SMatthew G. Knepley     PetscScalar *x        = NULL;
1197cb1e1211SMatthew G Knepley     PetscReal    elemDiff = 0.0;
11989c3cf19fSMatthew G. Knepley     PetscInt     qc       = 0;
1199cb1e1211SMatthew G Knepley 
12009566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
12019566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x));
1202cb1e1211SMatthew G Knepley 
120315496722SMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
120415496722SMatthew G. Knepley       PetscObject  obj;
120515496722SMatthew G. Knepley       PetscClassId id;
1206c110b1eeSGeoffrey Irving       void *const  ctx = ctxs ? ctxs[field] : NULL;
120715496722SMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1208cb1e1211SMatthew G Knepley 
12099566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
12109566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
12119371c9d4SSatish Balay       if (id == PETSCFE_CLASSID) {
12129371c9d4SSatish Balay         PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
12139371c9d4SSatish Balay         PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
12149371c9d4SSatish Balay       } else if (id == PETSCFV_CLASSID) {
12159371c9d4SSatish Balay         PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
12169371c9d4SSatish Balay         Nb = 1;
12179371c9d4SSatish Balay       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1218cb1e1211SMatthew G Knepley       if (debug) {
1219cb1e1211SMatthew G Knepley         char title[1024];
122063a3b9bcSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
12219566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1222cb1e1211SMatthew G Knepley       }
12237318780aSToby Isaac       for (q = 0; q < Nq; ++q) {
12242a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1225d0609cedSBarry Smith         PetscErrorCode ierr;
12262a4e142eSMatthew G. Knepley 
12272a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
12282a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q * coordDim * coordDim];
12292a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
12302a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
123163a3b9bcSJacob Faibussowitsch         PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", point %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1232d3a7d86cSMatthew G. Knepley         if (transform) {
1233d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * Nq];
12349566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx));
1235d3a7d86cSMatthew G. Knepley         } else {
1236d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * q];
1237d3a7d86cSMatthew G. Knepley         }
12389566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal, Nc));
1239ca3d3a14SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, Nc, funcVal, ctx);
1240e735a8a9SMatthew G. Knepley         if (ierr) {
12419566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
12429566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
12439566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
1244e735a8a9SMatthew G. Knepley         }
12459566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
12469566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant));
12479566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant));
12482df84da0SMatthew G. Knepley         else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
124915496722SMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1250beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
12519371c9d4SSatish Balay           if (debug)
12529371c9d4SSatish Balay             PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g (%g, %g)\n", c, field, fc, (double)(coordDim > 0 ? coords[coordDim * q] : 0.), (double)(coordDim > 1 ? coords[coordDim * q + 1] : 0.), (double)(coordDim > 2 ? coords[coordDim * q + 2] : 0.),
12539371c9d4SSatish Balay                                   (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q]), (double)PetscRealPart(interpolant[fc]), (double)PetscRealPart(funcVal[fc])));
12544bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
1255cb1e1211SMatthew G Knepley         }
1256cb1e1211SMatthew G Knepley       }
12579c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
1258beaa55a6SMatthew G. Knepley       qc += Nc;
1259cb1e1211SMatthew G Knepley     }
12609566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
12619566063dSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
1262cb1e1211SMatthew G Knepley     localDiff += elemDiff;
1263cb1e1211SMatthew G Knepley   }
12649566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
12651c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
1266cb1e1211SMatthew G Knepley   *diff = PetscSqrtReal(*diff);
1267cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
1268cb1e1211SMatthew G Knepley }
1269cb1e1211SMatthew G Knepley 
12709371c9d4SSatish Balay PetscErrorCode DMComputeL2GradientDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, const PetscReal n[], PetscReal *diff) {
12710f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex *)dm->data)->printL2;
1272ca3d3a14SMatthew G. Knepley   DM               tdm;
1273cb1e1211SMatthew G Knepley   PetscSection     section;
127440e14135SMatthew G. Knepley   PetscQuadrature  quad;
1275ca3d3a14SMatthew G. Knepley   Vec              localX, tv;
12769c3cf19fSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
12772a4e142eSMatthew G. Knepley   const PetscReal *quadWeights;
12784bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
12794bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
128040e14135SMatthew G. Knepley   PetscReal        localDiff = 0.0;
1281485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, qNc = 0, Nq = 0, numFields, numComponents = 0, cStart, cEnd, c, field, fieldOffset;
1282ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1283cb1e1211SMatthew G Knepley 
1284cb1e1211SMatthew G Knepley   PetscFunctionBegin;
12859566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
12869566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
12874bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
12889566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
12899566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
12909566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
12919566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
12929566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
12939566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
12949566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
12959566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1296652b88e8SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
12970f2d7e86SMatthew G. Knepley     PetscFE  fe;
129840e14135SMatthew G. Knepley     PetscInt Nc;
1299652b88e8SMatthew G. Knepley 
13009566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe));
13019566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
13029566063dSJacob Faibussowitsch     PetscCall(PetscFEGetNumComponents(fe, &Nc));
130340e14135SMatthew G. Knepley     numComponents += Nc;
1304652b88e8SMatthew G. Knepley   }
13059566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
130663a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
13079566063dSJacob Faibussowitsch   /* PetscCall(DMProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX)); */
13089566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents, &funcVal, coordDim * Nq, &coords, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ, numComponents * coordDim, &interpolant, Nq, &fegeom.detJ));
13099566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
131040e14135SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
131140e14135SMatthew G. Knepley     PetscScalar *x        = NULL;
131240e14135SMatthew G. Knepley     PetscReal    elemDiff = 0.0;
13139c3cf19fSMatthew G. Knepley     PetscInt     qc       = 0;
1314652b88e8SMatthew G. Knepley 
13159566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
13169566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x));
131740e14135SMatthew G. Knepley 
13189c3cf19fSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
13190f2d7e86SMatthew G. Knepley       PetscFE     fe;
132051259fa3SMatthew G. Knepley       void *const ctx = ctxs ? ctxs[field] : NULL;
13219c3cf19fSMatthew G. Knepley       PetscInt    Nb, Nc, q, fc;
132240e14135SMatthew G. Knepley 
13239566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, (PetscObject *)&fe));
13249566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
13259566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
132640e14135SMatthew G. Knepley       if (debug) {
132740e14135SMatthew G. Knepley         char title[1024];
13289566063dSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
13299566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1330652b88e8SMatthew G. Knepley       }
13319c3cf19fSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
13322a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1333d0609cedSBarry Smith         PetscErrorCode ierr;
13342a4e142eSMatthew G. Knepley 
13352a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
13362a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q * coordDim * coordDim];
13372a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
13382a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
13392df84da0SMatthew G. Knepley         PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1340d3a7d86cSMatthew G. Knepley         if (transform) {
1341d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * Nq];
13429566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim * q], PETSC_TRUE, coordDim, &coords[coordDim * q], gcoords, dm->transformCtx));
1343d3a7d86cSMatthew G. Knepley         } else {
1344d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim * q];
1345d3a7d86cSMatthew G. Knepley         }
13469566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal, Nc));
13474bee2e38SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, n, Nc, funcVal, ctx);
1348e735a8a9SMatthew G. Knepley         if (ierr) {
13499566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
13509566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
13519566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ));
1352e735a8a9SMatthew G. Knepley         }
13539566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
13549566063dSJacob Faibussowitsch         PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[fieldOffset], &qgeom, q, interpolant));
13554bee2e38SMatthew G. Knepley         /* Overwrite with the dot product if the normal is given */
13564bee2e38SMatthew G. Knepley         if (n) {
13574bee2e38SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
13584bee2e38SMatthew G. Knepley             PetscScalar sum = 0.0;
13594bee2e38SMatthew G. Knepley             PetscInt    d;
13604bee2e38SMatthew G. Knepley             for (d = 0; d < dim; ++d) sum += interpolant[fc * dim + d] * n[d];
13614bee2e38SMatthew G. Knepley             interpolant[fc] = sum;
13624bee2e38SMatthew G. Knepley           }
13634bee2e38SMatthew G. Knepley         }
13649c3cf19fSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1365beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
136663a3b9bcSJacob Faibussowitsch           if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " fieldDer %" PetscInt_FMT ",%" PetscInt_FMT " diff %g\n", c, field, fc, (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q])));
13674bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
136840e14135SMatthew G. Knepley         }
136940e14135SMatthew G. Knepley       }
13709c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
13719c3cf19fSMatthew G. Knepley       qc += Nc;
137240e14135SMatthew G. Knepley     }
13739566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
137463a3b9bcSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
137540e14135SMatthew G. Knepley     localDiff += elemDiff;
137640e14135SMatthew G. Knepley   }
13779566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal, coords, fegeom.J, fegeom.invJ, interpolant, fegeom.detJ));
13789566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
13791c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
138040e14135SMatthew G. Knepley   *diff = PetscSqrtReal(*diff);
1381cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
1382cb1e1211SMatthew G Knepley }
1383cb1e1211SMatthew G Knepley 
13849371c9d4SSatish Balay PetscErrorCode DMComputeL2FieldDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff) {
13850f09c10fSMatthew G. Knepley   const PetscInt debug = ((DM_Plex *)dm->data)->printL2;
1386ca3d3a14SMatthew G. Knepley   DM             tdm;
1387083401c6SMatthew G. Knepley   DMLabel        depthLabel;
138873d901b8SMatthew G. Knepley   PetscSection   section;
1389ca3d3a14SMatthew G. Knepley   Vec            localX, tv;
139073d901b8SMatthew G. Knepley   PetscReal     *localDiff;
1391083401c6SMatthew G. Knepley   PetscInt       dim, depth, dE, Nf, f, Nds, s;
1392ca3d3a14SMatthew G. Knepley   PetscBool      transform;
139373d901b8SMatthew G. Knepley 
139473d901b8SMatthew G. Knepley   PetscFunctionBegin;
13959566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
13969566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dE));
13979566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
13989566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
13999566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
14009566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
14019566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
14029566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
14039566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
14049566063dSJacob Faibussowitsch   PetscCall(DMLabelGetNumValues(depthLabel, &depth));
1405083401c6SMatthew G. Knepley 
14069566063dSJacob Faibussowitsch   PetscCall(VecSet(localX, 0.0));
14079566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
14089566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
14099566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
14109566063dSJacob Faibussowitsch   PetscCall(DMGetNumDS(dm, &Nds));
14119566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(Nf, &localDiff));
1412083401c6SMatthew G. Knepley   for (s = 0; s < Nds; ++s) {
1413083401c6SMatthew G. Knepley     PetscDS          ds;
1414083401c6SMatthew G. Knepley     DMLabel          label;
1415083401c6SMatthew G. Knepley     IS               fieldIS, pointIS;
1416083401c6SMatthew G. Knepley     const PetscInt  *fields, *points = NULL;
1417083401c6SMatthew G. Knepley     PetscQuadrature  quad;
1418083401c6SMatthew G. Knepley     const PetscReal *quadPoints, *quadWeights;
1419083401c6SMatthew G. Knepley     PetscFEGeom      fegeom;
1420083401c6SMatthew G. Knepley     PetscReal       *coords, *gcoords;
1421083401c6SMatthew G. Knepley     PetscScalar     *funcVal, *interpolant;
14225fedec97SMatthew G. Knepley     PetscBool        isCohesive;
1423083401c6SMatthew G. Knepley     PetscInt         qNc, Nq, totNc, cStart = 0, cEnd, c, dsNf;
142473d901b8SMatthew G. Knepley 
14259566063dSJacob Faibussowitsch     PetscCall(DMGetRegionNumDS(dm, s, &label, &fieldIS, &ds));
14269566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(fieldIS, &fields));
14279566063dSJacob Faibussowitsch     PetscCall(PetscDSIsCohesive(ds, &isCohesive));
14289566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(ds, &dsNf));
14299566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalComponents(ds, &totNc));
14309566063dSJacob Faibussowitsch     PetscCall(PetscDSGetQuadrature(ds, &quad));
14319566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
143263a3b9bcSJacob Faibussowitsch     PetscCheck(!(qNc != 1) || !(qNc != totNc), PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, totNc);
14339566063dSJacob Faibussowitsch     PetscCall(PetscCalloc6(totNc, &funcVal, totNc, &interpolant, dE * (Nq + 1), &coords, Nq, &fegeom.detJ, dE * dE * Nq, &fegeom.J, dE * dE * Nq, &fegeom.invJ));
1434083401c6SMatthew G. Knepley     if (!label) {
14359566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1436083401c6SMatthew G. Knepley     } else {
14379566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(label, 1, &pointIS));
14389566063dSJacob Faibussowitsch       PetscCall(ISGetLocalSize(pointIS, &cEnd));
14399566063dSJacob Faibussowitsch       PetscCall(ISGetIndices(pointIS, &points));
1440083401c6SMatthew G. Knepley     }
144173d901b8SMatthew G. Knepley     for (c = cStart; c < cEnd; ++c) {
1442083401c6SMatthew G. Knepley       const PetscInt  cell = points ? points[c] : c;
144373d901b8SMatthew G. Knepley       PetscScalar    *x    = NULL;
14445fedec97SMatthew G. Knepley       const PetscInt *cone;
14455fedec97SMatthew G. Knepley       PetscInt        qc = 0, fOff = 0, dep;
144673d901b8SMatthew G. Knepley 
14479566063dSJacob Faibussowitsch       PetscCall(DMLabelGetValue(depthLabel, cell, &dep));
1448083401c6SMatthew G. Knepley       if (dep != depth - 1) continue;
14495fedec97SMatthew G. Knepley       if (isCohesive) {
14509566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cell, &cone));
14519566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cone[0], quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
145296959cd1SMatthew G. Knepley       } else {
14539566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
145496959cd1SMatthew G. Knepley       }
14559566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, localX, cell, NULL, &x));
14565fedec97SMatthew G. Knepley       for (f = 0; f < dsNf; ++f) {
145715496722SMatthew G. Knepley         PetscObject  obj;
145815496722SMatthew G. Knepley         PetscClassId id;
1459083401c6SMatthew G. Knepley         void *const  ctx = ctxs ? ctxs[fields[f]] : NULL;
146015496722SMatthew G. Knepley         PetscInt     Nb, Nc, q, fc;
146115496722SMatthew G. Knepley         PetscReal    elemDiff = 0.0;
14625fedec97SMatthew G. Knepley         PetscBool    cohesive;
146315496722SMatthew G. Knepley 
14649566063dSJacob Faibussowitsch         PetscCall(PetscDSGetCohesive(ds, f, &cohesive));
14655fedec97SMatthew G. Knepley         if (isCohesive && !cohesive) continue;
14669566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
14679566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
14689371c9d4SSatish Balay         if (id == PETSCFE_CLASSID) {
14699371c9d4SSatish Balay           PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
14709371c9d4SSatish Balay           PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
14719371c9d4SSatish Balay         } else if (id == PETSCFV_CLASSID) {
14729371c9d4SSatish Balay           PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
14739371c9d4SSatish Balay           Nb = 1;
14749371c9d4SSatish Balay         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
147573d901b8SMatthew G. Knepley         if (debug) {
147673d901b8SMatthew G. Knepley           char title[1024];
147763a3b9bcSJacob Faibussowitsch           PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, fields[f]));
14789566063dSJacob Faibussowitsch           PetscCall(DMPrintCellVector(cell, title, Nb, &x[fOff]));
147973d901b8SMatthew G. Knepley         }
14807318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
14812a4e142eSMatthew G. Knepley           PetscFEGeom    qgeom;
1482d0609cedSBarry Smith           PetscErrorCode ierr;
14832a4e142eSMatthew G. Knepley 
14842a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
1485083401c6SMatthew G. Knepley           qgeom.J        = &fegeom.J[q * dE * dE];
1486083401c6SMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * dE * dE];
14872a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
148863a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for cell %" PetscInt_FMT ", quadrature point %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
1489d3a7d86cSMatthew G. Knepley           if (transform) {
1490083401c6SMatthew G. Knepley             gcoords = &coords[dE * Nq];
14919566063dSJacob Faibussowitsch             PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[dE * q], PETSC_TRUE, dE, &coords[dE * q], gcoords, dm->transformCtx));
1492d3a7d86cSMatthew G. Knepley           } else {
1493083401c6SMatthew G. Knepley             gcoords = &coords[dE * q];
1494d3a7d86cSMatthew G. Knepley           }
14952df84da0SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) funcVal[fc] = 0.;
1496083401c6SMatthew G. Knepley           ierr = (*funcs[fields[f]])(dE, time, gcoords, Nc, funcVal, ctx);
1497e735a8a9SMatthew G. Knepley           if (ierr) {
14989566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
14999566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
15009566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
1501e735a8a9SMatthew G. Knepley           }
15029566063dSJacob Faibussowitsch           if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[dE * q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
150396959cd1SMatthew G. Knepley           /* Call once for each face, except for lagrange field */
15049566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fOff], &qgeom, q, interpolant));
15059566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fOff], q, interpolant));
150663a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
150715496722SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1508beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
15099371c9d4SSatish Balay             if (debug)
15109371c9d4SSatish Balay               PetscCall(PetscPrintf(PETSC_COMM_SELF, "    cell %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g\n", cell, fields[f], fc, (double)(dE > 0 ? coords[dE * q] : 0.), (double)(dE > 1 ? coords[dE * q + 1] : 0.), (double)(dE > 2 ? coords[dE * q + 2] : 0.),
15119371c9d4SSatish Balay                                     (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q])));
15124bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
151373d901b8SMatthew G. Knepley           }
151473d901b8SMatthew G. Knepley         }
1515083401c6SMatthew G. Knepley         fOff += Nb;
15169c3cf19fSMatthew G. Knepley         qc += Nc;
1517083401c6SMatthew G. Knepley         localDiff[fields[f]] += elemDiff;
151863a3b9bcSJacob Faibussowitsch         if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  cell %" PetscInt_FMT " field %" PetscInt_FMT " cum diff %g\n", cell, fields[f], (double)localDiff[fields[f]]));
151973d901b8SMatthew G. Knepley       }
15209566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
1521083401c6SMatthew G. Knepley     }
1522083401c6SMatthew G. Knepley     if (label) {
15239566063dSJacob Faibussowitsch       PetscCall(ISRestoreIndices(pointIS, &points));
15249566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
1525083401c6SMatthew G. Knepley     }
15269566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(fieldIS, &fields));
15279566063dSJacob Faibussowitsch     PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
152873d901b8SMatthew G. Knepley   }
15299566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
15301c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(localDiff, diff, Nf, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
15319566063dSJacob Faibussowitsch   PetscCall(PetscFree(localDiff));
1532083401c6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) diff[f] = PetscSqrtReal(diff[f]);
153373d901b8SMatthew G. Knepley   PetscFunctionReturn(0);
153473d901b8SMatthew G. Knepley }
153573d901b8SMatthew G. Knepley 
1536e729f68cSMatthew G. Knepley /*@C
1537e729f68cSMatthew G. Knepley   DMPlexComputeL2DiffVec - This function computes the cellwise L_2 difference between a function u and an FEM interpolant solution u_h, and stores it in a Vec.
1538e729f68cSMatthew G. Knepley 
1539d083f849SBarry Smith   Collective on dm
1540c0f8e1fdSMatthew G. Knepley 
1541e729f68cSMatthew G. Knepley   Input Parameters:
1542e729f68cSMatthew G. Knepley + dm    - The DM
15430163fd50SMatthew G. Knepley . time  - The time
1544ca3eba1bSToby Isaac . funcs - The functions to evaluate for each field component: NULL means that component does not contribute to error calculation
1545e729f68cSMatthew G. Knepley . ctxs  - Optional array of contexts to pass to each function, or NULL.
1546e729f68cSMatthew G. Knepley - X     - The coefficient vector u_h
1547e729f68cSMatthew G. Knepley 
1548e729f68cSMatthew G. Knepley   Output Parameter:
1549e729f68cSMatthew G. Knepley . D - A Vec which holds the difference ||u - u_h||_2 for each cell
1550e729f68cSMatthew G. Knepley 
1551e729f68cSMatthew G. Knepley   Level: developer
1552e729f68cSMatthew G. Knepley 
1553db781477SPatrick Sanan .seealso: `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
1554e729f68cSMatthew G. Knepley @*/
15559371c9d4SSatish Balay PetscErrorCode DMPlexComputeL2DiffVec(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, Vec D) {
1556e729f68cSMatthew G. Knepley   PetscSection     section;
1557e729f68cSMatthew G. Knepley   PetscQuadrature  quad;
1558e729f68cSMatthew G. Knepley   Vec              localX;
15594bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
1560e729f68cSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
15614bee2e38SMatthew G. Knepley   PetscReal       *coords;
1562e729f68cSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1563485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, c, field, fieldOffset;
1564e729f68cSMatthew G. Knepley 
1565e729f68cSMatthew G. Knepley   PetscFunctionBegin;
15669566063dSJacob Faibussowitsch   PetscCall(VecSet(D, 0.0));
15679566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
15689566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
15699566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
15709566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
15719566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
15729566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
15739566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
15749566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
1575e729f68cSMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
1576e729f68cSMatthew G. Knepley     PetscObject  obj;
1577e729f68cSMatthew G. Knepley     PetscClassId id;
1578e729f68cSMatthew G. Knepley     PetscInt     Nc;
1579e729f68cSMatthew G. Knepley 
15809566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
15819566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1582e729f68cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1583e729f68cSMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
1584e729f68cSMatthew G. Knepley 
15859566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
15869566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
1587e729f68cSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
1588e729f68cSMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
1589e729f68cSMatthew G. Knepley 
15909566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
15919566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
159263a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1593e729f68cSMatthew G. Knepley     numComponents += Nc;
1594e729f68cSMatthew G. Knepley   }
15959566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
159663a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
15979566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents, &funcVal, numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ));
15989566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1599e729f68cSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
1600e729f68cSMatthew G. Knepley     PetscScalar *x        = NULL;
16016f288a59SMatthew G. Knepley     PetscScalar  elemDiff = 0.0;
16029c3cf19fSMatthew G. Knepley     PetscInt     qc       = 0;
1603e729f68cSMatthew G. Knepley 
16049566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
16059566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x));
1606e729f68cSMatthew G. Knepley 
1607e729f68cSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
1608e729f68cSMatthew G. Knepley       PetscObject  obj;
1609e729f68cSMatthew G. Knepley       PetscClassId id;
1610e729f68cSMatthew G. Knepley       void *const  ctx = ctxs ? ctxs[field] : NULL;
1611e729f68cSMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1612e729f68cSMatthew G. Knepley 
16139566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
16149566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
16159371c9d4SSatish Balay       if (id == PETSCFE_CLASSID) {
16169371c9d4SSatish Balay         PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
16179371c9d4SSatish Balay         PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
16189371c9d4SSatish Balay       } else if (id == PETSCFV_CLASSID) {
16199371c9d4SSatish Balay         PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
16209371c9d4SSatish Balay         Nb = 1;
16219371c9d4SSatish Balay       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
162223f34ed2SToby Isaac       if (funcs[field]) {
16237318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
16242a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
16252a4e142eSMatthew G. Knepley 
16262a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
16272a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q * coordDim * coordDim];
16282a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
16292a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
163063a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
16319566063dSJacob Faibussowitsch           PetscCall((*funcs[field])(coordDim, time, &coords[q * coordDim], Nc, funcVal, ctx));
1632c3e24edfSBarry Smith #if defined(needs_fix_with_return_code_argument)
1633e735a8a9SMatthew G. Knepley           if (ierr) {
16349566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
16359566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
16369566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
1637e735a8a9SMatthew G. Knepley           }
1638c3e24edfSBarry Smith #endif
16399566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[fieldOffset], &qgeom, q, interpolant));
16409566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV)obj, &x[fieldOffset], q, interpolant));
164163a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1642e729f68cSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1643beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q * qNc + (qNc == 1 ? 0 : qc + fc)];
16444bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc])) * wt * fegeom.detJ[q];
1645e729f68cSMatthew G. Knepley           }
1646e729f68cSMatthew G. Knepley         }
164723f34ed2SToby Isaac       }
1648beaa55a6SMatthew G. Knepley       fieldOffset += Nb;
16499c3cf19fSMatthew G. Knepley       qc += Nc;
1650e729f68cSMatthew G. Knepley     }
16519566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
16529566063dSJacob Faibussowitsch     PetscCall(VecSetValue(D, c - cStart, elemDiff, INSERT_VALUES));
1653e729f68cSMatthew G. Knepley   }
16549566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
16559566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
16569566063dSJacob Faibussowitsch   PetscCall(VecSqrtAbs(D));
1657e729f68cSMatthew G. Knepley   PetscFunctionReturn(0);
1658e729f68cSMatthew G. Knepley }
1659e729f68cSMatthew G. Knepley 
16605f0b18bfSMatthew G. Knepley /*@
16615f0b18bfSMatthew G. Knepley   DMPlexComputeClementInterpolant - This function computes the L2 projection of the cellwise values of a function u onto P1, and stores it in a Vec.
16625f0b18bfSMatthew G. Knepley 
16635f0b18bfSMatthew G. Knepley   Collective on dm
16645f0b18bfSMatthew G. Knepley 
16655f0b18bfSMatthew G. Knepley   Input Parameters:
16665f0b18bfSMatthew G. Knepley + dm - The DM
16675f0b18bfSMatthew G. Knepley - locX  - The coefficient vector u_h
16685f0b18bfSMatthew G. Knepley 
16695f0b18bfSMatthew G. Knepley   Output Parameter:
16705f0b18bfSMatthew G. Knepley . locC - A Vec which holds the Clement interpolant of the function
16715f0b18bfSMatthew G. Knepley 
16725f0b18bfSMatthew G. Knepley   Notes:
16735f0b18bfSMatthew G. Knepley   u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| where |T_i| is the cell volume
16745f0b18bfSMatthew G. Knepley 
16755f0b18bfSMatthew G. Knepley   Level: developer
16765f0b18bfSMatthew G. Knepley 
1677db781477SPatrick Sanan .seealso: `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
16785f0b18bfSMatthew G. Knepley @*/
16799371c9d4SSatish Balay PetscErrorCode DMPlexComputeClementInterpolant(DM dm, Vec locX, Vec locC) {
16805f0b18bfSMatthew G. Knepley   PetscInt         debug = ((DM_Plex *)dm->data)->printFEM;
16815f0b18bfSMatthew G. Knepley   DM               dmc;
16825f0b18bfSMatthew G. Knepley   PetscQuadrature  quad;
16835f0b18bfSMatthew G. Knepley   PetscScalar     *interpolant, *valsum;
16845f0b18bfSMatthew G. Knepley   PetscFEGeom      fegeom;
16855f0b18bfSMatthew G. Knepley   PetscReal       *coords;
16865f0b18bfSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
16875f0b18bfSMatthew G. Knepley   PetscInt         dim, cdim, Nf, f, Nc = 0, Nq, qNc, cStart, cEnd, vStart, vEnd, v;
16885f0b18bfSMatthew G. Knepley 
16895f0b18bfSMatthew G. Knepley   PetscFunctionBegin;
16909566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
16919566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmc));
16929566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
16939566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
16949566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
16955f0b18bfSMatthew G. Knepley   fegeom.dimEmbed = cdim;
16969566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
16975f0b18bfSMatthew G. Knepley   PetscCheck(Nf > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
16985f0b18bfSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
16995f0b18bfSMatthew G. Knepley     PetscObject  obj;
17005f0b18bfSMatthew G. Knepley     PetscClassId id;
17015f0b18bfSMatthew G. Knepley     PetscInt     fNc;
17025f0b18bfSMatthew G. Knepley 
17039566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, f, NULL, &obj));
17049566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
17055f0b18bfSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
17065f0b18bfSMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
17075f0b18bfSMatthew G. Knepley 
17089566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
17099566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &fNc));
17105f0b18bfSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
17115f0b18bfSMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
17125f0b18bfSMatthew G. Knepley 
17139566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
17149566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &fNc));
171563a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
17165f0b18bfSMatthew G. Knepley     Nc += fNc;
17175f0b18bfSMatthew G. Knepley   }
17189566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
171963a3b9bcSJacob Faibussowitsch   PetscCheck(qNc == 1, PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " > 1", qNc);
17209566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(Nc * 2, &valsum, Nc, &interpolant, cdim * Nq, &coords, Nq, &fegeom.detJ, cdim * cdim * Nq, &fegeom.J, cdim * cdim * Nq, &fegeom.invJ));
17219566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
17229566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
17235f0b18bfSMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
17245f0b18bfSMatthew G. Knepley     PetscScalar volsum = 0.0;
17255f0b18bfSMatthew G. Knepley     PetscInt   *star   = NULL;
17265f0b18bfSMatthew G. Knepley     PetscInt    starSize, st, fc;
17275f0b18bfSMatthew G. Knepley 
17289566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(valsum, Nc));
17299566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
17305f0b18bfSMatthew G. Knepley     for (st = 0; st < starSize * 2; st += 2) {
17315f0b18bfSMatthew G. Knepley       const PetscInt cell = star[st];
17325f0b18bfSMatthew G. Knepley       PetscScalar   *val  = &valsum[Nc];
17335f0b18bfSMatthew G. Knepley       PetscScalar   *x    = NULL;
17345f0b18bfSMatthew G. Knepley       PetscReal      vol  = 0.0;
17355f0b18bfSMatthew G. Knepley       PetscInt       foff = 0;
17365f0b18bfSMatthew G. Knepley 
17375f0b18bfSMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
17389566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
17399566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
17405f0b18bfSMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
17415f0b18bfSMatthew G. Knepley         PetscObject  obj;
17425f0b18bfSMatthew G. Knepley         PetscClassId id;
17435f0b18bfSMatthew G. Knepley         PetscInt     Nb, fNc, q;
17445f0b18bfSMatthew G. Knepley 
17459566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(val, Nc));
17469566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, f, NULL, &obj));
17479566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
17489371c9d4SSatish Balay         if (id == PETSCFE_CLASSID) {
17499371c9d4SSatish Balay           PetscCall(PetscFEGetNumComponents((PetscFE)obj, &fNc));
17509371c9d4SSatish Balay           PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
17519371c9d4SSatish Balay         } else if (id == PETSCFV_CLASSID) {
17529371c9d4SSatish Balay           PetscCall(PetscFVGetNumComponents((PetscFV)obj, &fNc));
17539371c9d4SSatish Balay           Nb = 1;
17549371c9d4SSatish Balay         } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
17555f0b18bfSMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
17565f0b18bfSMatthew G. Knepley           const PetscReal wt = quadWeights[q] * fegeom.detJ[q];
17575f0b18bfSMatthew G. Knepley           PetscFEGeom     qgeom;
17585f0b18bfSMatthew G. Knepley 
17595f0b18bfSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
17605f0b18bfSMatthew G. Knepley           qgeom.J        = &fegeom.J[q * cdim * cdim];
17615f0b18bfSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * cdim * cdim];
17625f0b18bfSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
176363a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
17649566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE)obj, &x[foff], &qgeom, q, interpolant));
176563a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
17665f0b18bfSMatthew G. Knepley           for (fc = 0; fc < fNc; ++fc) val[foff + fc] += interpolant[fc] * wt;
17675f0b18bfSMatthew G. Knepley           vol += wt;
17685f0b18bfSMatthew G. Knepley         }
17695f0b18bfSMatthew G. Knepley         foff += Nb;
17705f0b18bfSMatthew G. Knepley       }
17719566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
17725f0b18bfSMatthew G. Knepley       for (fc = 0; fc < Nc; ++fc) valsum[fc] += val[fc];
17735f0b18bfSMatthew G. Knepley       volsum += vol;
17745f0b18bfSMatthew G. Knepley       if (debug) {
17759566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " value: [", v, cell));
17765f0b18bfSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
17779566063dSJacob Faibussowitsch           if (fc) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
17789566063dSJacob Faibussowitsch           PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(val[fc])));
17795f0b18bfSMatthew G. Knepley         }
17809566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
17815f0b18bfSMatthew G. Knepley       }
17825f0b18bfSMatthew G. Knepley     }
17835f0b18bfSMatthew G. Knepley     for (fc = 0; fc < Nc; ++fc) valsum[fc] /= volsum;
17849566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
17859566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmc, NULL, locC, v, valsum, INSERT_VALUES));
17865f0b18bfSMatthew G. Knepley   }
17879566063dSJacob Faibussowitsch   PetscCall(PetscFree6(valsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
17885f0b18bfSMatthew G. Knepley   PetscFunctionReturn(0);
17895f0b18bfSMatthew G. Knepley }
17905f0b18bfSMatthew G. Knepley 
17915f0b18bfSMatthew G. Knepley /*@
17921555c271SMatthew G. Knepley   DMPlexComputeGradientClementInterpolant - This function computes the L2 projection of the cellwise gradient of a function u onto P1, and stores it in a Vec.
17931555c271SMatthew G. Knepley 
1794d083f849SBarry Smith   Collective on dm
1795c0f8e1fdSMatthew G. Knepley 
17961555c271SMatthew G. Knepley   Input Parameters:
17971555c271SMatthew G. Knepley + dm - The DM
17985f0b18bfSMatthew G. Knepley - locX  - The coefficient vector u_h
17991555c271SMatthew G. Knepley 
18001555c271SMatthew G. Knepley   Output Parameter:
18011555c271SMatthew G. Knepley . locC - A Vec which holds the Clement interpolant of the gradient
18021555c271SMatthew G. Knepley 
180395452b02SPatrick Sanan   Notes:
18041555c271SMatthew G. Knepley   \nabla u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| \nabla u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| where |T_i| is the cell volume
18051555c271SMatthew G. Knepley 
18061555c271SMatthew G. Knepley   Level: developer
18071555c271SMatthew G. Knepley 
1808db781477SPatrick Sanan .seealso: `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
18091555c271SMatthew G. Knepley @*/
18109371c9d4SSatish Balay PetscErrorCode DMPlexComputeGradientClementInterpolant(DM dm, Vec locX, Vec locC) {
1811db1066baSMatthew G. Knepley   DM_Plex         *mesh  = (DM_Plex *)dm->data;
1812db1066baSMatthew G. Knepley   PetscInt         debug = mesh->printFEM;
18131555c271SMatthew G. Knepley   DM               dmC;
18141555c271SMatthew G. Knepley   PetscQuadrature  quad;
18151555c271SMatthew G. Knepley   PetscScalar     *interpolant, *gradsum;
18164bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
18174bee2e38SMatthew G. Knepley   PetscReal       *coords;
18181555c271SMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1819485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, vStart, vEnd, v, field, fieldOffset;
18201555c271SMatthew G. Knepley 
18211555c271SMatthew G. Knepley   PetscFunctionBegin;
18229566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
18239566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmC));
18249566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
18259566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
18269566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
18274bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
18289566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
18295f80ce2aSJacob Faibussowitsch   PetscCheck(numFields, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
18301555c271SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
18311555c271SMatthew G. Knepley     PetscObject  obj;
18321555c271SMatthew G. Knepley     PetscClassId id;
18331555c271SMatthew G. Knepley     PetscInt     Nc;
18341555c271SMatthew G. Knepley 
18359566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
18369566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
18371555c271SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
18381555c271SMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
18391555c271SMatthew G. Knepley 
18409566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
18419566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
18421555c271SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
18431555c271SMatthew G. Knepley       PetscFV fv = (PetscFV)obj;
18441555c271SMatthew G. Knepley 
18459566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
18469566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
184763a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
18481555c271SMatthew G. Knepley     numComponents += Nc;
18491555c271SMatthew G. Knepley   }
18509566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
185163a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents), PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
18529566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(coordDim * numComponents * 2, &gradsum, coordDim * numComponents, &interpolant, coordDim * Nq, &coords, Nq, &fegeom.detJ, coordDim * coordDim * Nq, &fegeom.J, coordDim * coordDim * Nq, &fegeom.invJ));
18539566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
18549566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
18551555c271SMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
18561555c271SMatthew G. Knepley     PetscScalar volsum = 0.0;
18571555c271SMatthew G. Knepley     PetscInt   *star   = NULL;
18581555c271SMatthew G. Knepley     PetscInt    starSize, st, d, fc;
18591555c271SMatthew G. Knepley 
18609566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(gradsum, coordDim * numComponents));
18619566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
18621555c271SMatthew G. Knepley     for (st = 0; st < starSize * 2; st += 2) {
18631555c271SMatthew G. Knepley       const PetscInt cell = star[st];
18641555c271SMatthew G. Knepley       PetscScalar   *grad = &gradsum[coordDim * numComponents];
18651555c271SMatthew G. Knepley       PetscScalar   *x    = NULL;
18661555c271SMatthew G. Knepley       PetscReal      vol  = 0.0;
18671555c271SMatthew G. Knepley 
18681555c271SMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
18699566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
18709566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
18711555c271SMatthew G. Knepley       for (field = 0, fieldOffset = 0; field < numFields; ++field) {
18721555c271SMatthew G. Knepley         PetscObject  obj;
18731555c271SMatthew G. Knepley         PetscClassId id;
18741555c271SMatthew G. Knepley         PetscInt     Nb, Nc, q, qc = 0;
18751555c271SMatthew G. Knepley 
18769566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(grad, coordDim * numComponents));
18779566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, field, NULL, &obj));
18789566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
18799371c9d4SSatish Balay         if (id == PETSCFE_CLASSID) {
18809371c9d4SSatish Balay           PetscCall(PetscFEGetNumComponents((PetscFE)obj, &Nc));
18819371c9d4SSatish Balay           PetscCall(PetscFEGetDimension((PetscFE)obj, &Nb));
18829371c9d4SSatish Balay         } else if (id == PETSCFV_CLASSID) {
18839371c9d4SSatish Balay           PetscCall(PetscFVGetNumComponents((PetscFV)obj, &Nc));
18849371c9d4SSatish Balay           Nb = 1;
18859371c9d4SSatish Balay         } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
18861555c271SMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
18872a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
18882a4e142eSMatthew G. Knepley 
18892a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
18902a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q * coordDim * coordDim];
18912a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q * coordDim * coordDim];
18922a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
189363a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
18949566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolateGradient_Static((PetscFE)obj, 1, &x[fieldOffset], &qgeom, q, interpolant));
189563a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
18961555c271SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
189753d2db2dSJoe Wallwork             const PetscReal wt = quadWeights[q * qNc + qc];
18981555c271SMatthew G. Knepley 
18994bee2e38SMatthew G. Knepley             for (d = 0; d < coordDim; ++d) grad[fc * coordDim + d] += interpolant[fc * dim + d] * wt * fegeom.detJ[q];
19001555c271SMatthew G. Knepley           }
19014bee2e38SMatthew G. Knepley           vol += quadWeights[q * qNc] * fegeom.detJ[q];
19021555c271SMatthew G. Knepley         }
19031555c271SMatthew G. Knepley         fieldOffset += Nb;
19041555c271SMatthew G. Knepley         qc += Nc;
19051555c271SMatthew G. Knepley       }
19069566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
1907f8527842SMatthew G. Knepley       for (fc = 0; fc < numComponents; ++fc) {
19089371c9d4SSatish Balay         for (d = 0; d < coordDim; ++d) { gradsum[fc * coordDim + d] += grad[fc * coordDim + d]; }
1909f8527842SMatthew G. Knepley       }
1910f8527842SMatthew G. Knepley       volsum += vol;
1911db1066baSMatthew G. Knepley       if (debug) {
19129566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " gradient: [", v, cell));
19131555c271SMatthew G. Knepley         for (fc = 0; fc < numComponents; ++fc) {
19141555c271SMatthew G. Knepley           for (d = 0; d < coordDim; ++d) {
19159566063dSJacob Faibussowitsch             if (fc || d > 0) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
19169566063dSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(grad[fc * coordDim + d])));
19171555c271SMatthew G. Knepley           }
19181555c271SMatthew G. Knepley         }
19199566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
1920db1066baSMatthew G. Knepley       }
19211555c271SMatthew G. Knepley     }
19221555c271SMatthew G. Knepley     for (fc = 0; fc < numComponents; ++fc) {
19231555c271SMatthew G. Knepley       for (d = 0; d < coordDim; ++d) gradsum[fc * coordDim + d] /= volsum;
19241555c271SMatthew G. Knepley     }
19259566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
19269566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmC, NULL, locC, v, gradsum, INSERT_VALUES));
19271555c271SMatthew G. Knepley   }
19289566063dSJacob Faibussowitsch   PetscCall(PetscFree6(gradsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
19291555c271SMatthew G. Knepley   PetscFunctionReturn(0);
19301555c271SMatthew G. Knepley }
19311555c271SMatthew G. Knepley 
19329371c9d4SSatish Balay static PetscErrorCode DMPlexComputeIntegral_Internal(DM dm, Vec X, PetscInt cStart, PetscInt cEnd, PetscScalar *cintegral, void *user) {
1933338f77d5SMatthew G. Knepley   DM                 dmAux = NULL;
193461aaff12SToby Isaac   PetscDS            prob, probAux = NULL;
193573d901b8SMatthew G. Knepley   PetscSection       section, sectionAux;
1936338f77d5SMatthew G. Knepley   Vec                locX, locA;
1937c330f8ffSToby Isaac   PetscInt           dim, numCells = cEnd - cStart, c, f;
1938c330f8ffSToby Isaac   PetscBool          useFVM = PETSC_FALSE;
1939338f77d5SMatthew G. Knepley   /* DS */
1940338f77d5SMatthew G. Knepley   PetscInt           Nf, totDim, *uOff, *uOff_x, numConstants;
1941338f77d5SMatthew G. Knepley   PetscInt           NfAux, totDimAux, *aOff;
1942338f77d5SMatthew G. Knepley   PetscScalar       *u, *a;
1943338f77d5SMatthew G. Knepley   const PetscScalar *constants;
1944338f77d5SMatthew G. Knepley   /* Geometry */
1945c330f8ffSToby Isaac   PetscFEGeom       *cgeomFEM;
1946338f77d5SMatthew G. Knepley   DM                 dmGrad;
1947c330f8ffSToby Isaac   PetscQuadrature    affineQuad      = NULL;
1948338f77d5SMatthew G. Knepley   Vec                cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL;
1949b5a3613cSMatthew G. Knepley   PetscFVCellGeom   *cgeomFVM;
1950338f77d5SMatthew G. Knepley   const PetscScalar *lgrad;
1951b7260050SToby Isaac   PetscInt           maxDegree;
1952c330f8ffSToby Isaac   DMField            coordField;
1953c330f8ffSToby Isaac   IS                 cellIS;
195473d901b8SMatthew G. Knepley 
195573d901b8SMatthew G. Knepley   PetscFunctionBegin;
19569566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
19579566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
19589566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
19599566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
1960338f77d5SMatthew G. Knepley   /* Determine which discretizations we have */
1961b5a3613cSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
1962b5a3613cSMatthew G. Knepley     PetscObject  obj;
1963b5a3613cSMatthew G. Knepley     PetscClassId id;
1964b5a3613cSMatthew G. Knepley 
19659566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
19669566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1967338f77d5SMatthew G. Knepley     if (id == PETSCFV_CLASSID) useFVM = PETSC_TRUE;
1968338f77d5SMatthew G. Knepley   }
1969338f77d5SMatthew G. Knepley   /* Get local solution with boundary values */
19709566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &locX));
19719566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
19729566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
19739566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
1974338f77d5SMatthew G. Knepley   /* Read DS information */
19759566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
19769566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
19779566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
19789566063dSJacob Faibussowitsch   PetscCall(ISCreateStride(PETSC_COMM_SELF, numCells, cStart, 1, &cellIS));
19799566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
1980338f77d5SMatthew G. Knepley   /* Read Auxiliary DS information */
19819566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
19829a2a23afSMatthew G. Knepley   if (locA) {
19839566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
19849566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
19859566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
19869566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
19879566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
19889566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
1989338f77d5SMatthew G. Knepley   }
1990338f77d5SMatthew G. Knepley   /* Allocate data  arrays */
19919566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(numCells * totDim, &u));
19929566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
1993338f77d5SMatthew G. Knepley   /* Read out geometry */
19949566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
19959566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
1996b7260050SToby Isaac   if (maxDegree <= 1) {
19979566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad));
1998*48a46eb9SPierre Jolivet     if (affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &cgeomFEM));
1999b5a3613cSMatthew G. Knepley   }
2000b5a3613cSMatthew G. Knepley   if (useFVM) {
2001338f77d5SMatthew G. Knepley     PetscFV   fv = NULL;
2002b5a3613cSMatthew G. Knepley     Vec       grad;
2003b5a3613cSMatthew G. Knepley     PetscInt  fStart, fEnd;
2004b5a3613cSMatthew G. Knepley     PetscBool compGrad;
2005b5a3613cSMatthew G. Knepley 
2006338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
2007338f77d5SMatthew G. Knepley       PetscObject  obj;
2008338f77d5SMatthew G. Knepley       PetscClassId id;
2009338f77d5SMatthew G. Knepley 
20109566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
20119566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
20129371c9d4SSatish Balay       if (id == PETSCFV_CLASSID) {
20139371c9d4SSatish Balay         fv = (PetscFV)obj;
20149371c9d4SSatish Balay         break;
20159371c9d4SSatish Balay       }
2016338f77d5SMatthew G. Knepley     }
20179566063dSJacob Faibussowitsch     PetscCall(PetscFVGetComputeGradients(fv, &compGrad));
20189566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, PETSC_TRUE));
20199566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGeometryFVM(dm, &cellGeometryFVM, &faceGeometryFVM));
20209566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGradientFVM(dm, fv, faceGeometryFVM, cellGeometryFVM, &dmGrad));
20219566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, compGrad));
20229566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM));
2023b5a3613cSMatthew G. Knepley     /* Reconstruct and limit cell gradients */
20249566063dSJacob Faibussowitsch     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
20259566063dSJacob Faibussowitsch     PetscCall(DMGetGlobalVector(dmGrad, &grad));
20269566063dSJacob Faibussowitsch     PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
2027b5a3613cSMatthew G. Knepley     /* Communicate gradient values */
20289566063dSJacob Faibussowitsch     PetscCall(DMGetLocalVector(dmGrad, &locGrad));
20299566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
20309566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
20319566063dSJacob Faibussowitsch     PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
2032b5a3613cSMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
20339566063dSJacob Faibussowitsch     PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, 0.0, faceGeometryFVM, cellGeometryFVM, locGrad));
20349566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
2035b5a3613cSMatthew G. Knepley   }
2036338f77d5SMatthew G. Knepley   /* Read out data from inputs */
203773d901b8SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
203873d901b8SMatthew G. Knepley     PetscScalar *x = NULL;
203973d901b8SMatthew G. Knepley     PetscInt     i;
204073d901b8SMatthew G. Knepley 
20419566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, section, locX, c, NULL, &x));
20420f2d7e86SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i];
20439566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, section, locX, c, NULL, &x));
204473d901b8SMatthew G. Knepley     if (dmAux) {
20459566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, locA, c, NULL, &x));
20460f2d7e86SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i];
20479566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, locA, c, NULL, &x));
204873d901b8SMatthew G. Knepley     }
204973d901b8SMatthew G. Knepley   }
2050338f77d5SMatthew G. Knepley   /* Do integration for each field */
205173d901b8SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
2052c1f031eeSMatthew G. Knepley     PetscObject  obj;
2053c1f031eeSMatthew G. Knepley     PetscClassId id;
2054c1f031eeSMatthew G. Knepley     PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
205573d901b8SMatthew G. Knepley 
20569566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
20579566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
2058c1f031eeSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
2059c1f031eeSMatthew G. Knepley       PetscFE         fe = (PetscFE)obj;
2060c1f031eeSMatthew G. Knepley       PetscQuadrature q;
2061c330f8ffSToby Isaac       PetscFEGeom    *chunkGeom = NULL;
2062c1f031eeSMatthew G. Knepley       PetscInt        Nq, Nb;
2063c1f031eeSMatthew G. Knepley 
20649566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
20659566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &q));
20669566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
20679566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
2068c1f031eeSMatthew G. Knepley       blockSize = Nb * Nq;
206973d901b8SMatthew G. Knepley       batchSize = numBlocks * blockSize;
20709566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
207173d901b8SMatthew G. Knepley       numChunks = numCells / (numBatches * batchSize);
207273d901b8SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
207373d901b8SMatthew G. Knepley       Nr        = numCells % (numBatches * batchSize);
207473d901b8SMatthew G. Knepley       offset    = numCells - Nr;
2075*48a46eb9SPierre Jolivet       if (!affineQuad) PetscCall(DMFieldCreateFEGeom(coordField, cellIS, q, PETSC_FALSE, &cgeomFEM));
20769566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
20779566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrate(prob, f, Ne, chunkGeom, u, probAux, a, cintegral));
20789566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &chunkGeom));
20799566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrate(prob, f, Nr, chunkGeom, &u[offset * totDim], probAux, &a[offset * totDimAux], &cintegral[offset * Nf]));
20809566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &chunkGeom));
2081*48a46eb9SPierre Jolivet       if (!affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM));
2082c1f031eeSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
2083c1f031eeSMatthew G. Knepley       PetscInt       foff;
2084420e96edSMatthew G. Knepley       PetscPointFunc obj_func;
2085b69edc29SMatthew G. Knepley       PetscScalar    lint;
2086c1f031eeSMatthew G. Knepley 
20879566063dSJacob Faibussowitsch       PetscCall(PetscDSGetObjective(prob, f, &obj_func));
20889566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, f, &foff));
2089c1f031eeSMatthew G. Knepley       if (obj_func) {
2090c1f031eeSMatthew G. Knepley         for (c = 0; c < numCells; ++c) {
2091b5a3613cSMatthew G. Knepley           PetscScalar *u_x;
2092b5a3613cSMatthew G. Knepley 
20939566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, c, lgrad, &u_x));
2094338f77d5SMatthew G. Knepley           obj_func(dim, Nf, NfAux, uOff, uOff_x, &u[totDim * c + foff], NULL, u_x, aOff, NULL, &a[totDimAux * c], NULL, NULL, 0.0, cgeomFVM[c].centroid, numConstants, constants, &lint);
2095338f77d5SMatthew G. Knepley           cintegral[c * Nf + f] += PetscRealPart(lint) * cgeomFVM[c].volume;
209673d901b8SMatthew G. Knepley         }
2097c1f031eeSMatthew G. Knepley       }
209863a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
2099c1f031eeSMatthew G. Knepley   }
2100338f77d5SMatthew G. Knepley   /* Cleanup data arrays */
2101b5a3613cSMatthew G. Knepley   if (useFVM) {
21029566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
21039566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM));
21049566063dSJacob Faibussowitsch     PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
21059566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&faceGeometryFVM));
21069566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&cellGeometryFVM));
21079566063dSJacob Faibussowitsch     PetscCall(DMDestroy(&dmGrad));
2108b5a3613cSMatthew G. Knepley   }
21099566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscFree(a));
21109566063dSJacob Faibussowitsch   PetscCall(PetscFree(u));
2111338f77d5SMatthew G. Knepley   /* Cleanup */
2112*48a46eb9SPierre Jolivet   if (affineQuad) PetscCall(PetscFEGeomDestroy(&cgeomFEM));
21139566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&affineQuad));
21149566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cellIS));
21159566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &locX));
2116338f77d5SMatthew G. Knepley   PetscFunctionReturn(0);
2117338f77d5SMatthew G. Knepley }
2118338f77d5SMatthew G. Knepley 
2119338f77d5SMatthew G. Knepley /*@
2120338f77d5SMatthew G. Knepley   DMPlexComputeIntegralFEM - Form the integral over the domain from the global input X using pointwise functions specified by the user
2121338f77d5SMatthew G. Knepley 
2122338f77d5SMatthew G. Knepley   Input Parameters:
2123338f77d5SMatthew G. Knepley + dm - The mesh
2124338f77d5SMatthew G. Knepley . X  - Global input vector
2125338f77d5SMatthew G. Knepley - user - The user context
2126338f77d5SMatthew G. Knepley 
2127338f77d5SMatthew G. Knepley   Output Parameter:
2128338f77d5SMatthew G. Knepley . integral - Integral for each field
2129338f77d5SMatthew G. Knepley 
2130338f77d5SMatthew G. Knepley   Level: developer
2131338f77d5SMatthew G. Knepley 
2132db781477SPatrick Sanan .seealso: `DMPlexSNESComputeResidualFEM()`
2133338f77d5SMatthew G. Knepley @*/
21349371c9d4SSatish Balay PetscErrorCode DMPlexComputeIntegralFEM(DM dm, Vec X, PetscScalar *integral, void *user) {
2135338f77d5SMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dm->data;
2136b8feb594SMatthew G. Knepley   PetscScalar *cintegral, *lintegral;
2137412e9a14SMatthew G. Knepley   PetscInt     Nf, f, cellHeight, cStart, cEnd, cell;
2138338f77d5SMatthew G. Knepley 
2139338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2140338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2141338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2142dadcf809SJacob Faibussowitsch   PetscValidScalarPointer(integral, 3);
21439566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0));
21449566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
21459566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
21469566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2147338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
21489566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &lintegral, (cEnd - cStart) * Nf, &cintegral));
21499566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user));
2150338f77d5SMatthew G. Knepley   /* Sum up values */
2151338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2152338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2153338f77d5SMatthew G. Knepley 
21549566063dSJacob Faibussowitsch     if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf]));
2155b8feb594SMatthew G. Knepley     for (f = 0; f < Nf; ++f) lintegral[f] += cintegral[c * Nf + f];
2156338f77d5SMatthew G. Knepley   }
21571c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(lintegral, integral, Nf, MPIU_SCALAR, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
215873d901b8SMatthew G. Knepley   if (mesh->printFEM) {
21599566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "Integral:"));
21609566063dSJacob Faibussowitsch     for (f = 0; f < Nf; ++f) PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), " %g", (double)PetscRealPart(integral[f])));
21619566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "\n"));
216273d901b8SMatthew G. Knepley   }
21639566063dSJacob Faibussowitsch   PetscCall(PetscFree2(lintegral, cintegral));
21649566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0));
2165338f77d5SMatthew G. Knepley   PetscFunctionReturn(0);
2166338f77d5SMatthew G. Knepley }
2167338f77d5SMatthew G. Knepley 
2168338f77d5SMatthew G. Knepley /*@
2169338f77d5SMatthew G. Knepley   DMPlexComputeCellwiseIntegralFEM - Form the vector of cellwise integrals F from the global input X using pointwise functions specified by the user
2170338f77d5SMatthew G. Knepley 
2171338f77d5SMatthew G. Knepley   Input Parameters:
2172338f77d5SMatthew G. Knepley + dm - The mesh
2173338f77d5SMatthew G. Knepley . X  - Global input vector
2174338f77d5SMatthew G. Knepley - user - The user context
2175338f77d5SMatthew G. Knepley 
2176338f77d5SMatthew G. Knepley   Output Parameter:
2177338f77d5SMatthew G. Knepley . integral - Cellwise integrals for each field
2178338f77d5SMatthew G. Knepley 
2179338f77d5SMatthew G. Knepley   Level: developer
2180338f77d5SMatthew G. Knepley 
2181db781477SPatrick Sanan .seealso: `DMPlexSNESComputeResidualFEM()`
2182338f77d5SMatthew G. Knepley @*/
21839371c9d4SSatish Balay PetscErrorCode DMPlexComputeCellwiseIntegralFEM(DM dm, Vec X, Vec F, void *user) {
2184338f77d5SMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dm->data;
2185338f77d5SMatthew G. Knepley   DM           dmF;
2186338f77d5SMatthew G. Knepley   PetscSection sectionF;
2187338f77d5SMatthew G. Knepley   PetscScalar *cintegral, *af;
2188412e9a14SMatthew G. Knepley   PetscInt     Nf, f, cellHeight, cStart, cEnd, cell;
2189338f77d5SMatthew G. Knepley 
2190338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2191338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2192338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2193338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(F, VEC_CLASSID, 3);
21949566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0));
21959566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
21969566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
21979566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2198338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
21999566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1((cEnd - cStart) * Nf, &cintegral));
22009566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user));
2201338f77d5SMatthew G. Knepley   /* Put values in F*/
22029566063dSJacob Faibussowitsch   PetscCall(VecGetDM(F, &dmF));
22039566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmF, &sectionF));
22049566063dSJacob Faibussowitsch   PetscCall(VecGetArray(F, &af));
2205338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2206338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2207338f77d5SMatthew G. Knepley     PetscInt       dof, off;
2208338f77d5SMatthew G. Knepley 
22099566063dSJacob Faibussowitsch     if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c * Nf]));
22109566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetDof(sectionF, cell, &dof));
22119566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetOffset(sectionF, cell, &off));
221263a3b9bcSJacob Faibussowitsch     PetscCheck(dof == Nf, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of cell dofs %" PetscInt_FMT " != %" PetscInt_FMT, dof, Nf);
2213338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) af[off + f] = cintegral[c * Nf + f];
2214338f77d5SMatthew G. Knepley   }
22159566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(F, &af));
22169566063dSJacob Faibussowitsch   PetscCall(PetscFree(cintegral));
22179566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0));
221873d901b8SMatthew G. Knepley   PetscFunctionReturn(0);
221973d901b8SMatthew G. Knepley }
222073d901b8SMatthew G. Knepley 
22219371c9d4SSatish Balay static PetscErrorCode DMPlexComputeBdIntegral_Internal(DM dm, Vec locX, IS pointIS, void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *fintegral, void *user) {
22229b6f715bSMatthew G. Knepley   DM                 plex = NULL, plexA = NULL;
2223a6e0b375SMatthew G. Knepley   DMEnclosureType    encAux;
22249b6f715bSMatthew G. Knepley   PetscDS            prob, probAux       = NULL;
22259b6f715bSMatthew G. Knepley   PetscSection       section, sectionAux = NULL;
22269b6f715bSMatthew G. Knepley   Vec                locA = NULL;
22279b6f715bSMatthew G. Knepley   DMField            coordField;
22289b6f715bSMatthew G. Knepley   PetscInt           Nf, totDim, *uOff, *uOff_x;
22299b6f715bSMatthew G. Knepley   PetscInt           NfAux = 0, totDimAux = 0, *aOff = NULL;
22309b6f715bSMatthew G. Knepley   PetscScalar       *u, *a = NULL;
223164c72086SMatthew G. Knepley   const PetscScalar *constants;
22329b6f715bSMatthew G. Knepley   PetscInt           numConstants, f;
223364c72086SMatthew G. Knepley 
223464c72086SMatthew G. Knepley   PetscFunctionBegin;
22359566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
22369566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
22379566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
22389566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
22399566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
224064c72086SMatthew G. Knepley   /* Determine which discretizations we have */
22419b6f715bSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
224264c72086SMatthew G. Knepley     PetscObject  obj;
224364c72086SMatthew G. Knepley     PetscClassId id;
224464c72086SMatthew G. Knepley 
22459566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
22469566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
224763a3b9bcSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID, PetscObjectComm((PetscObject)dm), PETSC_ERR_SUP, "Not supported for FVM (field %" PetscInt_FMT ")", f);
224864c72086SMatthew G. Knepley   }
224964c72086SMatthew G. Knepley   /* Read DS information */
22509566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
22519566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
22529566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
22539566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
225464c72086SMatthew G. Knepley   /* Read Auxiliary DS information */
22559566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
22569b6f715bSMatthew G. Knepley   if (locA) {
22579b6f715bSMatthew G. Knepley     DM dmAux;
22589b6f715bSMatthew G. Knepley 
22599566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
22609566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
22619566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
22629566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
22639566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
22649566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
22659566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
22669566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
226764c72086SMatthew G. Knepley   }
22689b6f715bSMatthew G. Knepley   /* Integrate over points */
22699b6f715bSMatthew G. Knepley   {
22709b6f715bSMatthew G. Knepley     PetscFEGeom    *fgeom, *chunkGeom = NULL;
2271b7260050SToby Isaac     PetscInt        maxDegree;
22729b6f715bSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
22739b6f715bSMatthew G. Knepley     const PetscInt *points;
22749b6f715bSMatthew G. Knepley     PetscInt        numFaces, face, Nq, field;
22759b6f715bSMatthew G. Knepley     PetscInt        numChunks, chunkSize, chunk, Nr, offset;
227664c72086SMatthew G. Knepley 
22779566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
22789566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
22799566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(numFaces * totDim, &u, locA ? numFaces * totDimAux : 0, &a));
22809566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
228164c72086SMatthew G. Knepley     for (field = 0; field < Nf; ++field) {
228264c72086SMatthew G. Knepley       PetscFE fe;
228364c72086SMatthew G. Knepley 
22849566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fe));
22859566063dSJacob Faibussowitsch       if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
22869b6f715bSMatthew G. Knepley       if (!qGeom) {
22879566063dSJacob Faibussowitsch         PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
22889566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)qGeom));
22899b6f715bSMatthew G. Knepley       }
22909566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
22919566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
22929b6f715bSMatthew G. Knepley       for (face = 0; face < numFaces; ++face) {
2293f15274beSMatthew Knepley         const PetscInt point = points[face], *support;
22949b6f715bSMatthew G. Knepley         PetscScalar   *x     = NULL;
2295f15274beSMatthew Knepley         PetscInt       i;
22969b6f715bSMatthew G. Knepley 
22979566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, point, &support));
22989566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
22999b6f715bSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i];
23009566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
23019b6f715bSMatthew G. Knepley         if (locA) {
23029b6f715bSMatthew G. Knepley           PetscInt subp;
23039566063dSJacob Faibussowitsch           PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
23049566063dSJacob Faibussowitsch           PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
23059b6f715bSMatthew G. Knepley           for (i = 0; i < totDimAux; ++i) a[f * totDimAux + i] = x[i];
23069566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
23079b6f715bSMatthew G. Knepley         }
23089b6f715bSMatthew G. Knepley       }
23099b6f715bSMatthew G. Knepley       /* Get blocking */
23109b6f715bSMatthew G. Knepley       {
23119b6f715bSMatthew G. Knepley         PetscQuadrature q;
23129b6f715bSMatthew G. Knepley         PetscInt        numBatches, batchSize, numBlocks, blockSize;
23139b6f715bSMatthew G. Knepley         PetscInt        Nq, Nb;
23149b6f715bSMatthew G. Knepley 
23159566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
23169566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &q));
23179566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
23189566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
231964c72086SMatthew G. Knepley         blockSize = Nb * Nq;
232064c72086SMatthew G. Knepley         batchSize = numBlocks * blockSize;
23219b6f715bSMatthew G. Knepley         chunkSize = numBatches * batchSize;
23229566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
23239b6f715bSMatthew G. Knepley         numChunks = numFaces / chunkSize;
23249b6f715bSMatthew G. Knepley         Nr        = numFaces % chunkSize;
232564c72086SMatthew G. Knepley         offset    = numFaces - Nr;
232664c72086SMatthew G. Knepley       }
23279b6f715bSMatthew G. Knepley       /* Do integration for each field */
23289b6f715bSMatthew G. Knepley       for (chunk = 0; chunk < numChunks; ++chunk) {
23299566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(fgeom, chunk * chunkSize, (chunk + 1) * chunkSize, &chunkGeom));
23309566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateBd(prob, field, func, chunkSize, chunkGeom, u, probAux, a, fintegral));
23319566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
233264c72086SMatthew G. Knepley       }
23339566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
23349566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBd(prob, field, func, Nr, chunkGeom, &u[offset * totDim], probAux, a ? &a[offset * totDimAux] : NULL, &fintegral[offset * Nf]));
23359566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
233664c72086SMatthew G. Knepley       /* Cleanup data arrays */
23379566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
23389566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&qGeom));
23399566063dSJacob Faibussowitsch       PetscCall(PetscFree2(u, a));
23409566063dSJacob Faibussowitsch       PetscCall(ISRestoreIndices(pointIS, &points));
234164c72086SMatthew G. Knepley     }
234264c72086SMatthew G. Knepley   }
23439566063dSJacob Faibussowitsch   if (plex) PetscCall(DMDestroy(&plex));
23449566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
23459b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
23469b6f715bSMatthew G. Knepley }
23479b6f715bSMatthew G. Knepley 
23489b6f715bSMatthew G. Knepley /*@
23499b6f715bSMatthew G. Knepley   DMPlexComputeBdIntegral - Form the integral over the specified boundary from the global input X using pointwise functions specified by the user
23509b6f715bSMatthew G. Knepley 
23519b6f715bSMatthew G. Knepley   Input Parameters:
23529b6f715bSMatthew G. Knepley + dm      - The mesh
23539b6f715bSMatthew G. Knepley . X       - Global input vector
23549b6f715bSMatthew G. Knepley . label   - The boundary DMLabel
23559b6f715bSMatthew G. Knepley . numVals - The number of label values to use, or PETSC_DETERMINE for all values
23569b6f715bSMatthew G. Knepley . vals    - The label values to use, or PETSC_NULL for all values
235767b8a455SSatish Balay . func    - The function to integrate along the boundary
23589b6f715bSMatthew G. Knepley - user    - The user context
23599b6f715bSMatthew G. Knepley 
23609b6f715bSMatthew G. Knepley   Output Parameter:
23619b6f715bSMatthew G. Knepley . integral - Integral for each field
23629b6f715bSMatthew G. Knepley 
23639b6f715bSMatthew G. Knepley   Level: developer
23649b6f715bSMatthew G. Knepley 
2365db781477SPatrick Sanan .seealso: `DMPlexComputeIntegralFEM()`, `DMPlexComputeBdResidualFEM()`
23669b6f715bSMatthew G. Knepley @*/
23679371c9d4SSatish Balay PetscErrorCode DMPlexComputeBdIntegral(DM dm, Vec X, DMLabel label, PetscInt numVals, const PetscInt vals[], void (*func)(PetscInt, PetscInt, PetscInt, const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[], PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]), PetscScalar *integral, void *user) {
23689b6f715bSMatthew G. Knepley   Vec          locX;
23699b6f715bSMatthew G. Knepley   PetscSection section;
23709b6f715bSMatthew G. Knepley   DMLabel      depthLabel;
23719b6f715bSMatthew G. Knepley   IS           facetIS;
23729b6f715bSMatthew G. Knepley   PetscInt     dim, Nf, f, v;
23739b6f715bSMatthew G. Knepley 
23749b6f715bSMatthew G. Knepley   PetscFunctionBegin;
23759b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
23769b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
23779b6f715bSMatthew G. Knepley   PetscValidPointer(label, 3);
2378dadcf809SJacob Faibussowitsch   if (vals) PetscValidIntPointer(vals, 5);
2379dadcf809SJacob Faibussowitsch   PetscValidScalarPointer(integral, 7);
23809566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM, dm, 0, 0, 0));
23819566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
23829566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
23839566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
23849566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
23859566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
23869b6f715bSMatthew G. Knepley   /* Get local solution with boundary values */
23879566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &locX));
23889566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
23899566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
23909566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
23919b6f715bSMatthew G. Knepley   /* Loop over label values */
23929566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(integral, Nf));
23939b6f715bSMatthew G. Knepley   for (v = 0; v < numVals; ++v) {
23949b6f715bSMatthew G. Knepley     IS           pointIS;
23959b6f715bSMatthew G. Knepley     PetscInt     numFaces, face;
23969b6f715bSMatthew G. Knepley     PetscScalar *fintegral;
23979b6f715bSMatthew G. Knepley 
23989566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, vals[v], &pointIS));
23999b6f715bSMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
24009b6f715bSMatthew G. Knepley     {
24019b6f715bSMatthew G. Knepley       IS isectIS;
24029b6f715bSMatthew G. Knepley 
24039b6f715bSMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
24049566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
24059566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
24069b6f715bSMatthew G. Knepley       pointIS = isectIS;
24079b6f715bSMatthew G. Knepley     }
24089566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
24099566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(numFaces * Nf, &fintegral));
24109566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdIntegral_Internal(dm, locX, pointIS, func, fintegral, user));
24119b6f715bSMatthew G. Knepley     /* Sum point contributions into integral */
24129371c9d4SSatish Balay     for (f = 0; f < Nf; ++f)
24139371c9d4SSatish Balay       for (face = 0; face < numFaces; ++face) integral[f] += fintegral[face * Nf + f];
24149566063dSJacob Faibussowitsch     PetscCall(PetscFree(fintegral));
24159566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
24169b6f715bSMatthew G. Knepley   }
24179566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &locX));
24189566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
24199566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM, dm, 0, 0, 0));
242064c72086SMatthew G. Knepley   PetscFunctionReturn(0);
242164c72086SMatthew G. Knepley }
242264c72086SMatthew G. Knepley 
2423d69c5d34SMatthew G. Knepley /*@
2424cf51de39SMatthew G. Knepley   DMPlexComputeInterpolatorNested - Form the local portion of the interpolation matrix I from the coarse DM to a uniformly refined DM.
2425d69c5d34SMatthew G. Knepley 
2426d69c5d34SMatthew G. Knepley   Input Parameters:
2427cf51de39SMatthew G. Knepley + dmc  - The coarse mesh
2428cf51de39SMatthew G. Knepley . dmf  - The fine mesh
2429cf51de39SMatthew G. Knepley . isRefined - Flag indicating regular refinement, rather than the same topology
2430d69c5d34SMatthew G. Knepley - user - The user context
2431d69c5d34SMatthew G. Knepley 
2432d69c5d34SMatthew G. Knepley   Output Parameter:
2433934789fcSMatthew G. Knepley . In  - The interpolation matrix
2434d69c5d34SMatthew G. Knepley 
2435d69c5d34SMatthew G. Knepley   Level: developer
2436d69c5d34SMatthew G. Knepley 
2437db781477SPatrick Sanan .seealso: `DMPlexComputeInterpolatorGeneral()`, `DMPlexComputeJacobianFEM()`
2438d69c5d34SMatthew G. Knepley @*/
24399371c9d4SSatish Balay PetscErrorCode DMPlexComputeInterpolatorNested(DM dmc, DM dmf, PetscBool isRefined, Mat In, void *user) {
2440d69c5d34SMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dmc->data;
2441d69c5d34SMatthew G. Knepley   const char  *name = "Interpolator";
2442d69c5d34SMatthew G. Knepley   PetscFE     *feRef;
244397c42addSMatthew G. Knepley   PetscFV     *fvRef;
2444d69c5d34SMatthew G. Knepley   PetscSection fsection, fglobalSection;
2445d69c5d34SMatthew G. Knepley   PetscSection csection, cglobalSection;
2446d69c5d34SMatthew G. Knepley   PetscScalar *elemMat;
2447485ad865SMatthew G. Knepley   PetscInt     dim, Nf, f, fieldI, fieldJ, offsetI, offsetJ, cStart, cEnd, c;
24482ea9c922SToby Isaac   PetscInt     cTotDim = 0, rTotDim = 0;
2449d69c5d34SMatthew G. Knepley 
2450d69c5d34SMatthew G. Knepley   PetscFunctionBegin;
24519566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
24529566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
24539566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
24549566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
24559566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
24569566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
24579566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
24589566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
24599566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &feRef, Nf, &fvRef));
2460d69c5d34SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
24612ea9c922SToby Isaac     PetscObject  obj, objc;
24622ea9c922SToby Isaac     PetscClassId id, idc;
24632ea9c922SToby Isaac     PetscInt     rNb = 0, Nc = 0, cNb = 0;
2464d69c5d34SMatthew G. Knepley 
24659566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmf, f, NULL, &obj));
24669566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
246797c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
246897c42addSMatthew G. Knepley       PetscFE fe = (PetscFE)obj;
246997c42addSMatthew G. Knepley 
2470cf51de39SMatthew G. Knepley       if (isRefined) {
24719566063dSJacob Faibussowitsch         PetscCall(PetscFERefine(fe, &feRef[f]));
2472cf51de39SMatthew G. Knepley       } else {
24739566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)fe));
2474cf51de39SMatthew G. Knepley         feRef[f] = fe;
2475cf51de39SMatthew G. Knepley       }
24769566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &rNb));
24779566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
247897c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
247997c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV)obj;
248097c42addSMatthew G. Knepley       PetscDualSpace Q;
248197c42addSMatthew G. Knepley 
2482cf51de39SMatthew G. Knepley       if (isRefined) {
24839566063dSJacob Faibussowitsch         PetscCall(PetscFVRefine(fv, &fvRef[f]));
2484cf51de39SMatthew G. Knepley       } else {
24859566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)fv));
2486cf51de39SMatthew G. Knepley         fvRef[f] = fv;
2487cf51de39SMatthew G. Knepley       }
24889566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
24899566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &rNb));
24909566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
24919566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
249297c42addSMatthew G. Knepley     }
24939566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmc, f, NULL, &objc));
24949566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(objc, &idc));
24952ea9c922SToby Isaac     if (idc == PETSCFE_CLASSID) {
24962ea9c922SToby Isaac       PetscFE fe = (PetscFE)objc;
24972ea9c922SToby Isaac 
24989566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &cNb));
24992ea9c922SToby Isaac     } else if (id == PETSCFV_CLASSID) {
25002ea9c922SToby Isaac       PetscFV        fv = (PetscFV)obj;
25012ea9c922SToby Isaac       PetscDualSpace Q;
25022ea9c922SToby Isaac 
25039566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
25049566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &cNb));
2505d69c5d34SMatthew G. Knepley     }
25062ea9c922SToby Isaac     rTotDim += rNb;
25072ea9c922SToby Isaac     cTotDim += cNb;
25082ea9c922SToby Isaac   }
25099566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(rTotDim * cTotDim, &elemMat));
25109566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, rTotDim * cTotDim));
2511d69c5d34SMatthew G. Knepley   for (fieldI = 0, offsetI = 0; fieldI < Nf; ++fieldI) {
2512d69c5d34SMatthew G. Knepley     PetscDualSpace   Qref;
2513d69c5d34SMatthew G. Knepley     PetscQuadrature  f;
2514d69c5d34SMatthew G. Knepley     const PetscReal *qpoints, *qweights;
2515d69c5d34SMatthew G. Knepley     PetscReal       *points;
2516d69c5d34SMatthew G. Knepley     PetscInt         npoints = 0, Nc, Np, fpdim, i, k, p, d;
2517d69c5d34SMatthew G. Knepley 
2518d69c5d34SMatthew G. Knepley     /* Compose points from all dual basis functionals */
251997c42addSMatthew G. Knepley     if (feRef[fieldI]) {
25209566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[fieldI], &Qref));
25219566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[fieldI], &Nc));
252297c42addSMatthew G. Knepley     } else {
25239566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[fieldI], &Qref));
25249566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[fieldI], &Nc));
252597c42addSMatthew G. Knepley     }
25269566063dSJacob Faibussowitsch     PetscCall(PetscDualSpaceGetDimension(Qref, &fpdim));
2527d69c5d34SMatthew G. Knepley     for (i = 0; i < fpdim; ++i) {
25289566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
25299566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, NULL, NULL));
2530d69c5d34SMatthew G. Knepley       npoints += Np;
2531d69c5d34SMatthew G. Knepley     }
25329566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(npoints * dim, &points));
2533d69c5d34SMatthew G. Knepley     for (i = 0, k = 0; i < fpdim; ++i) {
25349566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
25359566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL));
25369371c9d4SSatish Balay       for (p = 0; p < Np; ++p, ++k)
25379371c9d4SSatish Balay         for (d = 0; d < dim; ++d) points[k * dim + d] = qpoints[p * dim + d];
2538d69c5d34SMatthew G. Knepley     }
2539d69c5d34SMatthew G. Knepley 
2540d69c5d34SMatthew G. Knepley     for (fieldJ = 0, offsetJ = 0; fieldJ < Nf; ++fieldJ) {
254197c42addSMatthew G. Knepley       PetscObject  obj;
254297c42addSMatthew G. Knepley       PetscClassId id;
25439c3cf19fSMatthew G. Knepley       PetscInt     NcJ = 0, cpdim = 0, j, qNc;
2544d69c5d34SMatthew G. Knepley 
25459566063dSJacob Faibussowitsch       PetscCall(DMGetField(dmc, fieldJ, NULL, &obj));
25469566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
254797c42addSMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
254897c42addSMatthew G. Knepley         PetscFE         fe = (PetscFE)obj;
2549ef0bb6c7SMatthew G. Knepley         PetscTabulation T  = NULL;
2550d69c5d34SMatthew G. Knepley 
2551d69c5d34SMatthew G. Knepley         /* Evaluate basis at points */
25529566063dSJacob Faibussowitsch         PetscCall(PetscFEGetNumComponents(fe, &NcJ));
25539566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &cpdim));
2554ffe73a53SMatthew G. Knepley         /* For now, fields only interpolate themselves */
2555ffe73a53SMatthew G. Knepley         if (fieldI == fieldJ) {
255663a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
25579566063dSJacob Faibussowitsch           PetscCall(PetscFECreateTabulation(fe, 1, npoints, points, 0, &T));
2558d69c5d34SMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
25599566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
25609566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
256163a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
2562d69c5d34SMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
256336a6d9c0SMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
2564d172c84bSMatthew G. Knepley                 /*
2565d172c84bSMatthew G. Knepley                    cTotDim:            Total columns in element interpolation matrix, sum of number of dual basis functionals in each field
2566d172c84bSMatthew G. Knepley                    offsetI, offsetJ:   Offsets into the larger element interpolation matrix for different fields
2567d172c84bSMatthew G. Knepley                    fpdim, i, cpdim, j: Dofs for fine and coarse grids, correspond to dual space basis functionals
2568d172c84bSMatthew G. Knepley                    qNC, Nc, Ncj, c:    Number of components in this field
2569d172c84bSMatthew G. Knepley                    Np, p:              Number of quad points in the fine grid functional i
2570d172c84bSMatthew G. Knepley                    k:                  i*Np + p, overall point number for the interpolation
2571d172c84bSMatthew G. Knepley                 */
2572ef0bb6c7SMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += T->T[0][k * cpdim * NcJ + j * Nc + c] * qweights[p * qNc + c];
257336a6d9c0SMatthew G. Knepley               }
2574d69c5d34SMatthew G. Knepley             }
2575d69c5d34SMatthew G. Knepley           }
25769566063dSJacob Faibussowitsch           PetscCall(PetscTabulationDestroy(&T));
2577ffe73a53SMatthew G. Knepley         }
257897c42addSMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
257997c42addSMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
258097c42addSMatthew G. Knepley 
258197c42addSMatthew G. Knepley         /* Evaluate constant function at points */
25829566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &NcJ));
258397c42addSMatthew G. Knepley         cpdim = 1;
258497c42addSMatthew G. Knepley         /* For now, fields only interpolate themselves */
258597c42addSMatthew G. Knepley         if (fieldI == fieldJ) {
258663a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
258797c42addSMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
25889566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
25899566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
259063a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
259197c42addSMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
259297c42addSMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
2593458eb97cSMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i) * cTotDim + offsetJ + j] += 1.0 * qweights[p * qNc + c];
259497c42addSMatthew G. Knepley               }
259597c42addSMatthew G. Knepley             }
259697c42addSMatthew G. Knepley           }
259797c42addSMatthew G. Knepley         }
259897c42addSMatthew G. Knepley       }
2599d172c84bSMatthew G. Knepley       offsetJ += cpdim;
2600d69c5d34SMatthew G. Knepley     }
2601d172c84bSMatthew G. Knepley     offsetI += fpdim;
26029566063dSJacob Faibussowitsch     PetscCall(PetscFree(points));
2603d69c5d34SMatthew G. Knepley   }
26049566063dSJacob Faibussowitsch   if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(0, name, rTotDim, cTotDim, elemMat));
26057f5b169aSMatthew G. Knepley   /* Preallocate matrix */
26067f5b169aSMatthew G. Knepley   {
2607c094ef40SMatthew G. Knepley     Mat          preallocator;
2608c094ef40SMatthew G. Knepley     PetscScalar *vals;
2609c094ef40SMatthew G. Knepley     PetscInt    *cellCIndices, *cellFIndices;
2610c094ef40SMatthew G. Knepley     PetscInt     locRows, locCols, cell;
26117f5b169aSMatthew G. Knepley 
26129566063dSJacob Faibussowitsch     PetscCall(MatGetLocalSize(In, &locRows, &locCols));
26139566063dSJacob Faibussowitsch     PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &preallocator));
26149566063dSJacob Faibussowitsch     PetscCall(MatSetType(preallocator, MATPREALLOCATOR));
26159566063dSJacob Faibussowitsch     PetscCall(MatSetSizes(preallocator, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE));
26169566063dSJacob Faibussowitsch     PetscCall(MatSetUp(preallocator));
26179566063dSJacob Faibussowitsch     PetscCall(PetscCalloc3(rTotDim * cTotDim, &vals, cTotDim, &cellCIndices, rTotDim, &cellFIndices));
26187f5b169aSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
2619cf51de39SMatthew G. Knepley       if (isRefined) {
26209566063dSJacob Faibussowitsch         PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, cell, cellCIndices, cellFIndices));
26219566063dSJacob Faibussowitsch         PetscCall(MatSetValues(preallocator, rTotDim, cellFIndices, cTotDim, cellCIndices, vals, INSERT_VALUES));
2622cf51de39SMatthew G. Knepley       } else {
26239566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, preallocator, cell, vals, INSERT_VALUES));
2624cf51de39SMatthew G. Knepley       }
26257f5b169aSMatthew G. Knepley     }
26269566063dSJacob Faibussowitsch     PetscCall(PetscFree3(vals, cellCIndices, cellFIndices));
26279566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(preallocator, MAT_FINAL_ASSEMBLY));
26289566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(preallocator, MAT_FINAL_ASSEMBLY));
26299566063dSJacob Faibussowitsch     PetscCall(MatPreallocatorPreallocate(preallocator, PETSC_TRUE, In));
26309566063dSJacob Faibussowitsch     PetscCall(MatDestroy(&preallocator));
26317f5b169aSMatthew G. Knepley   }
26327f5b169aSMatthew G. Knepley   /* Fill matrix */
26339566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(In));
2634d69c5d34SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
2635cf51de39SMatthew G. Knepley     if (isRefined) {
26369566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosureRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES));
2637cf51de39SMatthew G. Knepley     } else {
26389566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES));
2639cf51de39SMatthew G. Knepley     }
2640d69c5d34SMatthew G. Knepley   }
26419566063dSJacob Faibussowitsch   for (f = 0; f < Nf; ++f) PetscCall(PetscFEDestroy(&feRef[f]));
26429566063dSJacob Faibussowitsch   PetscCall(PetscFree2(feRef, fvRef));
26439566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
26449566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
26459566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
26469318fe57SMatthew G. Knepley   if (mesh->printFEM > 1) {
26479566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)In), "%s:\n", name));
26489566063dSJacob Faibussowitsch     PetscCall(MatChop(In, 1.0e-10));
26499566063dSJacob Faibussowitsch     PetscCall(MatView(In, NULL));
2650d69c5d34SMatthew G. Knepley   }
26519566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
2652d69c5d34SMatthew G. Knepley   PetscFunctionReturn(0);
2653d69c5d34SMatthew G. Knepley }
26546c73c22cSMatthew G. Knepley 
26559371c9d4SSatish Balay PetscErrorCode DMPlexComputeMassMatrixNested(DM dmc, DM dmf, Mat mass, void *user) {
2656bd041c0cSMatthew G. Knepley   SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_SUP, "Laziness");
2657bd041c0cSMatthew G. Knepley }
2658bd041c0cSMatthew G. Knepley 
265968132eb9SMatthew G. Knepley /*@
266068132eb9SMatthew G. Knepley   DMPlexComputeInterpolatorGeneral - Form the local portion of the interpolation matrix I from the coarse DM to a non-nested fine DM.
266168132eb9SMatthew G. Knepley 
266268132eb9SMatthew G. Knepley   Input Parameters:
266368132eb9SMatthew G. Knepley + dmf  - The fine mesh
266468132eb9SMatthew G. Knepley . dmc  - The coarse mesh
266568132eb9SMatthew G. Knepley - user - The user context
266668132eb9SMatthew G. Knepley 
266768132eb9SMatthew G. Knepley   Output Parameter:
266868132eb9SMatthew G. Knepley . In  - The interpolation matrix
266968132eb9SMatthew G. Knepley 
267068132eb9SMatthew G. Knepley   Level: developer
267168132eb9SMatthew G. Knepley 
2672db781477SPatrick Sanan .seealso: `DMPlexComputeInterpolatorNested()`, `DMPlexComputeJacobianFEM()`
267368132eb9SMatthew G. Knepley @*/
26749371c9d4SSatish Balay PetscErrorCode DMPlexComputeInterpolatorGeneral(DM dmc, DM dmf, Mat In, void *user) {
267564e98e1dSMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dmf->data;
267664e98e1dSMatthew G. Knepley   const char  *name = "Interpolator";
26774ef9d792SMatthew G. Knepley   PetscDS      prob;
2678a0806964SMatthew G. Knepley   Mat          interp;
2679a0806964SMatthew G. Knepley   PetscSection fsection, globalFSection;
2680a0806964SMatthew G. Knepley   PetscSection csection, globalCSection;
2681a0806964SMatthew G. Knepley   PetscInt     locRows, locCols;
26824ef9d792SMatthew G. Knepley   PetscReal   *x, *v0, *J, *invJ, detJ;
26834ef9d792SMatthew G. Knepley   PetscReal   *v0c, *Jc, *invJc, detJc;
26844ef9d792SMatthew G. Knepley   PetscScalar *elemMat;
2685a0806964SMatthew G. Knepley   PetscInt     dim, Nf, field, totDim, cStart, cEnd, cell, ccell, s;
26864ef9d792SMatthew G. Knepley 
26874ef9d792SMatthew G. Knepley   PetscFunctionBegin;
26889566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
26899566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
26909566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
26919566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
26929566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
26939566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ));
26949566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc));
26959566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
26969566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
26979566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
26989566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
2699a0806964SMatthew G. Knepley   PetscCall(DMPlexGetSimplexOrBoxCells(dmf, 0, &cStart, &cEnd));
27009566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
27019566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
27024ef9d792SMatthew G. Knepley 
2703a0806964SMatthew G. Knepley   PetscCall(MatGetLocalSize(In, &locRows, &locCols));
2704a0806964SMatthew G. Knepley   PetscCall(MatCreate(PetscObjectComm((PetscObject)In), &interp));
2705a0806964SMatthew G. Knepley   PetscCall(MatSetType(interp, MATPREALLOCATOR));
2706a0806964SMatthew G. Knepley   PetscCall(MatSetSizes(interp, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE));
2707a0806964SMatthew G. Knepley   PetscCall(MatSetUp(interp));
2708a0806964SMatthew G. Knepley   for (s = 0; s < 2; ++s) {
27094ef9d792SMatthew G. Knepley     for (field = 0; field < Nf; ++field) {
27104ef9d792SMatthew G. Knepley       PetscObject      obj;
27114ef9d792SMatthew G. Knepley       PetscClassId     id;
2712c0d7054bSMatthew G. Knepley       PetscDualSpace   Q = NULL;
2713ef0bb6c7SMatthew G. Knepley       PetscTabulation  T = NULL;
27144ef9d792SMatthew G. Knepley       PetscQuadrature  f;
27154ef9d792SMatthew G. Knepley       const PetscReal *qpoints, *qweights;
2716d0f6233fSMatthew G. Knepley       PetscInt         Nc, qNc, Np, fpdim, off, i, d;
27174ef9d792SMatthew G. Knepley 
2718d0f6233fSMatthew G. Knepley       PetscCall(PetscDSGetFieldOffset(prob, field, &off));
27199566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, &obj));
27209566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
27214ef9d792SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
27224ef9d792SMatthew G. Knepley         PetscFE fe = (PetscFE)obj;
27234ef9d792SMatthew G. Knepley 
27249566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDualSpace(fe, &Q));
27259566063dSJacob Faibussowitsch         PetscCall(PetscFEGetNumComponents(fe, &Nc));
2726a0806964SMatthew G. Knepley         if (s) PetscCall(PetscFECreateTabulation(fe, 1, 1, x, 0, &T));
27274ef9d792SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
27284ef9d792SMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
27294ef9d792SMatthew G. Knepley 
27309566063dSJacob Faibussowitsch         PetscCall(PetscFVGetDualSpace(fv, &Q));
27314ef9d792SMatthew G. Knepley         Nc = 1;
273263a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject)dmc), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
27339566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &fpdim));
27344ef9d792SMatthew G. Knepley       /* For each fine grid cell */
27354ef9d792SMatthew G. Knepley       for (cell = cStart; cell < cEnd; ++cell) {
27364ef9d792SMatthew G. Knepley         PetscInt *findices, *cindices;
27374ef9d792SMatthew G. Knepley         PetscInt  numFIndices, numCIndices;
27384ef9d792SMatthew G. Knepley 
27399566063dSJacob Faibussowitsch         PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
27409566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
2741d0f6233fSMatthew G. Knepley         PetscCheck(numFIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %" PetscInt_FMT " != %" PetscInt_FMT " dual basis vecs", numFIndices, totDim);
27424ef9d792SMatthew G. Knepley         for (i = 0; i < fpdim; ++i) {
27434ef9d792SMatthew G. Knepley           Vec                pointVec;
27444ef9d792SMatthew G. Knepley           PetscScalar       *pV;
274512111d7cSToby Isaac           PetscSF            coarseCellSF = NULL;
27463a93e3b7SToby Isaac           const PetscSFNode *coarseCells;
2747d0f6233fSMatthew G. Knepley           PetscInt           numCoarseCells, cpdim, row = findices[i + off], q, c, j;
27484ef9d792SMatthew G. Knepley 
27494ef9d792SMatthew G. Knepley           /* Get points from the dual basis functional quadrature */
27509566063dSJacob Faibussowitsch           PetscCall(PetscDualSpaceGetFunctional(Q, i, &f));
27519566063dSJacob Faibussowitsch           PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, &qpoints, &qweights));
275263a3b9bcSJacob Faibussowitsch           PetscCheck(qNc == Nc, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, Nc);
27539566063dSJacob Faibussowitsch           PetscCall(VecCreateSeq(PETSC_COMM_SELF, Np * dim, &pointVec));
27549566063dSJacob Faibussowitsch           PetscCall(VecSetBlockSize(pointVec, dim));
27559566063dSJacob Faibussowitsch           PetscCall(VecGetArray(pointVec, &pV));
27564ef9d792SMatthew G. Knepley           for (q = 0; q < Np; ++q) {
2757c330f8ffSToby Isaac             const PetscReal xi0[3] = {-1., -1., -1.};
2758c330f8ffSToby Isaac 
27594ef9d792SMatthew G. Knepley             /* Transform point to real space */
2760c330f8ffSToby Isaac             CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x);
27614ef9d792SMatthew G. Knepley             for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d];
27624ef9d792SMatthew G. Knepley           }
27639566063dSJacob Faibussowitsch           PetscCall(VecRestoreArray(pointVec, &pV));
27644ef9d792SMatthew G. Knepley           /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
27651555c271SMatthew G. Knepley           /* OPT: Read this out from preallocation information */
27669566063dSJacob Faibussowitsch           PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
27674ef9d792SMatthew G. Knepley           /* Update preallocation info */
27689566063dSJacob Faibussowitsch           PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
27695f80ce2aSJacob Faibussowitsch           PetscCheck(numCoarseCells == Np, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located");
27709566063dSJacob Faibussowitsch           PetscCall(VecGetArray(pointVec, &pV));
27714ef9d792SMatthew G. Knepley           for (ccell = 0; ccell < numCoarseCells; ++ccell) {
2772826eb36dSMatthew G. Knepley             PetscReal       pVReal[3];
2773c330f8ffSToby Isaac             const PetscReal xi0[3] = {-1., -1., -1.};
2774826eb36dSMatthew G. Knepley 
27759566063dSJacob Faibussowitsch             PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
2776a0806964SMatthew G. Knepley             if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetDimension((PetscFE)obj, &cpdim));
2777a0806964SMatthew G. Knepley             else cpdim = 1;
2778a0806964SMatthew G. Knepley 
2779a0806964SMatthew G. Knepley             if (s) {
27804ef9d792SMatthew G. Knepley               /* Transform points from real space to coarse reference space */
27819566063dSJacob Faibussowitsch               PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
2782e2d86523SMatthew G. Knepley               for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]);
2783c330f8ffSToby Isaac               CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
27844ef9d792SMatthew G. Knepley 
27854ef9d792SMatthew G. Knepley               if (id == PETSCFE_CLASSID) {
27864ef9d792SMatthew G. Knepley                 /* Evaluate coarse basis on contained point */
2787a0806964SMatthew G. Knepley                 PetscCall(PetscFEComputeTabulation((PetscFE)obj, 1, x, 0, T));
27889566063dSJacob Faibussowitsch                 PetscCall(PetscArrayzero(elemMat, cpdim));
27894ef9d792SMatthew G. Knepley                 /* Get elemMat entries by multiplying by weight */
27904ef9d792SMatthew G. Knepley                 for (j = 0; j < cpdim; ++j) {
2791ef0bb6c7SMatthew G. Knepley                   for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * qweights[ccell * qNc + c];
27924ef9d792SMatthew G. Knepley                 }
27934ef9d792SMatthew G. Knepley               } else {
27944ef9d792SMatthew G. Knepley                 for (j = 0; j < cpdim; ++j) {
27959c3cf19fSMatthew G. Knepley                   for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * qweights[ccell * qNc + c];
27964ef9d792SMatthew G. Knepley                 }
27974ef9d792SMatthew G. Knepley               }
27989566063dSJacob Faibussowitsch               if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
2799a0806964SMatthew G. Knepley             }
2800a0806964SMatthew G. Knepley             /* Update interpolator */
2801d0f6233fSMatthew G. Knepley             PetscCheck(numCIndices == totDim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, totDim);
2802a0806964SMatthew G. Knepley             PetscCall(MatSetValues(interp, 1, &row, cpdim, &cindices[off], elemMat, INSERT_VALUES));
28039566063dSJacob Faibussowitsch             PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
28044ef9d792SMatthew G. Knepley           }
28059566063dSJacob Faibussowitsch           PetscCall(VecRestoreArray(pointVec, &pV));
28069566063dSJacob Faibussowitsch           PetscCall(PetscSFDestroy(&coarseCellSF));
28079566063dSJacob Faibussowitsch           PetscCall(VecDestroy(&pointVec));
28084ef9d792SMatthew G. Knepley         }
28099566063dSJacob Faibussowitsch         PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
28104ef9d792SMatthew G. Knepley       }
2811a0806964SMatthew G. Knepley       if (s && id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
2812a0806964SMatthew G. Knepley     }
2813a0806964SMatthew G. Knepley     if (!s) {
2814a0806964SMatthew G. Knepley       PetscCall(MatAssemblyBegin(interp, MAT_FINAL_ASSEMBLY));
2815a0806964SMatthew G. Knepley       PetscCall(MatAssemblyEnd(interp, MAT_FINAL_ASSEMBLY));
2816a0806964SMatthew G. Knepley       PetscCall(MatPreallocatorPreallocate(interp, PETSC_TRUE, In));
2817a0806964SMatthew G. Knepley       PetscCall(MatDestroy(&interp));
2818a0806964SMatthew G. Knepley       interp = In;
2819a0806964SMatthew G. Knepley     }
28204ef9d792SMatthew G. Knepley   }
28219566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0, J, invJ));
28229566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c, Jc, invJc));
28239566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
28249566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
28259566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
28269566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM, dmc, dmf, 0, 0));
28274ef9d792SMatthew G. Knepley   PetscFunctionReturn(0);
28284ef9d792SMatthew G. Knepley }
28294ef9d792SMatthew G. Knepley 
283046fa42a0SMatthew G. Knepley /*@
2831bd041c0cSMatthew G. Knepley   DMPlexComputeMassMatrixGeneral - Form the local portion of the mass matrix M from the coarse DM to a non-nested fine DM.
2832bd041c0cSMatthew G. Knepley 
2833bd041c0cSMatthew G. Knepley   Input Parameters:
2834bd041c0cSMatthew G. Knepley + dmf  - The fine mesh
2835bd041c0cSMatthew G. Knepley . dmc  - The coarse mesh
2836bd041c0cSMatthew G. Knepley - user - The user context
2837bd041c0cSMatthew G. Knepley 
2838bd041c0cSMatthew G. Knepley   Output Parameter:
2839bd041c0cSMatthew G. Knepley . mass  - The mass matrix
2840bd041c0cSMatthew G. Knepley 
2841bd041c0cSMatthew G. Knepley   Level: developer
2842bd041c0cSMatthew G. Knepley 
2843db781477SPatrick Sanan .seealso: `DMPlexComputeMassMatrixNested()`, `DMPlexComputeInterpolatorNested()`, `DMPlexComputeInterpolatorGeneral()`, `DMPlexComputeJacobianFEM()`
2844bd041c0cSMatthew G. Knepley @*/
28459371c9d4SSatish Balay PetscErrorCode DMPlexComputeMassMatrixGeneral(DM dmc, DM dmf, Mat mass, void *user) {
2846bd041c0cSMatthew G. Knepley   DM_Plex     *mesh = (DM_Plex *)dmf->data;
2847bd041c0cSMatthew G. Knepley   const char  *name = "Mass Matrix";
2848bd041c0cSMatthew G. Knepley   PetscDS      prob;
2849bd041c0cSMatthew G. Knepley   PetscSection fsection, csection, globalFSection, globalCSection;
2850e8f14785SLisandro Dalcin   PetscHSetIJ  ht;
2851bd041c0cSMatthew G. Knepley   PetscLayout  rLayout;
2852bd041c0cSMatthew G. Knepley   PetscInt    *dnz, *onz;
2853bd041c0cSMatthew G. Knepley   PetscInt     locRows, rStart, rEnd;
2854bd041c0cSMatthew G. Knepley   PetscReal   *x, *v0, *J, *invJ, detJ;
2855bd041c0cSMatthew G. Knepley   PetscReal   *v0c, *Jc, *invJc, detJc;
2856bd041c0cSMatthew G. Knepley   PetscScalar *elemMat;
2857bd041c0cSMatthew G. Knepley   PetscInt     dim, Nf, field, totDim, cStart, cEnd, cell, ccell;
2858bd041c0cSMatthew G. Knepley 
2859bd041c0cSMatthew G. Knepley   PetscFunctionBegin;
28609566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
28619566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
28629566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
28639566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
28649566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0, dim * dim, &J, dim * dim, &invJ));
28659566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim, &v0c, dim * dim, &Jc, dim * dim, &invJc));
28669566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
28679566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
28689566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
28699566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
28709566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd));
28719566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
28729566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
2873bd041c0cSMatthew G. Knepley 
28749566063dSJacob Faibussowitsch   PetscCall(MatGetLocalSize(mass, &locRows, NULL));
28759566063dSJacob Faibussowitsch   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject)mass), &rLayout));
28769566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetLocalSize(rLayout, locRows));
28779566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetBlockSize(rLayout, 1));
28789566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetUp(rLayout));
28799566063dSJacob Faibussowitsch   PetscCall(PetscLayoutGetRange(rLayout, &rStart, &rEnd));
28809566063dSJacob Faibussowitsch   PetscCall(PetscLayoutDestroy(&rLayout));
28819566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(locRows, &dnz, locRows, &onz));
28829566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJCreate(&ht));
2883bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
2884bd041c0cSMatthew G. Knepley     PetscObject      obj;
2885bd041c0cSMatthew G. Knepley     PetscClassId     id;
2886bd041c0cSMatthew G. Knepley     PetscQuadrature  quad;
2887bd041c0cSMatthew G. Knepley     const PetscReal *qpoints;
2888bd041c0cSMatthew G. Knepley     PetscInt         Nq, Nc, i, d;
2889bd041c0cSMatthew G. Knepley 
28909566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
28919566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
28929566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad));
28939566063dSJacob Faibussowitsch     else PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad));
28949566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, NULL));
2895bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
2896bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
2897bd041c0cSMatthew G. Knepley       Vec                pointVec;
2898bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
2899bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
2900bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
2901bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, q, c;
2902bd041c0cSMatthew G. Knepley       PetscInt          *findices, *cindices;
2903bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
2904bd041c0cSMatthew G. Knepley 
29059566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
29069566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
2907bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
29089566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec));
29099566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
29109566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
2911bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
2912c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
2913c330f8ffSToby Isaac 
2914bd041c0cSMatthew G. Knepley         /* Transform point to real space */
2915c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x);
2916bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d];
2917bd041c0cSMatthew G. Knepley       }
29189566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
2919bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
29209566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
29219566063dSJacob Faibussowitsch       PetscCall(PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view"));
2922bd041c0cSMatthew G. Knepley       /* Update preallocation info */
29239566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
29245f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located");
2925bd041c0cSMatthew G. Knepley       {
2926e8f14785SLisandro Dalcin         PetscHashIJKey key;
2927e8f14785SLisandro Dalcin         PetscBool      missing;
2928bd041c0cSMatthew G. Knepley 
2929bd041c0cSMatthew G. Knepley         for (i = 0; i < numFIndices; ++i) {
2930e8f14785SLisandro Dalcin           key.i = findices[i];
2931e8f14785SLisandro Dalcin           if (key.i >= 0) {
2932bd041c0cSMatthew G. Knepley             /* Get indices for coarse elements */
2933bd041c0cSMatthew G. Knepley             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
29349566063dSJacob Faibussowitsch               PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
2935bd041c0cSMatthew G. Knepley               for (c = 0; c < numCIndices; ++c) {
2936e8f14785SLisandro Dalcin                 key.j = cindices[c];
2937e8f14785SLisandro Dalcin                 if (key.j < 0) continue;
29389566063dSJacob Faibussowitsch                 PetscCall(PetscHSetIJQueryAdd(ht, key, &missing));
2939bd041c0cSMatthew G. Knepley                 if (missing) {
2940e8f14785SLisandro Dalcin                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i - rStart];
2941e8f14785SLisandro Dalcin                   else ++onz[key.i - rStart];
2942bd041c0cSMatthew G. Knepley                 }
2943bd041c0cSMatthew G. Knepley               }
29449566063dSJacob Faibussowitsch               PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
2945bd041c0cSMatthew G. Knepley             }
2946bd041c0cSMatthew G. Knepley           }
2947bd041c0cSMatthew G. Knepley         }
2948bd041c0cSMatthew G. Knepley       }
29499566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
29509566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
29519566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
2952bd041c0cSMatthew G. Knepley     }
2953bd041c0cSMatthew G. Knepley   }
29549566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJDestroy(&ht));
29559566063dSJacob Faibussowitsch   PetscCall(MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL));
29569566063dSJacob Faibussowitsch   PetscCall(MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR, PETSC_TRUE));
29579566063dSJacob Faibussowitsch   PetscCall(PetscFree2(dnz, onz));
2958bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
2959bd041c0cSMatthew G. Knepley     PetscObject      obj;
2960bd041c0cSMatthew G. Knepley     PetscClassId     id;
2961ef0bb6c7SMatthew G. Knepley     PetscTabulation  T, Tfine;
2962bd041c0cSMatthew G. Knepley     PetscQuadrature  quad;
2963bd041c0cSMatthew G. Knepley     const PetscReal *qpoints, *qweights;
2964bd041c0cSMatthew G. Knepley     PetscInt         Nq, Nc, i, d;
2965bd041c0cSMatthew G. Knepley 
29669566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
29679566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
2968ef0bb6c7SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
29699566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature((PetscFE)obj, &quad));
29709566063dSJacob Faibussowitsch       PetscCall(PetscFEGetCellTabulation((PetscFE)obj, 1, &Tfine));
29719566063dSJacob Faibussowitsch       PetscCall(PetscFECreateTabulation((PetscFE)obj, 1, 1, x, 0, &T));
2972ef0bb6c7SMatthew G. Knepley     } else {
29739566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature((PetscFV)obj, &quad));
2974ef0bb6c7SMatthew G. Knepley     }
29759566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, &qweights));
2976bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
2977bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
2978bd041c0cSMatthew G. Knepley       Vec                pointVec;
2979bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
2980bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
2981bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
2982bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, cpdim, q, c, j;
2983bd041c0cSMatthew G. Knepley       PetscInt          *findices, *cindices;
2984bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
2985bd041c0cSMatthew G. Knepley 
29869566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
29879566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
2988bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
29899566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq * dim, &pointVec));
29909566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
29919566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
2992bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
2993c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
2994c330f8ffSToby Isaac 
2995bd041c0cSMatthew G. Knepley         /* Transform point to real space */
2996c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q * dim], x);
2997bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q * dim + d] = x[d];
2998bd041c0cSMatthew G. Knepley       }
29999566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
3000bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
30019566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
3002bd041c0cSMatthew G. Knepley       /* Update matrix */
30039566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
30045f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Not all closure points located");
30059566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3006bd041c0cSMatthew G. Knepley       for (ccell = 0; ccell < numCoarseCells; ++ccell) {
3007bd041c0cSMatthew G. Knepley         PetscReal       pVReal[3];
3008c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3009c330f8ffSToby Isaac 
30109566063dSJacob Faibussowitsch         PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3011bd041c0cSMatthew G. Knepley         /* Transform points from real space to coarse reference space */
30129566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
3013bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell * dim + d]);
3014c330f8ffSToby Isaac         CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
3015bd041c0cSMatthew G. Knepley 
3016bd041c0cSMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
3017bd041c0cSMatthew G. Knepley           PetscFE fe = (PetscFE)obj;
3018bd041c0cSMatthew G. Knepley 
3019bd041c0cSMatthew G. Knepley           /* Evaluate coarse basis on contained point */
30209566063dSJacob Faibussowitsch           PetscCall(PetscFEGetDimension(fe, &cpdim));
30219566063dSJacob Faibussowitsch           PetscCall(PetscFEComputeTabulation(fe, 1, x, 0, T));
3022bd041c0cSMatthew G. Knepley           /* Get elemMat entries by multiplying by weight */
3023bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
30249566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3025bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3026ef0bb6c7SMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j * Nc + c] * Tfine->T[0][(ccell * numFIndices + i) * Nc + c] * qweights[ccell * Nc + c] * detJ;
3027bd041c0cSMatthew G. Knepley             }
3028bd041c0cSMatthew G. Knepley             /* Update interpolator */
30299566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
303063a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
30319566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3032bd041c0cSMatthew G. Knepley           }
3033bd041c0cSMatthew G. Knepley         } else {
3034bd041c0cSMatthew G. Knepley           cpdim = 1;
3035bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
30369566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3037bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3038bd041c0cSMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0 * 1.0 * qweights[ccell * Nc + c] * detJ;
3039bd041c0cSMatthew G. Knepley             }
3040bd041c0cSMatthew G. Knepley             /* Update interpolator */
30419566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
304263a3b9bcSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "Nq: %" PetscInt_FMT " %" PetscInt_FMT " Nf: %" PetscInt_FMT " %" PetscInt_FMT " Nc: %" PetscInt_FMT " %" PetscInt_FMT "\n", ccell, Nq, i, numFIndices, j, numCIndices));
304363a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
30449566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3045bd041c0cSMatthew G. Knepley           }
3046bd041c0cSMatthew G. Knepley         }
30479566063dSJacob Faibussowitsch         PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3048bd041c0cSMatthew G. Knepley       }
30499566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
30509566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
30519566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
30529566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
3053bd041c0cSMatthew G. Knepley     }
30549566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
3055bd041c0cSMatthew G. Knepley   }
30569566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0, J, invJ));
30579566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c, Jc, invJc));
30589566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
30599566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY));
30609566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY));
3061bd041c0cSMatthew G. Knepley   PetscFunctionReturn(0);
3062bd041c0cSMatthew G. Knepley }
3063bd041c0cSMatthew G. Knepley 
3064bd041c0cSMatthew G. Knepley /*@
306546fa42a0SMatthew G. Knepley   DMPlexComputeInjectorFEM - Compute a mapping from coarse unknowns to fine unknowns
306646fa42a0SMatthew G. Knepley 
306746fa42a0SMatthew G. Knepley   Input Parameters:
306846fa42a0SMatthew G. Knepley + dmc  - The coarse mesh
306946fa42a0SMatthew G. Knepley - dmf  - The fine mesh
307046fa42a0SMatthew G. Knepley - user - The user context
307146fa42a0SMatthew G. Knepley 
307246fa42a0SMatthew G. Knepley   Output Parameter:
307346fa42a0SMatthew G. Knepley . sc   - The mapping
307446fa42a0SMatthew G. Knepley 
307546fa42a0SMatthew G. Knepley   Level: developer
307646fa42a0SMatthew G. Knepley 
3077db781477SPatrick Sanan .seealso: `DMPlexComputeInterpolatorNested()`, `DMPlexComputeJacobianFEM()`
307846fa42a0SMatthew G. Knepley @*/
30799371c9d4SSatish Balay PetscErrorCode DMPlexComputeInjectorFEM(DM dmc, DM dmf, VecScatter *sc, void *user) {
3080e9d4ef1bSMatthew G. Knepley   PetscDS      prob;
30817c927364SMatthew G. Knepley   PetscFE     *feRef;
308297c42addSMatthew G. Knepley   PetscFV     *fvRef;
30837c927364SMatthew G. Knepley   Vec          fv, cv;
30847c927364SMatthew G. Knepley   IS           fis, cis;
30857c927364SMatthew G. Knepley   PetscSection fsection, fglobalSection, csection, cglobalSection;
30867c927364SMatthew G. Knepley   PetscInt    *cmap, *cellCIndices, *cellFIndices, *cindices, *findices;
3087485ad865SMatthew G. Knepley   PetscInt     cTotDim, fTotDim = 0, Nf, f, field, cStart, cEnd, c, dim, d, startC, endC, offsetC, offsetF, m;
30886f3d3cbcSMatthew G. Knepley   PetscBool   *needAvg;
30897c927364SMatthew G. Knepley 
30907c927364SMatthew G. Knepley   PetscFunctionBegin;
30919566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InjectorFEM, dmc, dmf, 0, 0));
30929566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
30939566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
30949566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
30959566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
30969566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
30979566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
30989566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
30999566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
31009566063dSJacob Faibussowitsch   PetscCall(PetscCalloc3(Nf, &feRef, Nf, &fvRef, Nf, &needAvg));
31017c927364SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
310297c42addSMatthew G. Knepley     PetscObject  obj;
310397c42addSMatthew G. Knepley     PetscClassId id;
3104aa7890ccSMatthew G. Knepley     PetscInt     fNb = 0, Nc = 0;
31057c927364SMatthew G. Knepley 
31069566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
31079566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
310897c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
310997c42addSMatthew G. Knepley       PetscFE    fe = (PetscFE)obj;
31106f3d3cbcSMatthew G. Knepley       PetscSpace sp;
31119b2fc754SMatthew G. Knepley       PetscInt   maxDegree;
311297c42addSMatthew G. Knepley 
31139566063dSJacob Faibussowitsch       PetscCall(PetscFERefine(fe, &feRef[f]));
31149566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &fNb));
31159566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
31169566063dSJacob Faibussowitsch       PetscCall(PetscFEGetBasisSpace(fe, &sp));
31179566063dSJacob Faibussowitsch       PetscCall(PetscSpaceGetDegree(sp, NULL, &maxDegree));
31189b2fc754SMatthew G. Knepley       if (!maxDegree) needAvg[f] = PETSC_TRUE;
311997c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
312097c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV)obj;
312197c42addSMatthew G. Knepley       PetscDualSpace Q;
312297c42addSMatthew G. Knepley 
31239566063dSJacob Faibussowitsch       PetscCall(PetscFVRefine(fv, &fvRef[f]));
31249566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
31259566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &fNb));
31269566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
31276f3d3cbcSMatthew G. Knepley       needAvg[f] = PETSC_TRUE;
312897c42addSMatthew G. Knepley     }
3129d172c84bSMatthew G. Knepley     fTotDim += fNb;
31307c927364SMatthew G. Knepley   }
31319566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &cTotDim));
31329566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(cTotDim, &cmap));
31337c927364SMatthew G. Knepley   for (field = 0, offsetC = 0, offsetF = 0; field < Nf; ++field) {
31347c927364SMatthew G. Knepley     PetscFE        feC;
313597c42addSMatthew G. Knepley     PetscFV        fvC;
31367c927364SMatthew G. Knepley     PetscDualSpace QF, QC;
3137d172c84bSMatthew G. Knepley     PetscInt       order = -1, NcF, NcC, fpdim, cpdim;
31387c927364SMatthew G. Knepley 
313997c42addSMatthew G. Knepley     if (feRef[field]) {
31409566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&feC));
31419566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feC, &NcC));
31429566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[field], &NcF));
31439566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[field], &QF));
31449566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetOrder(QF, &order));
31459566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
31469566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feC, &QC));
31479566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
314897c42addSMatthew G. Knepley     } else {
31499566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *)&fvC));
31509566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvC, &NcC));
31519566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[field], &NcF));
31529566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[field], &QF));
31539566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
31549566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvC, &QC));
31559566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
315697c42addSMatthew G. Knepley     }
315763a3b9bcSJacob Faibussowitsch     PetscCheck(NcF == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, NcF, NcC);
31587c927364SMatthew G. Knepley     for (c = 0; c < cpdim; ++c) {
31597c927364SMatthew G. Knepley       PetscQuadrature  cfunc;
3160d172c84bSMatthew G. Knepley       const PetscReal *cqpoints, *cqweights;
3161d172c84bSMatthew G. Knepley       PetscInt         NqcC, NpC;
316297c42addSMatthew G. Knepley       PetscBool        found = PETSC_FALSE;
31637c927364SMatthew G. Knepley 
31649566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(QC, c, &cfunc));
31659566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(cfunc, NULL, &NqcC, &NpC, &cqpoints, &cqweights));
316663a3b9bcSJacob Faibussowitsch       PetscCheck(NqcC == NcC, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcC, NcC);
31671dca8a05SBarry Smith       PetscCheck(NpC == 1 || !feRef[field], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Do not know how to do injection for moments");
31687c927364SMatthew G. Knepley       for (f = 0; f < fpdim; ++f) {
31697c927364SMatthew G. Knepley         PetscQuadrature  ffunc;
3170d172c84bSMatthew G. Knepley         const PetscReal *fqpoints, *fqweights;
31717c927364SMatthew G. Knepley         PetscReal        sum = 0.0;
3172d172c84bSMatthew G. Knepley         PetscInt         NqcF, NpF;
31737c927364SMatthew G. Knepley 
31749566063dSJacob Faibussowitsch         PetscCall(PetscDualSpaceGetFunctional(QF, f, &ffunc));
31759566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(ffunc, NULL, &NqcF, &NpF, &fqpoints, &fqweights));
317663a3b9bcSJacob Faibussowitsch         PetscCheck(NqcF == NcF, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcF, NcF);
31777c927364SMatthew G. Knepley         if (NpC != NpF) continue;
31787c927364SMatthew G. Knepley         for (d = 0; d < dim; ++d) sum += PetscAbsReal(cqpoints[d] - fqpoints[d]);
31797c927364SMatthew G. Knepley         if (sum > 1.0e-9) continue;
3180d172c84bSMatthew G. Knepley         for (d = 0; d < NcC; ++d) sum += PetscAbsReal(cqweights[d] * fqweights[d]);
3181d172c84bSMatthew G. Knepley         if (sum < 1.0e-9) continue;
3182d172c84bSMatthew G. Knepley         cmap[offsetC + c] = offsetF + f;
318397c42addSMatthew G. Knepley         found             = PETSC_TRUE;
31847c927364SMatthew G. Knepley         break;
31857c927364SMatthew G. Knepley       }
318697c42addSMatthew G. Knepley       if (!found) {
318797c42addSMatthew G. Knepley         /* TODO We really want the average here, but some asshole put VecScatter in the interface */
3188d172c84bSMatthew G. Knepley         if (fvRef[field] || (feRef[field] && order == 0)) {
3189d172c84bSMatthew G. Knepley           cmap[offsetC + c] = offsetF + 0;
319097c42addSMatthew G. Knepley         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate matching functional for injection");
319197c42addSMatthew G. Knepley       }
31927c927364SMatthew G. Knepley     }
3193d172c84bSMatthew G. Knepley     offsetC += cpdim;
3194d172c84bSMatthew G. Knepley     offsetF += fpdim;
31957c927364SMatthew G. Knepley   }
31969371c9d4SSatish Balay   for (f = 0; f < Nf; ++f) {
31979371c9d4SSatish Balay     PetscCall(PetscFEDestroy(&feRef[f]));
31989371c9d4SSatish Balay     PetscCall(PetscFVDestroy(&fvRef[f]));
31999371c9d4SSatish Balay   }
32009566063dSJacob Faibussowitsch   PetscCall(PetscFree3(feRef, fvRef, needAvg));
32017c927364SMatthew G. Knepley 
32029566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmf, &fv));
32039566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmc, &cv));
32049566063dSJacob Faibussowitsch   PetscCall(VecGetOwnershipRange(cv, &startC, &endC));
32059566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(cglobalSection, &m));
32069566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(cTotDim, &cellCIndices, fTotDim, &cellFIndices));
32079566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m, &cindices));
32089566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m, &findices));
32097c927364SMatthew G. Knepley   for (d = 0; d < m; ++d) cindices[d] = findices[d] = -1;
32107c927364SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
32119566063dSJacob Faibussowitsch     PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, c, cellCIndices, cellFIndices));
32127c927364SMatthew G. Knepley     for (d = 0; d < cTotDim; ++d) {
32130bd915a7SMatthew G. Knepley       if ((cellCIndices[d] < startC) || (cellCIndices[d] >= endC)) continue;
321463a3b9bcSJacob Faibussowitsch       PetscCheck(!(findices[cellCIndices[d] - startC] >= 0) || !(findices[cellCIndices[d] - startC] != cellFIndices[cmap[d]]), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Coarse dof %" PetscInt_FMT " maps to both %" PetscInt_FMT " and %" PetscInt_FMT, cindices[cellCIndices[d] - startC], findices[cellCIndices[d] - startC], cellFIndices[cmap[d]]);
32157c927364SMatthew G. Knepley       cindices[cellCIndices[d] - startC] = cellCIndices[d];
32167c927364SMatthew G. Knepley       findices[cellCIndices[d] - startC] = cellFIndices[cmap[d]];
32177c927364SMatthew G. Knepley     }
32187c927364SMatthew G. Knepley   }
32199566063dSJacob Faibussowitsch   PetscCall(PetscFree(cmap));
32209566063dSJacob Faibussowitsch   PetscCall(PetscFree2(cellCIndices, cellFIndices));
32217c927364SMatthew G. Knepley 
32229566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis));
32239566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis));
32249566063dSJacob Faibussowitsch   PetscCall(VecScatterCreate(cv, cis, fv, fis, sc));
32259566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cis));
32269566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&fis));
32279566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmf, &fv));
32289566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmc, &cv));
32299566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InjectorFEM, dmc, dmf, 0, 0));
3230cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
3231cb1e1211SMatthew G Knepley }
3232a1cf66bbSMatthew G. Knepley 
32332f856554SMatthew G. Knepley /*@C
32342f856554SMatthew G. Knepley   DMPlexGetCellFields - Retrieve the field values values for a chunk of cells
32352f856554SMatthew G. Knepley 
32362f856554SMatthew G. Knepley   Input Parameters:
32372f856554SMatthew G. Knepley + dm     - The DM
32382f856554SMatthew G. Knepley . cellIS - The cells to include
32392f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
32402f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
32412f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
32422f856554SMatthew G. Knepley 
32432f856554SMatthew G. Knepley   Output Parameters:
32442f856554SMatthew G. Knepley + u   - The field coefficients
32452f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
32462f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
32472f856554SMatthew G. Knepley 
32482f856554SMatthew G. Knepley   Level: developer
32492f856554SMatthew G. Knepley 
3250db781477SPatrick Sanan .seealso: `DMPlexGetFaceFields()`
32512f856554SMatthew G. Knepley @*/
32529371c9d4SSatish Balay PetscErrorCode DMPlexGetCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a) {
32532f856554SMatthew G. Knepley   DM              plex, plexA = NULL;
3254a6e0b375SMatthew G. Knepley   DMEnclosureType encAux;
32552f856554SMatthew G. Knepley   PetscSection    section, sectionAux;
32562f856554SMatthew G. Knepley   PetscDS         prob;
32572f856554SMatthew G. Knepley   const PetscInt *cells;
32582f856554SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, totDim, totDimAux, c;
32592f856554SMatthew G. Knepley 
32602f856554SMatthew G. Knepley   PetscFunctionBegin;
32612f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3262064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
3263064a246eSJacob Faibussowitsch   if (locX_t) { PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4); }
3264064a246eSJacob Faibussowitsch   if (locA) { PetscValidHeaderSpecific(locA, VEC_CLASSID, 5); }
3265064a246eSJacob Faibussowitsch   PetscValidPointer(u, 6);
3266064a246eSJacob Faibussowitsch   PetscValidPointer(u_t, 7);
3267064a246eSJacob Faibussowitsch   PetscValidPointer(a, 8);
32689566063dSJacob Faibussowitsch   PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE));
32699566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
32709566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
32719566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob));
32729566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
32732f856554SMatthew G. Knepley   if (locA) {
32742f856554SMatthew G. Knepley     DM      dmAux;
32752f856554SMatthew G. Knepley     PetscDS probAux;
32762f856554SMatthew G. Knepley 
32779566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
32789566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
32799566063dSJacob Faibussowitsch     PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE));
32809566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
32819566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
32829566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
32832f856554SMatthew G. Knepley   }
32842f856554SMatthew G. Knepley   numCells = cEnd - cStart;
32859566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u));
32869371c9d4SSatish Balay   if (locX_t) PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, u_t));
32879371c9d4SSatish Balay   else { *u_t = NULL; }
32889371c9d4SSatish Balay   if (locA) PetscCall(DMGetWorkArray(dm, numCells * totDimAux, MPIU_SCALAR, a));
32899371c9d4SSatish Balay   else { *a = NULL; }
32902f856554SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
32912f856554SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
32922f856554SMatthew G. Knepley     const PetscInt cind = c - cStart;
32932f856554SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL, *ul = *u, *ul_t = *u_t, *al = *a;
32942f856554SMatthew G. Knepley     PetscInt       i;
32952f856554SMatthew G. Knepley 
32969566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x));
32972f856554SMatthew G. Knepley     for (i = 0; i < totDim; ++i) ul[cind * totDim + i] = x[i];
32989566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x));
32992f856554SMatthew G. Knepley     if (locX_t) {
33009566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t));
33012f856554SMatthew G. Knepley       for (i = 0; i < totDim; ++i) ul_t[cind * totDim + i] = x_t[i];
33029566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t));
33032f856554SMatthew G. Knepley     }
33042f856554SMatthew G. Knepley     if (locA) {
33052f856554SMatthew G. Knepley       PetscInt subcell;
33069566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell));
33079566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, NULL, &x));
33082f856554SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) al[cind * totDimAux + i] = x[i];
33099566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, NULL, &x));
33102f856554SMatthew G. Knepley     }
33112f856554SMatthew G. Knepley   }
33129566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
33139566063dSJacob Faibussowitsch   if (locA) PetscCall(DMDestroy(&plexA));
33149566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
33152f856554SMatthew G. Knepley   PetscFunctionReturn(0);
33162f856554SMatthew G. Knepley }
33172f856554SMatthew G. Knepley 
33182f856554SMatthew G. Knepley /*@C
33192f856554SMatthew G. Knepley   DMPlexRestoreCellFields - Restore the field values values for a chunk of cells
33202f856554SMatthew G. Knepley 
33212f856554SMatthew G. Knepley   Input Parameters:
33222f856554SMatthew G. Knepley + dm     - The DM
33232f856554SMatthew G. Knepley . cellIS - The cells to include
33242f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
33252f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
33262f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
33272f856554SMatthew G. Knepley 
33282f856554SMatthew G. Knepley   Output Parameters:
33292f856554SMatthew G. Knepley + u   - The field coefficients
33302f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
33312f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
33322f856554SMatthew G. Knepley 
33332f856554SMatthew G. Knepley   Level: developer
33342f856554SMatthew G. Knepley 
3335db781477SPatrick Sanan .seealso: `DMPlexGetFaceFields()`
33362f856554SMatthew G. Knepley @*/
33379371c9d4SSatish Balay PetscErrorCode DMPlexRestoreCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a) {
33382f856554SMatthew G. Knepley   PetscFunctionBegin;
33399566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u));
33409566063dSJacob Faibussowitsch   if (locX_t) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u_t));
33419566063dSJacob Faibussowitsch   if (locA) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, a));
33422f856554SMatthew G. Knepley   PetscFunctionReturn(0);
33432f856554SMatthew G. Knepley }
33442f856554SMatthew G. Knepley 
3345faff4f7fSMatthew G. Knepley /*
3346faff4f7fSMatthew G. Knepley   Get the auxiliary field vectors for the negative side (s = 0) and positive side (s = 1) of the interfaace
3347faff4f7fSMatthew G. Knepley */
33489371c9d4SSatish Balay static PetscErrorCode DMPlexGetHybridAuxFields(DM dm, DM dmAux[], PetscDS dsAux[], IS cellIS, Vec locA[], PetscScalar *a[]) {
334904c51a94SMatthew G. Knepley   DM              plexA[2];
3350148442b3SMatthew G. Knepley   DMEnclosureType encAux[2];
335104c51a94SMatthew G. Knepley   PetscSection    sectionAux[2];
33526528b96dSMatthew G. Knepley   const PetscInt *cells;
3353148442b3SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, c, s, totDimAux[2];
33546528b96dSMatthew G. Knepley 
33556528b96dSMatthew G. Knepley   PetscFunctionBegin;
3356148442b3SMatthew G. Knepley   PetscValidPointer(locA, 5);
335704c51a94SMatthew G. Knepley   if (!locA[0] || !locA[1]) PetscFunctionReturn(0);
3358148442b3SMatthew G. Knepley   PetscValidPointer(dmAux, 2);
3359148442b3SMatthew G. Knepley   PetscValidPointer(dsAux, 3);
3360148442b3SMatthew G. Knepley   PetscValidPointer(a, 6);
33619566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
33626528b96dSMatthew G. Knepley   numCells = cEnd - cStart;
3363148442b3SMatthew G. Knepley   for (s = 0; s < 2; ++s) {
3364148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(dmAux[s], DM_CLASSID, 2);
3365148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(dsAux[s], PETSCDS_CLASSID, 3);
3366148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(locA[s], VEC_CLASSID, 5);
33679566063dSJacob Faibussowitsch     PetscCall(DMPlexConvertPlex(dmAux[s], &plexA[s], PETSC_FALSE));
33689566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux[s], dm, &encAux[s]));
33699566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux[s], &sectionAux[s]));
33709566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[s], &totDimAux[s]));
33719566063dSJacob Faibussowitsch     PetscCall(DMGetWorkArray(dmAux[s], numCells * totDimAux[s], MPIU_SCALAR, &a[s]));
337204c51a94SMatthew G. Knepley   }
3373148442b3SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
33746528b96dSMatthew G. Knepley     const PetscInt  cell = cells ? cells[c] : c;
33756528b96dSMatthew G. Knepley     const PetscInt  cind = c - cStart;
33766528b96dSMatthew G. Knepley     const PetscInt *cone, *ornt;
33776528b96dSMatthew G. Knepley 
33789566063dSJacob Faibussowitsch     PetscCall(DMPlexGetCone(dm, cell, &cone));
33799566063dSJacob Faibussowitsch     PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
3380148442b3SMatthew G. Knepley     for (s = 0; s < 2; ++s) {
33815fedec97SMatthew G. Knepley       const PetscInt *support;
3382148442b3SMatthew G. Knepley       PetscScalar    *x = NULL, *al = a[s];
3383148442b3SMatthew G. Knepley       const PetscInt  tdA = totDimAux[s];
33845fedec97SMatthew G. Knepley       PetscInt        ssize, scell;
3385148442b3SMatthew G. Knepley       PetscInt        subface, Na, i;
33866528b96dSMatthew G. Knepley 
33879566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, cone[s], &support));
33889566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupportSize(dm, cone[s], &ssize));
338963a3b9bcSJacob Faibussowitsch       PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[s], cell, ssize);
33905fedec97SMatthew G. Knepley       if (support[0] == cell) scell = support[1];
33915fedec97SMatthew G. Knepley       else if (support[1] == cell) scell = support[0];
339263a3b9bcSJacob Faibussowitsch       else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[s], cell);
33935fedec97SMatthew G. Knepley 
33949566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(plexA[s], dm, encAux[s], scell, &subface));
33959566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexA[s], sectionAux[s], locA[s], subface, &Na, &x));
33966528b96dSMatthew G. Knepley       for (i = 0; i < Na; ++i) al[cind * tdA + i] = x[i];
33979566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexA[s], sectionAux[s], locA[s], subface, &Na, &x));
33986528b96dSMatthew G. Knepley     }
33996528b96dSMatthew G. Knepley   }
34009566063dSJacob Faibussowitsch   for (s = 0; s < 2; ++s) PetscCall(DMDestroy(&plexA[s]));
34019566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
34026528b96dSMatthew G. Knepley   PetscFunctionReturn(0);
34036528b96dSMatthew G. Knepley }
34046528b96dSMatthew G. Knepley 
34059371c9d4SSatish Balay static PetscErrorCode DMPlexRestoreHybridAuxFields(DM dmAux[], PetscDS dsAux[], IS cellIS, Vec locA[], PetscScalar *a[]) {
34066528b96dSMatthew G. Knepley   PetscFunctionBegin;
340704c51a94SMatthew G. Knepley   if (!locA[0] || !locA[1]) PetscFunctionReturn(0);
34089566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dmAux[0], 0, MPIU_SCALAR, &a[0]));
34099566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dmAux[1], 0, MPIU_SCALAR, &a[1]));
34106528b96dSMatthew G. Knepley   PetscFunctionReturn(0);
34116528b96dSMatthew G. Knepley }
34126528b96dSMatthew G. Knepley 
34132f856554SMatthew G. Knepley /*@C
34142f856554SMatthew G. Knepley   DMPlexGetFaceFields - Retrieve the field values values for a chunk of faces
34152f856554SMatthew G. Knepley 
34162f856554SMatthew G. Knepley   Input Parameters:
34172f856554SMatthew G. Knepley + dm     - The DM
34182f856554SMatthew G. Knepley . fStart - The first face to include
34192f856554SMatthew G. Knepley . fEnd   - The first face to exclude
34202f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
34212f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
34222f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
34232f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
34242f856554SMatthew G. Knepley - locaGrad - A local vector with field gradients, or NULL
34252f856554SMatthew G. Knepley 
34262f856554SMatthew G. Knepley   Output Parameters:
34272f856554SMatthew G. Knepley + Nface - The number of faces with field values
34282f856554SMatthew G. Knepley . uL - The field values at the left side of the face
34292f856554SMatthew G. Knepley - uR - The field values at the right side of the face
34302f856554SMatthew G. Knepley 
34312f856554SMatthew G. Knepley   Level: developer
34322f856554SMatthew G. Knepley 
3433db781477SPatrick Sanan .seealso: `DMPlexGetCellFields()`
34342f856554SMatthew G. Knepley @*/
34359371c9d4SSatish Balay PetscErrorCode DMPlexGetFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR) {
34362f856554SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad = NULL;
34372f856554SMatthew G. Knepley   PetscSection       section;
34382f856554SMatthew G. Knepley   PetscDS            prob;
34392f856554SMatthew G. Knepley   DMLabel            ghostLabel;
34402f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom, *x, *lgrad;
34412f856554SMatthew G. Knepley   PetscBool         *isFE;
34422f856554SMatthew G. Knepley   PetscInt           dim, Nf, f, Nc, numFaces = fEnd - fStart, iface, face;
34432f856554SMatthew G. Knepley 
34442f856554SMatthew G. Knepley   PetscFunctionBegin;
34452f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
34462f856554SMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 4);
34472f856554SMatthew G. Knepley   if (locX_t) { PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 5); }
34482f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 6);
34492f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 7);
34502f856554SMatthew G. Knepley   if (locGrad) { PetscValidHeaderSpecific(locGrad, VEC_CLASSID, 8); }
3451064a246eSJacob Faibussowitsch   PetscValidPointer(uL, 10);
3452064a246eSJacob Faibussowitsch   PetscValidPointer(uR, 11);
34539566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
34549566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
34559566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
34569566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
34579566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalComponents(prob, &Nc));
34589566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(Nf, &isFE));
34592f856554SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
34602f856554SMatthew G. Knepley     PetscObject  obj;
34612f856554SMatthew G. Knepley     PetscClassId id;
34622f856554SMatthew G. Knepley 
34639566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
34649566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
34659371c9d4SSatish Balay     if (id == PETSCFE_CLASSID) {
34669371c9d4SSatish Balay       isFE[f] = PETSC_TRUE;
34679371c9d4SSatish Balay     } else if (id == PETSCFV_CLASSID) {
34689371c9d4SSatish Balay       isFE[f] = PETSC_FALSE;
34699371c9d4SSatish Balay     } else {
34709371c9d4SSatish Balay       isFE[f] = PETSC_FALSE;
34719371c9d4SSatish Balay     }
34722f856554SMatthew G. Knepley   }
34739566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
34749566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(locX, &x));
34759566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
34769566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
34779566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
34789566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
34792f856554SMatthew G. Knepley   if (locGrad) {
34809566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locGrad, &dmGrad));
34819566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
34822f856554SMatthew G. Knepley   }
34839566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uL));
34849566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces * Nc, MPIU_SCALAR, uR));
34852f856554SMatthew G. Knepley   /* Right now just eat the extra work for FE (could make a cell loop) */
34862f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
34872f856554SMatthew G. Knepley     const PetscInt  *cells;
34882f856554SMatthew G. Knepley     PetscFVFaceGeom *fg;
34892f856554SMatthew G. Knepley     PetscFVCellGeom *cgL, *cgR;
34902f856554SMatthew G. Knepley     PetscScalar     *xL, *xR, *gL, *gR;
34912f856554SMatthew G. Knepley     PetscScalar     *uLl = *uL, *uRl = *uR;
34922f856554SMatthew G. Knepley     PetscInt         ghost, nsupp, nchild;
34932f856554SMatthew G. Knepley 
34949566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
34959566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
34969566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
34972f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
34989566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
34999566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
35009566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
35019566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
35022f856554SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
35032f856554SMatthew G. Knepley       PetscInt off;
35042f856554SMatthew G. Knepley 
35059566063dSJacob Faibussowitsch       PetscCall(PetscDSGetComponentOffset(prob, f, &off));
35062f856554SMatthew G. Knepley       if (isFE[f]) {
35072f856554SMatthew G. Knepley         const PetscInt *cone;
35082f856554SMatthew G. Knepley         PetscInt        comp, coneSizeL, coneSizeR, faceLocL, faceLocR, ldof, rdof, d;
35092f856554SMatthew G. Knepley 
35102f856554SMatthew G. Knepley         xL = xR = NULL;
35119566063dSJacob Faibussowitsch         PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
35129566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **)&xL));
35139566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **)&xR));
35149566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[0], &cone));
35159566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[0], &coneSizeL));
35169371c9d4SSatish Balay         for (faceLocL = 0; faceLocL < coneSizeL; ++faceLocL)
35179371c9d4SSatish Balay           if (cone[faceLocL] == face) break;
35189566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[1], &cone));
35199566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[1], &coneSizeR));
35209371c9d4SSatish Balay         for (faceLocR = 0; faceLocR < coneSizeR; ++faceLocR)
35219371c9d4SSatish Balay           if (cone[faceLocR] == face) break;
35221dca8a05SBarry Smith         PetscCheck(faceLocL != coneSizeL || faceLocR != coneSizeR, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %" PetscInt_FMT " in cone of cell %" PetscInt_FMT " or cell %" PetscInt_FMT, face, cells[0], cells[1]);
35232f856554SMatthew G. Knepley         /* Check that FEM field has values in the right cell (sometimes its an FV ghost cell) */
35242f856554SMatthew G. Knepley         /* TODO: this is a hack that might not be right for nonconforming */
35252f856554SMatthew G. Knepley         if (faceLocL < coneSizeL) {
35269566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocL, xL, &uLl[iface * Nc + off]));
35279566063dSJacob Faibussowitsch           if (rdof == ldof && faceLocR < coneSizeR) PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off]));
35282f856554SMatthew G. Knepley           else {
35299371c9d4SSatish Balay             for (d = 0; d < comp; ++d) uRl[iface * Nc + off + d] = uLl[iface * Nc + off + d];
35309371c9d4SSatish Balay           }
35319371c9d4SSatish Balay         } else {
35329566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface * Nc + off]));
35339566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
35342f856554SMatthew G. Knepley           for (d = 0; d < comp; ++d) uLl[iface * Nc + off + d] = uRl[iface * Nc + off + d];
35352f856554SMatthew G. Knepley         }
35369566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **)&xL));
35379566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **)&xR));
35382f856554SMatthew G. Knepley       } else {
35392f856554SMatthew G. Knepley         PetscFV  fv;
35402f856554SMatthew G. Knepley         PetscInt numComp, c;
35412f856554SMatthew G. Knepley 
35429566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, (PetscObject *)&fv));
35439566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &numComp));
35449566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[0], f, x, &xL));
35459566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[1], f, x, &xR));
35462f856554SMatthew G. Knepley         if (dmGrad) {
35472f856554SMatthew G. Knepley           PetscReal dxL[3], dxR[3];
35482f856554SMatthew G. Knepley 
35499566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], lgrad, &gL));
35509566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[1], lgrad, &gR));
35512f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgL->centroid, fg->centroid, dxL);
35522f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgR->centroid, fg->centroid, dxR);
35532f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
35542f856554SMatthew G. Knepley             uLl[iface * Nc + off + c] = xL[c] + DMPlex_DotD_Internal(dim, &gL[c * dim], dxL);
35552f856554SMatthew G. Knepley             uRl[iface * Nc + off + c] = xR[c] + DMPlex_DotD_Internal(dim, &gR[c * dim], dxR);
35562f856554SMatthew G. Knepley           }
35572f856554SMatthew G. Knepley         } else {
35582f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
35592f856554SMatthew G. Knepley             uLl[iface * Nc + off + c] = xL[c];
35602f856554SMatthew G. Knepley             uRl[iface * Nc + off + c] = xR[c];
35612f856554SMatthew G. Knepley           }
35622f856554SMatthew G. Knepley         }
35632f856554SMatthew G. Knepley       }
35642f856554SMatthew G. Knepley     }
35652f856554SMatthew G. Knepley     ++iface;
35662f856554SMatthew G. Knepley   }
35672f856554SMatthew G. Knepley   *Nface = iface;
35689566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(locX, &x));
35699566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
35709566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
3571*48a46eb9SPierre Jolivet   if (locGrad) PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
35729566063dSJacob Faibussowitsch   PetscCall(PetscFree(isFE));
35732f856554SMatthew G. Knepley   PetscFunctionReturn(0);
35742f856554SMatthew G. Knepley }
35752f856554SMatthew G. Knepley 
35762f856554SMatthew G. Knepley /*@C
35772f856554SMatthew G. Knepley   DMPlexRestoreFaceFields - Restore the field values values for a chunk of faces
35782f856554SMatthew G. Knepley 
35792f856554SMatthew G. Knepley   Input Parameters:
35802f856554SMatthew G. Knepley + dm     - The DM
35812f856554SMatthew G. Knepley . fStart - The first face to include
35822f856554SMatthew G. Knepley . fEnd   - The first face to exclude
35832f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
35842f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
35852f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
35862f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
35872f856554SMatthew G. Knepley - locaGrad - A local vector with field gradients, or NULL
35882f856554SMatthew G. Knepley 
35892f856554SMatthew G. Knepley   Output Parameters:
35902f856554SMatthew G. Knepley + Nface - The number of faces with field values
35912f856554SMatthew G. Knepley . uL - The field values at the left side of the face
35922f856554SMatthew G. Knepley - uR - The field values at the right side of the face
35932f856554SMatthew G. Knepley 
35942f856554SMatthew G. Knepley   Level: developer
35952f856554SMatthew G. Knepley 
3596db781477SPatrick Sanan .seealso: `DMPlexGetFaceFields()`
35972f856554SMatthew G. Knepley @*/
35989371c9d4SSatish Balay PetscErrorCode DMPlexRestoreFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR) {
35992f856554SMatthew G. Knepley   PetscFunctionBegin;
36009566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uL));
36019566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uR));
36022f856554SMatthew G. Knepley   PetscFunctionReturn(0);
36032f856554SMatthew G. Knepley }
36042f856554SMatthew G. Knepley 
36052f856554SMatthew G. Knepley /*@C
36062f856554SMatthew G. Knepley   DMPlexGetFaceGeometry - Retrieve the geometric values for a chunk of faces
36072f856554SMatthew G. Knepley 
36082f856554SMatthew G. Knepley   Input Parameters:
36092f856554SMatthew G. Knepley + dm     - The DM
36102f856554SMatthew G. Knepley . fStart - The first face to include
36112f856554SMatthew G. Knepley . fEnd   - The first face to exclude
36122f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
36132f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
36142f856554SMatthew G. Knepley 
36152f856554SMatthew G. Knepley   Output Parameters:
36162f856554SMatthew G. Knepley + Nface - The number of faces with field values
36172f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
36182f856554SMatthew G. Knepley - vol   - The cell volume
36192f856554SMatthew G. Knepley 
36202f856554SMatthew G. Knepley   Level: developer
36212f856554SMatthew G. Knepley 
3622db781477SPatrick Sanan .seealso: `DMPlexGetCellFields()`
36232f856554SMatthew G. Knepley @*/
36249371c9d4SSatish Balay PetscErrorCode DMPlexGetFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol) {
36252f856554SMatthew G. Knepley   DM                 dmFace, dmCell;
36262f856554SMatthew G. Knepley   DMLabel            ghostLabel;
36272f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom;
36282f856554SMatthew G. Knepley   PetscInt           dim, numFaces = fEnd - fStart, iface, face;
36292f856554SMatthew G. Knepley 
36302f856554SMatthew G. Knepley   PetscFunctionBegin;
36312f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
36322f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 4);
36332f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 5);
3634064a246eSJacob Faibussowitsch   PetscValidPointer(fgeom, 7);
3635064a246eSJacob Faibussowitsch   PetscValidPointer(vol, 8);
36369566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
36379566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
36389566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
36399566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
36409566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
36419566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
36429566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(numFaces, fgeom));
36439566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces * 2, MPIU_SCALAR, vol));
36442f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
36452f856554SMatthew G. Knepley     const PetscInt  *cells;
36462f856554SMatthew G. Knepley     PetscFVFaceGeom *fg;
36472f856554SMatthew G. Knepley     PetscFVCellGeom *cgL, *cgR;
36482f856554SMatthew G. Knepley     PetscFVFaceGeom *fgeoml = *fgeom;
36492f856554SMatthew G. Knepley     PetscReal       *voll   = *vol;
36502f856554SMatthew G. Knepley     PetscInt         ghost, d, nchild, nsupp;
36512f856554SMatthew G. Knepley 
36529566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
36539566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
36549566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
36552f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
36569566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
36579566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
36589566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
36599566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
36602f856554SMatthew G. Knepley     for (d = 0; d < dim; ++d) {
36612f856554SMatthew G. Knepley       fgeoml[iface].centroid[d] = fg->centroid[d];
36622f856554SMatthew G. Knepley       fgeoml[iface].normal[d]   = fg->normal[d];
36632f856554SMatthew G. Knepley     }
36642f856554SMatthew G. Knepley     voll[iface * 2 + 0] = cgL->volume;
36652f856554SMatthew G. Knepley     voll[iface * 2 + 1] = cgR->volume;
36662f856554SMatthew G. Knepley     ++iface;
36672f856554SMatthew G. Knepley   }
36682f856554SMatthew G. Knepley   *Nface = iface;
36699566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
36709566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
36712f856554SMatthew G. Knepley   PetscFunctionReturn(0);
36722f856554SMatthew G. Knepley }
36732f856554SMatthew G. Knepley 
36742f856554SMatthew G. Knepley /*@C
36752f856554SMatthew G. Knepley   DMPlexRestoreFaceGeometry - Restore the field values values for a chunk of faces
36762f856554SMatthew G. Knepley 
36772f856554SMatthew G. Knepley   Input Parameters:
36782f856554SMatthew G. Knepley + dm     - The DM
36792f856554SMatthew G. Knepley . fStart - The first face to include
36802f856554SMatthew G. Knepley . fEnd   - The first face to exclude
36812f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
36822f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
36832f856554SMatthew G. Knepley 
36842f856554SMatthew G. Knepley   Output Parameters:
36852f856554SMatthew G. Knepley + Nface - The number of faces with field values
36862f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
36872f856554SMatthew G. Knepley - vol   - The cell volume
36882f856554SMatthew G. Knepley 
36892f856554SMatthew G. Knepley   Level: developer
36902f856554SMatthew G. Knepley 
3691db781477SPatrick Sanan .seealso: `DMPlexGetFaceFields()`
36922f856554SMatthew G. Knepley @*/
36939371c9d4SSatish Balay PetscErrorCode DMPlexRestoreFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol) {
36942f856554SMatthew G. Knepley   PetscFunctionBegin;
36959566063dSJacob Faibussowitsch   PetscCall(PetscFree(*fgeom));
36969566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_REAL, vol));
36972f856554SMatthew G. Knepley   PetscFunctionReturn(0);
36982f856554SMatthew G. Knepley }
36992f856554SMatthew G. Knepley 
37009371c9d4SSatish Balay PetscErrorCode DMSNESGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom) {
3701a1cf66bbSMatthew G. Knepley   char           composeStr[33] = {0};
3702a1cf66bbSMatthew G. Knepley   PetscObjectId  id;
3703a1cf66bbSMatthew G. Knepley   PetscContainer container;
3704a1cf66bbSMatthew G. Knepley 
3705a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
37069566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad, &id));
370763a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMSNESGetFEGeom_%" PetscInt64_FMT "\n", id));
37089566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject)pointIS, composeStr, (PetscObject *)&container));
3709a1cf66bbSMatthew G. Knepley   if (container) {
37109566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **)geom));
3711a1cf66bbSMatthew G. Knepley   } else {
37129566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom));
37139566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF, &container));
37149566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *)*geom));
37159566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom));
37169566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject)pointIS, composeStr, (PetscObject)container));
37179566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
3718a1cf66bbSMatthew G. Knepley   }
3719a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
3720a1cf66bbSMatthew G. Knepley }
3721a1cf66bbSMatthew G. Knepley 
37229371c9d4SSatish Balay PetscErrorCode DMSNESRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom) {
3723a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
3724a1cf66bbSMatthew G. Knepley   *geom = NULL;
3725a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
3726a1cf66bbSMatthew G. Knepley }
3727a1cf66bbSMatthew G. Knepley 
37289371c9d4SSatish Balay PetscErrorCode DMPlexComputeResidual_Patch_Internal(DM dm, PetscSection section, IS cellIS, PetscReal t, Vec locX, Vec locX_t, Vec locF, void *user) {
372992d50984SMatthew G. Knepley   DM_Plex        *mesh       = (DM_Plex *)dm->data;
373092d50984SMatthew G. Knepley   const char     *name       = "Residual";
373192d50984SMatthew G. Knepley   DM              dmAux      = NULL;
373292d50984SMatthew G. Knepley   DMLabel         ghostLabel = NULL;
373392d50984SMatthew G. Knepley   PetscDS         prob       = NULL;
373492d50984SMatthew G. Knepley   PetscDS         probAux    = NULL;
373592d50984SMatthew G. Knepley   PetscBool       useFEM     = PETSC_FALSE;
373692d50984SMatthew G. Knepley   PetscBool       isImplicit = (locX_t || t == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
373792d50984SMatthew G. Knepley   DMField         coordField = NULL;
3738c0006e53SPatrick Farrell   Vec             locA;
3739c0006e53SPatrick Farrell   PetscScalar    *u = NULL, *u_t, *a, *uL = NULL, *uR = NULL;
374092d50984SMatthew G. Knepley   IS              chunkIS;
374192d50984SMatthew G. Knepley   const PetscInt *cells;
374292d50984SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
3743364207b6SKarl Rupp   PetscInt        Nf, f, totDim, totDimAux, numChunks, cellChunkSize, chunk, fStart, fEnd;
374492d50984SMatthew G. Knepley   PetscInt        maxDegree = PETSC_MAX_INT;
374506ad1575SMatthew G. Knepley   PetscFormKey    key;
374692d50984SMatthew G. Knepley   PetscQuadrature affineQuad = NULL, *quads = NULL;
374792d50984SMatthew G. Knepley   PetscFEGeom    *affineGeom = NULL, **geoms = NULL;
374892d50984SMatthew G. Knepley 
374992d50984SMatthew G. Knepley   PetscFunctionBegin;
37509566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0));
375192d50984SMatthew G. Knepley   /* FEM+FVM */
375292d50984SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
37539566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
37549566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
37559566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
37569566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
37579566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
375892d50984SMatthew G. Knepley   if (locA) {
37599566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
37609566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
37619566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
376292d50984SMatthew G. Knepley   }
376392d50984SMatthew G. Knepley   /* 2: Get geometric data */
376492d50984SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
376592d50984SMatthew G. Knepley     PetscObject  obj;
376692d50984SMatthew G. Knepley     PetscClassId id;
376792d50984SMatthew G. Knepley     PetscBool    fimp;
376892d50984SMatthew G. Knepley 
37699566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(prob, f, &fimp));
377092d50984SMatthew G. Knepley     if (isImplicit != fimp) continue;
37719566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
37729566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
377392d50984SMatthew G. Knepley     if (id == PETSCFE_CLASSID) { useFEM = PETSC_TRUE; }
37745f80ce2aSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Use of FVM with PCPATCH not yet implemented");
377592d50984SMatthew G. Knepley   }
377692d50984SMatthew G. Knepley   if (useFEM) {
37779566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
37789566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
377992d50984SMatthew G. Knepley     if (maxDegree <= 1) {
37809566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad));
3781*48a46eb9SPierre Jolivet       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
378292d50984SMatthew G. Knepley     } else {
37839566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
378492d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
378592d50984SMatthew G. Knepley         PetscObject  obj;
378692d50984SMatthew G. Knepley         PetscClassId id;
378792d50984SMatthew G. Knepley         PetscBool    fimp;
378892d50984SMatthew G. Knepley 
37899566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(prob, f, &fimp));
379092d50984SMatthew G. Knepley         if (isImplicit != fimp) continue;
37919566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
37929566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
379392d50984SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
379492d50984SMatthew G. Knepley           PetscFE fe = (PetscFE)obj;
379592d50984SMatthew G. Knepley 
37969566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
37979566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
37989566063dSJacob Faibussowitsch           PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
379992d50984SMatthew G. Knepley         }
380092d50984SMatthew G. Knepley       }
380192d50984SMatthew G. Knepley     }
380292d50984SMatthew G. Knepley   }
380392d50984SMatthew G. Knepley   /* Loop over chunks */
38049566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
38059566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
38069566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
380792d50984SMatthew G. Knepley   numCells      = cEnd - cStart;
380892d50984SMatthew G. Knepley   numChunks     = 1;
380992d50984SMatthew G. Knepley   cellChunkSize = numCells / numChunks;
381092d50984SMatthew G. Knepley   numChunks     = PetscMin(1, numCells);
38116528b96dSMatthew G. Knepley   key.label     = NULL;
38126528b96dSMatthew G. Knepley   key.value     = 0;
381306ad1575SMatthew G. Knepley   key.part      = 0;
381492d50984SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
3815c0006e53SPatrick Farrell     PetscScalar     *elemVec, *fluxL = NULL, *fluxR = NULL;
3816c0006e53SPatrick Farrell     PetscReal       *vol   = NULL;
3817c0006e53SPatrick Farrell     PetscFVFaceGeom *fgeom = NULL;
381892d50984SMatthew G. Knepley     PetscInt         cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
3819c0006e53SPatrick Farrell     PetscInt         numFaces = 0;
382092d50984SMatthew G. Knepley 
382192d50984SMatthew G. Knepley     /* Extract field coefficients */
382292d50984SMatthew G. Knepley     if (useFEM) {
38239566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
38249566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
38259566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
38269566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells * totDim));
382792d50984SMatthew G. Knepley     }
382892d50984SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
382992d50984SMatthew G. Knepley     /* Loop over fields */
383092d50984SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
383192d50984SMatthew G. Knepley       PetscObject  obj;
383292d50984SMatthew G. Knepley       PetscClassId id;
383392d50984SMatthew G. Knepley       PetscBool    fimp;
383492d50984SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
383592d50984SMatthew G. Knepley 
38366528b96dSMatthew G. Knepley       key.field = f;
38379566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(prob, f, &fimp));
383892d50984SMatthew G. Knepley       if (isImplicit != fimp) continue;
38399566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
38409566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
384192d50984SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
384292d50984SMatthew G. Knepley         PetscFE         fe        = (PetscFE)obj;
384392d50984SMatthew G. Knepley         PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[f];
384492d50984SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
384592d50984SMatthew G. Knepley         PetscQuadrature quad      = affineQuad ? affineQuad : quads[f];
384692d50984SMatthew G. Knepley         PetscInt        Nq, Nb;
384792d50984SMatthew G. Knepley 
38489566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
38499566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
38509566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
385192d50984SMatthew G. Knepley         blockSize = Nb;
385292d50984SMatthew G. Knepley         batchSize = numBlocks * blockSize;
38539566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
385492d50984SMatthew G. Knepley         numChunks = numCells / (numBatches * batchSize);
385592d50984SMatthew G. Knepley         Ne        = numChunks * numBatches * batchSize;
385692d50984SMatthew G. Knepley         Nr        = numCells % (numBatches * batchSize);
385792d50984SMatthew G. Knepley         offset    = numCells - Nr;
385892d50984SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
385992d50984SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
38609566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom));
38619566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(prob, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
38629566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom));
38639566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(prob, key, Nr, chunkGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, &elemVec[offset * totDim]));
38649566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom));
386592d50984SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
386692d50984SMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
386792d50984SMatthew G. Knepley 
386892d50984SMatthew G. Knepley         Ne = numFaces;
386992d50984SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
387092d50984SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
38719566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
387263a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
387392d50984SMatthew G. Knepley     }
387492d50984SMatthew G. Knepley     /* Loop over domain */
387592d50984SMatthew G. Knepley     if (useFEM) {
387692d50984SMatthew G. Knepley       /* Add elemVec to locX */
387792d50984SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
387892d50984SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
387992d50984SMatthew G. Knepley         const PetscInt cind = c - cStart;
388092d50984SMatthew G. Knepley 
38819566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim]));
388292d50984SMatthew G. Knepley         if (ghostLabel) {
388392d50984SMatthew G. Knepley           PetscInt ghostVal;
388492d50984SMatthew G. Knepley 
38859566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
388692d50984SMatthew G. Knepley           if (ghostVal > 0) continue;
388792d50984SMatthew G. Knepley         }
38889566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES));
388992d50984SMatthew G. Knepley       }
389092d50984SMatthew G. Knepley     }
389192d50984SMatthew G. Knepley     /* Handle time derivative */
389292d50984SMatthew G. Knepley     if (locX_t) {
389392d50984SMatthew G. Knepley       PetscScalar *x_t, *fa;
389492d50984SMatthew G. Knepley 
38959566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
38969566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
389792d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
389892d50984SMatthew G. Knepley         PetscFV      fv;
389992d50984SMatthew G. Knepley         PetscObject  obj;
390092d50984SMatthew G. Knepley         PetscClassId id;
390192d50984SMatthew G. Knepley         PetscInt     pdim, d;
390292d50984SMatthew G. Knepley 
39039566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
39049566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
390592d50984SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
390692d50984SMatthew G. Knepley         fv = (PetscFV)obj;
39079566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
390892d50984SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
390992d50984SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
391092d50984SMatthew G. Knepley           PetscScalar   *u_t, *r;
391192d50984SMatthew G. Knepley 
391292d50984SMatthew G. Knepley           if (ghostLabel) {
391392d50984SMatthew G. Knepley             PetscInt ghostVal;
391492d50984SMatthew G. Knepley 
39159566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
391692d50984SMatthew G. Knepley             if (ghostVal > 0) continue;
391792d50984SMatthew G. Knepley           }
39189566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
39199566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
392092d50984SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
392192d50984SMatthew G. Knepley         }
392292d50984SMatthew G. Knepley       }
39239566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
39249566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
392592d50984SMatthew G. Knepley     }
392692d50984SMatthew G. Knepley     if (useFEM) {
39279566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
39289566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
392992d50984SMatthew G. Knepley     }
393092d50984SMatthew G. Knepley   }
39319566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
39329566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
393392d50984SMatthew G. Knepley   /* TODO Could include boundary residual here (see DMPlexComputeResidual_Internal) */
393492d50984SMatthew G. Knepley   if (useFEM) {
393592d50984SMatthew G. Knepley     if (maxDegree <= 1) {
39369566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
39379566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
393892d50984SMatthew G. Knepley     } else {
393992d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
39409566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
39419566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
394292d50984SMatthew G. Knepley       }
39439566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads, geoms));
394492d50984SMatthew G. Knepley     }
394592d50984SMatthew G. Knepley   }
39469566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0));
394792d50984SMatthew G. Knepley   PetscFunctionReturn(0);
394892d50984SMatthew G. Knepley }
394992d50984SMatthew G. Knepley 
3950a1cf66bbSMatthew G. Knepley /*
3951a1cf66bbSMatthew G. Knepley   We always assemble JacP, and if the matrix is different from Jac and two different sets of point functions are provided, we also assemble Jac
3952a1cf66bbSMatthew G. Knepley 
3953a1cf66bbSMatthew G. Knepley   X   - The local solution vector
3954a5b23f4aSJose E. Roman   X_t - The local solution time derivative vector, or NULL
3955a1cf66bbSMatthew G. Knepley */
39569371c9d4SSatish Balay PetscErrorCode DMPlexComputeJacobian_Patch_Internal(DM dm, PetscSection section, PetscSection globalSection, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *ctx) {
3957a1cf66bbSMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *)dm->data;
3958a1cf66bbSMatthew G. Knepley   const char     *name = "Jacobian", *nameP = "JacobianPre";
3959a1cf66bbSMatthew G. Knepley   DM              dmAux = NULL;
3960a1cf66bbSMatthew G. Knepley   PetscDS         prob, probAux = NULL;
3961a1cf66bbSMatthew G. Knepley   PetscSection    sectionAux = NULL;
3962a1cf66bbSMatthew G. Knepley   Vec             A;
3963a1cf66bbSMatthew G. Knepley   DMField         coordField;
3964a1cf66bbSMatthew G. Knepley   PetscFEGeom    *cgeomFEM;
3965a1cf66bbSMatthew G. Knepley   PetscQuadrature qGeom = NULL;
3966a1cf66bbSMatthew G. Knepley   Mat             J = Jac, JP = JacP;
3967a1cf66bbSMatthew G. Knepley   PetscScalar    *work, *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL, *elemMatD = NULL;
3968e432b41dSStefano Zampini   PetscBool       hasJac, hasPrec, hasDyn, assembleJac, *isFE, hasFV = PETSC_FALSE;
3969a1cf66bbSMatthew G. Knepley   const PetscInt *cells;
397006ad1575SMatthew G. Knepley   PetscFormKey    key;
39719b2fc754SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ, maxDegree, numCells, cStart, cEnd, numChunks, chunkSize, chunk, totDim, totDimAux = 0, sz, wsz, off = 0, offCell = 0;
3972a1cf66bbSMatthew G. Knepley 
3973a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
39749566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
39759566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
39769566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
39779566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
39789566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &A));
39799a2a23afSMatthew G. Knepley   if (A) {
39809566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
39819566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
39829566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
3983a1cf66bbSMatthew G. Knepley   }
3984a1cf66bbSMatthew G. Knepley   /* Get flags */
39859566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
39869566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, Nf, MPIU_BOOL, &isFE));
3987a1cf66bbSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
3988a1cf66bbSMatthew G. Knepley     PetscObject  disc;
3989a1cf66bbSMatthew G. Knepley     PetscClassId id;
39909566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &disc));
39919566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(disc, &id));
39929371c9d4SSatish Balay     if (id == PETSCFE_CLASSID) {
39939371c9d4SSatish Balay       isFE[fieldI] = PETSC_TRUE;
39949371c9d4SSatish Balay     } else if (id == PETSCFV_CLASSID) {
39959371c9d4SSatish Balay       hasFV        = PETSC_TRUE;
39969371c9d4SSatish Balay       isFE[fieldI] = PETSC_FALSE;
39979371c9d4SSatish Balay     }
3998a1cf66bbSMatthew G. Knepley   }
39999566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
40009566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
40019566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
4002a1cf66bbSMatthew G. Knepley   assembleJac = hasJac && hasPrec && (Jac != JacP) ? PETSC_TRUE : PETSC_FALSE;
4003a1cf66bbSMatthew G. Knepley   hasDyn      = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
40049566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE)); /* No allocated space for FV stuff, so ignore the zero entries */
40059566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
40069566063dSJacob Faibussowitsch   if (probAux) PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
4007a1cf66bbSMatthew G. Knepley   /* Compute batch sizes */
4008a1cf66bbSMatthew G. Knepley   if (isFE[0]) {
4009a1cf66bbSMatthew G. Knepley     PetscFE         fe;
4010a1cf66bbSMatthew G. Knepley     PetscQuadrature q;
4011a1cf66bbSMatthew G. Knepley     PetscInt        numQuadPoints, numBatches, batchSize, numBlocks, blockSize, Nb;
4012a1cf66bbSMatthew G. Knepley 
40139566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe));
40149566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &q));
40159566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(q, NULL, NULL, &numQuadPoints, NULL, NULL));
40169566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
40179566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
4018a1cf66bbSMatthew G. Knepley     blockSize = Nb * numQuadPoints;
4019a1cf66bbSMatthew G. Knepley     batchSize = numBlocks * blockSize;
4020a1cf66bbSMatthew G. Knepley     chunkSize = numBatches * batchSize;
4021a1cf66bbSMatthew G. Knepley     numChunks = numCells / chunkSize + numCells % chunkSize;
40229566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
4023a1cf66bbSMatthew G. Knepley   } else {
4024a1cf66bbSMatthew G. Knepley     chunkSize = numCells;
4025a1cf66bbSMatthew G. Knepley     numChunks = 1;
4026a1cf66bbSMatthew G. Knepley   }
4027a1cf66bbSMatthew G. Knepley   /* Get work space */
4028a1cf66bbSMatthew G. Knepley   wsz = (((X ? 1 : 0) + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize;
40299566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, wsz, MPIU_SCALAR, &work));
40309566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(work, wsz));
4031a1cf66bbSMatthew G. Knepley   off      = 0;
4032a1cf66bbSMatthew G. Knepley   u        = X ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL;
4033a1cf66bbSMatthew G. Knepley   u_t      = X_t ? (sz = chunkSize * totDim, off += sz, work + off - sz) : NULL;
4034a1cf66bbSMatthew G. Knepley   a        = dmAux ? (sz = chunkSize * totDimAux, off += sz, work + off - sz) : NULL;
4035a1cf66bbSMatthew G. Knepley   elemMat  = hasJac ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL;
4036a1cf66bbSMatthew G. Knepley   elemMatP = hasPrec ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL;
4037a1cf66bbSMatthew G. Knepley   elemMatD = hasDyn ? (sz = chunkSize * totDim * totDim, off += sz, work + off - sz) : NULL;
403863a3b9bcSJacob Faibussowitsch   PetscCheck(off == wsz, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error is workspace size %" PetscInt_FMT " should be %" PetscInt_FMT, off, wsz);
4039a1cf66bbSMatthew G. Knepley   /* Setup geometry */
40409566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
40419566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
40429566063dSJacob Faibussowitsch   if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
4043a1cf66bbSMatthew G. Knepley   if (!qGeom) {
4044a1cf66bbSMatthew G. Knepley     PetscFE fe;
4045a1cf66bbSMatthew G. Knepley 
40469566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *)&fe));
40479566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &qGeom));
40489566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)qGeom));
4049a1cf66bbSMatthew G. Knepley   }
40509566063dSJacob Faibussowitsch   PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
4051a1cf66bbSMatthew G. Knepley   /* Compute volume integrals */
40529566063dSJacob Faibussowitsch   if (assembleJac) PetscCall(MatZeroEntries(J));
40539566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(JP));
40546528b96dSMatthew G. Knepley   key.label = NULL;
40556528b96dSMatthew G. Knepley   key.value = 0;
405606ad1575SMatthew G. Knepley   key.part  = 0;
4057a1cf66bbSMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk, offCell += chunkSize) {
4058a1cf66bbSMatthew G. Knepley     const PetscInt Ncell = PetscMin(chunkSize, numCells - offCell);
4059a1cf66bbSMatthew G. Knepley     PetscInt       c;
4060a1cf66bbSMatthew G. Knepley 
4061a1cf66bbSMatthew G. Knepley     /* Extract values */
4062a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4063a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c + offCell] : c + offCell;
4064a1cf66bbSMatthew G. Knepley       PetscScalar   *x = NULL, *x_t = NULL;
4065a1cf66bbSMatthew G. Knepley       PetscInt       i;
4066a1cf66bbSMatthew G. Knepley 
4067a1cf66bbSMatthew G. Knepley       if (X) {
40689566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x));
4069a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[c * totDim + i] = x[i];
40709566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x));
4071a1cf66bbSMatthew G. Knepley       }
4072a1cf66bbSMatthew G. Knepley       if (X_t) {
40739566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t));
4074a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[c * totDim + i] = x_t[i];
40759566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t));
4076a1cf66bbSMatthew G. Knepley       }
4077a1cf66bbSMatthew G. Knepley       if (dmAux) {
40789566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, A, cell, NULL, &x));
4079a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[c * totDimAux + i] = x[i];
40809566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, A, cell, NULL, &x));
4081a1cf66bbSMatthew G. Knepley       }
4082a1cf66bbSMatthew G. Knepley     }
4083a1cf66bbSMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
4084a1cf66bbSMatthew G. Knepley       PetscFE fe;
40859566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
4086a1cf66bbSMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
40876528b96dSMatthew G. Knepley         key.field = fieldI * Nf + fieldJ;
40889566063dSJacob Faibussowitsch         if (hasJac) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMat));
40899566063dSJacob Faibussowitsch         if (hasPrec) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatP));
40909566063dSJacob Faibussowitsch         if (hasDyn) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatD));
4091a1cf66bbSMatthew G. Knepley       }
4092a1cf66bbSMatthew G. Knepley       /* For finite volume, add the identity */
4093a1cf66bbSMatthew G. Knepley       if (!isFE[fieldI]) {
4094a1cf66bbSMatthew G. Knepley         PetscFV  fv;
4095a1cf66bbSMatthew G. Knepley         PetscInt eOffset = 0, Nc, fc, foff;
4096a1cf66bbSMatthew G. Knepley 
40979566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(prob, fieldI, &foff));
40989566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv));
40999566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &Nc));
4100a1cf66bbSMatthew G. Knepley         for (c = 0; c < chunkSize; ++c, eOffset += totDim * totDim) {
4101a1cf66bbSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
4102a1cf66bbSMatthew G. Knepley             const PetscInt i = foff + fc;
4103a1cf66bbSMatthew G. Knepley             if (hasJac) { elemMat[eOffset + i * totDim + i] = 1.0; }
4104a1cf66bbSMatthew G. Knepley             if (hasPrec) { elemMatP[eOffset + i * totDim + i] = 1.0; }
4105a1cf66bbSMatthew G. Knepley           }
4106a1cf66bbSMatthew G. Knepley         }
4107a1cf66bbSMatthew G. Knepley       }
4108a1cf66bbSMatthew G. Knepley     }
4109a1cf66bbSMatthew G. Knepley     /*   Add contribution from X_t */
41109371c9d4SSatish Balay     if (hasDyn) {
41119371c9d4SSatish Balay       for (c = 0; c < chunkSize * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
41129371c9d4SSatish Balay     }
4113a1cf66bbSMatthew G. Knepley     /* Insert values into matrix */
4114a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4115a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c + offCell] : c + offCell;
4116a1cf66bbSMatthew G. Knepley       if (mesh->printFEM > 1) {
41179566063dSJacob Faibussowitsch         if (hasJac) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[(c - cStart) * totDim * totDim]));
41189566063dSJacob Faibussowitsch         if (hasPrec) PetscCall(DMPrintCellMatrix(cell, nameP, totDim, totDim, &elemMatP[(c - cStart) * totDim * totDim]));
4119a1cf66bbSMatthew G. Knepley       }
41209566063dSJacob Faibussowitsch       if (assembleJac) PetscCall(DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES));
41219566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JP, cell, &elemMat[(c - cStart) * totDim * totDim], ADD_VALUES));
4122a1cf66bbSMatthew G. Knepley     }
4123a1cf66bbSMatthew G. Knepley   }
4124a1cf66bbSMatthew G. Knepley   /* Cleanup */
41259566063dSJacob Faibussowitsch   PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
41269566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&qGeom));
41279566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
41289566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, Nf, MPIU_BOOL, &isFE));
41299566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, ((1 + (X_t ? 1 : 0) + (dmAux ? 1 : 0)) * totDim + ((hasJac ? 1 : 0) + (hasPrec ? 1 : 0) + (hasDyn ? 1 : 0)) * totDim * totDim) * chunkSize, MPIU_SCALAR, &work));
4130a1cf66bbSMatthew G. Knepley   /* Compute boundary integrals */
41319566063dSJacob Faibussowitsch   /* PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, ctx)); */
4132a1cf66bbSMatthew G. Knepley   /* Assemble matrix */
41339371c9d4SSatish Balay   if (assembleJac) {
41349371c9d4SSatish Balay     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
41359371c9d4SSatish Balay     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
41369371c9d4SSatish Balay   }
41379371c9d4SSatish Balay   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
41389371c9d4SSatish Balay   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
41399566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
4140a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
4141a1cf66bbSMatthew G. Knepley }
41423e9753d6SMatthew G. Knepley 
41433e9753d6SMatthew G. Knepley /******** FEM Assembly Function ********/
41443e9753d6SMatthew G. Knepley 
41459371c9d4SSatish Balay static PetscErrorCode DMConvertPlex_Internal(DM dm, DM *plex, PetscBool copy) {
41463e9753d6SMatthew G. Knepley   PetscBool isPlex;
41473e9753d6SMatthew G. Knepley 
41483e9753d6SMatthew G. Knepley   PetscFunctionBegin;
41499566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &isPlex));
41503e9753d6SMatthew G. Knepley   if (isPlex) {
41513e9753d6SMatthew G. Knepley     *plex = dm;
41529566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)dm));
41533e9753d6SMatthew G. Knepley   } else {
41549566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject)dm, "dm_plex", (PetscObject *)plex));
41553e9753d6SMatthew G. Knepley     if (!*plex) {
41569566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm, DMPLEX, plex));
41579566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject)dm, "dm_plex", (PetscObject)*plex));
41581baa6e33SBarry Smith       if (copy) PetscCall(DMCopyAuxiliaryVec(dm, *plex));
41593e9753d6SMatthew G. Knepley     } else {
41609566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)*plex));
41613e9753d6SMatthew G. Knepley     }
41623e9753d6SMatthew G. Knepley   }
41633e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
41643e9753d6SMatthew G. Knepley }
41653e9753d6SMatthew G. Knepley 
41663e9753d6SMatthew G. Knepley /*@
41673e9753d6SMatthew G. Knepley   DMPlexGetGeometryFVM - Return precomputed geometric data
41683e9753d6SMatthew G. Knepley 
41693e9753d6SMatthew G. Knepley   Collective on DM
41703e9753d6SMatthew G. Knepley 
41713e9753d6SMatthew G. Knepley   Input Parameter:
41723e9753d6SMatthew G. Knepley . dm - The DM
41733e9753d6SMatthew G. Knepley 
41743e9753d6SMatthew G. Knepley   Output Parameters:
41753e9753d6SMatthew G. Knepley + facegeom - The values precomputed from face geometry
41763e9753d6SMatthew G. Knepley . cellgeom - The values precomputed from cell geometry
41773e9753d6SMatthew G. Knepley - minRadius - The minimum radius over the mesh of an inscribed sphere in a cell
41783e9753d6SMatthew G. Knepley 
41793e9753d6SMatthew G. Knepley   Level: developer
41803e9753d6SMatthew G. Knepley 
4181db781477SPatrick Sanan .seealso: `DMTSSetRHSFunctionLocal()`
41823e9753d6SMatthew G. Knepley @*/
41839371c9d4SSatish Balay PetscErrorCode DMPlexGetGeometryFVM(DM dm, Vec *facegeom, Vec *cellgeom, PetscReal *minRadius) {
41843e9753d6SMatthew G. Knepley   DM plex;
41853e9753d6SMatthew G. Knepley 
41863e9753d6SMatthew G. Knepley   PetscFunctionBegin;
41873e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
41889566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE));
41899566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, NULL, cellgeom, facegeom, NULL));
41909566063dSJacob Faibussowitsch   if (minRadius) PetscCall(DMPlexGetMinRadius(plex, minRadius));
41919566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
41923e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
41933e9753d6SMatthew G. Knepley }
41943e9753d6SMatthew G. Knepley 
41953e9753d6SMatthew G. Knepley /*@
41963e9753d6SMatthew G. Knepley   DMPlexGetGradientDM - Return gradient data layout
41973e9753d6SMatthew G. Knepley 
41983e9753d6SMatthew G. Knepley   Collective on DM
41993e9753d6SMatthew G. Knepley 
42003e9753d6SMatthew G. Knepley   Input Parameters:
42013e9753d6SMatthew G. Knepley + dm - The DM
42023e9753d6SMatthew G. Knepley - fv - The PetscFV
42033e9753d6SMatthew G. Knepley 
42043e9753d6SMatthew G. Knepley   Output Parameter:
42053e9753d6SMatthew G. Knepley . dmGrad - The layout for gradient values
42063e9753d6SMatthew G. Knepley 
42073e9753d6SMatthew G. Knepley   Level: developer
42083e9753d6SMatthew G. Knepley 
4209db781477SPatrick Sanan .seealso: `DMPlexGetGeometryFVM()`
42103e9753d6SMatthew G. Knepley @*/
42119371c9d4SSatish Balay PetscErrorCode DMPlexGetGradientDM(DM dm, PetscFV fv, DM *dmGrad) {
42123e9753d6SMatthew G. Knepley   DM        plex;
42133e9753d6SMatthew G. Knepley   PetscBool computeGradients;
42143e9753d6SMatthew G. Knepley 
42153e9753d6SMatthew G. Knepley   PetscFunctionBegin;
42163e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
42173e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(fv, PETSCFV_CLASSID, 2);
42183e9753d6SMatthew G. Knepley   PetscValidPointer(dmGrad, 3);
42199566063dSJacob Faibussowitsch   PetscCall(PetscFVGetComputeGradients(fv, &computeGradients));
42209371c9d4SSatish Balay   if (!computeGradients) {
42219371c9d4SSatish Balay     *dmGrad = NULL;
42229371c9d4SSatish Balay     PetscFunctionReturn(0);
42239371c9d4SSatish Balay   }
42249566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm, &plex, PETSC_TRUE));
42259566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, fv, NULL, NULL, dmGrad));
42269566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
42273e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
42283e9753d6SMatthew G. Knepley }
42293e9753d6SMatthew G. Knepley 
42309371c9d4SSatish Balay static PetscErrorCode DMPlexComputeBdResidual_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF, DMField coordField, IS facetIS) {
42313e9753d6SMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *)dm->data;
42323e9753d6SMatthew G. Knepley   DM              plex = NULL, plexA = NULL;
42333e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
42343e9753d6SMatthew G. Knepley   PetscDS         prob, probAux       = NULL;
42353e9753d6SMatthew G. Knepley   PetscSection    section, sectionAux = NULL;
42363e9753d6SMatthew G. Knepley   Vec             locA = NULL;
42373e9753d6SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t = NULL, *a = NULL, *elemVec = NULL;
42383e9753d6SMatthew G. Knepley   PetscInt        totDim, totDimAux = 0;
42393e9753d6SMatthew G. Knepley 
42403e9753d6SMatthew G. Knepley   PetscFunctionBegin;
42419566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
42429566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
42439566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
42449566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
42459566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
42463e9753d6SMatthew G. Knepley   if (locA) {
42473e9753d6SMatthew G. Knepley     DM dmAux;
42483e9753d6SMatthew G. Knepley 
42499566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
42509566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
42519566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
42529566063dSJacob Faibussowitsch     PetscCall(DMGetDS(plexA, &probAux));
42539566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
42549566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
42553e9753d6SMatthew G. Knepley   }
42560c290148SMatthew G. Knepley   {
42573e9753d6SMatthew G. Knepley     PetscFEGeom    *fgeom;
42583e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
42593e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
42603e9753d6SMatthew G. Knepley     IS              pointIS;
42613e9753d6SMatthew G. Knepley     const PetscInt *points;
42623e9753d6SMatthew G. Knepley     PetscInt        numFaces, face, Nq;
42633e9753d6SMatthew G. Knepley 
42649566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(key.label, key.value, &pointIS));
42650c290148SMatthew G. Knepley     if (!pointIS) goto end; /* No points with that id on this process */
42663e9753d6SMatthew G. Knepley     {
42673e9753d6SMatthew G. Knepley       IS isectIS;
42683e9753d6SMatthew G. Knepley 
42693e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
42709566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
42719566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
42723e9753d6SMatthew G. Knepley       pointIS = isectIS;
42733e9753d6SMatthew G. Knepley     }
42749566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
42759566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
42769566063dSJacob Faibussowitsch     PetscCall(PetscMalloc4(numFaces * totDim, &u, locX_t ? numFaces * totDim : 0, &u_t, numFaces * totDim, &elemVec, locA ? numFaces * totDimAux : 0, &a));
42779566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
4278*48a46eb9SPierre Jolivet     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
42793e9753d6SMatthew G. Knepley     if (!qGeom) {
42803e9753d6SMatthew G. Knepley       PetscFE fe;
42813e9753d6SMatthew G. Knepley 
42829566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe));
42839566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
42849566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
42853e9753d6SMatthew G. Knepley     }
42869566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
42879566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
42883e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
4289f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
42903e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
4291f15274beSMatthew Knepley       PetscInt       i;
42923e9753d6SMatthew G. Knepley 
42939566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
42949566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
42953e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i];
42969566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
42973e9753d6SMatthew G. Knepley       if (locX_t) {
42989566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
42993e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i];
43009566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
43013e9753d6SMatthew G. Knepley       }
43023e9753d6SMatthew G. Knepley       if (locA) {
43033e9753d6SMatthew G. Knepley         PetscInt subp;
43043e9753d6SMatthew G. Knepley 
43059566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
43069566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
43073e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i];
43089566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
43093e9753d6SMatthew G. Knepley       }
43103e9753d6SMatthew G. Knepley     }
43119566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(elemVec, numFaces * totDim));
43123e9753d6SMatthew G. Knepley     {
43133e9753d6SMatthew G. Knepley       PetscFE      fe;
43143e9753d6SMatthew G. Knepley       PetscInt     Nb;
43153e9753d6SMatthew G. Knepley       PetscFEGeom *chunkGeom = NULL;
43163e9753d6SMatthew G. Knepley       /* Conforming batches */
43173e9753d6SMatthew G. Knepley       PetscInt     numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
43183e9753d6SMatthew G. Knepley       /* Remainder */
43193e9753d6SMatthew G. Knepley       PetscInt     Nr, offset;
43203e9753d6SMatthew G. Knepley 
43219566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *)&fe));
43229566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
43239566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
43243e9753d6SMatthew G. Knepley       /* TODO: documentation is unclear about what is going on with these numbers: how should Nb / Nq factor in ? */
43253e9753d6SMatthew G. Knepley       blockSize = Nb;
43263e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
43279566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
43283e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches * batchSize);
43293e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
43303e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches * batchSize);
43313e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
43329566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom));
43339566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
43349566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
43359566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
43369566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Nr, chunkGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, a ? &a[offset * totDimAux] : NULL, t, &elemVec[offset * totDim]));
43379566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
43383e9753d6SMatthew G. Knepley     }
43393e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
43403e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
43413e9753d6SMatthew G. Knepley 
43429566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(point, "BdResidual", totDim, &elemVec[face * totDim]));
43439566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
43449566063dSJacob Faibussowitsch       PetscCall(DMPlexVecSetClosure(plex, NULL, locF, support[0], &elemVec[face * totDim], ADD_ALL_VALUES));
43453e9753d6SMatthew G. Knepley     }
43469566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
43479566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
43489566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
43499566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
43509566063dSJacob Faibussowitsch     PetscCall(PetscFree4(u, u_t, elemVec, a));
43513e9753d6SMatthew G. Knepley   }
43520c290148SMatthew G. Knepley end:
43539566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
43549566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexA));
43553e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
43563e9753d6SMatthew G. Knepley }
43573e9753d6SMatthew G. Knepley 
43589371c9d4SSatish Balay PetscErrorCode DMPlexComputeBdResidualSingle(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF) {
43593e9753d6SMatthew G. Knepley   DMField  coordField;
43603e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
43613e9753d6SMatthew G. Knepley   IS       facetIS;
43623e9753d6SMatthew G. Knepley   PetscInt dim;
43633e9753d6SMatthew G. Knepley 
43643e9753d6SMatthew G. Knepley   PetscFunctionBegin;
43659566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
43669566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
43679566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
43689566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
43699566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS));
43709566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
43713e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
43723e9753d6SMatthew G. Knepley }
43733e9753d6SMatthew G. Knepley 
43749371c9d4SSatish Balay PetscErrorCode DMPlexComputeBdResidual_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user) {
43753e9753d6SMatthew G. Knepley   PetscDS  prob;
43763e9753d6SMatthew G. Knepley   PetscInt numBd, bd;
43773e9753d6SMatthew G. Knepley   DMField  coordField = NULL;
43783e9753d6SMatthew G. Knepley   IS       facetIS    = NULL;
43793e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
43803e9753d6SMatthew G. Knepley   PetscInt dim;
43813e9753d6SMatthew G. Knepley 
43823e9753d6SMatthew G. Knepley   PetscFunctionBegin;
43839566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
43849566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
43859566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
43869566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
43879566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
43883e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
438945480ffeSMatthew G. Knepley     PetscWeakForm           wf;
43903e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
43913e9753d6SMatthew G. Knepley     DMLabel                 label;
43923e9753d6SMatthew G. Knepley     const PetscInt         *values;
43930c290148SMatthew G. Knepley     PetscInt                field, numValues, v;
43943e9753d6SMatthew G. Knepley     PetscObject             obj;
43953e9753d6SMatthew G. Knepley     PetscClassId            id;
43960c290148SMatthew G. Knepley     PetscFormKey            key;
43973e9753d6SMatthew G. Knepley 
43989566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &field, NULL, NULL, NULL, NULL, NULL));
43999566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
44009566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
44013e9753d6SMatthew G. Knepley     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
44023e9753d6SMatthew G. Knepley     if (!facetIS) {
44033e9753d6SMatthew G. Knepley       DMLabel  depthLabel;
44043e9753d6SMatthew G. Knepley       PetscInt dim;
44053e9753d6SMatthew G. Knepley 
44069566063dSJacob Faibussowitsch       PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
44079566063dSJacob Faibussowitsch       PetscCall(DMGetDimension(dm, &dim));
44089566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
44093e9753d6SMatthew G. Knepley     }
44109566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
44110c290148SMatthew G. Knepley     for (v = 0; v < numValues; ++v) {
44120c290148SMatthew G. Knepley       key.label = label;
44130c290148SMatthew G. Knepley       key.value = values[v];
44140c290148SMatthew G. Knepley       key.field = field;
44150c290148SMatthew G. Knepley       key.part  = 0;
44169566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS));
44170c290148SMatthew G. Knepley     }
44183e9753d6SMatthew G. Knepley   }
44199566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
44203e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
44213e9753d6SMatthew G. Knepley }
44223e9753d6SMatthew G. Knepley 
44239371c9d4SSatish Balay PetscErrorCode DMPlexComputeResidual_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user) {
44243e9753d6SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *)dm->data;
44253e9753d6SMatthew G. Knepley   const char      *name       = "Residual";
44263e9753d6SMatthew G. Knepley   DM               dmAux      = NULL;
44273e9753d6SMatthew G. Knepley   DM               dmGrad     = NULL;
44283e9753d6SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
44296528b96dSMatthew G. Knepley   PetscDS          ds         = NULL;
44306528b96dSMatthew G. Knepley   PetscDS          dsAux      = NULL;
44313e9753d6SMatthew G. Knepley   PetscSection     section    = NULL;
44323e9753d6SMatthew G. Knepley   PetscBool        useFEM     = PETSC_FALSE;
44333e9753d6SMatthew G. Knepley   PetscBool        useFVM     = PETSC_FALSE;
44343e9753d6SMatthew G. Knepley   PetscBool        isImplicit = (locX_t || time == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
44353e9753d6SMatthew G. Knepley   PetscFV          fvm        = NULL;
44363e9753d6SMatthew G. Knepley   PetscFVCellGeom *cgeomFVM   = NULL;
44373e9753d6SMatthew G. Knepley   PetscFVFaceGeom *fgeomFVM   = NULL;
44383e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
44393e9753d6SMatthew G. Knepley   Vec              locA, cellGeometryFVM = NULL, faceGeometryFVM = NULL, grad, locGrad = NULL;
44403e9753d6SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a, *uL, *uR;
44413e9753d6SMatthew G. Knepley   IS               chunkIS;
44423e9753d6SMatthew G. Knepley   const PetscInt  *cells;
44433e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
44443e9753d6SMatthew G. Knepley   PetscInt         Nf, f, totDim, totDimAux, numChunks, cellChunkSize, faceChunkSize, chunk, fStart, fEnd;
44453e9753d6SMatthew G. Knepley   PetscInt         maxDegree  = PETSC_MAX_INT;
44463e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
44473e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
44483e9753d6SMatthew G. Knepley 
44493e9753d6SMatthew G. Knepley   PetscFunctionBegin;
4450e04ae0b4SMatthew G. Knepley   if (!cellIS) PetscFunctionReturn(0);
44519566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0));
44523e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
44533e9753d6SMatthew G. Knepley   /* TODO The FVM geometry is over-manipulated. Make the precalc functions return exactly what we need */
44543e9753d6SMatthew G. Knepley   /* FEM+FVM */
44559566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
44569566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
44573e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
44589566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
44599566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
44609566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds));
44619566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
44629566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
44639566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
44643e9753d6SMatthew G. Knepley   if (locA) {
44653e9753d6SMatthew G. Knepley     PetscInt subcell;
44669566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
44671059d808SMatthew G. Knepley     PetscCall(DMGetEnclosurePoint(dmAux, dm, DM_ENC_UNKNOWN, cells ? cells[cStart] : cStart, &subcell));
44689566063dSJacob Faibussowitsch     PetscCall(DMGetCellDS(dmAux, subcell, &dsAux));
44699566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux));
44703e9753d6SMatthew G. Knepley   }
44713e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
44723e9753d6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
44733e9753d6SMatthew G. Knepley     PetscObject  obj;
44743e9753d6SMatthew G. Knepley     PetscClassId id;
44753e9753d6SMatthew G. Knepley     PetscBool    fimp;
44763e9753d6SMatthew G. Knepley 
44779566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(ds, f, &fimp));
44783e9753d6SMatthew G. Knepley     if (isImplicit != fimp) continue;
44799566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(ds, f, &obj));
44809566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
44813e9753d6SMatthew G. Knepley     if (id == PETSCFE_CLASSID) { useFEM = PETSC_TRUE; }
44829371c9d4SSatish Balay     if (id == PETSCFV_CLASSID) {
44839371c9d4SSatish Balay       useFVM = PETSC_TRUE;
44849371c9d4SSatish Balay       fvm    = (PetscFV)obj;
44859371c9d4SSatish Balay     }
44863e9753d6SMatthew G. Knepley   }
44873e9753d6SMatthew G. Knepley   if (useFEM) {
44889566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
44899566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
44903e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
44919566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &affineQuad));
4492*48a46eb9SPierre Jolivet       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
44933e9753d6SMatthew G. Knepley     } else {
44949566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
44953e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
44963e9753d6SMatthew G. Knepley         PetscObject  obj;
44973e9753d6SMatthew G. Knepley         PetscClassId id;
44983e9753d6SMatthew G. Knepley         PetscBool    fimp;
44993e9753d6SMatthew G. Knepley 
45009566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(ds, f, &fimp));
45013e9753d6SMatthew G. Knepley         if (isImplicit != fimp) continue;
45029566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
45039566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
45043e9753d6SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
45053e9753d6SMatthew G. Knepley           PetscFE fe = (PetscFE)obj;
45063e9753d6SMatthew G. Knepley 
45079566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
45089566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
45099566063dSJacob Faibussowitsch           PetscCall(DMSNESGetFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
45103e9753d6SMatthew G. Knepley         }
45113e9753d6SMatthew G. Knepley       }
45123e9753d6SMatthew G. Knepley     }
45133e9753d6SMatthew G. Knepley   }
45143e9753d6SMatthew G. Knepley   if (useFVM) {
45159566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL));
45169566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(faceGeometryFVM, (const PetscScalar **)&fgeomFVM));
45179566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **)&cgeomFVM));
45183e9753d6SMatthew G. Knepley     /* Reconstruct and limit cell gradients */
45199566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGradientDM(dm, fvm, &dmGrad));
45203e9753d6SMatthew G. Knepley     if (dmGrad) {
45219566063dSJacob Faibussowitsch       PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
45229566063dSJacob Faibussowitsch       PetscCall(DMGetGlobalVector(dmGrad, &grad));
45239566063dSJacob Faibussowitsch       PetscCall(DMPlexReconstructGradients_Internal(dm, fvm, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
45243e9753d6SMatthew G. Knepley       /* Communicate gradient values */
45259566063dSJacob Faibussowitsch       PetscCall(DMGetLocalVector(dmGrad, &locGrad));
45269566063dSJacob Faibussowitsch       PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
45279566063dSJacob Faibussowitsch       PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
45289566063dSJacob Faibussowitsch       PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
45293e9753d6SMatthew G. Knepley     }
45303e9753d6SMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
45319566063dSJacob Faibussowitsch     PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, time, faceGeometryFVM, cellGeometryFVM, locGrad));
45323e9753d6SMatthew G. Knepley   }
45333e9753d6SMatthew G. Knepley   /* Loop over chunks */
45349566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
45353e9753d6SMatthew G. Knepley   numCells      = cEnd - cStart;
45363e9753d6SMatthew G. Knepley   numChunks     = 1;
45373e9753d6SMatthew G. Knepley   cellChunkSize = numCells / numChunks;
45383e9753d6SMatthew G. Knepley   faceChunkSize = (fEnd - fStart) / numChunks;
45393e9753d6SMatthew G. Knepley   numChunks     = PetscMin(1, numCells);
45403e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
45413e9753d6SMatthew G. Knepley     PetscScalar     *elemVec, *fluxL, *fluxR;
45423e9753d6SMatthew G. Knepley     PetscReal       *vol;
45433e9753d6SMatthew G. Knepley     PetscFVFaceGeom *fgeom;
45443e9753d6SMatthew G. Knepley     PetscInt         cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
45453e9753d6SMatthew G. Knepley     PetscInt         fS = fStart + chunk * faceChunkSize, fE = PetscMin(fS + faceChunkSize, fEnd), numFaces = 0, face;
45463e9753d6SMatthew G. Knepley 
45473e9753d6SMatthew G. Knepley     /* Extract field coefficients */
45483e9753d6SMatthew G. Knepley     if (useFEM) {
45499566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
45509566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
45519566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
45529566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells * totDim));
45533e9753d6SMatthew G. Knepley     }
45543e9753d6SMatthew G. Knepley     if (useFVM) {
45559566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
45569566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
45579566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL));
45589566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR));
45599566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxL, numFaces * totDim));
45609566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxR, numFaces * totDim));
45613e9753d6SMatthew G. Knepley     }
45623e9753d6SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
45633e9753d6SMatthew G. Knepley     /* Loop over fields */
45643e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
45653e9753d6SMatthew G. Knepley       PetscObject  obj;
45663e9753d6SMatthew G. Knepley       PetscClassId id;
45673e9753d6SMatthew G. Knepley       PetscBool    fimp;
45683e9753d6SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
45693e9753d6SMatthew G. Knepley 
45706528b96dSMatthew G. Knepley       key.field = f;
45719566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(ds, f, &fimp));
45723e9753d6SMatthew G. Knepley       if (isImplicit != fimp) continue;
45739566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, &obj));
45749566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
45753e9753d6SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
45763e9753d6SMatthew G. Knepley         PetscFE         fe        = (PetscFE)obj;
45773e9753d6SMatthew G. Knepley         PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[f];
45783e9753d6SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
45793e9753d6SMatthew G. Knepley         PetscQuadrature quad      = affineQuad ? affineQuad : quads[f];
45803e9753d6SMatthew G. Knepley         PetscInt        Nq, Nb;
45813e9753d6SMatthew G. Knepley 
45829566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
45839566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
45849566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
45853e9753d6SMatthew G. Knepley         blockSize = Nb;
45863e9753d6SMatthew G. Knepley         batchSize = numBlocks * blockSize;
45879566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
45883e9753d6SMatthew G. Knepley         numChunks = numCells / (numBatches * batchSize);
45893e9753d6SMatthew G. Knepley         Ne        = numChunks * numBatches * batchSize;
45903e9753d6SMatthew G. Knepley         Nr        = numCells % (numBatches * batchSize);
45913e9753d6SMatthew G. Knepley         offset    = numCells - Nr;
45923e9753d6SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
45933e9753d6SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
45949566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom));
45959566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(ds, key, Ne, chunkGeom, u, u_t, dsAux, a, t, elemVec));
45969566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &chunkGeom));
45979566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(ds, key, Nr, chunkGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux, &a[offset * totDimAux], t, &elemVec[offset * totDim]));
45989566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &chunkGeom));
45993e9753d6SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
46003e9753d6SMatthew G. Knepley         PetscFV fv = (PetscFV)obj;
46013e9753d6SMatthew G. Knepley 
46023e9753d6SMatthew G. Knepley         Ne = numFaces;
46033e9753d6SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
46043e9753d6SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
46059566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, ds, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
460663a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
46073e9753d6SMatthew G. Knepley     }
46083e9753d6SMatthew G. Knepley     /* Loop over domain */
46093e9753d6SMatthew G. Knepley     if (useFEM) {
46103e9753d6SMatthew G. Knepley       /* Add elemVec to locX */
46113e9753d6SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
46123e9753d6SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
46133e9753d6SMatthew G. Knepley         const PetscInt cind = c - cStart;
46143e9753d6SMatthew G. Knepley 
46159566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim]));
46163e9753d6SMatthew G. Knepley         if (ghostLabel) {
46173e9753d6SMatthew G. Knepley           PetscInt ghostVal;
46183e9753d6SMatthew G. Knepley 
46199566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
46203e9753d6SMatthew G. Knepley           if (ghostVal > 0) continue;
46213e9753d6SMatthew G. Knepley         }
46229566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES));
46233e9753d6SMatthew G. Knepley       }
46243e9753d6SMatthew G. Knepley     }
46253e9753d6SMatthew G. Knepley     if (useFVM) {
46263e9753d6SMatthew G. Knepley       PetscScalar *fa;
46273e9753d6SMatthew G. Knepley       PetscInt     iface;
46283e9753d6SMatthew G. Knepley 
46299566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
46303e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
46313e9753d6SMatthew G. Knepley         PetscFV      fv;
46323e9753d6SMatthew G. Knepley         PetscObject  obj;
46333e9753d6SMatthew G. Knepley         PetscClassId id;
46343e9753d6SMatthew G. Knepley         PetscInt     foff, pdim;
46353e9753d6SMatthew G. Knepley 
46369566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
46379566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(ds, f, &foff));
46389566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
46393e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
46403e9753d6SMatthew G. Knepley         fv = (PetscFV)obj;
46419566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
46423e9753d6SMatthew G. Knepley         /* Accumulate fluxes to cells */
46433e9753d6SMatthew G. Knepley         for (face = fS, iface = 0; face < fE; ++face) {
46443e9753d6SMatthew G. Knepley           const PetscInt *scells;
46453e9753d6SMatthew G. Knepley           PetscScalar    *fL = NULL, *fR = NULL;
46463e9753d6SMatthew G. Knepley           PetscInt        ghost, d, nsupp, nchild;
46473e9753d6SMatthew G. Knepley 
46489566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
46499566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
46509566063dSJacob Faibussowitsch           PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
46513e9753d6SMatthew G. Knepley           if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
46529566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupport(dm, face, &scells));
46539566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, scells[0], &ghost));
46549566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[0], f, fa, &fL));
46559566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, scells[1], &ghost));
46569566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[1], f, fa, &fR));
46573e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) {
46583e9753d6SMatthew G. Knepley             if (fL) fL[d] -= fluxL[iface * totDim + foff + d];
46593e9753d6SMatthew G. Knepley             if (fR) fR[d] += fluxR[iface * totDim + foff + d];
46603e9753d6SMatthew G. Knepley           }
46613e9753d6SMatthew G. Knepley           ++iface;
46623e9753d6SMatthew G. Knepley         }
46633e9753d6SMatthew G. Knepley       }
46649566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
46653e9753d6SMatthew G. Knepley     }
46663e9753d6SMatthew G. Knepley     /* Handle time derivative */
46673e9753d6SMatthew G. Knepley     if (locX_t) {
46683e9753d6SMatthew G. Knepley       PetscScalar *x_t, *fa;
46693e9753d6SMatthew G. Knepley 
46709566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
46719566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
46723e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
46733e9753d6SMatthew G. Knepley         PetscFV      fv;
46743e9753d6SMatthew G. Knepley         PetscObject  obj;
46753e9753d6SMatthew G. Knepley         PetscClassId id;
46763e9753d6SMatthew G. Knepley         PetscInt     pdim, d;
46773e9753d6SMatthew G. Knepley 
46789566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
46799566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
46803e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
46813e9753d6SMatthew G. Knepley         fv = (PetscFV)obj;
46829566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
46833e9753d6SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
46843e9753d6SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
46853e9753d6SMatthew G. Knepley           PetscScalar   *u_t, *r;
46863e9753d6SMatthew G. Knepley 
46873e9753d6SMatthew G. Knepley           if (ghostLabel) {
46883e9753d6SMatthew G. Knepley             PetscInt ghostVal;
46893e9753d6SMatthew G. Knepley 
46909566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
46913e9753d6SMatthew G. Knepley             if (ghostVal > 0) continue;
46923e9753d6SMatthew G. Knepley           }
46939566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
46949566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
46953e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
46963e9753d6SMatthew G. Knepley         }
46973e9753d6SMatthew G. Knepley       }
46989566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
46999566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
47003e9753d6SMatthew G. Knepley     }
47013e9753d6SMatthew G. Knepley     if (useFEM) {
47029566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
47039566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
47043e9753d6SMatthew G. Knepley     }
47053e9753d6SMatthew G. Knepley     if (useFVM) {
47069566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
47079566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
47089566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxL));
47099566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces * totDim, MPIU_SCALAR, &fluxR));
47109566063dSJacob Faibussowitsch       if (dmGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
47113e9753d6SMatthew G. Knepley     }
47123e9753d6SMatthew G. Knepley   }
47139566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
47149566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
47153e9753d6SMatthew G. Knepley 
47163e9753d6SMatthew G. Knepley   if (useFEM) {
47179566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdResidual_Internal(dm, locX, locX_t, t, locF, user));
47183e9753d6SMatthew G. Knepley 
47193e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
47209566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
47219566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
47223e9753d6SMatthew G. Knepley     } else {
47233e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
47249566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
47259566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
47263e9753d6SMatthew G. Knepley       }
47279566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads, geoms));
47283e9753d6SMatthew G. Knepley     }
47293e9753d6SMatthew G. Knepley   }
47303e9753d6SMatthew G. Knepley 
47313e9753d6SMatthew G. Knepley   /* FEM */
47323e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
47333e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
47343e9753d6SMatthew G. Knepley   /* 3: Handle boundary values */
47353e9753d6SMatthew G. Knepley   /* 4: Loop over domain */
47363e9753d6SMatthew G. Knepley   /*   Extract coefficients */
47373e9753d6SMatthew G. Knepley   /* Loop over fields */
47383e9753d6SMatthew G. Knepley   /*   Set tiling for FE*/
47393e9753d6SMatthew G. Knepley   /*   Integrate FE residual to get elemVec */
47403e9753d6SMatthew G. Knepley   /*     Loop over subdomain */
47413e9753d6SMatthew G. Knepley   /*       Loop over quad points */
47423e9753d6SMatthew G. Knepley   /*         Transform coords to real space */
47433e9753d6SMatthew G. Knepley   /*         Evaluate field and aux fields at point */
47443e9753d6SMatthew G. Knepley   /*         Evaluate residual at point */
47453e9753d6SMatthew G. Knepley   /*         Transform residual to real space */
47463e9753d6SMatthew G. Knepley   /*       Add residual to elemVec */
47473e9753d6SMatthew G. Knepley   /* Loop over domain */
47483e9753d6SMatthew G. Knepley   /*   Add elemVec to locX */
47493e9753d6SMatthew G. Knepley 
47503e9753d6SMatthew G. Knepley   /* FVM */
47513e9753d6SMatthew G. Knepley   /* Get geometric data */
47523e9753d6SMatthew G. Knepley   /* If using gradients */
47533e9753d6SMatthew G. Knepley   /*   Compute gradient data */
47543e9753d6SMatthew G. Knepley   /*   Loop over domain faces */
47553e9753d6SMatthew G. Knepley   /*     Count computational faces */
47563e9753d6SMatthew G. Knepley   /*     Reconstruct cell gradient */
47573e9753d6SMatthew G. Knepley   /*   Loop over domain cells */
47583e9753d6SMatthew G. Knepley   /*     Limit cell gradients */
47593e9753d6SMatthew G. Knepley   /* Handle boundary values */
47603e9753d6SMatthew G. Knepley   /* Loop over domain faces */
47613e9753d6SMatthew G. Knepley   /*   Read out field, centroid, normal, volume for each side of face */
47623e9753d6SMatthew G. Knepley   /* Riemann solve over faces */
47633e9753d6SMatthew G. Knepley   /* Loop over domain faces */
47643e9753d6SMatthew G. Knepley   /*   Accumulate fluxes to cells */
47653e9753d6SMatthew G. Knepley   /* TODO Change printFEM to printDisc here */
47663e9753d6SMatthew G. Knepley   if (mesh->printFEM) {
47673e9753d6SMatthew G. Knepley     Vec          locFbc;
47683e9753d6SMatthew G. Knepley     PetscInt     pStart, pEnd, p, maxDof;
47693e9753d6SMatthew G. Knepley     PetscScalar *zeroes;
47703e9753d6SMatthew G. Knepley 
47719566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(locF, &locFbc));
47729566063dSJacob Faibussowitsch     PetscCall(VecCopy(locF, locFbc));
47739566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetChart(section, &pStart, &pEnd));
47749566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetMaxDof(section, &maxDof));
47759566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(maxDof, &zeroes));
4776*48a46eb9SPierre Jolivet     for (p = pStart; p < pEnd; p++) PetscCall(VecSetValuesSection(locFbc, section, p, zeroes, INSERT_BC_VALUES));
47779566063dSJacob Faibussowitsch     PetscCall(PetscFree(zeroes));
47789566063dSJacob Faibussowitsch     PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc));
47799566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&locFbc));
47803e9753d6SMatthew G. Knepley   }
47819566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0));
47823e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
47833e9753d6SMatthew G. Knepley }
47843e9753d6SMatthew G. Knepley 
47856528b96dSMatthew G. Knepley /*
47866528b96dSMatthew G. Knepley   1) Allow multiple kernels for BdResidual for hybrid DS
47876528b96dSMatthew G. Knepley 
47886528b96dSMatthew G. Knepley   DONE 2) Get out dsAux for either side at the same time as cohesive cell dsAux
47896528b96dSMatthew G. Knepley 
47906528b96dSMatthew G. Knepley   DONE 3) Change DMGetCellFields() to get different aux data a[] for each side
47916528b96dSMatthew G. Knepley      - I think I just need to replace a[] with the closure from each face
47926528b96dSMatthew G. Knepley 
47936528b96dSMatthew G. Knepley   4) Run both kernels for each non-hybrid field with correct dsAux, and then hybrid field as before
47946528b96dSMatthew G. Knepley */
47959371c9d4SSatish Balay PetscErrorCode DMPlexComputeResidual_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user) {
47963e9753d6SMatthew G. Knepley   DM_Plex        *mesh       = (DM_Plex *)dm->data;
47973e9753d6SMatthew G. Knepley   const char     *name       = "Hybrid Residual";
479804c51a94SMatthew G. Knepley   DM              dmAux[3]   = {NULL, NULL, NULL};
47993e9753d6SMatthew G. Knepley   DMLabel         ghostLabel = NULL;
48006528b96dSMatthew G. Knepley   PetscDS         ds         = NULL;
48016528b96dSMatthew G. Knepley   PetscDS         dsAux[3]   = {NULL, NULL, NULL};
480204c51a94SMatthew G. Knepley   Vec             locA[3]    = {NULL, NULL, NULL};
48033e9753d6SMatthew G. Knepley   PetscSection    section    = NULL;
48043e9753d6SMatthew G. Knepley   DMField         coordField = NULL;
48056528b96dSMatthew G. Knepley   PetscScalar    *u          = NULL, *u_t, *a[3];
48063e9753d6SMatthew G. Knepley   PetscScalar    *elemVec;
48073e9753d6SMatthew G. Knepley   IS              chunkIS;
48083e9753d6SMatthew G. Knepley   const PetscInt *cells;
48093e9753d6SMatthew G. Knepley   PetscInt       *faces;
48103e9753d6SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
48116528b96dSMatthew G. Knepley   PetscInt        Nf, f, totDim, totDimAux[3], numChunks, cellChunkSize, chunk;
48123e9753d6SMatthew G. Knepley   PetscInt        maxDegree  = PETSC_MAX_INT;
48133e9753d6SMatthew G. Knepley   PetscQuadrature affineQuad = NULL, *quads = NULL;
48143e9753d6SMatthew G. Knepley   PetscFEGeom    *affineGeom = NULL, **geoms = NULL;
48153e9753d6SMatthew G. Knepley 
48163e9753d6SMatthew G. Knepley   PetscFunctionBegin;
48175fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
48185fedec97SMatthew G. Knepley     const char *name;
48199566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name));
482063a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
48215fedec97SMatthew G. Knepley   }
48229566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM, dm, 0, 0, 0));
48233e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
48243e9753d6SMatthew G. Knepley   /* FEM */
48259566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
48269566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
48273e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
48289566063dSJacob Faibussowitsch   PetscCall(DMGetSection(dm, &section));
48299566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
48301059d808SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds));
48319566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
48329566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
48339566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
483404c51a94SMatthew G. Knepley   if (locA[2]) {
48351059d808SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
48361059d808SMatthew G. Knepley 
48379566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
48381059d808SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2]));
48399566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
48406528b96dSMatthew G. Knepley     {
48416528b96dSMatthew G. Knepley       const PetscInt *cone;
48426528b96dSMatthew G. Knepley       PetscInt        c;
48436528b96dSMatthew G. Knepley 
48441059d808SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
48456528b96dSMatthew G. Knepley       for (c = 0; c < 2; ++c) {
48466528b96dSMatthew G. Knepley         const PetscInt *support;
48476528b96dSMatthew G. Knepley         PetscInt        ssize, s;
48486528b96dSMatthew G. Knepley 
48499566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
48509566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
48511059d808SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
48521059d808SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
48531059d808SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
48541059d808SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
48559566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
48569566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
485704c51a94SMatthew G. Knepley         else { dmAux[c] = dmAux[2]; }
48589566063dSJacob Faibussowitsch         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c]));
48599566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
48606528b96dSMatthew G. Knepley       }
48616528b96dSMatthew G. Knepley     }
48623e9753d6SMatthew G. Knepley   }
48633e9753d6SMatthew G. Knepley   /* 2: Setup geometric data */
48649566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
48659566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
48663e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
48679566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
48683e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
48693e9753d6SMatthew G. Knepley       PetscFE fe;
48703e9753d6SMatthew G. Knepley 
48719566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
48723e9753d6SMatthew G. Knepley       if (fe) {
48739566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
48749566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)quads[f]));
48753e9753d6SMatthew G. Knepley       }
48763e9753d6SMatthew G. Knepley     }
48773e9753d6SMatthew G. Knepley   }
48783e9753d6SMatthew G. Knepley   /* Loop over chunks */
48793e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
48803e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize);
4881a4158a15SMatthew G. Knepley   PetscCall(PetscCalloc1(1 * cellChunkSize, &faces));
4882a4158a15SMatthew G. Knepley   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 1 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS));
48833e9753d6SMatthew G. Knepley   /* Extract field coefficients */
48843e9753d6SMatthew G. Knepley   /* NOTE This needs the end cap faces to have identical orientations */
48859566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
48869566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHybridAuxFields(dm, dmAux, dsAux, cellIS, locA, a));
48879566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, cellChunkSize * totDim, MPIU_SCALAR, &elemVec));
48883e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
48893e9753d6SMatthew G. Knepley     PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
48903e9753d6SMatthew G. Knepley 
48919566063dSJacob Faibussowitsch     PetscCall(PetscMemzero(elemVec, cellChunkSize * totDim * sizeof(PetscScalar)));
48923e9753d6SMatthew G. Knepley     /* Get faces */
48933e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
48943e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
48953e9753d6SMatthew G. Knepley       const PetscInt *cone;
48969566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(dm, cell, &cone));
4897a4158a15SMatthew G. Knepley       faces[0 * cellChunkSize + (c - cS)] = cone[0];
4898a4158a15SMatthew G. Knepley       /*faces[1*cellChunkSize+(c-cS)] = cone[1];*/
48993e9753d6SMatthew G. Knepley     }
4900a4158a15SMatthew G. Knepley     PetscCall(ISGeneralSetIndices(chunkIS, 1 * cellChunkSize, faces, PETSC_USE_POINTER));
49013e9753d6SMatthew G. Knepley     /* Get geometric data */
49023e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
49039566063dSJacob Faibussowitsch       if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad));
49049566063dSJacob Faibussowitsch       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom));
49053e9753d6SMatthew G. Knepley     } else {
49063e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
49079566063dSJacob Faibussowitsch         if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]));
49083e9753d6SMatthew G. Knepley       }
49093e9753d6SMatthew G. Knepley     }
49103e9753d6SMatthew G. Knepley     /* Loop over fields */
49113e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
49123e9753d6SMatthew G. Knepley       PetscFE         fe;
49133e9753d6SMatthew G. Knepley       PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[f];
4914148442b3SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
49153e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
49163e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
49175fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
49183e9753d6SMatthew G. Knepley 
49199566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
49203e9753d6SMatthew G. Knepley       if (!fe) continue;
49219566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
49229566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
49239566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
49243e9753d6SMatthew G. Knepley       blockSize = Nb;
49253e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
49269566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
49273e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches * batchSize);
49283e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
49293e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches * batchSize);
49303e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
49319566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom));
49329566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &remGeom));
49339566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, f, &isCohesiveField));
49345fedec97SMatthew G. Knepley       chunkGeom->isCohesive = remGeom->isCohesive = PETSC_TRUE;
49356528b96dSMatthew G. Knepley       key[0].field                                = f;
49366528b96dSMatthew G. Knepley       key[1].field                                = f;
49375fedec97SMatthew G. Knepley       key[2].field                                = f;
49389566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, elemVec));
49399566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[0], 0, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux[0], &a[0][offset * totDimAux[0]], t, &elemVec[offset * totDim]));
49409566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, elemVec));
49419566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[1], 1, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux[1], &a[1][offset * totDimAux[1]], t, &elemVec[offset * totDim]));
49429566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, elemVec));
49439566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[2], 2, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux[2], &a[2][offset * totDimAux[2]], t, &elemVec[offset * totDim]));
49449566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &remGeom));
49459566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom, 0, offset, &chunkGeom));
49463e9753d6SMatthew G. Knepley     }
49473e9753d6SMatthew G. Knepley     /* Add elemVec to locX */
49483e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
49493e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
49503e9753d6SMatthew G. Knepley       const PetscInt cind = c - cStart;
49513e9753d6SMatthew G. Knepley 
49529566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind * totDim]));
49533e9753d6SMatthew G. Knepley       if (ghostLabel) {
49543e9753d6SMatthew G. Knepley         PetscInt ghostVal;
49553e9753d6SMatthew G. Knepley 
49569566063dSJacob Faibussowitsch         PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
49573e9753d6SMatthew G. Knepley         if (ghostVal > 0) continue;
49583e9753d6SMatthew G. Knepley       }
49599566063dSJacob Faibussowitsch       PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind * totDim], ADD_ALL_VALUES));
49603e9753d6SMatthew G. Knepley     }
49613e9753d6SMatthew G. Knepley   }
49629566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
49639566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreHybridAuxFields(dmAux, dsAux, cellIS, locA, a));
49649566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, numCells * totDim, MPIU_SCALAR, &elemVec));
49659566063dSJacob Faibussowitsch   PetscCall(PetscFree(faces));
49669566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&chunkIS));
49679566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
49683e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
49699566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
49709566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&affineQuad));
49713e9753d6SMatthew G. Knepley   } else {
49723e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
49739566063dSJacob Faibussowitsch       if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
49749566063dSJacob Faibussowitsch       if (quads) PetscCall(PetscQuadratureDestroy(&quads[f]));
49753e9753d6SMatthew G. Knepley     }
49769566063dSJacob Faibussowitsch     PetscCall(PetscFree2(quads, geoms));
49773e9753d6SMatthew G. Knepley   }
49789566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM, dm, 0, 0, 0));
49793e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
49803e9753d6SMatthew G. Knepley }
49813e9753d6SMatthew G. Knepley 
49829371c9d4SSatish Balay PetscErrorCode DMPlexComputeBdJacobian_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP, DMField coordField, IS facetIS) {
49833e9753d6SMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *)dm->data;
49843e9753d6SMatthew G. Knepley   DM              plex = NULL, plexA = NULL, tdm;
49853e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
49863e9753d6SMatthew G. Knepley   PetscDS         prob, probAux       = NULL;
49873e9753d6SMatthew G. Knepley   PetscSection    section, sectionAux = NULL;
4988e432b41dSStefano Zampini   PetscSection    globalSection;
49893e9753d6SMatthew G. Knepley   Vec             locA = NULL, tv;
49903e9753d6SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL;
49913e9753d6SMatthew G. Knepley   PetscInt        v;
49923e9753d6SMatthew G. Knepley   PetscInt        Nf, totDim, totDimAux = 0;
4993e432b41dSStefano Zampini   PetscBool       transform;
49943e9753d6SMatthew G. Knepley 
49953e9753d6SMatthew G. Knepley   PetscFunctionBegin;
49969566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
49979566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
49989566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
49999566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
50009566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
50019566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
50029566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
50039566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
50049566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, label, values[0], 0, &locA));
50053e9753d6SMatthew G. Knepley   if (locA) {
50063e9753d6SMatthew G. Knepley     DM dmAux;
50073e9753d6SMatthew G. Knepley 
50089566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
50099566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
50109566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
50119566063dSJacob Faibussowitsch     PetscCall(DMGetDS(plexA, &probAux));
50129566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
50139566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
50143e9753d6SMatthew G. Knepley   }
50153e9753d6SMatthew G. Knepley 
50169566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
50173e9753d6SMatthew G. Knepley   for (v = 0; v < numValues; ++v) {
50183e9753d6SMatthew G. Knepley     PetscFEGeom    *fgeom;
50193e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
50203e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
50213e9753d6SMatthew G. Knepley     IS              pointIS;
50223e9753d6SMatthew G. Knepley     const PetscInt *points;
502306ad1575SMatthew G. Knepley     PetscFormKey    key;
50243e9753d6SMatthew G. Knepley     PetscInt        numFaces, face, Nq;
50253e9753d6SMatthew G. Knepley 
502645480ffeSMatthew G. Knepley     key.label = label;
502745480ffeSMatthew G. Knepley     key.value = values[v];
502806ad1575SMatthew G. Knepley     key.part  = 0;
50299566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
50303e9753d6SMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
50313e9753d6SMatthew G. Knepley     {
50323e9753d6SMatthew G. Knepley       IS isectIS;
50333e9753d6SMatthew G. Knepley 
50343e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a prior if one is a superset of the other */
50359566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
50369566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
50373e9753d6SMatthew G. Knepley       pointIS = isectIS;
50383e9753d6SMatthew G. Knepley     }
50399566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
50409566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
50419566063dSJacob Faibussowitsch     PetscCall(PetscMalloc4(numFaces * totDim, &u, locX_t ? numFaces * totDim : 0, &u_t, numFaces * totDim * totDim, &elemMat, locA ? numFaces * totDimAux : 0, &a));
50429566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
5043*48a46eb9SPierre Jolivet     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
50443e9753d6SMatthew G. Knepley     if (!qGeom) {
50453e9753d6SMatthew G. Knepley       PetscFE fe;
50463e9753d6SMatthew G. Knepley 
50479566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
50489566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
50499566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
50503e9753d6SMatthew G. Knepley     }
50519566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
50529566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
50533e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
5054f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
50553e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
5056f15274beSMatthew Knepley       PetscInt       i;
50573e9753d6SMatthew G. Knepley 
50589566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
50599566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
50603e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face * totDim + i] = x[i];
50619566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
50623e9753d6SMatthew G. Knepley       if (locX_t) {
50639566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
50643e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face * totDim + i] = x[i];
50659566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
50663e9753d6SMatthew G. Knepley       }
50673e9753d6SMatthew G. Knepley       if (locA) {
50683e9753d6SMatthew G. Knepley         PetscInt subp;
50699566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
50709566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
50713e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face * totDimAux + i] = x[i];
50729566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
50733e9753d6SMatthew G. Knepley       }
50743e9753d6SMatthew G. Knepley     }
50759566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(elemMat, numFaces * totDim * totDim));
50763e9753d6SMatthew G. Knepley     {
50773e9753d6SMatthew G. Knepley       PetscFE      fe;
50783e9753d6SMatthew G. Knepley       PetscInt     Nb;
50793e9753d6SMatthew G. Knepley       /* Conforming batches */
50803e9753d6SMatthew G. Knepley       PetscInt     numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
50813e9753d6SMatthew G. Knepley       /* Remainder */
50823e9753d6SMatthew G. Knepley       PetscFEGeom *chunkGeom = NULL;
50833e9753d6SMatthew G. Knepley       PetscInt     fieldJ, Nr, offset;
50843e9753d6SMatthew G. Knepley 
50859566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
50869566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
50879566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
50883e9753d6SMatthew G. Knepley       blockSize = Nb;
50893e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
50909566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
50913e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches * batchSize);
50923e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
50933e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches * batchSize);
50943e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
50959566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, 0, offset, &chunkGeom));
50963e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
509745480ffeSMatthew G. Knepley         key.field = fieldI * Nf + fieldJ;
50989566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateBdJacobian(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
50993e9753d6SMatthew G. Knepley       }
51009566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
51013e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
510245480ffeSMatthew G. Knepley         key.field = fieldI * Nf + fieldJ;
51039566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateBdJacobian(prob, wf, key, Nr, chunkGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, a ? &a[offset * totDimAux] : NULL, t, X_tShift, &elemMat[offset * totDim * totDim]));
51043e9753d6SMatthew G. Knepley       }
51059566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
51063e9753d6SMatthew G. Knepley     }
51073e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
51083e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
51093e9753d6SMatthew G. Knepley 
51103e9753d6SMatthew G. Knepley       /* Transform to global basis before insertion in Jacobian */
51119566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
51129566063dSJacob Faibussowitsch       if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMat[face * totDim * totDim]));
51139566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face * totDim * totDim]));
51149566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, support[0], &elemMat[face * totDim * totDim], ADD_VALUES));
51153e9753d6SMatthew G. Knepley     }
51169566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
51179566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
51189566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
51199566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
51209566063dSJacob Faibussowitsch     PetscCall(PetscFree4(u, u_t, elemMat, a));
51213e9753d6SMatthew G. Knepley   }
51229566063dSJacob Faibussowitsch   if (plex) PetscCall(DMDestroy(&plex));
51239566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
51243e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
51253e9753d6SMatthew G. Knepley }
51263e9753d6SMatthew G. Knepley 
51279371c9d4SSatish Balay PetscErrorCode DMPlexComputeBdJacobianSingle(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP) {
51283e9753d6SMatthew G. Knepley   DMField  coordField;
51293e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
51303e9753d6SMatthew G. Knepley   IS       facetIS;
51313e9753d6SMatthew G. Knepley   PetscInt dim;
51323e9753d6SMatthew G. Knepley 
51333e9753d6SMatthew G. Knepley   PetscFunctionBegin;
51349566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
51359566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
51369566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
51379566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
51389566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, field, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS));
51399566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
51403e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
51413e9753d6SMatthew G. Knepley }
51423e9753d6SMatthew G. Knepley 
51439371c9d4SSatish Balay PetscErrorCode DMPlexComputeBdJacobian_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP, void *user) {
51443e9753d6SMatthew G. Knepley   PetscDS  prob;
51453e9753d6SMatthew G. Knepley   PetscInt dim, numBd, bd;
51463e9753d6SMatthew G. Knepley   DMLabel  depthLabel;
51473e9753d6SMatthew G. Knepley   DMField  coordField = NULL;
51483e9753d6SMatthew G. Knepley   IS       facetIS;
51493e9753d6SMatthew G. Knepley 
51503e9753d6SMatthew G. Knepley   PetscFunctionBegin;
51519566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
51529566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
51539566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
51549566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
51559566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
51569566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
51573e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
515845480ffeSMatthew G. Knepley     PetscWeakForm           wf;
51593e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
51603e9753d6SMatthew G. Knepley     DMLabel                 label;
51613e9753d6SMatthew G. Knepley     const PetscInt         *values;
51623e9753d6SMatthew G. Knepley     PetscInt                fieldI, numValues;
51633e9753d6SMatthew G. Knepley     PetscObject             obj;
51643e9753d6SMatthew G. Knepley     PetscClassId            id;
51653e9753d6SMatthew G. Knepley 
51669566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &fieldI, NULL, NULL, NULL, NULL, NULL));
51679566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &obj));
51689566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
51693e9753d6SMatthew G. Knepley     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
51709566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, fieldI, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS));
51713e9753d6SMatthew G. Knepley   }
51729566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
51733e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
51743e9753d6SMatthew G. Knepley }
51753e9753d6SMatthew G. Knepley 
51769371c9d4SSatish Balay PetscErrorCode DMPlexComputeJacobian_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *user) {
51773e9753d6SMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *)dm->data;
51783e9753d6SMatthew G. Knepley   const char     *name  = "Jacobian";
51799a2a23afSMatthew G. Knepley   DM              dmAux = NULL, plex, tdm;
51803e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
51813e9753d6SMatthew G. Knepley   Vec             A, tv;
51823e9753d6SMatthew G. Knepley   DMField         coordField;
51833e9753d6SMatthew G. Knepley   PetscDS         prob, probAux = NULL;
5184e432b41dSStefano Zampini   PetscSection    section, globalSection, sectionAux;
51853e9753d6SMatthew G. Knepley   PetscScalar    *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL;
51863e9753d6SMatthew G. Knepley   const PetscInt *cells;
51873e9753d6SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
518828351e22SJed Brown   PetscInt        totDim, totDimAux = 0, cStart, cEnd, numCells, c;
5189e04ae0b4SMatthew G. Knepley   PetscBool       hasJac = PETSC_FALSE, hasPrec = PETSC_FALSE, hasDyn, hasFV = PETSC_FALSE, transform;
51903e9753d6SMatthew G. Knepley 
51913e9753d6SMatthew G. Knepley   PetscFunctionBegin;
5192e04ae0b4SMatthew G. Knepley   if (!cellIS) goto end;
51939566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
51949566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
51959566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
51969566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
51979566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
51989566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
51999566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
52009566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
52019566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob));
52029566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
52039566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
52049566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
52059566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
52063e9753d6SMatthew G. Knepley   /* user passed in the same matrix, avoid double contributions and
52073e9753d6SMatthew G. Knepley      only assemble the Jacobian */
52083e9753d6SMatthew G. Knepley   if (hasJac && Jac == JacP) hasPrec = PETSC_FALSE;
52099566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
52103e9753d6SMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
52119566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
52129a2a23afSMatthew G. Knepley   if (A) {
52139566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
52149566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
52159566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plex));
52169566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plex, &sectionAux));
52179566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
52189566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
52193e9753d6SMatthew G. Knepley   }
52209566063dSJacob Faibussowitsch   PetscCall(PetscMalloc5(numCells * totDim, &u, X_t ? numCells * totDim : 0, &u_t, hasJac ? numCells * totDim * totDim : 0, &elemMat, hasPrec ? numCells * totDim * totDim : 0, &elemMatP, hasDyn ? numCells * totDim * totDim : 0, &elemMatD));
52219566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
52229566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
52233e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
52243e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
52253e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
52263e9753d6SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL;
52273e9753d6SMatthew G. Knepley     PetscInt       i;
52283e9753d6SMatthew G. Knepley 
52299566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x));
52303e9753d6SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i];
52319566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x));
52323e9753d6SMatthew G. Knepley     if (X_t) {
52339566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t));
52343e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i];
52359566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t));
52363e9753d6SMatthew G. Knepley     }
52373e9753d6SMatthew G. Knepley     if (dmAux) {
52383e9753d6SMatthew G. Knepley       PetscInt subcell;
52399566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
52409566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x));
52413e9753d6SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i];
52429566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x));
52433e9753d6SMatthew G. Knepley     }
52443e9753d6SMatthew G. Knepley   }
52459566063dSJacob Faibussowitsch   if (hasJac) PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim));
52469566063dSJacob Faibussowitsch   if (hasPrec) PetscCall(PetscArrayzero(elemMatP, numCells * totDim * totDim));
52479566063dSJacob Faibussowitsch   if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim));
52483e9753d6SMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
52493e9753d6SMatthew G. Knepley     PetscClassId    id;
52503e9753d6SMatthew G. Knepley     PetscFE         fe;
52513e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
52523e9753d6SMatthew G. Knepley     PetscInt        Nb;
52533e9753d6SMatthew G. Knepley     /* Conforming batches */
52543e9753d6SMatthew G. Knepley     PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
52553e9753d6SMatthew G. Knepley     /* Remainder */
52563e9753d6SMatthew G. Knepley     PetscInt        Nr, offset, Nq;
52573e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
52583e9753d6SMatthew G. Knepley     PetscFEGeom    *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
52593e9753d6SMatthew G. Knepley 
52609566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
52619566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId((PetscObject)fe, &id));
52629371c9d4SSatish Balay     if (id == PETSCFV_CLASSID) {
52639371c9d4SSatish Balay       hasFV = PETSC_TRUE;
52649371c9d4SSatish Balay       continue;
52659371c9d4SSatish Balay     }
52669566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
52679566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
52689566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
5269*48a46eb9SPierre Jolivet     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
52703e9753d6SMatthew G. Knepley     if (!qGeom) {
52719566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &qGeom));
52729566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
52733e9753d6SMatthew G. Knepley     }
52749566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
52759566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
52763e9753d6SMatthew G. Knepley     blockSize = Nb;
52773e9753d6SMatthew G. Knepley     batchSize = numBlocks * blockSize;
52789566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
52793e9753d6SMatthew G. Knepley     numChunks = numCells / (numBatches * batchSize);
52803e9753d6SMatthew G. Knepley     Ne        = numChunks * numBatches * batchSize;
52813e9753d6SMatthew G. Knepley     Nr        = numCells % (numBatches * batchSize);
52823e9753d6SMatthew G. Knepley     offset    = numCells - Nr;
52839566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
52849566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom));
52853e9753d6SMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
52866528b96dSMatthew G. Knepley       key.field = fieldI * Nf + fieldJ;
52873e9753d6SMatthew G. Knepley       if (hasJac) {
52889566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
52899566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMat[offset * totDim * totDim]));
52903e9753d6SMatthew G. Knepley       }
52913e9753d6SMatthew G. Knepley       if (hasPrec) {
52929566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatP));
52939566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMatP[offset * totDim * totDim]));
52943e9753d6SMatthew G. Knepley       }
52953e9753d6SMatthew G. Knepley       if (hasDyn) {
52969566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
52979566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMatD[offset * totDim * totDim]));
52983e9753d6SMatthew G. Knepley       }
52993e9753d6SMatthew G. Knepley     }
53009566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom));
53019566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom));
53029566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
53039566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
53043e9753d6SMatthew G. Knepley   }
53053e9753d6SMatthew G. Knepley   /*   Add contribution from X_t */
53069371c9d4SSatish Balay   if (hasDyn) {
53079371c9d4SSatish Balay     for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
53089371c9d4SSatish Balay   }
53093e9753d6SMatthew G. Knepley   if (hasFV) {
53103e9753d6SMatthew G. Knepley     PetscClassId id;
53113e9753d6SMatthew G. Knepley     PetscFV      fv;
53123e9753d6SMatthew G. Knepley     PetscInt     offsetI, NcI, NbI = 1, fc, f;
53133e9753d6SMatthew G. Knepley 
53143e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
53159566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fv));
53169566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, fieldI, &offsetI));
53179566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId((PetscObject)fv, &id));
53183e9753d6SMatthew G. Knepley       if (id != PETSCFV_CLASSID) continue;
53193e9753d6SMatthew G. Knepley       /* Put in the identity */
53209566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &NcI));
53213e9753d6SMatthew G. Knepley       for (c = cStart; c < cEnd; ++c) {
53223e9753d6SMatthew G. Knepley         const PetscInt cind    = c - cStart;
53233e9753d6SMatthew G. Knepley         const PetscInt eOffset = cind * totDim * totDim;
53243e9753d6SMatthew G. Knepley         for (fc = 0; fc < NcI; ++fc) {
53253e9753d6SMatthew G. Knepley           for (f = 0; f < NbI; ++f) {
53263e9753d6SMatthew G. Knepley             const PetscInt i = offsetI + f * NcI + fc;
53273e9753d6SMatthew G. Knepley             if (hasPrec) {
53283e9753d6SMatthew G. Knepley               if (hasJac) { elemMat[eOffset + i * totDim + i] = 1.0; }
53293e9753d6SMatthew G. Knepley               elemMatP[eOffset + i * totDim + i] = 1.0;
53309371c9d4SSatish Balay             } else {
53319371c9d4SSatish Balay               elemMat[eOffset + i * totDim + i] = 1.0;
53329371c9d4SSatish Balay             }
53333e9753d6SMatthew G. Knepley           }
53343e9753d6SMatthew G. Knepley         }
53353e9753d6SMatthew G. Knepley       }
53363e9753d6SMatthew G. Knepley     }
53373e9753d6SMatthew G. Knepley     /* No allocated space for FV stuff, so ignore the zero entries */
53389566063dSJacob Faibussowitsch     PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE));
53393e9753d6SMatthew G. Knepley   }
53403e9753d6SMatthew G. Knepley   /* Insert values into matrix */
53413e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
53423e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
53433e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
53443e9753d6SMatthew G. Knepley 
53453e9753d6SMatthew G. Knepley     /* Transform to global basis before insertion in Jacobian */
53469566063dSJacob Faibussowitsch     if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, cell, PETSC_TRUE, totDim, &elemMat[cind * totDim * totDim]));
53473e9753d6SMatthew G. Knepley     if (hasPrec) {
53483e9753d6SMatthew G. Knepley       if (hasJac) {
53499566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
53509566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[cind * totDim * totDim], ADD_VALUES));
53513e9753d6SMatthew G. Knepley       }
53529566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind * totDim * totDim]));
53539566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMatP[cind * totDim * totDim], ADD_VALUES));
53543e9753d6SMatthew G. Knepley     } else {
53553e9753d6SMatthew G. Knepley       if (hasJac) {
53569566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
53579566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMat[cind * totDim * totDim], ADD_VALUES));
53583e9753d6SMatthew G. Knepley       }
53593e9753d6SMatthew G. Knepley     }
53603e9753d6SMatthew G. Knepley   }
53619566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
53629566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
53639566063dSJacob Faibussowitsch   PetscCall(PetscFree5(u, u_t, elemMat, elemMatP, elemMatD));
53643e9753d6SMatthew G. Knepley   if (dmAux) {
53659566063dSJacob Faibussowitsch     PetscCall(PetscFree(a));
53669566063dSJacob Faibussowitsch     PetscCall(DMDestroy(&plex));
53673e9753d6SMatthew G. Knepley   }
53683e9753d6SMatthew G. Knepley   /* Compute boundary integrals */
53699566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, user));
53703e9753d6SMatthew G. Knepley   /* Assemble matrix */
53719371c9d4SSatish Balay end : {
5372e04ae0b4SMatthew G. Knepley   PetscBool assOp = hasJac && hasPrec ? PETSC_TRUE : PETSC_FALSE, gassOp;
5373e04ae0b4SMatthew G. Knepley 
5374e04ae0b4SMatthew G. Knepley   PetscCallMPI(MPI_Allreduce(&assOp, &gassOp, 1, MPIU_BOOL, MPI_LOR, PetscObjectComm((PetscObject)dm)));
53753e9753d6SMatthew G. Knepley   if (hasJac && hasPrec) {
53769566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
53779566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
53783e9753d6SMatthew G. Knepley   }
5379e04ae0b4SMatthew G. Knepley }
53809566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
53819566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
53829566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
53833e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
53843e9753d6SMatthew G. Knepley }
53853e9753d6SMatthew G. Knepley 
53869371c9d4SSatish Balay PetscErrorCode DMPlexComputeJacobian_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user) {
53873e9753d6SMatthew G. Knepley   DM_Plex        *mesh          = (DM_Plex *)dm->data;
53883e9753d6SMatthew G. Knepley   const char     *name          = "Hybrid Jacobian";
5389148442b3SMatthew G. Knepley   DM              dmAux[3]      = {NULL, NULL, NULL};
5390148442b3SMatthew G. Knepley   DMLabel         ghostLabel    = NULL;
53913e9753d6SMatthew G. Knepley   DM              plex          = NULL;
53923e9753d6SMatthew G. Knepley   DM              plexA         = NULL;
5393148442b3SMatthew G. Knepley   PetscDS         ds            = NULL;
5394148442b3SMatthew G. Knepley   PetscDS         dsAux[3]      = {NULL, NULL, NULL};
5395148442b3SMatthew G. Knepley   Vec             locA[3]       = {NULL, NULL, NULL};
53963e9753d6SMatthew G. Knepley   PetscSection    section       = NULL;
5397148442b3SMatthew G. Knepley   PetscSection    sectionAux[3] = {NULL, NULL, NULL};
53983e9753d6SMatthew G. Knepley   DMField         coordField    = NULL;
5399148442b3SMatthew G. Knepley   PetscScalar    *u             = NULL, *u_t, *a[3];
54003e9753d6SMatthew G. Knepley   PetscScalar    *elemMat, *elemMatP;
5401e432b41dSStefano Zampini   PetscSection    globalSection;
54023e9753d6SMatthew G. Knepley   IS              chunkIS;
54033e9753d6SMatthew G. Knepley   const PetscInt *cells;
54043e9753d6SMatthew G. Knepley   PetscInt       *faces;
54053e9753d6SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells;
5406148442b3SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ, totDim, totDimAux[3], numChunks, cellChunkSize, chunk;
54073e9753d6SMatthew G. Knepley   PetscInt        maxDegree  = PETSC_MAX_INT;
54083e9753d6SMatthew G. Knepley   PetscQuadrature affineQuad = NULL, *quads = NULL;
54093e9753d6SMatthew G. Knepley   PetscFEGeom    *affineGeom = NULL, **geoms = NULL;
5410e432b41dSStefano Zampini   PetscBool       hasBdJac, hasBdPrec;
54113e9753d6SMatthew G. Knepley 
54123e9753d6SMatthew G. Knepley   PetscFunctionBegin;
54135fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
54145fedec97SMatthew G. Knepley     const char *name;
54159566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject)key[0].label, &name));
541663a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
54175fedec97SMatthew G. Knepley   }
54189566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
54199566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
54209566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
54219566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
54229566063dSJacob Faibussowitsch   PetscCall(DMGetSection(dm, &section));
54239566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
54249566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
54251059d808SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds));
54269566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
54279566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
54289566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobian(ds, &hasBdJac));
54299566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobianPreconditioner(ds, &hasBdPrec));
54309566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
5431148442b3SMatthew G. Knepley   if (locA[2]) {
54321059d808SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
54331059d808SMatthew G. Knepley 
54349566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
54359566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux[2], DMPLEX, &plexA));
54369566063dSJacob Faibussowitsch     PetscCall(DMGetSection(dmAux[2], &sectionAux[2]));
54371059d808SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2]));
54389566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
5439148442b3SMatthew G. Knepley     {
5440148442b3SMatthew G. Knepley       const PetscInt *cone;
5441148442b3SMatthew G. Knepley       PetscInt        c;
5442148442b3SMatthew G. Knepley 
54431059d808SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
5444148442b3SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
5445148442b3SMatthew G. Knepley         const PetscInt *support;
5446148442b3SMatthew G. Knepley         PetscInt        ssize, s;
5447148442b3SMatthew G. Knepley 
54489566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
54499566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
54501059d808SMatthew G. Knepley         PetscCheck(ssize == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
54511059d808SMatthew G. Knepley         if (support[0] == cellStart) s = 1;
54521059d808SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
54531059d808SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
54549566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
54559566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
5456148442b3SMatthew G. Knepley         else { dmAux[c] = dmAux[2]; }
54579566063dSJacob Faibussowitsch         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c]));
54589566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
5459148442b3SMatthew G. Knepley       }
5460148442b3SMatthew G. Knepley     }
54613e9753d6SMatthew G. Knepley   }
54629566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
54639566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
54643e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
54653e9753d6SMatthew G. Knepley     PetscInt f;
54669566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
54673e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
54683e9753d6SMatthew G. Knepley       PetscFE fe;
54693e9753d6SMatthew G. Knepley 
54709566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
54713e9753d6SMatthew G. Knepley       if (fe) {
54729566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
54739566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject)quads[f]));
54743e9753d6SMatthew G. Knepley       }
54753e9753d6SMatthew G. Knepley     }
54763e9753d6SMatthew G. Knepley   }
54773e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
54783e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal)numCells) / cellChunkSize);
5479a4158a15SMatthew G. Knepley   PetscCall(PetscCalloc1(1 * cellChunkSize, &faces));
5480a4158a15SMatthew G. Knepley   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 1 * cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS));
54819566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
54829566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHybridAuxFields(dm, dmAux, dsAux, cellIS, locA, a));
54839566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMat));
54849566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatP));
54853e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
54863e9753d6SMatthew G. Knepley     PetscInt cS = cStart + chunk * cellChunkSize, cE = PetscMin(cS + cellChunkSize, cEnd), numCells = cE - cS, c;
54873e9753d6SMatthew G. Knepley 
54889566063dSJacob Faibussowitsch     if (hasBdJac) PetscCall(PetscMemzero(elemMat, numCells * totDim * totDim * sizeof(PetscScalar)));
54899566063dSJacob Faibussowitsch     if (hasBdPrec) PetscCall(PetscMemzero(elemMatP, numCells * totDim * totDim * sizeof(PetscScalar)));
54903e9753d6SMatthew G. Knepley     /* Get faces */
54913e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
54923e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
54933e9753d6SMatthew G. Knepley       const PetscInt *cone;
54949566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(plex, cell, &cone));
5495a4158a15SMatthew G. Knepley       faces[0 * cellChunkSize + (c - cS)] = cone[0];
5496a4158a15SMatthew G. Knepley       /*faces[2*cellChunkSize+(c-cS)] = cone[1];*/
54973e9753d6SMatthew G. Knepley     }
5498a4158a15SMatthew G. Knepley     PetscCall(ISGeneralSetIndices(chunkIS, 1 * cellChunkSize, faces, PETSC_USE_POINTER));
54993e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
55009566063dSJacob Faibussowitsch       if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad));
55019566063dSJacob Faibussowitsch       if (affineQuad) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom));
55023e9753d6SMatthew G. Knepley     } else {
55033e9753d6SMatthew G. Knepley       PetscInt f;
55043e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
55059566063dSJacob Faibussowitsch         if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]));
55063e9753d6SMatthew G. Knepley       }
55073e9753d6SMatthew G. Knepley     }
55083e9753d6SMatthew G. Knepley 
55093e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
55103e9753d6SMatthew G. Knepley       PetscFE         feI;
55113e9753d6SMatthew G. Knepley       PetscFEGeom    *geom      = affineGeom ? affineGeom : geoms[fieldI];
55123e9753d6SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
55133e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[fieldI];
55143e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
55155fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
55163e9753d6SMatthew G. Knepley 
55179566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *)&feI));
55183e9753d6SMatthew G. Knepley       if (!feI) continue;
55199566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(feI, NULL, &numBlocks, NULL, &numBatches));
55209566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
55219566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feI, &Nb));
55223e9753d6SMatthew G. Knepley       blockSize = Nb;
55233e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
55249566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(feI, blockSize, numBlocks, batchSize, numBatches));
55253e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches * batchSize);
55263e9753d6SMatthew G. Knepley       Ne        = numChunks * numBatches * batchSize;
55273e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches * batchSize);
55283e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
55299566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom, 0, offset, &chunkGeom));
55309566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom, offset, numCells, &remGeom));
55319566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, fieldI, &isCohesiveField));
55323e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
55333e9753d6SMatthew G. Knepley         PetscFE feJ;
55343e9753d6SMatthew G. Knepley 
55359566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, fieldJ, (PetscObject *)&feJ));
55363e9753d6SMatthew G. Knepley         if (!feJ) continue;
5537148442b3SMatthew G. Knepley         key[0].field = fieldI * Nf + fieldJ;
5538148442b3SMatthew G. Knepley         key[1].field = fieldI * Nf + fieldJ;
55395fedec97SMatthew G. Knepley         key[2].field = fieldI * Nf + fieldJ;
5540148442b3SMatthew G. Knepley         if (hasBdJac) {
55419566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMat));
55429566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[0], 0, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux[0], &a[0][offset * totDimAux[0]], t, X_tShift, &elemMat[offset * totDim * totDim]));
55439566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMat));
55449566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[1], 1, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux[1], &a[1][offset * totDimAux[1]], t, X_tShift, &elemMat[offset * totDim * totDim]));
5545148442b3SMatthew G. Knepley         }
5546148442b3SMatthew G. Knepley         if (hasBdPrec) {
55479566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatP));
55489566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[0], 0, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux[0], &a[0][offset * totDimAux[0]], t, X_tShift, &elemMatP[offset * totDim * totDim]));
55499566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatP));
55509566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[1], 1, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux[1], &a[1][offset * totDimAux[1]], t, X_tShift, &elemMatP[offset * totDim * totDim]));
5551148442b3SMatthew G. Knepley         }
55525fedec97SMatthew G. Knepley         if (hasBdJac) {
55539566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMat));
55549566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[2], 2, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux[2], &a[2][offset * totDimAux[2]], t, X_tShift, &elemMat[offset * totDim * totDim]));
5555148442b3SMatthew G. Knepley         }
55565fedec97SMatthew G. Knepley         if (hasBdPrec) {
55579566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatP));
55589566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[2], 2, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, dsAux[2], &a[2][offset * totDimAux[2]], t, X_tShift, &elemMatP[offset * totDim * totDim]));
55593e9753d6SMatthew G. Knepley         }
55603e9753d6SMatthew G. Knepley       }
55619566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom, offset, numCells, &remGeom));
55629566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom, 0, offset, &chunkGeom));
55633e9753d6SMatthew G. Knepley     }
55643e9753d6SMatthew G. Knepley     /* Insert values into matrix */
55653e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
55663e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
55673e9753d6SMatthew G. Knepley       const PetscInt cind = c - cS;
55683e9753d6SMatthew G. Knepley 
55693e9753d6SMatthew G. Knepley       if (hasBdPrec) {
55703e9753d6SMatthew G. Knepley         if (hasBdJac) {
55719566063dSJacob Faibussowitsch           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
55729566063dSJacob Faibussowitsch           PetscCall(DMPlexMatSetClosure(plex, section, globalSection, Jac, cell, &elemMat[cind * totDim * totDim], ADD_VALUES));
55733e9753d6SMatthew G. Knepley         }
55749566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind * totDim * totDim]));
55759566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMatP[cind * totDim * totDim], ADD_VALUES));
55763e9753d6SMatthew G. Knepley       } else if (hasBdJac) {
55779566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
55789566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMat[cind * totDim * totDim], ADD_VALUES));
55793e9753d6SMatthew G. Knepley       }
55803e9753d6SMatthew G. Knepley     }
55813e9753d6SMatthew G. Knepley   }
55829566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
55839566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreHybridAuxFields(dmAux, dsAux, cellIS, locA, a));
55849566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, hasBdJac ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMat));
55859566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize * totDim * totDim : 0, MPIU_SCALAR, &elemMatP));
55869566063dSJacob Faibussowitsch   PetscCall(PetscFree(faces));
55879566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&chunkIS));
55889566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
55893e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
55909566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, affineQuad, PETSC_FALSE, &affineGeom));
55919566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&affineQuad));
55923e9753d6SMatthew G. Knepley   } else {
55933e9753d6SMatthew G. Knepley     PetscInt f;
55943e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
55959566063dSJacob Faibussowitsch       if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, quads[f], PETSC_FALSE, &geoms[f]));
55969566063dSJacob Faibussowitsch       if (quads) PetscCall(PetscQuadratureDestroy(&quads[f]));
55973e9753d6SMatthew G. Knepley     }
55989566063dSJacob Faibussowitsch     PetscCall(PetscFree2(quads, geoms));
55993e9753d6SMatthew G. Knepley   }
56009566063dSJacob Faibussowitsch   if (dmAux[2]) PetscCall(DMDestroy(&plexA));
56019566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
56029566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
56033e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
56043e9753d6SMatthew G. Knepley }
56058e3a2eefSMatthew G. Knepley 
56068e3a2eefSMatthew G. Knepley /*
56078e3a2eefSMatthew G. Knepley   DMPlexComputeJacobian_Action_Internal - Form the local portion of the Jacobian action Z = J(X) Y at the local solution X using pointwise functions specified by the user.
56088e3a2eefSMatthew G. Knepley 
56098e3a2eefSMatthew G. Knepley   Input Parameters:
56108e3a2eefSMatthew G. Knepley + dm     - The mesh
56118e3a2eefSMatthew G. Knepley . key    - The PetscWeakFormKey indcating where integration should happen
561206ad1575SMatthew G. Knepley . cellIS - The cells to integrate over
56138e3a2eefSMatthew G. Knepley . t      - The time
56148e3a2eefSMatthew G. Knepley . X_tShift - The multiplier for the Jacobian with repsect to X_t
56158e3a2eefSMatthew G. Knepley . X      - Local solution vector
56168e3a2eefSMatthew G. Knepley . X_t    - Time-derivative of the local solution vector
56178e3a2eefSMatthew G. Knepley . Y      - Local input vector
561806ad1575SMatthew G. Knepley - user   - the user context
56198e3a2eefSMatthew G. Knepley 
56208e3a2eefSMatthew G. Knepley   Output Parameter:
56218e3a2eefSMatthew G. Knepley . Z - Local output vector
56228e3a2eefSMatthew G. Knepley 
56238e3a2eefSMatthew G. Knepley   Note:
56248e3a2eefSMatthew G. Knepley   We form the residual one batch of elements at a time. This allows us to offload work onto an accelerator,
56258e3a2eefSMatthew G. Knepley   like a GPU, or vectorize on a multicore machine.
56268e3a2eefSMatthew G. Knepley */
56279371c9d4SSatish Balay PetscErrorCode DMPlexComputeJacobian_Action_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Vec Y, Vec Z, void *user) {
56288e3a2eefSMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *)dm->data;
56298e3a2eefSMatthew G. Knepley   const char     *name  = "Jacobian";
56308e3a2eefSMatthew G. Knepley   DM              dmAux = NULL, plex, plexAux = NULL;
56318e3a2eefSMatthew G. Knepley   DMEnclosureType encAux;
56328e3a2eefSMatthew G. Knepley   Vec             A;
56338e3a2eefSMatthew G. Knepley   DMField         coordField;
56348e3a2eefSMatthew G. Knepley   PetscDS         prob, probAux = NULL;
56358e3a2eefSMatthew G. Knepley   PetscQuadrature quad;
56368e3a2eefSMatthew G. Knepley   PetscSection    section, globalSection, sectionAux;
56378e3a2eefSMatthew G. Knepley   PetscScalar    *elemMat, *elemMatD, *u, *u_t, *a = NULL, *y, *z;
56388e3a2eefSMatthew G. Knepley   const PetscInt *cells;
56398e3a2eefSMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
56408e3a2eefSMatthew G. Knepley   PetscInt        totDim, totDimAux = 0, cStart, cEnd, numCells, c;
56418e3a2eefSMatthew G. Knepley   PetscBool       hasDyn;
56428e3a2eefSMatthew G. Knepley 
56438e3a2eefSMatthew G. Knepley   PetscFunctionBegin;
56449566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM, dm, 0, 0, 0));
56459566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
56468e3a2eefSMatthew G. Knepley   if (!cellIS) {
56478e3a2eefSMatthew G. Knepley     PetscInt depth;
56488e3a2eefSMatthew G. Knepley 
56499566063dSJacob Faibussowitsch     PetscCall(DMPlexGetDepth(plex, &depth));
56509566063dSJacob Faibussowitsch     PetscCall(DMGetStratumIS(plex, "dim", depth, &cellIS));
56519566063dSJacob Faibussowitsch     if (!cellIS) PetscCall(DMGetStratumIS(plex, "depth", depth, &cellIS));
56528e3a2eefSMatthew G. Knepley   } else {
56539566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)cellIS));
56548e3a2eefSMatthew G. Knepley   }
56559566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
56569566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
56579566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
56589566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
56599566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob));
56609566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
56619566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
56629566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
56638e3a2eefSMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
56649566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
56658e3a2eefSMatthew G. Knepley   if (A) {
56669566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
56679566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
56689566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexAux));
56699566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexAux, &sectionAux));
56709566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
56719566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
56728e3a2eefSMatthew G. Knepley   }
56739566063dSJacob Faibussowitsch   PetscCall(VecSet(Z, 0.0));
56749566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numCells * totDim, &u, X_t ? numCells * totDim : 0, &u_t, numCells * totDim * totDim, &elemMat, hasDyn ? numCells * totDim * totDim : 0, &elemMatD, numCells * totDim, &y, totDim, &z));
56759566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells * totDimAux, &a));
56769566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
56778e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
56788e3a2eefSMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
56798e3a2eefSMatthew G. Knepley     const PetscInt cind = c - cStart;
56808e3a2eefSMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL;
56818e3a2eefSMatthew G. Knepley     PetscInt       i;
56828e3a2eefSMatthew G. Knepley 
56839566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, X, cell, NULL, &x));
56848e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind * totDim + i] = x[i];
56859566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, X, cell, NULL, &x));
56868e3a2eefSMatthew G. Knepley     if (X_t) {
56879566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, X_t, cell, NULL, &x_t));
56888e3a2eefSMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind * totDim + i] = x_t[i];
56899566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, X_t, cell, NULL, &x_t));
56908e3a2eefSMatthew G. Knepley     }
56918e3a2eefSMatthew G. Knepley     if (dmAux) {
56928e3a2eefSMatthew G. Knepley       PetscInt subcell;
56939566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
56949566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexAux, sectionAux, A, subcell, NULL, &x));
56958e3a2eefSMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind * totDimAux + i] = x[i];
56969566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexAux, sectionAux, A, subcell, NULL, &x));
56978e3a2eefSMatthew G. Knepley     }
56989566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, Y, cell, NULL, &x));
56998e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) y[cind * totDim + i] = x[i];
57009566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, Y, cell, NULL, &x));
57018e3a2eefSMatthew G. Knepley   }
57029566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, numCells * totDim * totDim));
57039566063dSJacob Faibussowitsch   if (hasDyn) PetscCall(PetscArrayzero(elemMatD, numCells * totDim * totDim));
57048e3a2eefSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
57058e3a2eefSMatthew G. Knepley     PetscFE         fe;
57068e3a2eefSMatthew G. Knepley     PetscInt        Nb;
57078e3a2eefSMatthew G. Knepley     /* Conforming batches */
57088e3a2eefSMatthew G. Knepley     PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
57098e3a2eefSMatthew G. Knepley     /* Remainder */
57108e3a2eefSMatthew G. Knepley     PetscInt        Nr, offset, Nq;
57118e3a2eefSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
57128e3a2eefSMatthew G. Knepley     PetscInt        maxDegree;
57138e3a2eefSMatthew G. Knepley     PetscFEGeom    *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
57148e3a2eefSMatthew G. Knepley 
57159566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *)&fe));
57169566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
57179566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
57189566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
57199566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
57209566063dSJacob Faibussowitsch     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
57218e3a2eefSMatthew G. Knepley     if (!qGeom) {
57229566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &qGeom));
57239566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
57248e3a2eefSMatthew G. Knepley     }
57259566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
57269566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
57278e3a2eefSMatthew G. Knepley     blockSize = Nb;
57288e3a2eefSMatthew G. Knepley     batchSize = numBlocks * blockSize;
57299566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
57308e3a2eefSMatthew G. Knepley     numChunks = numCells / (numBatches * batchSize);
57318e3a2eefSMatthew G. Knepley     Ne        = numChunks * numBatches * batchSize;
57328e3a2eefSMatthew G. Knepley     Nr        = numCells % (numBatches * batchSize);
57338e3a2eefSMatthew G. Knepley     offset    = numCells - Nr;
57349566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, 0, offset, &chunkGeom));
57359566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM, offset, numCells, &remGeom));
57368e3a2eefSMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
57378e3a2eefSMatthew G. Knepley       key.field = fieldI * Nf + fieldJ;
57389566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
57399566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMat[offset * totDim * totDim]));
57408e3a2eefSMatthew G. Knepley       if (hasDyn) {
57419566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
57429566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset * totDim], u_t ? &u_t[offset * totDim] : NULL, probAux, &a[offset * totDimAux], t, X_tShift, &elemMatD[offset * totDim * totDim]));
57438e3a2eefSMatthew G. Knepley       }
57448e3a2eefSMatthew G. Knepley     }
57459566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, offset, numCells, &remGeom));
57469566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM, 0, offset, &chunkGeom));
57479566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
57489566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
57498e3a2eefSMatthew G. Knepley   }
57508e3a2eefSMatthew G. Knepley   if (hasDyn) {
57518e3a2eefSMatthew G. Knepley     for (c = 0; c < numCells * totDim * totDim; ++c) elemMat[c] += X_tShift * elemMatD[c];
57528e3a2eefSMatthew G. Knepley   }
57538e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
57548e3a2eefSMatthew G. Knepley     const PetscInt     cell = cells ? cells[c] : c;
57558e3a2eefSMatthew G. Knepley     const PetscInt     cind = c - cStart;
57568e3a2eefSMatthew G. Knepley     const PetscBLASInt M = totDim, one = 1;
57578e3a2eefSMatthew G. Knepley     const PetscScalar  a = 1.0, b = 0.0;
57588e3a2eefSMatthew G. Knepley 
5759792fecdfSBarry Smith     PetscCallBLAS("BLASgemv", BLASgemv_("N", &M, &M, &a, &elemMat[cind * totDim * totDim], &M, &y[cind * totDim], &one, &b, z, &one));
57608e3a2eefSMatthew G. Knepley     if (mesh->printFEM > 1) {
57619566063dSJacob Faibussowitsch       PetscCall(DMPrintCellMatrix(c, name, totDim, totDim, &elemMat[cind * totDim * totDim]));
57629566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Y", totDim, &y[cind * totDim]));
57639566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Z", totDim, z));
57648e3a2eefSMatthew G. Knepley     }
57659566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dm, section, Z, cell, z, ADD_VALUES));
57668e3a2eefSMatthew G. Knepley   }
57679566063dSJacob Faibussowitsch   PetscCall(PetscFree6(u, u_t, elemMat, elemMatD, y, z));
57688e3a2eefSMatthew G. Knepley   if (mesh->printFEM) {
57699566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)Z), "Z:\n"));
57709566063dSJacob Faibussowitsch     PetscCall(VecView(Z, NULL));
57718e3a2eefSMatthew G. Knepley   }
57721059d808SMatthew G. Knepley   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
57739566063dSJacob Faibussowitsch   PetscCall(PetscFree(a));
57749566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cellIS));
57759566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexAux));
57769566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
57779566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM, dm, 0, 0, 0));
57788e3a2eefSMatthew G. Knepley   PetscFunctionReturn(0);
57798e3a2eefSMatthew G. Knepley }
5780