xref: /petsc/src/dm/impls/plex/plexfem.c (revision 1059d80888ed9eed9fdcb10c2c9fd40e2f93029a)
1af0996ceSBarry Smith #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2da97024aSMatthew G. Knepley #include <petscsf.h>
3cb1e1211SMatthew G Knepley 
48e3a2eefSMatthew G. Knepley #include <petscblaslapack.h>
5e8f14785SLisandro Dalcin #include <petsc/private/hashsetij.h>
6af0996ceSBarry Smith #include <petsc/private/petscfeimpl.h>
7af0996ceSBarry Smith #include <petsc/private/petscfvimpl.h>
8a0845e3aSMatthew G. Knepley 
95f0b18bfSMatthew G. Knepley PetscBool Clementcite = PETSC_FALSE;
105f0b18bfSMatthew G. Knepley const char ClementCitation[] = "@article{clement1975approximation,\n"
115f0b18bfSMatthew G. Knepley                                "  title   = {Approximation by finite element functions using local regularization},\n"
125f0b18bfSMatthew G. Knepley                                "  author  = {Philippe Cl{\\'e}ment},\n"
135f0b18bfSMatthew G. Knepley                                "  journal = {Revue fran{\\c{c}}aise d'automatique, informatique, recherche op{\\'e}rationnelle. Analyse num{\\'e}rique},\n"
145f0b18bfSMatthew G. Knepley                                "  volume  = {9},\n"
155f0b18bfSMatthew G. Knepley                                "  number  = {R2},\n"
165f0b18bfSMatthew G. Knepley                                "  pages   = {77--84},\n"
175f0b18bfSMatthew G. Knepley                                "  year    = {1975}\n}\n";
185f0b18bfSMatthew G. Knepley 
192f856554SMatthew G. Knepley static PetscErrorCode DMPlexConvertPlex(DM dm, DM *plex, PetscBool copy)
202f856554SMatthew G. Knepley {
212f856554SMatthew G. Knepley   PetscBool      isPlex;
222f856554SMatthew G. Knepley 
232f856554SMatthew G. Knepley   PetscFunctionBegin;
249566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject) dm, DMPLEX, &isPlex));
252f856554SMatthew G. Knepley   if (isPlex) {
262f856554SMatthew G. Knepley     *plex = dm;
279566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject) dm));
282f856554SMatthew G. Knepley   } else {
299566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject) dm, "dm_plex", (PetscObject *) plex));
302f856554SMatthew G. Knepley     if (!*plex) {
319566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm, DMPLEX, plex));
329566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject) dm, "dm_plex", (PetscObject) *plex));
332f856554SMatthew G. Knepley       if (copy) {
342f856554SMatthew G. Knepley         DMSubDomainHookLink link;
359a2a23afSMatthew G. Knepley 
369566063dSJacob Faibussowitsch         PetscCall(DMCopyAuxiliaryVec(dm, *plex));
379a2a23afSMatthew G. Knepley         /* Run the subdomain hook (this will copy the DMSNES/DMTS) */
382f856554SMatthew G. Knepley         for (link = dm->subdomainhook; link; link = link->next) {
399566063dSJacob Faibussowitsch           if (link->ddhook) PetscCall((*link->ddhook)(dm, *plex, link->ctx));
402f856554SMatthew G. Knepley         }
412f856554SMatthew G. Knepley       }
422f856554SMatthew G. Knepley     } else {
439566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject) *plex));
442f856554SMatthew G. Knepley     }
452f856554SMatthew G. Knepley   }
462f856554SMatthew G. Knepley   PetscFunctionReturn(0);
472f856554SMatthew G. Knepley }
482f856554SMatthew G. Knepley 
499b6f715bSMatthew G. Knepley static PetscErrorCode PetscContainerUserDestroy_PetscFEGeom (void *ctx)
509b6f715bSMatthew G. Knepley {
519b6f715bSMatthew G. Knepley   PetscFEGeom *geom = (PetscFEGeom *) ctx;
529b6f715bSMatthew G. Knepley 
539b6f715bSMatthew G. Knepley   PetscFunctionBegin;
549566063dSJacob Faibussowitsch   PetscCall(PetscFEGeomDestroy(&geom));
559b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
569b6f715bSMatthew G. Knepley }
579b6f715bSMatthew G. Knepley 
589b6f715bSMatthew G. Knepley static PetscErrorCode DMPlexGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
599b6f715bSMatthew G. Knepley {
609b6f715bSMatthew G. Knepley   char            composeStr[33] = {0};
619b6f715bSMatthew G. Knepley   PetscObjectId   id;
629b6f715bSMatthew G. Knepley   PetscContainer  container;
639b6f715bSMatthew G. Knepley 
649b6f715bSMatthew G. Knepley   PetscFunctionBegin;
659566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad,&id));
6663a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMPlexGetFEGeom_%" PetscInt64_FMT "\n", id));
679566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject) pointIS, composeStr, (PetscObject *) &container));
689b6f715bSMatthew G. Knepley   if (container) {
699566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **) geom));
709b6f715bSMatthew G. Knepley   } else {
719566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom));
729566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF,&container));
739566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *) *geom));
749566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom));
759566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject) pointIS, composeStr, (PetscObject) container));
769566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
779b6f715bSMatthew G. Knepley   }
789b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
799b6f715bSMatthew G. Knepley }
809b6f715bSMatthew G. Knepley 
819b6f715bSMatthew G. Knepley static PetscErrorCode DMPlexRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
829b6f715bSMatthew G. Knepley {
839b6f715bSMatthew G. Knepley   PetscFunctionBegin;
849b6f715bSMatthew G. Knepley   *geom = NULL;
859b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
869b6f715bSMatthew G. Knepley }
879b6f715bSMatthew G. Knepley 
8846fa42a0SMatthew G. Knepley /*@
8946fa42a0SMatthew G. Knepley   DMPlexGetScale - Get the scale for the specified fundamental unit
9046fa42a0SMatthew G. Knepley 
9146fa42a0SMatthew G. Knepley   Not collective
9246fa42a0SMatthew G. Knepley 
934165533cSJose E. Roman   Input Parameters:
9446fa42a0SMatthew G. Knepley + dm   - the DM
9546fa42a0SMatthew G. Knepley - unit - The SI unit
9646fa42a0SMatthew G. Knepley 
974165533cSJose E. Roman   Output Parameter:
9846fa42a0SMatthew G. Knepley . scale - The value used to scale all quantities with this unit
9946fa42a0SMatthew G. Knepley 
10046fa42a0SMatthew G. Knepley   Level: advanced
10146fa42a0SMatthew G. Knepley 
102db781477SPatrick Sanan .seealso: `DMPlexSetScale()`, `PetscUnit`
10346fa42a0SMatthew G. Knepley @*/
104cb1e1211SMatthew G Knepley PetscErrorCode DMPlexGetScale(DM dm, PetscUnit unit, PetscReal *scale)
105cb1e1211SMatthew G Knepley {
106cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex*) dm->data;
107cb1e1211SMatthew G Knepley 
108cb1e1211SMatthew G Knepley   PetscFunctionBegin;
109cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
110dadcf809SJacob Faibussowitsch   PetscValidRealPointer(scale, 3);
111cb1e1211SMatthew G Knepley   *scale = mesh->scale[unit];
112cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
113cb1e1211SMatthew G Knepley }
114cb1e1211SMatthew G Knepley 
11546fa42a0SMatthew G. Knepley /*@
11646fa42a0SMatthew G. Knepley   DMPlexSetScale - Set the scale for the specified fundamental unit
11746fa42a0SMatthew G. Knepley 
11846fa42a0SMatthew G. Knepley   Not collective
11946fa42a0SMatthew G. Knepley 
1204165533cSJose E. Roman   Input Parameters:
12146fa42a0SMatthew G. Knepley + dm   - the DM
12246fa42a0SMatthew G. Knepley . unit - The SI unit
12346fa42a0SMatthew G. Knepley - scale - The value used to scale all quantities with this unit
12446fa42a0SMatthew G. Knepley 
12546fa42a0SMatthew G. Knepley   Level: advanced
12646fa42a0SMatthew G. Knepley 
127db781477SPatrick Sanan .seealso: `DMPlexGetScale()`, `PetscUnit`
12846fa42a0SMatthew G. Knepley @*/
129cb1e1211SMatthew G Knepley PetscErrorCode DMPlexSetScale(DM dm, PetscUnit unit, PetscReal scale)
130cb1e1211SMatthew G Knepley {
131cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex*) dm->data;
132cb1e1211SMatthew G Knepley 
133cb1e1211SMatthew G Knepley   PetscFunctionBegin;
134cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
135cb1e1211SMatthew G Knepley   mesh->scale[unit] = scale;
136cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
137cb1e1211SMatthew G Knepley }
138cb1e1211SMatthew G Knepley 
139327779f2SMatthew Knepley static PetscErrorCode DMPlexProjectRigidBody_Private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nc, PetscScalar *mode, void *ctx)
140026175e5SToby Isaac {
14112adca46SMatthew G. Knepley   const PetscInt eps[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
142026175e5SToby Isaac   PetscInt *ctxInt  = (PetscInt *) ctx;
143ad917190SMatthew G. Knepley   PetscInt  dim2    = ctxInt[0];
144026175e5SToby Isaac   PetscInt  d       = ctxInt[1];
145026175e5SToby Isaac   PetscInt  i, j, k = dim > 2 ? d - dim : d;
146026175e5SToby Isaac 
147ad917190SMatthew G. Knepley   PetscFunctionBegin;
14863a3b9bcSJacob Faibussowitsch   PetscCheck(dim == dim2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Input dimension %" PetscInt_FMT " does not match context dimension %" PetscInt_FMT, dim, dim2);
149026175e5SToby Isaac   for (i = 0; i < dim; i++) mode[i] = 0.;
150026175e5SToby Isaac   if (d < dim) {
15112adca46SMatthew G. Knepley     mode[d] = 1.; /* Translation along axis d */
152ad917190SMatthew G. Knepley   } else {
153026175e5SToby Isaac     for (i = 0; i < dim; i++) {
154026175e5SToby Isaac       for (j = 0; j < dim; j++) {
15512adca46SMatthew G. Knepley         mode[j] += eps[i][j][k]*X[i]; /* Rotation about axis d */
156026175e5SToby Isaac       }
157026175e5SToby Isaac     }
158026175e5SToby Isaac   }
159ad917190SMatthew G. Knepley   PetscFunctionReturn(0);
160026175e5SToby Isaac }
161026175e5SToby Isaac 
162cc4e42d9SMartin Diehl /*@
16312adca46SMatthew G. Knepley   DMPlexCreateRigidBody - For the default global section, create rigid body modes by function space interpolation
164cb1e1211SMatthew G Knepley 
165d083f849SBarry Smith   Collective on dm
166cb1e1211SMatthew G Knepley 
1674165533cSJose E. Roman   Input Parameters:
16856cf3b9cSMatthew G. Knepley + dm - the DM
16956cf3b9cSMatthew G. Knepley - field - The field number for the rigid body space, or 0 for the default
170cb1e1211SMatthew G Knepley 
1714165533cSJose E. Roman   Output Parameter:
172cb1e1211SMatthew G Knepley . sp - the null space
173cb1e1211SMatthew G Knepley 
17412adca46SMatthew G. Knepley   Note: This is necessary to provide a suitable coarse space for algebraic multigrid
175cb1e1211SMatthew G Knepley 
176cb1e1211SMatthew G Knepley   Level: advanced
177cb1e1211SMatthew G Knepley 
178db781477SPatrick Sanan .seealso: `MatNullSpaceCreate()`, `PCGAMG`
179cb1e1211SMatthew G Knepley @*/
18056cf3b9cSMatthew G. Knepley PetscErrorCode DMPlexCreateRigidBody(DM dm, PetscInt field, MatNullSpace *sp)
181cb1e1211SMatthew G Knepley {
18256cf3b9cSMatthew G. Knepley   PetscErrorCode (**func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *);
183cb1e1211SMatthew G Knepley   MPI_Comm          comm;
184026175e5SToby Isaac   Vec               mode[6];
185026175e5SToby Isaac   PetscSection      section, globalSection;
18656cf3b9cSMatthew G. Knepley   PetscInt          dim, dimEmbed, Nf, n, m, mmin, d, i, j;
187cb1e1211SMatthew G Knepley 
188cb1e1211SMatthew G Knepley   PetscFunctionBegin;
1899566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject) dm, &comm));
1909566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
1919566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
1929566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
19363a3b9bcSJacob Faibussowitsch   PetscCheck(!Nf || !(field < 0 || field >= Nf),comm, PETSC_ERR_ARG_OUTOFRANGE, "Field %" PetscInt_FMT " is not in [0, %" PetscInt_FMT ")", field, Nf);
19456cf3b9cSMatthew G. Knepley   if (dim == 1 && Nf < 2) {
1959566063dSJacob Faibussowitsch     PetscCall(MatNullSpaceCreate(comm, PETSC_TRUE, 0, NULL, sp));
196cb1e1211SMatthew G Knepley     PetscFunctionReturn(0);
197cb1e1211SMatthew G Knepley   }
1989566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
1999566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
2009566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
2019566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(Nf, &func));
202b247467aSMatthew G. Knepley   m    = (dim*(dim+1))/2;
2039566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
2049566063dSJacob Faibussowitsch   PetscCall(VecSetType(mode[0], dm->vectype));
2059566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
2069566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
2079566063dSJacob Faibussowitsch   PetscCall(VecGetSize(mode[0], &n));
208b247467aSMatthew G. Knepley   mmin = PetscMin(m, n);
20956cf3b9cSMatthew G. Knepley   func[field] = DMPlexProjectRigidBody_Private;
2109566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
211026175e5SToby Isaac   for (d = 0; d < m; d++) {
212330485fdSToby Isaac     PetscInt ctx[2];
213026175e5SToby Isaac     void    *voidctx = (void *) (&ctx[0]);
214cb1e1211SMatthew G Knepley 
2159d8fbdeaSMatthew G. Knepley     ctx[0] = dimEmbed;
216330485fdSToby Isaac     ctx[1] = d;
2179566063dSJacob Faibussowitsch     PetscCall(DMProjectFunction(dm, 0.0, func, &voidctx, INSERT_VALUES, mode[d]));
218cb1e1211SMatthew G Knepley   }
2193b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
220b50a2c0aSJacob Faibussowitsch   for (i = 0; i < mmin; ++i) {
221b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
222b50a2c0aSJacob Faibussowitsch 
2239566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
2249566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], mmin-i-1, mode+i+1, dots+i+1));
225b50a2c0aSJacob Faibussowitsch     for (j = i+1; j < mmin; ++j) {
226b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
2279566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
228b50a2c0aSJacob Faibussowitsch     }
229cb1e1211SMatthew G Knepley   }
2309566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, mmin, mode, sp));
2319566063dSJacob Faibussowitsch   for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i]));
2329566063dSJacob Faibussowitsch   PetscCall(PetscFree(func));
233cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
234cb1e1211SMatthew G Knepley }
235cb1e1211SMatthew G Knepley 
236cc4e42d9SMartin Diehl /*@
23712adca46SMatthew G. Knepley   DMPlexCreateRigidBodies - For the default global section, create rigid body modes by function space interpolation
23812adca46SMatthew G. Knepley 
239d083f849SBarry Smith   Collective on dm
24012adca46SMatthew G. Knepley 
2414165533cSJose E. Roman   Input Parameters:
24212adca46SMatthew G. Knepley + dm    - the DM
24312adca46SMatthew G. Knepley . nb    - The number of bodies
24412adca46SMatthew G. Knepley . label - The DMLabel marking each domain
24512adca46SMatthew G. Knepley . nids  - The number of ids per body
24612adca46SMatthew G. Knepley - ids   - An array of the label ids in sequence for each domain
24712adca46SMatthew G. Knepley 
2484165533cSJose E. Roman   Output Parameter:
24912adca46SMatthew G. Knepley . sp - the null space
25012adca46SMatthew G. Knepley 
25112adca46SMatthew G. Knepley   Note: This is necessary to provide a suitable coarse space for algebraic multigrid
25212adca46SMatthew G. Knepley 
25312adca46SMatthew G. Knepley   Level: advanced
25412adca46SMatthew G. Knepley 
255db781477SPatrick Sanan .seealso: `MatNullSpaceCreate()`
25612adca46SMatthew G. Knepley @*/
25712adca46SMatthew G. Knepley PetscErrorCode DMPlexCreateRigidBodies(DM dm, PetscInt nb, DMLabel label, const PetscInt nids[], const PetscInt ids[], MatNullSpace *sp)
25812adca46SMatthew G. Knepley {
25912adca46SMatthew G. Knepley   MPI_Comm       comm;
26012adca46SMatthew G. Knepley   PetscSection   section, globalSection;
26112adca46SMatthew G. Knepley   Vec           *mode;
26212adca46SMatthew G. Knepley   PetscScalar   *dots;
26312adca46SMatthew G. Knepley   PetscInt       dim, dimEmbed, n, m, b, d, i, j, off;
26412adca46SMatthew G. Knepley 
26512adca46SMatthew G. Knepley   PetscFunctionBegin;
2669566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2679566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
2689566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
2699566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
2709566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
2719566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
27212adca46SMatthew G. Knepley   m    = nb * (dim*(dim+1))/2;
2739566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(m, &mode, m, &dots));
2749566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
2759566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
2769566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
2779566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
27812adca46SMatthew G. Knepley   for (b = 0, off = 0; b < nb; ++b) {
27912adca46SMatthew G. Knepley     for (d = 0; d < m/nb; ++d) {
28012adca46SMatthew G. Knepley       PetscInt         ctx[2];
28112adca46SMatthew G. Knepley       PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private;
28212adca46SMatthew G. Knepley       void            *voidctx = (void *) (&ctx[0]);
28312adca46SMatthew G. Knepley 
28412adca46SMatthew G. Knepley       ctx[0] = dimEmbed;
28512adca46SMatthew G. Knepley       ctx[1] = d;
2869566063dSJacob Faibussowitsch       PetscCall(DMProjectFunctionLabel(dm, 0.0, label, nids[b], &ids[off], 0, NULL, &func, &voidctx, INSERT_VALUES, mode[d]));
28712adca46SMatthew G. Knepley       off   += nids[b];
28812adca46SMatthew G. Knepley     }
28912adca46SMatthew G. Knepley   }
2903b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
291606c1a1cSJacob Faibussowitsch   for (i = 0; i < m; ++i) {
292b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
2935a0e29b9SJacob Faibussowitsch 
2949566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
2959566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], m-i-1, mode+i+1, dots+i+1));
2965a0e29b9SJacob Faibussowitsch     for (j = i+1; j < m; ++j) {
297b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
2989566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
2995a0e29b9SJacob Faibussowitsch     }
30012adca46SMatthew G. Knepley   }
3019566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, m, mode, sp));
3029566063dSJacob Faibussowitsch   for (i = 0; i< m; ++i) PetscCall(VecDestroy(&mode[i]));
3039566063dSJacob Faibussowitsch   PetscCall(PetscFree2(mode, dots));
30412adca46SMatthew G. Knepley   PetscFunctionReturn(0);
30512adca46SMatthew G. Knepley }
30612adca46SMatthew G. Knepley 
307b29cfa1cSToby Isaac /*@
308b29cfa1cSToby Isaac   DMPlexSetMaxProjectionHeight - In DMPlexProjectXXXLocal() functions, the projected values of a basis function's dofs
309b29cfa1cSToby Isaac   are computed by associating the basis function with one of the mesh points in its transitively-closed support, and
310b29cfa1cSToby Isaac   evaluating the dual space basis of that point.  A basis function is associated with the point in its
311b29cfa1cSToby Isaac   transitively-closed support whose mesh height is highest (w.r.t. DAG height), but not greater than the maximum
312b29cfa1cSToby Isaac   projection height, which is set with this function.  By default, the maximum projection height is zero, which means
313b29cfa1cSToby Isaac   that only mesh cells are used to project basis functions.  A height of one, for example, evaluates a cell-interior
314b29cfa1cSToby Isaac   basis functions using its cells dual space basis, but all other basis functions with the dual space basis of a face.
315b29cfa1cSToby Isaac 
316b29cfa1cSToby Isaac   Input Parameters:
317b29cfa1cSToby Isaac + dm - the DMPlex object
318b29cfa1cSToby Isaac - height - the maximum projection height >= 0
319b29cfa1cSToby Isaac 
320b29cfa1cSToby Isaac   Level: advanced
321b29cfa1cSToby Isaac 
322db781477SPatrick Sanan .seealso: `DMPlexGetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()`
323b29cfa1cSToby Isaac @*/
324b29cfa1cSToby Isaac PetscErrorCode DMPlexSetMaxProjectionHeight(DM dm, PetscInt height)
325b29cfa1cSToby Isaac {
326b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *) dm->data;
327b29cfa1cSToby Isaac 
328b29cfa1cSToby Isaac   PetscFunctionBegin;
329b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
330b29cfa1cSToby Isaac   plex->maxProjectionHeight = height;
331b29cfa1cSToby Isaac   PetscFunctionReturn(0);
332b29cfa1cSToby Isaac }
333b29cfa1cSToby Isaac 
334b29cfa1cSToby Isaac /*@
335b29cfa1cSToby Isaac   DMPlexGetMaxProjectionHeight - Get the maximum height (w.r.t. DAG) of mesh points used to evaluate dual bases in
336b29cfa1cSToby Isaac   DMPlexProjectXXXLocal() functions.
337b29cfa1cSToby Isaac 
338b29cfa1cSToby Isaac   Input Parameters:
339b29cfa1cSToby Isaac . dm - the DMPlex object
340b29cfa1cSToby Isaac 
341b29cfa1cSToby Isaac   Output Parameters:
342b29cfa1cSToby Isaac . height - the maximum projection height
343b29cfa1cSToby Isaac 
344b29cfa1cSToby Isaac   Level: intermediate
345b29cfa1cSToby Isaac 
346db781477SPatrick Sanan .seealso: `DMPlexSetMaxProjectionHeight()`, `DMProjectFunctionLocal()`, `DMProjectFunctionLabelLocal()`
347b29cfa1cSToby Isaac @*/
348b29cfa1cSToby Isaac PetscErrorCode DMPlexGetMaxProjectionHeight(DM dm, PetscInt *height)
349b29cfa1cSToby Isaac {
350b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *) dm->data;
351b29cfa1cSToby Isaac 
352b29cfa1cSToby Isaac   PetscFunctionBegin;
353b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
354b29cfa1cSToby Isaac   *height = plex->maxProjectionHeight;
355b29cfa1cSToby Isaac   PetscFunctionReturn(0);
356b29cfa1cSToby Isaac }
357b29cfa1cSToby Isaac 
358ca3d3a14SMatthew G. Knepley typedef struct {
359ca3d3a14SMatthew G. Knepley   PetscReal    alpha; /* The first Euler angle, and in 2D the only one */
360ca3d3a14SMatthew G. Knepley   PetscReal    beta;  /* The second Euler angle */
361ca3d3a14SMatthew G. Knepley   PetscReal    gamma; /* The third Euler angle */
362ca3d3a14SMatthew G. Knepley   PetscInt     dim;   /* The dimension of R */
363ca3d3a14SMatthew G. Knepley   PetscScalar *R;     /* The rotation matrix, transforming a vector in the local basis to the global basis */
364ca3d3a14SMatthew G. Knepley   PetscScalar *RT;    /* The transposed rotation matrix, transforming a vector in the global basis to the local basis */
365ca3d3a14SMatthew G. Knepley } RotCtx;
366ca3d3a14SMatthew G. Knepley 
367ca3d3a14SMatthew G. Knepley /*
368ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
369ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
370ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
371ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
372ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
373ca3d3a14SMatthew G. Knepley */
374ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformSetUp_Rotation_Internal(DM dm, void *ctx)
375ca3d3a14SMatthew G. Knepley {
376ca3d3a14SMatthew G. Knepley   RotCtx        *rc  = (RotCtx *) ctx;
377ca3d3a14SMatthew G. Knepley   PetscInt       dim = rc->dim;
378ca3d3a14SMatthew G. Knepley   PetscReal      c1, s1, c2, s2, c3, s3;
379ca3d3a14SMatthew G. Knepley 
380ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
3819566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(PetscSqr(dim), &rc->R, PetscSqr(dim), &rc->RT));
382ca3d3a14SMatthew G. Knepley   switch (dim) {
383ca3d3a14SMatthew G. Knepley   case 2:
384ca3d3a14SMatthew G. Knepley     c1 = PetscCosReal(rc->alpha);s1 = PetscSinReal(rc->alpha);
385ca3d3a14SMatthew G. Knepley     rc->R[0] =  c1;rc->R[1] = s1;
386ca3d3a14SMatthew G. Knepley     rc->R[2] = -s1;rc->R[3] = c1;
3879566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
388b458e8f1SJose E. Roman     DMPlex_Transpose2D_Internal(rc->RT);
389ca3d3a14SMatthew G. Knepley     break;
390ca3d3a14SMatthew G. Knepley   case 3:
391ca3d3a14SMatthew G. Knepley     c1 = PetscCosReal(rc->alpha);s1 = PetscSinReal(rc->alpha);
392ca3d3a14SMatthew G. Knepley     c2 = PetscCosReal(rc->beta); s2 = PetscSinReal(rc->beta);
393ca3d3a14SMatthew G. Knepley     c3 = PetscCosReal(rc->gamma);s3 = PetscSinReal(rc->gamma);
394ca3d3a14SMatthew G. Knepley     rc->R[0] =  c1*c3 - c2*s1*s3;rc->R[1] =  c3*s1    + c1*c2*s3;rc->R[2] = s2*s3;
395ca3d3a14SMatthew G. Knepley     rc->R[3] = -c1*s3 - c2*c3*s1;rc->R[4] =  c1*c2*c3 - s1*s3;   rc->R[5] = c3*s2;
396ca3d3a14SMatthew G. Knepley     rc->R[6] =  s1*s2;           rc->R[7] = -c1*s2;              rc->R[8] = c2;
3979566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
398b458e8f1SJose E. Roman     DMPlex_Transpose3D_Internal(rc->RT);
399ca3d3a14SMatthew G. Knepley     break;
40063a3b9bcSJacob Faibussowitsch   default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " not supported", dim);
401ca3d3a14SMatthew G. Knepley   }
402ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
403ca3d3a14SMatthew G. Knepley }
404ca3d3a14SMatthew G. Knepley 
405ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformDestroy_Rotation_Internal(DM dm, void *ctx)
406ca3d3a14SMatthew G. Knepley {
407ca3d3a14SMatthew G. Knepley   RotCtx        *rc = (RotCtx *) ctx;
408ca3d3a14SMatthew G. Knepley 
409ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
4109566063dSJacob Faibussowitsch   PetscCall(PetscFree2(rc->R, rc->RT));
4119566063dSJacob Faibussowitsch   PetscCall(PetscFree(rc));
412ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
413ca3d3a14SMatthew G. Knepley }
414ca3d3a14SMatthew G. Knepley 
415ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformGetMatrix_Rotation_Internal(DM dm, const PetscReal x[], PetscBool l2g, const PetscScalar **A, void *ctx)
416ca3d3a14SMatthew G. Knepley {
417ca3d3a14SMatthew G. Knepley   RotCtx *rc = (RotCtx *) ctx;
418ca3d3a14SMatthew G. Knepley 
419ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
420ca3d3a14SMatthew G. Knepley   PetscValidPointer(ctx, 5);
421ca3d3a14SMatthew G. Knepley   if (l2g) {*A = rc->R;}
422ca3d3a14SMatthew G. Knepley   else     {*A = rc->RT;}
423ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
424ca3d3a14SMatthew G. Knepley }
425ca3d3a14SMatthew G. Knepley 
426ab6a9622SMatthew G. Knepley PetscErrorCode DMPlexBasisTransformApplyReal_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscReal *y, PetscReal *z, void *ctx)
427ab6a9622SMatthew G. Knepley {
428ec277c0fSMatthew G. Knepley   PetscFunctionBegin;
429ab6a9622SMatthew G. Knepley   #if defined(PETSC_USE_COMPLEX)
430ab6a9622SMatthew G. Knepley   switch (dim) {
431ab6a9622SMatthew G. Knepley     case 2:
432ab6a9622SMatthew G. Knepley     {
43327104ee2SJacob Faibussowitsch       PetscScalar yt[2] = {y[0], y[1]}, zt[2] = {0.0,0.0};
434ab6a9622SMatthew G. Knepley 
4359566063dSJacob Faibussowitsch       PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
436ab6a9622SMatthew G. Knepley       z[0] = PetscRealPart(zt[0]); z[1] = PetscRealPart(zt[1]);
437ab6a9622SMatthew G. Knepley     }
4389b4815dcSMatthew G. Knepley     break;
439ab6a9622SMatthew G. Knepley     case 3:
440ab6a9622SMatthew G. Knepley     {
44127104ee2SJacob Faibussowitsch       PetscScalar yt[3] = {y[0], y[1], y[2]}, zt[3] = {0.0,0.0,0.0};
442ab6a9622SMatthew G. Knepley 
4439566063dSJacob Faibussowitsch       PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
444ab6a9622SMatthew G. Knepley       z[0] = PetscRealPart(zt[0]); z[1] = PetscRealPart(zt[1]); z[2] = PetscRealPart(zt[2]);
445ab6a9622SMatthew G. Knepley     }
4469b4815dcSMatthew G. Knepley     break;
447ab6a9622SMatthew G. Knepley   }
448ab6a9622SMatthew G. Knepley   #else
4499566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, y, z, ctx));
450ab6a9622SMatthew G. Knepley   #endif
451ec277c0fSMatthew G. Knepley   PetscFunctionReturn(0);
452ab6a9622SMatthew G. Knepley }
453ab6a9622SMatthew G. Knepley 
454ec277c0fSMatthew G. Knepley PetscErrorCode DMPlexBasisTransformApply_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscScalar *y, PetscScalar *z, void *ctx)
455ca3d3a14SMatthew G. Knepley {
456ca3d3a14SMatthew G. Knepley   const PetscScalar *A;
457ca3d3a14SMatthew G. Knepley 
458ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
4599566063dSJacob Faibussowitsch   PetscCall((*dm->transformGetMatrix)(dm, x, l2g, &A, ctx));
460ca3d3a14SMatthew G. Knepley   switch (dim) {
4611fba962aSMatthew G. Knepley   case 2: DMPlex_Mult2D_Internal(A, 1, y, z);break;
4621fba962aSMatthew G. Knepley   case 3: DMPlex_Mult3D_Internal(A, 1, y, z);break;
463ca3d3a14SMatthew G. Knepley   }
464ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
465ca3d3a14SMatthew G. Knepley }
466ca3d3a14SMatthew G. Knepley 
467ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformField_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscInt f, PetscBool l2g, PetscScalar *a)
468ca3d3a14SMatthew G. Knepley {
469ca3d3a14SMatthew G. Knepley   PetscSection       ts;
470ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tva;
471ca3d3a14SMatthew G. Knepley   PetscInt           dof;
472ca3d3a14SMatthew G. Knepley 
473ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
4749566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
4759566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, p, f, &dof));
4769566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
4779566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, p, f, ta, &tva));
478ca3d3a14SMatthew G. Knepley   if (l2g) {
479ca3d3a14SMatthew G. Knepley     switch (dof) {
4801fba962aSMatthew G. Knepley     case 4: DMPlex_Mult2D_Internal(tva, 1, a, a);break;
4811fba962aSMatthew G. Knepley     case 9: DMPlex_Mult3D_Internal(tva, 1, a, a);break;
482ca3d3a14SMatthew G. Knepley     }
483ca3d3a14SMatthew G. Knepley   } else {
484ca3d3a14SMatthew G. Knepley     switch (dof) {
4851fba962aSMatthew G. Knepley     case 4: DMPlex_MultTranspose2D_Internal(tva, 1, a, a);break;
4861fba962aSMatthew G. Knepley     case 9: DMPlex_MultTranspose3D_Internal(tva, 1, a, a);break;
487ca3d3a14SMatthew G. Knepley     }
488ca3d3a14SMatthew G. Knepley   }
4899566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
490ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
491ca3d3a14SMatthew G. Knepley }
492ca3d3a14SMatthew G. Knepley 
493ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformFieldTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt pf, PetscInt f, PetscInt pg, PetscInt g, PetscBool l2g, PetscInt lda, PetscScalar *a)
494ca3d3a14SMatthew G. Knepley {
495ca3d3a14SMatthew G. Knepley   PetscSection       s, ts;
496ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tvaf, *tvag;
497ca3d3a14SMatthew G. Knepley   PetscInt           fdof, gdof, fpdof, gpdof;
498ca3d3a14SMatthew G. Knepley 
499ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
5009566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
5019566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
5029566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pf, f, &fpdof));
5039566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pg, g, &gpdof));
5049566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pf, f, &fdof));
5059566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pg, g, &gdof));
5069566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
5079566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pf, f, ta, &tvaf));
5089566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pg, g, ta, &tvag));
509ca3d3a14SMatthew G. Knepley   if (l2g) {
510ca3d3a14SMatthew G. Knepley     switch (fdof) {
511ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMult2D_Internal(tvaf, gpdof, lda, a, a);break;
512ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMult3D_Internal(tvaf, gpdof, lda, a, a);break;
513ca3d3a14SMatthew G. Knepley     }
514ca3d3a14SMatthew G. Knepley     switch (gdof) {
515ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultTransposeLeft2D_Internal(tvag, fpdof, lda, a, a);break;
516ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultTransposeLeft3D_Internal(tvag, fpdof, lda, a, a);break;
517ca3d3a14SMatthew G. Knepley     }
518ca3d3a14SMatthew G. Knepley   } else {
519ca3d3a14SMatthew G. Knepley     switch (fdof) {
520ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultTranspose2D_Internal(tvaf, gpdof, lda, a, a);break;
521ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultTranspose3D_Internal(tvaf, gpdof, lda, a, a);break;
522ca3d3a14SMatthew G. Knepley     }
523ca3d3a14SMatthew G. Knepley     switch (gdof) {
524ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultLeft2D_Internal(tvag, fpdof, lda, a, a);break;
525ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultLeft3D_Internal(tvag, fpdof, lda, a, a);break;
526ca3d3a14SMatthew G. Knepley     }
527ca3d3a14SMatthew G. Knepley   }
5289566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
529ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
530ca3d3a14SMatthew G. Knepley }
531ca3d3a14SMatthew G. Knepley 
532ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexBasisTransformPoint_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool fieldActive[], PetscBool l2g, PetscScalar *a)
533ca3d3a14SMatthew G. Knepley {
534ca3d3a14SMatthew G. Knepley   PetscSection    s;
535ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
536ca3d3a14SMatthew G. Knepley   IS              clPoints;
537ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
538ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
539ca3d3a14SMatthew G. Knepley   PetscInt        Nf, f, Np, cp, dof, d = 0;
540ca3d3a14SMatthew G. Knepley 
541ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5429566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
5439566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
5449566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
545ca3d3a14SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
546ca3d3a14SMatthew G. Knepley     for (cp = 0; cp < Np*2; cp += 2) {
5479566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cp], f, &dof));
548ca3d3a14SMatthew G. Knepley       if (!dof) continue;
5499566063dSJacob Faibussowitsch       if (fieldActive[f]) PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, points[cp], f, l2g, &a[d]));
550ca3d3a14SMatthew G. Knepley       d += dof;
551ca3d3a14SMatthew G. Knepley     }
552ca3d3a14SMatthew G. Knepley   }
5539566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
554ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
555ca3d3a14SMatthew G. Knepley }
556ca3d3a14SMatthew G. Knepley 
557ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexBasisTransformPointTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool l2g, PetscInt lda, PetscScalar *a)
558ca3d3a14SMatthew G. Knepley {
559ca3d3a14SMatthew G. Knepley   PetscSection    s;
560ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
561ca3d3a14SMatthew G. Knepley   IS              clPoints;
562ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
563ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
5648bdb3c71SMatthew G. Knepley   PetscInt        Nf, f, g, Np, cpf, cpg, fdof, gdof, r, c = 0;
565ca3d3a14SMatthew G. Knepley 
566ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5679566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
5689566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
5699566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
570ca3d3a14SMatthew G. Knepley   for (f = 0, r = 0; f < Nf; ++f) {
571ca3d3a14SMatthew G. Knepley     for (cpf = 0; cpf < Np*2; cpf += 2) {
5729566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cpf], f, &fdof));
573ca3d3a14SMatthew G. Knepley       for (g = 0, c = 0; g < Nf; ++g) {
574ca3d3a14SMatthew G. Knepley         for (cpg = 0; cpg < Np*2; cpg += 2) {
5759566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldDof(s, points[cpg], g, &gdof));
5769566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformFieldTensor_Internal(dm, tdm, tv, points[cpf], f, points[cpg], g, l2g, lda, &a[r*lda+c]));
577ca3d3a14SMatthew G. Knepley           c += gdof;
578ca3d3a14SMatthew G. Knepley         }
579ca3d3a14SMatthew G. Knepley       }
58063a3b9bcSJacob Faibussowitsch       PetscCheck(c == lda,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of columns %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
581ca3d3a14SMatthew G. Knepley       r += fdof;
582ca3d3a14SMatthew G. Knepley     }
583ca3d3a14SMatthew G. Knepley   }
58463a3b9bcSJacob Faibussowitsch   PetscCheck(r == lda,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of rows %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
5859566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
586ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
587ca3d3a14SMatthew G. Knepley }
588ca3d3a14SMatthew G. Knepley 
589ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransform_Internal(DM dm, Vec lv, PetscBool l2g)
590ca3d3a14SMatthew G. Knepley {
591ca3d3a14SMatthew G. Knepley   DM                 tdm;
592ca3d3a14SMatthew G. Knepley   Vec                tv;
593ca3d3a14SMatthew G. Knepley   PetscSection       ts, s;
594ca3d3a14SMatthew G. Knepley   const PetscScalar *ta;
595ca3d3a14SMatthew G. Knepley   PetscScalar       *a, *va;
596ca3d3a14SMatthew G. Knepley   PetscInt           pStart, pEnd, p, Nf, f;
597ca3d3a14SMatthew G. Knepley 
598ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5999566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
6009566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
6019566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
6029566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
6039566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetChart(s, &pStart, &pEnd));
6049566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
6059566063dSJacob Faibussowitsch   PetscCall(VecGetArray(lv, &a));
6069566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
607ca3d3a14SMatthew G. Knepley   for (p = pStart; p < pEnd; ++p) {
608ca3d3a14SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
6099566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalFieldRef(dm, p, f, a, &va));
6109566063dSJacob Faibussowitsch       PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, p, f, l2g, va));
611ca3d3a14SMatthew G. Knepley     }
612ca3d3a14SMatthew G. Knepley   }
6139566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(lv, &a));
6149566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
615ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
616ca3d3a14SMatthew G. Knepley }
617ca3d3a14SMatthew G. Knepley 
618ca3d3a14SMatthew G. Knepley /*@
619ca3d3a14SMatthew G. Knepley   DMPlexGlobalToLocalBasis - Transform the values in the given local vector from the global basis to the local basis
620ca3d3a14SMatthew G. Knepley 
621ca3d3a14SMatthew G. Knepley   Input Parameters:
622ca3d3a14SMatthew G. Knepley + dm - The DM
623ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the global basis
624ca3d3a14SMatthew G. Knepley 
625ca3d3a14SMatthew G. Knepley   Output Parameters:
626ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the local basis
627ca3d3a14SMatthew G. Knepley 
628c0f8e1fdSMatthew G. Knepley   Note: This method is only intended to be called inside DMGlobalToLocal(). It is unlikely that a user will have a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
629c0f8e1fdSMatthew G. Knepley 
630ca3d3a14SMatthew G. Knepley   Level: developer
631ca3d3a14SMatthew G. Knepley 
632db781477SPatrick Sanan .seealso: `DMPlexLocalToGlobalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()`
633ca3d3a14SMatthew G. Knepley @*/
634ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexGlobalToLocalBasis(DM dm, Vec lv)
635ca3d3a14SMatthew G. Knepley {
636ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
637ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
638ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
6399566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_FALSE));
640ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
641ca3d3a14SMatthew G. Knepley }
642ca3d3a14SMatthew G. Knepley 
643ca3d3a14SMatthew G. Knepley /*@
644ca3d3a14SMatthew G. Knepley   DMPlexLocalToGlobalBasis - Transform the values in the given local vector from the local basis to the global basis
645ca3d3a14SMatthew G. Knepley 
646ca3d3a14SMatthew G. Knepley   Input Parameters:
647ca3d3a14SMatthew G. Knepley + dm - The DM
648ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the local basis
649ca3d3a14SMatthew G. Knepley 
650ca3d3a14SMatthew G. Knepley   Output Parameters:
651ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the global basis
652ca3d3a14SMatthew G. Knepley 
653c0f8e1fdSMatthew G. Knepley   Note: This method is only intended to be called inside DMGlobalToLocal(). It is unlikely that a user would want a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
654c0f8e1fdSMatthew G. Knepley 
655ca3d3a14SMatthew G. Knepley   Level: developer
656ca3d3a14SMatthew G. Knepley 
657db781477SPatrick Sanan .seealso: `DMPlexGlobalToLocalBasis()`, `DMGetLocalSection()`, `DMPlexCreateBasisRotation()`
658ca3d3a14SMatthew G. Knepley @*/
659ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexLocalToGlobalBasis(DM dm, Vec lv)
660ca3d3a14SMatthew G. Knepley {
661ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
662ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
663ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
6649566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_TRUE));
665ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
666ca3d3a14SMatthew G. Knepley }
667ca3d3a14SMatthew G. Knepley 
668ca3d3a14SMatthew G. Knepley /*@
669ca3d3a14SMatthew G. Knepley   DMPlexCreateBasisRotation - Create an internal transformation from the global basis, used to specify boundary conditions
670ca3d3a14SMatthew G. Knepley     and global solutions, to a local basis, appropriate for discretization integrals and assembly.
671ca3d3a14SMatthew G. Knepley 
672ca3d3a14SMatthew G. Knepley   Input Parameters:
673ca3d3a14SMatthew G. Knepley + dm    - The DM
674ca3d3a14SMatthew G. Knepley . alpha - The first Euler angle, and in 2D the only one
675ca3d3a14SMatthew G. Knepley . beta  - The second Euler angle
676f0fc11ceSJed Brown - gamma - The third Euler angle
677ca3d3a14SMatthew G. Knepley 
678ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
679ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
680ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
681ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
682ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
683ca3d3a14SMatthew G. Knepley 
684ca3d3a14SMatthew G. Knepley   Level: developer
685ca3d3a14SMatthew G. Knepley 
686db781477SPatrick Sanan .seealso: `DMPlexGlobalToLocalBasis()`, `DMPlexLocalToGlobalBasis()`
687ca3d3a14SMatthew G. Knepley @*/
688ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexCreateBasisRotation(DM dm, PetscReal alpha, PetscReal beta, PetscReal gamma)
689ca3d3a14SMatthew G. Knepley {
690ca3d3a14SMatthew G. Knepley   RotCtx        *rc;
691ca3d3a14SMatthew G. Knepley   PetscInt       cdim;
692ca3d3a14SMatthew G. Knepley 
693362febeeSStefano Zampini   PetscFunctionBegin;
6949566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
6959566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(1, &rc));
696ca3d3a14SMatthew G. Knepley   dm->transformCtx       = rc;
697ca3d3a14SMatthew G. Knepley   dm->transformSetUp     = DMPlexBasisTransformSetUp_Rotation_Internal;
698ca3d3a14SMatthew G. Knepley   dm->transformDestroy   = DMPlexBasisTransformDestroy_Rotation_Internal;
699ca3d3a14SMatthew G. Knepley   dm->transformGetMatrix = DMPlexBasisTransformGetMatrix_Rotation_Internal;
700ca3d3a14SMatthew G. Knepley   rc->dim   = cdim;
701ca3d3a14SMatthew G. Knepley   rc->alpha = alpha;
702ca3d3a14SMatthew G. Knepley   rc->beta  = beta;
703ca3d3a14SMatthew G. Knepley   rc->gamma = gamma;
7049566063dSJacob Faibussowitsch   PetscCall((*dm->transformSetUp)(dm, dm->transformCtx));
7059566063dSJacob Faibussowitsch   PetscCall(DMConstructBasisTransform_Internal(dm));
706ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
707ca3d3a14SMatthew G. Knepley }
708ca3d3a14SMatthew G. Knepley 
709b278463cSMatthew G. Knepley /*@C
710ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssential - Insert boundary values into a local vector using a function of the coordinates
711b278463cSMatthew G. Knepley 
712b278463cSMatthew G. Knepley   Input Parameters:
713b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
714b278463cSMatthew G. Knepley . time   - The time
715b278463cSMatthew G. Knepley . field  - The field to constrain
7161c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
7171c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
718b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
719b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
720b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
721b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
722b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
723b278463cSMatthew G. Knepley 
724b278463cSMatthew G. Knepley   Output Parameter:
725b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
726b278463cSMatthew G. Knepley 
727b278463cSMatthew G. Knepley   Level: developer
728b278463cSMatthew G. Knepley 
729db781477SPatrick Sanan .seealso: `DMPlexInsertBoundaryValuesEssentialField()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()`
730b278463cSMatthew G. Knepley @*/
7311c531cf8SMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesEssential(DM dm, PetscReal time, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void *ctx, Vec locX)
73255f2e967SMatthew G. Knepley {
7330163fd50SMatthew G. Knepley   PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx);
73455f2e967SMatthew G. Knepley   void            **ctxs;
735d7ddef95SMatthew G. Knepley   PetscInt          numFields;
736d7ddef95SMatthew G. Knepley 
737d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
7389566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
7399566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields,&funcs,numFields,&ctxs));
740d7ddef95SMatthew G. Knepley   funcs[field] = func;
741d7ddef95SMatthew G. Knepley   ctxs[field]  = ctx;
7429566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_BC_VALUES, locX));
7439566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs,ctxs));
744d7ddef95SMatthew G. Knepley   PetscFunctionReturn(0);
745d7ddef95SMatthew G. Knepley }
746d7ddef95SMatthew G. Knepley 
747b278463cSMatthew G. Knepley /*@C
748ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialField - Insert boundary values into a local vector using a function of the coordinates and field data
749b278463cSMatthew G. Knepley 
750b278463cSMatthew G. Knepley   Input Parameters:
751b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
752b278463cSMatthew G. Knepley . time   - The time
753b278463cSMatthew G. Knepley . locU   - A local vector with the input solution values
754b278463cSMatthew G. Knepley . field  - The field to constrain
7551c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
7561c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
757b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
758b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
759b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
760b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
761b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
762b278463cSMatthew G. Knepley 
763b278463cSMatthew G. Knepley   Output Parameter:
764b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
765b278463cSMatthew G. Knepley 
766b278463cSMatthew G. Knepley   Level: developer
767b278463cSMatthew G. Knepley 
768db781477SPatrick Sanan .seealso: `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialBdField()`, `DMAddBoundary()`
769b278463cSMatthew G. Knepley @*/
7701c531cf8SMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesEssentialField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
771c60e475cSMatthew G. Knepley                                                         void (*func)(PetscInt, PetscInt, PetscInt,
772c60e475cSMatthew G. Knepley                                                                      const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
773c60e475cSMatthew G. Knepley                                                                      const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
77497b6e6e8SMatthew G. Knepley                                                                      PetscReal, const PetscReal[], PetscInt, const PetscScalar[],
775b278463cSMatthew G. Knepley                                                                      PetscScalar[]),
776b278463cSMatthew G. Knepley                                                         void *ctx, Vec locX)
777c60e475cSMatthew G. Knepley {
778c60e475cSMatthew G. Knepley   void (**funcs)(PetscInt, PetscInt, PetscInt,
779c60e475cSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
780c60e475cSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
78197b6e6e8SMatthew G. Knepley                  PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
782c60e475cSMatthew G. Knepley   void            **ctxs;
783c60e475cSMatthew G. Knepley   PetscInt          numFields;
784c60e475cSMatthew G. Knepley 
785c60e475cSMatthew G. Knepley   PetscFunctionBegin;
7869566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
7879566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields,&funcs,numFields,&ctxs));
788c60e475cSMatthew G. Knepley   funcs[field] = func;
789c60e475cSMatthew G. Knepley   ctxs[field]  = ctx;
7909566063dSJacob Faibussowitsch   PetscCall(DMProjectFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
7919566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs,ctxs));
792c60e475cSMatthew G. Knepley   PetscFunctionReturn(0);
793c60e475cSMatthew G. Knepley }
794c60e475cSMatthew G. Knepley 
795b278463cSMatthew G. Knepley /*@C
796ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialBdField - Insert boundary values into a local vector using a function of the coodinates and boundary field data
797ece3a9fcSMatthew G. Knepley 
798ece3a9fcSMatthew G. Knepley   Collective on dm
799ece3a9fcSMatthew G. Knepley 
800ece3a9fcSMatthew G. Knepley   Input Parameters:
801ece3a9fcSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
802ece3a9fcSMatthew G. Knepley . time   - The time
803ece3a9fcSMatthew G. Knepley . locU   - A local vector with the input solution values
804ece3a9fcSMatthew G. Knepley . field  - The field to constrain
805ece3a9fcSMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
806ece3a9fcSMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
807ece3a9fcSMatthew G. Knepley . label  - The DMLabel defining constrained points
808ece3a9fcSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
809ece3a9fcSMatthew G. Knepley . ids    - An array of ids for constrained points
810ece3a9fcSMatthew G. Knepley . func   - A pointwise function giving boundary values, the calling sequence is given in DMProjectBdFieldLabelLocal()
811ece3a9fcSMatthew G. Knepley - ctx    - An optional user context for bcFunc
812ece3a9fcSMatthew G. Knepley 
813ece3a9fcSMatthew G. Knepley   Output Parameter:
814ece3a9fcSMatthew G. Knepley . locX   - A local vector to receive the boundary values
815ece3a9fcSMatthew G. Knepley 
816ece3a9fcSMatthew G. Knepley   Level: developer
817ece3a9fcSMatthew G. Knepley 
818db781477SPatrick Sanan .seealso: `DMProjectBdFieldLabelLocal()`, `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()`
819ece3a9fcSMatthew G. Knepley @*/
820ece3a9fcSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesEssentialBdField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
821ece3a9fcSMatthew G. Knepley                                                           void (*func)(PetscInt, PetscInt, PetscInt,
822ece3a9fcSMatthew G. Knepley                                                                        const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
823ece3a9fcSMatthew G. Knepley                                                                        const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
824ece3a9fcSMatthew G. Knepley                                                                        PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[],
825ece3a9fcSMatthew G. Knepley                                                                        PetscScalar[]),
826ece3a9fcSMatthew G. Knepley                                                           void *ctx, Vec locX)
827ece3a9fcSMatthew G. Knepley {
828ece3a9fcSMatthew G. Knepley   void (**funcs)(PetscInt, PetscInt, PetscInt,
829ece3a9fcSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
830ece3a9fcSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
831ece3a9fcSMatthew G. Knepley                  PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
832ece3a9fcSMatthew G. Knepley   void            **ctxs;
833ece3a9fcSMatthew G. Knepley   PetscInt          numFields;
834ece3a9fcSMatthew G. Knepley 
835ece3a9fcSMatthew G. Knepley   PetscFunctionBegin;
8369566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
8379566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields,&funcs,numFields,&ctxs));
838ece3a9fcSMatthew G. Knepley   funcs[field] = func;
839ece3a9fcSMatthew G. Knepley   ctxs[field]  = ctx;
8409566063dSJacob Faibussowitsch   PetscCall(DMProjectBdFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
8419566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs,ctxs));
842ece3a9fcSMatthew G. Knepley   PetscFunctionReturn(0);
843ece3a9fcSMatthew G. Knepley }
844ece3a9fcSMatthew G. Knepley 
845ece3a9fcSMatthew G. Knepley /*@C
846b278463cSMatthew G. Knepley   DMPlexInsertBoundaryValuesRiemann - Insert boundary values into a local vector
847b278463cSMatthew G. Knepley 
848b278463cSMatthew G. Knepley   Input Parameters:
849b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
850b278463cSMatthew G. Knepley . time   - The time
851b278463cSMatthew G. Knepley . faceGeometry - A vector with the FVM face geometry information
852b278463cSMatthew G. Knepley . cellGeometry - A vector with the FVM cell geometry information
853b278463cSMatthew G. Knepley . Grad         - A vector with the FVM cell gradient information
854b278463cSMatthew G. Knepley . field  - The field to constrain
8551c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
8561c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
857b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
858b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
859b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
860b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
861b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
862b278463cSMatthew G. Knepley 
863b278463cSMatthew G. Knepley   Output Parameter:
864b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
865b278463cSMatthew G. Knepley 
866b278463cSMatthew G. Knepley   Note: This implementation currently ignores the numcomps/comps argument from DMAddBoundary()
867b278463cSMatthew G. Knepley 
868b278463cSMatthew G. Knepley   Level: developer
869b278463cSMatthew G. Knepley 
870db781477SPatrick Sanan .seealso: `DMPlexInsertBoundaryValuesEssential()`, `DMPlexInsertBoundaryValuesEssentialField()`, `DMAddBoundary()`
871b278463cSMatthew G. Knepley @*/
8721c531cf8SMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesRiemann(DM dm, PetscReal time, Vec faceGeometry, Vec cellGeometry, Vec Grad, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
873b278463cSMatthew G. Knepley                                                  PetscErrorCode (*func)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*), void *ctx, Vec locX)
874d7ddef95SMatthew G. Knepley {
87561f58d28SMatthew G. Knepley   PetscDS            prob;
876da97024aSMatthew G. Knepley   PetscSF            sf;
877d7ddef95SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad;
87820369375SToby Isaac   const PetscScalar *facegeom, *cellgeom = NULL, *grad;
879da97024aSMatthew G. Knepley   const PetscInt    *leaves;
880d7ddef95SMatthew G. Knepley   PetscScalar       *x, *fx;
881520b3818SMatthew G. Knepley   PetscInt           dim, nleaves, loc, fStart, fEnd, pdim, i;
8825f80ce2aSJacob Faibussowitsch   PetscErrorCode     ierru = 0;
883d7ddef95SMatthew G. Knepley 
884d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
8859566063dSJacob Faibussowitsch   PetscCall(DMGetPointSF(dm, &sf));
8869566063dSJacob Faibussowitsch   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
887da97024aSMatthew G. Knepley   nleaves = PetscMax(0, nleaves);
8889566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
8899566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
8909566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
8919566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
8929566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
89320369375SToby Isaac   if (cellGeometry) {
8949566063dSJacob Faibussowitsch     PetscCall(VecGetDM(cellGeometry, &dmCell));
8959566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
89620369375SToby Isaac   }
897d7ddef95SMatthew G. Knepley   if (Grad) {
898c0a6632aSMatthew G. Knepley     PetscFV fv;
899c0a6632aSMatthew G. Knepley 
9009566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *) &fv));
9019566063dSJacob Faibussowitsch     PetscCall(VecGetDM(Grad, &dmGrad));
9029566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(Grad, &grad));
9039566063dSJacob Faibussowitsch     PetscCall(PetscFVGetNumComponents(fv, &pdim));
9049566063dSJacob Faibussowitsch     PetscCall(DMGetWorkArray(dm, pdim, MPIU_SCALAR, &fx));
905d7ddef95SMatthew G. Knepley   }
9069566063dSJacob Faibussowitsch   PetscCall(VecGetArray(locX, &x));
907d7ddef95SMatthew G. Knepley   for (i = 0; i < numids; ++i) {
908d7ddef95SMatthew G. Knepley     IS              faceIS;
909d7ddef95SMatthew G. Knepley     const PetscInt *faces;
910d7ddef95SMatthew G. Knepley     PetscInt        numFaces, f;
911d7ddef95SMatthew G. Knepley 
9129566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, ids[i], &faceIS));
913d7ddef95SMatthew G. Knepley     if (!faceIS) continue; /* No points with that id on this process */
9149566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(faceIS, &numFaces));
9159566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(faceIS, &faces));
916d7ddef95SMatthew G. Knepley     for (f = 0; f < numFaces; ++f) {
917d7ddef95SMatthew G. Knepley       const PetscInt         face = faces[f], *cells;
918640bce14SSatish Balay       PetscFVFaceGeom        *fg;
919d7ddef95SMatthew G. Knepley 
920d7ddef95SMatthew G. Knepley       if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */
9219566063dSJacob Faibussowitsch       PetscCall(PetscFindInt(face, nleaves, (PetscInt *) leaves, &loc));
922da97024aSMatthew G. Knepley       if (loc >= 0) continue;
9239566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
9249566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, face, &cells));
925d7ddef95SMatthew G. Knepley       if (Grad) {
926640bce14SSatish Balay         PetscFVCellGeom       *cg;
927640bce14SSatish Balay         PetscScalar           *cx, *cgrad;
928d7ddef95SMatthew G. Knepley         PetscScalar           *xG;
929d7ddef95SMatthew G. Knepley         PetscReal              dx[3];
930d7ddef95SMatthew G. Knepley         PetscInt               d;
931d7ddef95SMatthew G. Knepley 
9329566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg));
9339566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &cx));
9349566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad));
9359566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
936d7ddef95SMatthew G. Knepley         DMPlex_WaxpyD_Internal(dim, -1, cg->centroid, fg->centroid, dx);
937d7ddef95SMatthew G. Knepley         for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DMPlex_DotD_Internal(dim, &cgrad[d*dim], dx);
9389566063dSJacob Faibussowitsch         PetscCall((*func)(time, fg->centroid, fg->normal, fx, xG, ctx));
939d7ddef95SMatthew G. Knepley       } else {
940640bce14SSatish Balay         PetscScalar *xI;
941d7ddef95SMatthew G. Knepley         PetscScalar *xG;
942d7ddef95SMatthew G. Knepley 
9439566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &xI));
9449566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
945e735a8a9SMatthew G. Knepley         ierru = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx);
946e735a8a9SMatthew G. Knepley         if (ierru) {
9479566063dSJacob Faibussowitsch           PetscCall(ISRestoreIndices(faceIS, &faces));
9489566063dSJacob Faibussowitsch           PetscCall(ISDestroy(&faceIS));
949e735a8a9SMatthew G. Knepley           goto cleanup;
950e735a8a9SMatthew G. Knepley         }
951d7ddef95SMatthew G. Knepley       }
952d7ddef95SMatthew G. Knepley     }
9539566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(faceIS, &faces));
9549566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&faceIS));
955d7ddef95SMatthew G. Knepley   }
956e735a8a9SMatthew G. Knepley   cleanup:
9579566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(locX, &x));
958d7ddef95SMatthew G. Knepley   if (Grad) {
9599566063dSJacob Faibussowitsch     PetscCall(DMRestoreWorkArray(dm, pdim, MPIU_SCALAR, &fx));
9609566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(Grad, &grad));
961d7ddef95SMatthew G. Knepley   }
9629566063dSJacob Faibussowitsch   if (cellGeometry) PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
9639566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
9649566063dSJacob Faibussowitsch   PetscCall(ierru);
965d7ddef95SMatthew G. Knepley   PetscFunctionReturn(0);
966d7ddef95SMatthew G. Knepley }
967d7ddef95SMatthew G. Knepley 
9680c364540SMatthew G. Knepley static PetscErrorCode zero(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nc, PetscScalar *u, void *ctx)
9690c364540SMatthew G. Knepley {
9700c364540SMatthew G. Knepley   PetscInt c;
9710c364540SMatthew G. Knepley   for (c = 0; c < Nc; ++c) u[c] = 0.0;
9720c364540SMatthew G. Knepley   return 0;
9730c364540SMatthew G. Knepley }
9740c364540SMatthew G. Knepley 
975f1d73a7aSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
976d7ddef95SMatthew G. Knepley {
9770c364540SMatthew G. Knepley   PetscObject    isZero;
978e5e52638SMatthew G. Knepley   PetscDS        prob;
979d7ddef95SMatthew G. Knepley   PetscInt       numBd, b;
98055f2e967SMatthew G. Knepley 
98155f2e967SMatthew G. Knepley   PetscFunctionBegin;
9829566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
9839566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
9849566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject) locX, "__Vec_bc_zero__", &isZero));
98555f2e967SMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
98645480ffeSMatthew G. Knepley     PetscWeakForm           wf;
987f971fd6bSMatthew G. Knepley     DMBoundaryConditionType type;
98845480ffeSMatthew G. Knepley     const char             *name;
989d7ddef95SMatthew G. Knepley     DMLabel                 label;
9901c531cf8SMatthew G. Knepley     PetscInt                field, Nc;
9911c531cf8SMatthew G. Knepley     const PetscInt         *comps;
992d7ddef95SMatthew G. Knepley     PetscObject             obj;
993d7ddef95SMatthew G. Knepley     PetscClassId            id;
99445480ffeSMatthew G. Knepley     void                  (*bvfunc)(void);
995d7ddef95SMatthew G. Knepley     PetscInt                numids;
996d7ddef95SMatthew G. Knepley     const PetscInt         *ids;
99755f2e967SMatthew G. Knepley     void                   *ctx;
99855f2e967SMatthew G. Knepley 
9999566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, &bvfunc, NULL, &ctx));
1000f971fd6bSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
10019566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
10029566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1003d7ddef95SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1004c60e475cSMatthew G. Knepley       switch (type) {
1005c60e475cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
1006c60e475cSMatthew G. Knepley       case DM_BC_ESSENTIAL:
100745480ffeSMatthew G. Knepley         {
100845480ffeSMatthew G. Knepley           PetscSimplePointFunc func = (PetscSimplePointFunc) bvfunc;
100945480ffeSMatthew G. Knepley 
101045480ffeSMatthew G. Knepley           if (isZero) func = zero;
10119566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelAddCells(dm,label));
10129566063dSJacob Faibussowitsch           PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func, ctx, locX));
10139566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelClearCells(dm,label));
101445480ffeSMatthew G. Knepley         }
1015c60e475cSMatthew G. Knepley         break;
1016c60e475cSMatthew G. Knepley       case DM_BC_ESSENTIAL_FIELD:
101745480ffeSMatthew G. Knepley         {
101845480ffeSMatthew G. Knepley           PetscPointFunc func = (PetscPointFunc) bvfunc;
101945480ffeSMatthew G. Knepley 
10209566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelAddCells(dm,label));
10219566063dSJacob Faibussowitsch           PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func, ctx, locX));
10229566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelClearCells(dm,label));
102345480ffeSMatthew G. Knepley         }
1024c60e475cSMatthew G. Knepley         break;
1025c60e475cSMatthew G. Knepley       default: break;
1026c60e475cSMatthew G. Knepley       }
1027d7ddef95SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
102845480ffeSMatthew G. Knepley       {
102945480ffeSMatthew G. Knepley         PetscErrorCode (*func)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*) = (PetscErrorCode (*)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*)) bvfunc;
103045480ffeSMatthew G. Knepley 
103143ea7facSMatthew G. Knepley         if (!faceGeomFVM) continue;
10329566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesRiemann(dm, time, faceGeomFVM, cellGeomFVM, gradFVM, field, Nc, comps, label, numids, ids, func, ctx, locX));
103345480ffeSMatthew G. Knepley       }
103463a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
103555f2e967SMatthew G. Knepley   }
103655f2e967SMatthew G. Knepley   PetscFunctionReturn(0);
103755f2e967SMatthew G. Knepley }
103855f2e967SMatthew G. Knepley 
103956cf3b9cSMatthew G. Knepley PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
104056cf3b9cSMatthew G. Knepley {
104156cf3b9cSMatthew G. Knepley   PetscObject    isZero;
104256cf3b9cSMatthew G. Knepley   PetscDS        prob;
104356cf3b9cSMatthew G. Knepley   PetscInt       numBd, b;
104456cf3b9cSMatthew G. Knepley 
104556cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
104656cf3b9cSMatthew G. Knepley   if (!locX) PetscFunctionReturn(0);
10479566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
10489566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
10499566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject) locX, "__Vec_bc_zero__", &isZero));
105056cf3b9cSMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
105145480ffeSMatthew G. Knepley     PetscWeakForm           wf;
105256cf3b9cSMatthew G. Knepley     DMBoundaryConditionType type;
105345480ffeSMatthew G. Knepley     const char             *name;
105456cf3b9cSMatthew G. Knepley     DMLabel                 label;
105556cf3b9cSMatthew G. Knepley     PetscInt                field, Nc;
105656cf3b9cSMatthew G. Knepley     const PetscInt         *comps;
105756cf3b9cSMatthew G. Knepley     PetscObject             obj;
105856cf3b9cSMatthew G. Knepley     PetscClassId            id;
105956cf3b9cSMatthew G. Knepley     PetscInt                numids;
106056cf3b9cSMatthew G. Knepley     const PetscInt         *ids;
106145480ffeSMatthew G. Knepley     void                  (*bvfunc)(void);
106256cf3b9cSMatthew G. Knepley     void                   *ctx;
106356cf3b9cSMatthew G. Knepley 
10649566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, NULL, &bvfunc, &ctx));
106556cf3b9cSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
10669566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
10679566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
106856cf3b9cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
106956cf3b9cSMatthew G. Knepley       switch (type) {
107056cf3b9cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
107156cf3b9cSMatthew G. Knepley       case DM_BC_ESSENTIAL:
107245480ffeSMatthew G. Knepley         {
107345480ffeSMatthew G. Knepley           PetscSimplePointFunc func_t = (PetscSimplePointFunc) bvfunc;
107445480ffeSMatthew G. Knepley 
107545480ffeSMatthew G. Knepley           if (isZero) func_t = zero;
10769566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelAddCells(dm,label));
10779566063dSJacob Faibussowitsch           PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
10789566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelClearCells(dm,label));
107945480ffeSMatthew G. Knepley         }
108056cf3b9cSMatthew G. Knepley         break;
108156cf3b9cSMatthew G. Knepley       case DM_BC_ESSENTIAL_FIELD:
108245480ffeSMatthew G. Knepley         {
108345480ffeSMatthew G. Knepley           PetscPointFunc func_t = (PetscPointFunc) bvfunc;
108445480ffeSMatthew G. Knepley 
10859566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelAddCells(dm,label));
10869566063dSJacob Faibussowitsch           PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
10879566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelClearCells(dm,label));
108845480ffeSMatthew G. Knepley         }
108956cf3b9cSMatthew G. Knepley         break;
109056cf3b9cSMatthew G. Knepley       default: break;
109156cf3b9cSMatthew G. Knepley       }
109263a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
109356cf3b9cSMatthew G. Knepley   }
109456cf3b9cSMatthew G. Knepley   PetscFunctionReturn(0);
109556cf3b9cSMatthew G. Knepley }
109656cf3b9cSMatthew G. Knepley 
1097f1d73a7aSMatthew G. Knepley /*@
1098f1d73a7aSMatthew G. Knepley   DMPlexInsertBoundaryValues - Puts coefficients which represent boundary values into the local solution vector
1099f1d73a7aSMatthew G. Knepley 
1100ed808b8fSJed Brown   Not Collective
1101ed808b8fSJed Brown 
1102f1d73a7aSMatthew G. Knepley   Input Parameters:
1103f1d73a7aSMatthew G. Knepley + dm - The DM
1104f1d73a7aSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
1105f1d73a7aSMatthew G. Knepley . time - The time
1106f1d73a7aSMatthew G. Knepley . faceGeomFVM - Face geometry data for FV discretizations
1107f1d73a7aSMatthew G. Knepley . cellGeomFVM - Cell geometry data for FV discretizations
1108f1d73a7aSMatthew G. Knepley - gradFVM - Gradient reconstruction data for FV discretizations
1109f1d73a7aSMatthew G. Knepley 
1110f1d73a7aSMatthew G. Knepley   Output Parameters:
1111f1d73a7aSMatthew G. Knepley . locX - Solution updated with boundary values
1112f1d73a7aSMatthew G. Knepley 
1113ed808b8fSJed Brown   Level: intermediate
1114f1d73a7aSMatthew G. Knepley 
1115db781477SPatrick Sanan .seealso: `DMProjectFunctionLabelLocal()`, `DMAddBoundary()`
1116f1d73a7aSMatthew G. Knepley @*/
1117f1d73a7aSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValues(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1118f1d73a7aSMatthew G. Knepley {
1119f1d73a7aSMatthew G. Knepley   PetscFunctionBegin;
1120f1d73a7aSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1121064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
1122064a246eSJacob Faibussowitsch   if (faceGeomFVM) {PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);}
1123064a246eSJacob Faibussowitsch   if (cellGeomFVM) {PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);}
1124064a246eSJacob Faibussowitsch   if (gradFVM)     {PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);}
1125cac4c232SBarry Smith   PetscTryMethod(dm,"DMPlexInsertBoundaryValues_C",(DM,PetscBool,Vec,PetscReal,Vec,Vec,Vec),(dm,insertEssential,locX,time,faceGeomFVM,cellGeomFVM,gradFVM));
1126f1d73a7aSMatthew G. Knepley   PetscFunctionReturn(0);
1127f1d73a7aSMatthew G. Knepley }
1128f1d73a7aSMatthew G. Knepley 
112956cf3b9cSMatthew G. Knepley /*@
1130a5b23f4aSJose E. Roman   DMPlexInsertTimeDerivativeBoundaryValues - Puts coefficients which represent boundary values of the time derivative into the local solution vector
113156cf3b9cSMatthew G. Knepley 
113256cf3b9cSMatthew G. Knepley   Input Parameters:
113356cf3b9cSMatthew G. Knepley + dm - The DM
113456cf3b9cSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
113556cf3b9cSMatthew G. Knepley . time - The time
113656cf3b9cSMatthew G. Knepley . faceGeomFVM - Face geometry data for FV discretizations
113756cf3b9cSMatthew G. Knepley . cellGeomFVM - Cell geometry data for FV discretizations
113856cf3b9cSMatthew G. Knepley - gradFVM - Gradient reconstruction data for FV discretizations
113956cf3b9cSMatthew G. Knepley 
114056cf3b9cSMatthew G. Knepley   Output Parameters:
114156cf3b9cSMatthew G. Knepley . locX_t - Solution updated with boundary values
114256cf3b9cSMatthew G. Knepley 
114356cf3b9cSMatthew G. Knepley   Level: developer
114456cf3b9cSMatthew G. Knepley 
1145db781477SPatrick Sanan .seealso: `DMProjectFunctionLabelLocal()`
114656cf3b9cSMatthew G. Knepley @*/
114756cf3b9cSMatthew G. Knepley PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues(DM dm, PetscBool insertEssential, Vec locX_t, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
114856cf3b9cSMatthew G. Knepley {
114956cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
115056cf3b9cSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1151064a246eSJacob Faibussowitsch   if (locX_t)      {PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 3);}
1152064a246eSJacob Faibussowitsch   if (faceGeomFVM) {PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);}
1153064a246eSJacob Faibussowitsch   if (cellGeomFVM) {PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);}
1154064a246eSJacob Faibussowitsch   if (gradFVM)     {PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);}
1155cac4c232SBarry Smith   PetscTryMethod(dm,"DMPlexInsertTimeDerviativeBoundaryValues_C",(DM,PetscBool,Vec,PetscReal,Vec,Vec,Vec),(dm,insertEssential,locX_t,time,faceGeomFVM,cellGeomFVM,gradFVM));
115656cf3b9cSMatthew G. Knepley   PetscFunctionReturn(0);
115756cf3b9cSMatthew G. Knepley }
115856cf3b9cSMatthew G. Knepley 
11590709b2feSToby Isaac PetscErrorCode DMComputeL2Diff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
1160cb1e1211SMatthew G Knepley {
1161574a98acSMatthew G. Knepley   Vec              localX;
1162574a98acSMatthew G. Knepley 
1163574a98acSMatthew G. Knepley   PetscFunctionBegin;
11649566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
11659566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, localX, time, NULL, NULL, NULL));
11669566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
11679566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
11689566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeL2DiffLocal(dm, time, funcs, ctxs, localX, diff));
11699566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
1170574a98acSMatthew G. Knepley   PetscFunctionReturn(0);
1171574a98acSMatthew G. Knepley }
1172574a98acSMatthew G. Knepley 
1173574a98acSMatthew G. Knepley /*@C
1174ca3d3a14SMatthew G. Knepley   DMComputeL2DiffLocal - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h.
1175574a98acSMatthew G. Knepley 
1176d083f849SBarry Smith   Collective on dm
1177c0f8e1fdSMatthew G. Knepley 
1178574a98acSMatthew G. Knepley   Input Parameters:
1179574a98acSMatthew G. Knepley + dm     - The DM
1180574a98acSMatthew G. Knepley . time   - The time
1181574a98acSMatthew G. Knepley . funcs  - The functions to evaluate for each field component
1182574a98acSMatthew G. Knepley . ctxs   - Optional array of contexts to pass to each function, or NULL.
1183574a98acSMatthew G. Knepley - localX - The coefficient vector u_h, a local vector
1184574a98acSMatthew G. Knepley 
1185574a98acSMatthew G. Knepley   Output Parameter:
1186574a98acSMatthew G. Knepley . diff - The diff ||u - u_h||_2
1187574a98acSMatthew G. Knepley 
1188574a98acSMatthew G. Knepley   Level: developer
1189574a98acSMatthew G. Knepley 
1190db781477SPatrick Sanan .seealso: `DMProjectFunction()`, `DMComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
1191574a98acSMatthew G. Knepley @*/
1192574a98acSMatthew G. Knepley PetscErrorCode DMPlexComputeL2DiffLocal(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec localX, PetscReal *diff)
1193574a98acSMatthew G. Knepley {
11940f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
1195ca3d3a14SMatthew G. Knepley   DM               tdm;
1196ca3d3a14SMatthew G. Knepley   Vec              tv;
1197cb1e1211SMatthew G Knepley   PetscSection     section;
1198c5bbbd5bSMatthew G. Knepley   PetscQuadrature  quad;
11994bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
120015496722SMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
12014bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
1202cb1e1211SMatthew G Knepley   PetscReal        localDiff = 0.0;
12037318780aSToby Isaac   const PetscReal *quadWeights;
1204412e9a14SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cellHeight, cStart, cEnd, c, field, fieldOffset;
1205ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1206cb1e1211SMatthew G Knepley 
1207cb1e1211SMatthew G Knepley   PetscFunctionBegin;
12089566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
12099566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
12102a4e142eSMatthew G. Knepley   fegeom.dimEmbed = coordDim;
12119566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
12129566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
12139566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
12149566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
12159566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1216cb1e1211SMatthew G Knepley   for (field = 0; field < numFields; ++field) {
121715496722SMatthew G. Knepley     PetscObject  obj;
121815496722SMatthew G. Knepley     PetscClassId id;
1219c5bbbd5bSMatthew G. Knepley     PetscInt     Nc;
1220c5bbbd5bSMatthew G. Knepley 
12219566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
12229566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
122315496722SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
122415496722SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
122515496722SMatthew G. Knepley 
12269566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
12279566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
122815496722SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
122915496722SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
123015496722SMatthew G. Knepley 
12319566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
12329566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
123363a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1234c5bbbd5bSMatthew G. Knepley     numComponents += Nc;
1235cb1e1211SMatthew G Knepley   }
12369566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
123763a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents),PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
12389566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents,&funcVal,numComponents,&interpolant,coordDim*Nq,&coords,Nq,&fegeom.detJ,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ));
12399566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
12409566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
1241cb1e1211SMatthew G Knepley   for (c = cStart; c < cEnd; ++c) {
1242a1e44745SMatthew G. Knepley     PetscScalar *x = NULL;
1243cb1e1211SMatthew G Knepley     PetscReal    elemDiff = 0.0;
12449c3cf19fSMatthew G. Knepley     PetscInt     qc = 0;
1245cb1e1211SMatthew G Knepley 
12469566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
12479566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x));
1248cb1e1211SMatthew G Knepley 
124915496722SMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
125015496722SMatthew G. Knepley       PetscObject  obj;
125115496722SMatthew G. Knepley       PetscClassId id;
1252c110b1eeSGeoffrey Irving       void * const ctx = ctxs ? ctxs[field] : NULL;
125315496722SMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1254cb1e1211SMatthew G Knepley 
12559566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
12569566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
12579566063dSJacob Faibussowitsch       if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &Nc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
12589566063dSJacob Faibussowitsch       else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &Nc));Nb = 1;}
125963a3b9bcSJacob Faibussowitsch       else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1260cb1e1211SMatthew G Knepley       if (debug) {
1261cb1e1211SMatthew G Knepley         char title[1024];
126263a3b9bcSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
12639566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1264cb1e1211SMatthew G Knepley       }
12657318780aSToby Isaac       for (q = 0; q < Nq; ++q) {
12662a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1267d0609cedSBarry Smith         PetscErrorCode ierr;
12682a4e142eSMatthew G. Knepley 
12692a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
12702a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q*coordDim*coordDim];
12712a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
12722a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
127363a3b9bcSJacob Faibussowitsch         PetscCheck(fegeom.detJ[q] > 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", point %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1274d3a7d86cSMatthew G. Knepley         if (transform) {
1275d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*Nq];
12769566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim*q], PETSC_TRUE, coordDim, &coords[coordDim*q], gcoords, dm->transformCtx));
1277d3a7d86cSMatthew G. Knepley         } else {
1278d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*q];
1279d3a7d86cSMatthew G. Knepley         }
12809566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal,Nc));
1281ca3d3a14SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, Nc, funcVal, ctx);
1282e735a8a9SMatthew G. Knepley         if (ierr) {
12839566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
12849566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
12859566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
1286e735a8a9SMatthew G. Knepley         }
12879566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim*q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
12889566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID)      PetscCall(PetscFEInterpolate_Static((PetscFE) obj, &x[fieldOffset], &qgeom, q, interpolant));
12899566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV) obj, &x[fieldOffset], q, interpolant));
12902df84da0SMatthew G. Knepley         else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
129115496722SMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1292beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
129363a3b9bcSJacob Faibussowitsch           if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g (%g, %g)\n", c, field, fc, (double)(coordDim > 0 ? coords[coordDim*q] : 0.), (double)(coordDim > 1 ? coords[coordDim*q+1] : 0.),(double)(coordDim > 2 ? coords[coordDim*q+2] : 0.), (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q]), (double)PetscRealPart(interpolant[fc]), (double)PetscRealPart(funcVal[fc])));
12944bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
1295cb1e1211SMatthew G Knepley         }
1296cb1e1211SMatthew G Knepley       }
12979c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
1298beaa55a6SMatthew G. Knepley       qc += Nc;
1299cb1e1211SMatthew G Knepley     }
13009566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
13019566063dSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
1302cb1e1211SMatthew G Knepley     localDiff += elemDiff;
1303cb1e1211SMatthew G Knepley   }
13049566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
13051c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
1306cb1e1211SMatthew G Knepley   *diff = PetscSqrtReal(*diff);
1307cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
1308cb1e1211SMatthew G Knepley }
1309cb1e1211SMatthew G Knepley 
1310b698f381SToby Isaac PetscErrorCode DMComputeL2GradientDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, const PetscReal n[], PetscReal *diff)
1311cb1e1211SMatthew G Knepley {
13120f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
1313ca3d3a14SMatthew G. Knepley   DM               tdm;
1314cb1e1211SMatthew G Knepley   PetscSection     section;
131540e14135SMatthew G. Knepley   PetscQuadrature  quad;
1316ca3d3a14SMatthew G. Knepley   Vec              localX, tv;
13179c3cf19fSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
13182a4e142eSMatthew G. Knepley   const PetscReal *quadWeights;
13194bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
13204bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
132140e14135SMatthew G. Knepley   PetscReal        localDiff = 0.0;
1322485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, qNc = 0, Nq = 0, numFields, numComponents = 0, cStart, cEnd, c, field, fieldOffset;
1323ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1324cb1e1211SMatthew G Knepley 
1325cb1e1211SMatthew G Knepley   PetscFunctionBegin;
13269566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
13279566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
13284bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
13299566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
13309566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
13319566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
13329566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
13339566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
13349566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
13359566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
13369566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1337652b88e8SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
13380f2d7e86SMatthew G. Knepley     PetscFE  fe;
133940e14135SMatthew G. Knepley     PetscInt Nc;
1340652b88e8SMatthew G. Knepley 
13419566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, (PetscObject *) &fe));
13429566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
13439566063dSJacob Faibussowitsch     PetscCall(PetscFEGetNumComponents(fe, &Nc));
134440e14135SMatthew G. Knepley     numComponents += Nc;
1345652b88e8SMatthew G. Knepley   }
13469566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
134763a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents),PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
13489566063dSJacob Faibussowitsch   /* PetscCall(DMProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX)); */
13499566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents,&funcVal,coordDim*Nq,&coords,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ,numComponents*coordDim,&interpolant,Nq,&fegeom.detJ));
13509566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
135140e14135SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
135240e14135SMatthew G. Knepley     PetscScalar *x = NULL;
135340e14135SMatthew G. Knepley     PetscReal    elemDiff = 0.0;
13549c3cf19fSMatthew G. Knepley     PetscInt     qc = 0;
1355652b88e8SMatthew G. Knepley 
13569566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
13579566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x));
135840e14135SMatthew G. Knepley 
13599c3cf19fSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
13600f2d7e86SMatthew G. Knepley       PetscFE          fe;
136151259fa3SMatthew G. Knepley       void * const     ctx = ctxs ? ctxs[field] : NULL;
13629c3cf19fSMatthew G. Knepley       PetscInt         Nb, Nc, q, fc;
136340e14135SMatthew G. Knepley 
13649566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, (PetscObject *) &fe));
13659566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
13669566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
136740e14135SMatthew G. Knepley       if (debug) {
136840e14135SMatthew G. Knepley         char title[1024];
13699566063dSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
13709566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1371652b88e8SMatthew G. Knepley       }
13729c3cf19fSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
13732a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1374d0609cedSBarry Smith         PetscErrorCode ierr;
13752a4e142eSMatthew G. Knepley 
13762a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
13772a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q*coordDim*coordDim];
13782a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
13792a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
13802df84da0SMatthew G. Knepley         PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1381d3a7d86cSMatthew G. Knepley         if (transform) {
1382d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*Nq];
13839566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim*q], PETSC_TRUE, coordDim, &coords[coordDim*q], gcoords, dm->transformCtx));
1384d3a7d86cSMatthew G. Knepley         } else {
1385d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*q];
1386d3a7d86cSMatthew G. Knepley         }
13879566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal,Nc));
13884bee2e38SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, n, Nc, funcVal, ctx);
1389e735a8a9SMatthew G. Knepley         if (ierr) {
13909566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
13919566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
13929566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal,coords,fegeom.J,fegeom.invJ,interpolant,fegeom.detJ));
1393e735a8a9SMatthew G. Knepley         }
13949566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim*q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
13959566063dSJacob Faibussowitsch         PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[fieldOffset], &qgeom, q, interpolant));
13964bee2e38SMatthew G. Knepley         /* Overwrite with the dot product if the normal is given */
13974bee2e38SMatthew G. Knepley         if (n) {
13984bee2e38SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
13994bee2e38SMatthew G. Knepley             PetscScalar sum = 0.0;
14004bee2e38SMatthew G. Knepley             PetscInt    d;
14014bee2e38SMatthew G. Knepley             for (d = 0; d < dim; ++d) sum += interpolant[fc*dim+d]*n[d];
14024bee2e38SMatthew G. Knepley             interpolant[fc] = sum;
14034bee2e38SMatthew G. Knepley           }
14044bee2e38SMatthew G. Knepley         }
14059c3cf19fSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1406beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
140763a3b9bcSJacob Faibussowitsch           if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " fieldDer %" PetscInt_FMT ",%" PetscInt_FMT " diff %g\n", c, field, fc, (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q])));
14084bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
140940e14135SMatthew G. Knepley         }
141040e14135SMatthew G. Knepley       }
14119c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
14129c3cf19fSMatthew G. Knepley       qc          += Nc;
141340e14135SMatthew G. Knepley     }
14149566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
141563a3b9bcSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
141640e14135SMatthew G. Knepley     localDiff += elemDiff;
141740e14135SMatthew G. Knepley   }
14189566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal,coords,fegeom.J,fegeom.invJ,interpolant,fegeom.detJ));
14199566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
14201c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
142140e14135SMatthew G. Knepley   *diff = PetscSqrtReal(*diff);
1422cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
1423cb1e1211SMatthew G Knepley }
1424cb1e1211SMatthew G Knepley 
1425c6eecec3SToby Isaac PetscErrorCode DMComputeL2FieldDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
142673d901b8SMatthew G. Knepley {
14270f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
1428ca3d3a14SMatthew G. Knepley   DM               tdm;
1429083401c6SMatthew G. Knepley   DMLabel          depthLabel;
143073d901b8SMatthew G. Knepley   PetscSection     section;
1431ca3d3a14SMatthew G. Knepley   Vec              localX, tv;
143273d901b8SMatthew G. Knepley   PetscReal       *localDiff;
1433083401c6SMatthew G. Knepley   PetscInt         dim, depth, dE, Nf, f, Nds, s;
1434ca3d3a14SMatthew G. Knepley   PetscBool        transform;
143573d901b8SMatthew G. Knepley 
143673d901b8SMatthew G. Knepley   PetscFunctionBegin;
14379566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
14389566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dE));
14399566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
14409566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
14419566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
14429566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
14439566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
14449566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
14459566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
14469566063dSJacob Faibussowitsch   PetscCall(DMLabelGetNumValues(depthLabel, &depth));
1447083401c6SMatthew G. Knepley 
14489566063dSJacob Faibussowitsch   PetscCall(VecSet(localX, 0.0));
14499566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
14509566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
14519566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
14529566063dSJacob Faibussowitsch   PetscCall(DMGetNumDS(dm, &Nds));
14539566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(Nf, &localDiff));
1454083401c6SMatthew G. Knepley   for (s = 0; s < Nds; ++s) {
1455083401c6SMatthew G. Knepley     PetscDS          ds;
1456083401c6SMatthew G. Knepley     DMLabel          label;
1457083401c6SMatthew G. Knepley     IS               fieldIS, pointIS;
1458083401c6SMatthew G. Knepley     const PetscInt  *fields, *points = NULL;
1459083401c6SMatthew G. Knepley     PetscQuadrature  quad;
1460083401c6SMatthew G. Knepley     const PetscReal *quadPoints, *quadWeights;
1461083401c6SMatthew G. Knepley     PetscFEGeom      fegeom;
1462083401c6SMatthew G. Knepley     PetscReal       *coords, *gcoords;
1463083401c6SMatthew G. Knepley     PetscScalar     *funcVal, *interpolant;
14645fedec97SMatthew G. Knepley     PetscBool        isCohesive;
1465083401c6SMatthew G. Knepley     PetscInt         qNc, Nq, totNc, cStart = 0, cEnd, c, dsNf;
146673d901b8SMatthew G. Knepley 
14679566063dSJacob Faibussowitsch     PetscCall(DMGetRegionNumDS(dm, s, &label, &fieldIS, &ds));
14689566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(fieldIS, &fields));
14699566063dSJacob Faibussowitsch     PetscCall(PetscDSIsCohesive(ds, &isCohesive));
14709566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(ds, &dsNf));
14719566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalComponents(ds, &totNc));
14729566063dSJacob Faibussowitsch     PetscCall(PetscDSGetQuadrature(ds, &quad));
14739566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
147463a3b9bcSJacob Faibussowitsch     PetscCheck(!(qNc != 1) || !(qNc != totNc),PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, totNc);
14759566063dSJacob Faibussowitsch     PetscCall(PetscCalloc6(totNc, &funcVal, totNc, &interpolant, dE*(Nq+1), &coords,Nq, &fegeom.detJ, dE*dE*Nq, &fegeom.J, dE*dE*Nq, &fegeom.invJ));
1476083401c6SMatthew G. Knepley     if (!label) {
14779566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1478083401c6SMatthew G. Knepley     } else {
14799566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(label, 1, &pointIS));
14809566063dSJacob Faibussowitsch       PetscCall(ISGetLocalSize(pointIS, &cEnd));
14819566063dSJacob Faibussowitsch       PetscCall(ISGetIndices(pointIS, &points));
1482083401c6SMatthew G. Knepley     }
148373d901b8SMatthew G. Knepley     for (c = cStart; c < cEnd; ++c) {
1484083401c6SMatthew G. Knepley       const PetscInt cell  = points ? points[c] : c;
148573d901b8SMatthew G. Knepley       PetscScalar    *x    = NULL;
14865fedec97SMatthew G. Knepley       const PetscInt *cone;
14875fedec97SMatthew G. Knepley       PetscInt        qc   = 0, fOff = 0, dep;
148873d901b8SMatthew G. Knepley 
14899566063dSJacob Faibussowitsch       PetscCall(DMLabelGetValue(depthLabel, cell, &dep));
1490083401c6SMatthew G. Knepley       if (dep != depth-1) continue;
14915fedec97SMatthew G. Knepley       if (isCohesive) {
14929566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cell, &cone));
14939566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cone[0], quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
149496959cd1SMatthew G. Knepley       } else {
14959566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
149696959cd1SMatthew G. Knepley       }
14979566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, localX, cell, NULL, &x));
14985fedec97SMatthew G. Knepley       for (f = 0; f < dsNf; ++f) {
149915496722SMatthew G. Knepley         PetscObject  obj;
150015496722SMatthew G. Knepley         PetscClassId id;
1501083401c6SMatthew G. Knepley         void * const ctx = ctxs ? ctxs[fields[f]] : NULL;
150215496722SMatthew G. Knepley         PetscInt     Nb, Nc, q, fc;
150315496722SMatthew G. Knepley         PetscReal    elemDiff = 0.0;
15045fedec97SMatthew G. Knepley         PetscBool    cohesive;
150515496722SMatthew G. Knepley 
15069566063dSJacob Faibussowitsch         PetscCall(PetscDSGetCohesive(ds, f, &cohesive));
15075fedec97SMatthew G. Knepley         if (isCohesive && !cohesive) continue;
15089566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
15099566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
15109566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &Nc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
15119566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &Nc));Nb = 1;}
151263a3b9bcSJacob Faibussowitsch         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
151373d901b8SMatthew G. Knepley         if (debug) {
151473d901b8SMatthew G. Knepley           char title[1024];
151563a3b9bcSJacob Faibussowitsch           PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, fields[f]));
15169566063dSJacob Faibussowitsch           PetscCall(DMPrintCellVector(cell, title, Nb, &x[fOff]));
151773d901b8SMatthew G. Knepley         }
15187318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
15192a4e142eSMatthew G. Knepley           PetscFEGeom    qgeom;
1520d0609cedSBarry Smith           PetscErrorCode ierr;
15212a4e142eSMatthew G. Knepley 
15222a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
1523083401c6SMatthew G. Knepley           qgeom.J        = &fegeom.J[q*dE*dE];
1524083401c6SMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*dE*dE];
15252a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
152663a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for cell %" PetscInt_FMT ", quadrature point %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
1527d3a7d86cSMatthew G. Knepley           if (transform) {
1528083401c6SMatthew G. Knepley             gcoords = &coords[dE*Nq];
15299566063dSJacob Faibussowitsch             PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[dE*q], PETSC_TRUE, dE, &coords[dE*q], gcoords, dm->transformCtx));
1530d3a7d86cSMatthew G. Knepley           } else {
1531083401c6SMatthew G. Knepley             gcoords = &coords[dE*q];
1532d3a7d86cSMatthew G. Knepley           }
15332df84da0SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) funcVal[fc] = 0.;
1534083401c6SMatthew G. Knepley           ierr = (*funcs[fields[f]])(dE, time, gcoords, Nc, funcVal, ctx);
1535e735a8a9SMatthew G. Knepley           if (ierr) {
15369566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
15379566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
15389566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
1539e735a8a9SMatthew G. Knepley           }
15409566063dSJacob Faibussowitsch           if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[dE*q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
154196959cd1SMatthew G. Knepley           /* Call once for each face, except for lagrange field */
15429566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID)      PetscCall(PetscFEInterpolate_Static((PetscFE) obj, &x[fOff], &qgeom, q, interpolant));
15439566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV) obj, &x[fOff], q, interpolant));
154463a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
154515496722SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1546beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
154763a3b9bcSJacob Faibussowitsch             if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    cell %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g\n", cell, fields[f], fc, (double)(dE > 0 ? coords[dE*q] : 0.), (double)(dE > 1 ? coords[dE*q+1] : 0.),(double)(dE > 2 ? coords[dE*q+2] : 0.), (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q])));
15484bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
154973d901b8SMatthew G. Knepley           }
155073d901b8SMatthew G. Knepley         }
1551083401c6SMatthew G. Knepley         fOff += Nb;
15529c3cf19fSMatthew G. Knepley         qc   += Nc;
1553083401c6SMatthew G. Knepley         localDiff[fields[f]] += elemDiff;
155463a3b9bcSJacob Faibussowitsch         if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  cell %" PetscInt_FMT " field %" PetscInt_FMT " cum diff %g\n", cell, fields[f], (double)localDiff[fields[f]]));
155573d901b8SMatthew G. Knepley       }
15569566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
1557083401c6SMatthew G. Knepley     }
1558083401c6SMatthew G. Knepley     if (label) {
15599566063dSJacob Faibussowitsch       PetscCall(ISRestoreIndices(pointIS, &points));
15609566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
1561083401c6SMatthew G. Knepley     }
15629566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(fieldIS, &fields));
15639566063dSJacob Faibussowitsch     PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
156473d901b8SMatthew G. Knepley   }
15659566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
15661c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(localDiff, diff, Nf, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
15679566063dSJacob Faibussowitsch   PetscCall(PetscFree(localDiff));
1568083401c6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) diff[f] = PetscSqrtReal(diff[f]);
156973d901b8SMatthew G. Knepley   PetscFunctionReturn(0);
157073d901b8SMatthew G. Knepley }
157173d901b8SMatthew G. Knepley 
1572e729f68cSMatthew G. Knepley /*@C
1573e729f68cSMatthew G. Knepley   DMPlexComputeL2DiffVec - This function computes the cellwise L_2 difference between a function u and an FEM interpolant solution u_h, and stores it in a Vec.
1574e729f68cSMatthew G. Knepley 
1575d083f849SBarry Smith   Collective on dm
1576c0f8e1fdSMatthew G. Knepley 
1577e729f68cSMatthew G. Knepley   Input Parameters:
1578e729f68cSMatthew G. Knepley + dm    - The DM
15790163fd50SMatthew G. Knepley . time  - The time
1580ca3eba1bSToby Isaac . funcs - The functions to evaluate for each field component: NULL means that component does not contribute to error calculation
1581e729f68cSMatthew G. Knepley . ctxs  - Optional array of contexts to pass to each function, or NULL.
1582e729f68cSMatthew G. Knepley - X     - The coefficient vector u_h
1583e729f68cSMatthew G. Knepley 
1584e729f68cSMatthew G. Knepley   Output Parameter:
1585e729f68cSMatthew G. Knepley . D - A Vec which holds the difference ||u - u_h||_2 for each cell
1586e729f68cSMatthew G. Knepley 
1587e729f68cSMatthew G. Knepley   Level: developer
1588e729f68cSMatthew G. Knepley 
1589db781477SPatrick Sanan .seealso: `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
1590e729f68cSMatthew G. Knepley @*/
15910163fd50SMatthew G. Knepley PetscErrorCode DMPlexComputeL2DiffVec(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, Vec D)
1592e729f68cSMatthew G. Knepley {
1593e729f68cSMatthew G. Knepley   PetscSection     section;
1594e729f68cSMatthew G. Knepley   PetscQuadrature  quad;
1595e729f68cSMatthew G. Knepley   Vec              localX;
15964bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
1597e729f68cSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
15984bee2e38SMatthew G. Knepley   PetscReal       *coords;
1599e729f68cSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1600485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, c, field, fieldOffset;
1601e729f68cSMatthew G. Knepley 
1602e729f68cSMatthew G. Knepley   PetscFunctionBegin;
16039566063dSJacob Faibussowitsch   PetscCall(VecSet(D, 0.0));
16049566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
16059566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
16069566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
16079566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
16089566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
16099566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
16109566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
16119566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
1612e729f68cSMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
1613e729f68cSMatthew G. Knepley     PetscObject  obj;
1614e729f68cSMatthew G. Knepley     PetscClassId id;
1615e729f68cSMatthew G. Knepley     PetscInt     Nc;
1616e729f68cSMatthew G. Knepley 
16179566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
16189566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1619e729f68cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1620e729f68cSMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
1621e729f68cSMatthew G. Knepley 
16229566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
16239566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
1624e729f68cSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
1625e729f68cSMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
1626e729f68cSMatthew G. Knepley 
16279566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
16289566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
162963a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1630e729f68cSMatthew G. Knepley     numComponents += Nc;
1631e729f68cSMatthew G. Knepley   }
16329566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
163363a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents),PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
16349566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents,&funcVal,numComponents,&interpolant,coordDim*Nq,&coords,Nq,&fegeom.detJ,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ));
16359566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1636e729f68cSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
1637e729f68cSMatthew G. Knepley     PetscScalar *x = NULL;
16386f288a59SMatthew G. Knepley     PetscScalar  elemDiff = 0.0;
16399c3cf19fSMatthew G. Knepley     PetscInt     qc = 0;
1640e729f68cSMatthew G. Knepley 
16419566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
16429566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x));
1643e729f68cSMatthew G. Knepley 
1644e729f68cSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
1645e729f68cSMatthew G. Knepley       PetscObject  obj;
1646e729f68cSMatthew G. Knepley       PetscClassId id;
1647e729f68cSMatthew G. Knepley       void * const ctx = ctxs ? ctxs[field] : NULL;
1648e729f68cSMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1649e729f68cSMatthew G. Knepley 
16509566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
16519566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
16529566063dSJacob Faibussowitsch       if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &Nc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
16539566063dSJacob Faibussowitsch       else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &Nc));Nb = 1;}
165463a3b9bcSJacob Faibussowitsch       else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
165523f34ed2SToby Isaac       if (funcs[field]) {
16567318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
16572a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
16582a4e142eSMatthew G. Knepley 
16592a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
16602a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q*coordDim*coordDim];
16612a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
16622a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
166363a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
16649566063dSJacob Faibussowitsch           PetscCall((*funcs[field])(coordDim, time, &coords[q*coordDim], Nc, funcVal, ctx));
1665c3e24edfSBarry Smith #if defined(needs_fix_with_return_code_argument)
1666e735a8a9SMatthew G. Knepley           if (ierr) {
16679566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
16689566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
16699566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
1670e735a8a9SMatthew G. Knepley           }
1671c3e24edfSBarry Smith #endif
16729566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID)      PetscCall(PetscFEInterpolate_Static((PetscFE) obj, &x[fieldOffset], &qgeom, q, interpolant));
16739566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV) obj, &x[fieldOffset], q, interpolant));
167463a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1675e729f68cSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1676beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
16774bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
1678e729f68cSMatthew G. Knepley           }
1679e729f68cSMatthew G. Knepley         }
168023f34ed2SToby Isaac       }
1681beaa55a6SMatthew G. Knepley       fieldOffset += Nb;
16829c3cf19fSMatthew G. Knepley       qc          += Nc;
1683e729f68cSMatthew G. Knepley     }
16849566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
16859566063dSJacob Faibussowitsch     PetscCall(VecSetValue(D, c - cStart, elemDiff, INSERT_VALUES));
1686e729f68cSMatthew G. Knepley   }
16879566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
16889566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
16899566063dSJacob Faibussowitsch   PetscCall(VecSqrtAbs(D));
1690e729f68cSMatthew G. Knepley   PetscFunctionReturn(0);
1691e729f68cSMatthew G. Knepley }
1692e729f68cSMatthew G. Knepley 
16935f0b18bfSMatthew G. Knepley /*@
16945f0b18bfSMatthew G. Knepley   DMPlexComputeClementInterpolant - This function computes the L2 projection of the cellwise values of a function u onto P1, and stores it in a Vec.
16955f0b18bfSMatthew G. Knepley 
16965f0b18bfSMatthew G. Knepley   Collective on dm
16975f0b18bfSMatthew G. Knepley 
16985f0b18bfSMatthew G. Knepley   Input Parameters:
16995f0b18bfSMatthew G. Knepley + dm - The DM
17005f0b18bfSMatthew G. Knepley - locX  - The coefficient vector u_h
17015f0b18bfSMatthew G. Knepley 
17025f0b18bfSMatthew G. Knepley   Output Parameter:
17035f0b18bfSMatthew G. Knepley . locC - A Vec which holds the Clement interpolant of the function
17045f0b18bfSMatthew G. Knepley 
17055f0b18bfSMatthew G. Knepley   Notes:
17065f0b18bfSMatthew G. Knepley   u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| where |T_i| is the cell volume
17075f0b18bfSMatthew G. Knepley 
17085f0b18bfSMatthew G. Knepley   Level: developer
17095f0b18bfSMatthew G. Knepley 
1710db781477SPatrick Sanan .seealso: `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
17115f0b18bfSMatthew G. Knepley @*/
17125f0b18bfSMatthew G. Knepley PetscErrorCode DMPlexComputeClementInterpolant(DM dm, Vec locX, Vec locC)
17135f0b18bfSMatthew G. Knepley {
17145f0b18bfSMatthew G. Knepley   PetscInt         debug = ((DM_Plex *) dm->data)->printFEM;
17155f0b18bfSMatthew G. Knepley   DM               dmc;
17165f0b18bfSMatthew G. Knepley   PetscQuadrature  quad;
17175f0b18bfSMatthew G. Knepley   PetscScalar     *interpolant, *valsum;
17185f0b18bfSMatthew G. Knepley   PetscFEGeom      fegeom;
17195f0b18bfSMatthew G. Knepley   PetscReal       *coords;
17205f0b18bfSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
17215f0b18bfSMatthew G. Knepley   PetscInt         dim, cdim, Nf, f, Nc = 0, Nq, qNc, cStart, cEnd, vStart, vEnd, v;
17225f0b18bfSMatthew G. Knepley 
17235f0b18bfSMatthew G. Knepley   PetscFunctionBegin;
17249566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
17259566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmc));
17269566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
17279566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
17289566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
17295f0b18bfSMatthew G. Knepley   fegeom.dimEmbed = cdim;
17309566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
17315f0b18bfSMatthew G. Knepley   PetscCheck(Nf > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
17325f0b18bfSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
17335f0b18bfSMatthew G. Knepley     PetscObject  obj;
17345f0b18bfSMatthew G. Knepley     PetscClassId id;
17355f0b18bfSMatthew G. Knepley     PetscInt     fNc;
17365f0b18bfSMatthew G. Knepley 
17379566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, f, NULL, &obj));
17389566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
17395f0b18bfSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
17405f0b18bfSMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
17415f0b18bfSMatthew G. Knepley 
17429566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
17439566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &fNc));
17445f0b18bfSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
17455f0b18bfSMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
17465f0b18bfSMatthew G. Knepley 
17479566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
17489566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &fNc));
174963a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
17505f0b18bfSMatthew G. Knepley     Nc += fNc;
17515f0b18bfSMatthew G. Knepley   }
17529566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
175363a3b9bcSJacob Faibussowitsch   PetscCheck(qNc == 1,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " > 1", qNc);
17549566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(Nc*2, &valsum, Nc, &interpolant, cdim*Nq, &coords, Nq, &fegeom.detJ, cdim*cdim*Nq, &fegeom.J, cdim*cdim*Nq, &fegeom.invJ));
17559566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
17569566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
17575f0b18bfSMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
17585f0b18bfSMatthew G. Knepley     PetscScalar volsum = 0.0;
17595f0b18bfSMatthew G. Knepley     PetscInt   *star   = NULL;
17605f0b18bfSMatthew G. Knepley     PetscInt    starSize, st, fc;
17615f0b18bfSMatthew G. Knepley 
17629566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(valsum, Nc));
17639566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
17645f0b18bfSMatthew G. Knepley     for (st = 0; st < starSize*2; st += 2) {
17655f0b18bfSMatthew G. Knepley       const PetscInt cell = star[st];
17665f0b18bfSMatthew G. Knepley       PetscScalar   *val  = &valsum[Nc];
17675f0b18bfSMatthew G. Knepley       PetscScalar   *x    = NULL;
17685f0b18bfSMatthew G. Knepley       PetscReal      vol  = 0.0;
17695f0b18bfSMatthew G. Knepley       PetscInt       foff = 0;
17705f0b18bfSMatthew G. Knepley 
17715f0b18bfSMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
17729566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
17739566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
17745f0b18bfSMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
17755f0b18bfSMatthew G. Knepley         PetscObject  obj;
17765f0b18bfSMatthew G. Knepley         PetscClassId id;
17775f0b18bfSMatthew G. Knepley         PetscInt     Nb, fNc, q;
17785f0b18bfSMatthew G. Knepley 
17799566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(val, Nc));
17809566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, f, NULL, &obj));
17819566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
17829566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &fNc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
17839566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &fNc));Nb = 1;}
178463a3b9bcSJacob Faibussowitsch         else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
17855f0b18bfSMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
17865f0b18bfSMatthew G. Knepley           const PetscReal wt = quadWeights[q]*fegeom.detJ[q];
17875f0b18bfSMatthew G. Knepley           PetscFEGeom     qgeom;
17885f0b18bfSMatthew G. Knepley 
17895f0b18bfSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
17905f0b18bfSMatthew G. Knepley           qgeom.J        = &fegeom.J[q*cdim*cdim];
17915f0b18bfSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*cdim*cdim];
17925f0b18bfSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
179363a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double) fegeom.detJ[q], cell, q);
17949566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE) obj, &x[foff], &qgeom, q, interpolant));
179563a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
17965f0b18bfSMatthew G. Knepley           for (fc = 0; fc < fNc; ++fc) val[foff+fc] += interpolant[fc]*wt;
17975f0b18bfSMatthew G. Knepley           vol += wt;
17985f0b18bfSMatthew G. Knepley         }
17995f0b18bfSMatthew G. Knepley         foff += Nb;
18005f0b18bfSMatthew G. Knepley       }
18019566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
18025f0b18bfSMatthew G. Knepley       for (fc = 0; fc < Nc; ++fc) valsum[fc] += val[fc];
18035f0b18bfSMatthew G. Knepley       volsum += vol;
18045f0b18bfSMatthew G. Knepley       if (debug) {
18059566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " value: [", v, cell));
18065f0b18bfSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
18079566063dSJacob Faibussowitsch           if (fc) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
18089566063dSJacob Faibussowitsch           PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double) PetscRealPart(val[fc])));
18095f0b18bfSMatthew G. Knepley         }
18109566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
18115f0b18bfSMatthew G. Knepley       }
18125f0b18bfSMatthew G. Knepley     }
18135f0b18bfSMatthew G. Knepley     for (fc = 0; fc < Nc; ++fc) valsum[fc] /= volsum;
18149566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
18159566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmc, NULL, locC, v, valsum, INSERT_VALUES));
18165f0b18bfSMatthew G. Knepley   }
18179566063dSJacob Faibussowitsch   PetscCall(PetscFree6(valsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
18185f0b18bfSMatthew G. Knepley   PetscFunctionReturn(0);
18195f0b18bfSMatthew G. Knepley }
18205f0b18bfSMatthew G. Knepley 
18215f0b18bfSMatthew G. Knepley /*@
18221555c271SMatthew G. Knepley   DMPlexComputeGradientClementInterpolant - This function computes the L2 projection of the cellwise gradient of a function u onto P1, and stores it in a Vec.
18231555c271SMatthew G. Knepley 
1824d083f849SBarry Smith   Collective on dm
1825c0f8e1fdSMatthew G. Knepley 
18261555c271SMatthew G. Knepley   Input Parameters:
18271555c271SMatthew G. Knepley + dm - The DM
18285f0b18bfSMatthew G. Knepley - locX  - The coefficient vector u_h
18291555c271SMatthew G. Knepley 
18301555c271SMatthew G. Knepley   Output Parameter:
18311555c271SMatthew G. Knepley . locC - A Vec which holds the Clement interpolant of the gradient
18321555c271SMatthew G. Knepley 
183395452b02SPatrick Sanan   Notes:
18341555c271SMatthew G. Knepley   \nabla u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| \nabla u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| where |T_i| is the cell volume
18351555c271SMatthew G. Knepley 
18361555c271SMatthew G. Knepley   Level: developer
18371555c271SMatthew G. Knepley 
1838db781477SPatrick Sanan .seealso: `DMProjectFunction()`, `DMComputeL2Diff()`, `DMPlexComputeL2FieldDiff()`, `DMComputeL2GradientDiff()`
18391555c271SMatthew G. Knepley @*/
18401555c271SMatthew G. Knepley PetscErrorCode DMPlexComputeGradientClementInterpolant(DM dm, Vec locX, Vec locC)
18411555c271SMatthew G. Knepley {
1842db1066baSMatthew G. Knepley   DM_Plex         *mesh  = (DM_Plex *) dm->data;
1843db1066baSMatthew G. Knepley   PetscInt         debug = mesh->printFEM;
18441555c271SMatthew G. Knepley   DM               dmC;
18451555c271SMatthew G. Knepley   PetscQuadrature  quad;
18461555c271SMatthew G. Knepley   PetscScalar     *interpolant, *gradsum;
18474bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
18484bee2e38SMatthew G. Knepley   PetscReal       *coords;
18491555c271SMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1850485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, vStart, vEnd, v, field, fieldOffset;
18511555c271SMatthew G. Knepley 
18521555c271SMatthew G. Knepley   PetscFunctionBegin;
18539566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
18549566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmC));
18559566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
18569566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
18579566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
18584bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
18599566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
18605f80ce2aSJacob Faibussowitsch   PetscCheck(numFields,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
18611555c271SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
18621555c271SMatthew G. Knepley     PetscObject  obj;
18631555c271SMatthew G. Knepley     PetscClassId id;
18641555c271SMatthew G. Knepley     PetscInt     Nc;
18651555c271SMatthew G. Knepley 
18669566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
18679566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
18681555c271SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
18691555c271SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
18701555c271SMatthew G. Knepley 
18719566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
18729566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
18731555c271SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
18741555c271SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
18751555c271SMatthew G. Knepley 
18769566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
18779566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
187863a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
18791555c271SMatthew G. Knepley     numComponents += Nc;
18801555c271SMatthew G. Knepley   }
18819566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
188263a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents),PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
18839566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(coordDim*numComponents*2,&gradsum,coordDim*numComponents,&interpolant,coordDim*Nq,&coords,Nq,&fegeom.detJ,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ));
18849566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
18859566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
18861555c271SMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
18871555c271SMatthew G. Knepley     PetscScalar volsum = 0.0;
18881555c271SMatthew G. Knepley     PetscInt   *star   = NULL;
18891555c271SMatthew G. Knepley     PetscInt    starSize, st, d, fc;
18901555c271SMatthew G. Knepley 
18919566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(gradsum, coordDim*numComponents));
18929566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
18931555c271SMatthew G. Knepley     for (st = 0; st < starSize*2; st += 2) {
18941555c271SMatthew G. Knepley       const PetscInt cell = star[st];
18951555c271SMatthew G. Knepley       PetscScalar   *grad = &gradsum[coordDim*numComponents];
18961555c271SMatthew G. Knepley       PetscScalar   *x    = NULL;
18971555c271SMatthew G. Knepley       PetscReal      vol  = 0.0;
18981555c271SMatthew G. Knepley 
18991555c271SMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
19009566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
19019566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
19021555c271SMatthew G. Knepley       for (field = 0, fieldOffset = 0; field < numFields; ++field) {
19031555c271SMatthew G. Knepley         PetscObject  obj;
19041555c271SMatthew G. Knepley         PetscClassId id;
19051555c271SMatthew G. Knepley         PetscInt     Nb, Nc, q, qc = 0;
19061555c271SMatthew G. Knepley 
19079566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(grad, coordDim*numComponents));
19089566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, field, NULL, &obj));
19099566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
19109566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &Nc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
19119566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &Nc));Nb = 1;}
191263a3b9bcSJacob Faibussowitsch         else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
19131555c271SMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
19142a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
19152a4e142eSMatthew G. Knepley 
19162a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
19172a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q*coordDim*coordDim];
19182a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
19192a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
192063a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
19219566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID)      PetscCall(PetscFEInterpolateGradient_Static((PetscFE) obj, 1, &x[fieldOffset], &qgeom, q, interpolant));
192263a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
19231555c271SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
192453d2db2dSJoe Wallwork             const PetscReal wt = quadWeights[q*qNc+qc];
19251555c271SMatthew G. Knepley 
19264bee2e38SMatthew G. Knepley             for (d = 0; d < coordDim; ++d) grad[fc*coordDim+d] += interpolant[fc*dim+d]*wt*fegeom.detJ[q];
19271555c271SMatthew G. Knepley           }
19284bee2e38SMatthew G. Knepley           vol += quadWeights[q*qNc]*fegeom.detJ[q];
19291555c271SMatthew G. Knepley         }
19301555c271SMatthew G. Knepley         fieldOffset += Nb;
19311555c271SMatthew G. Knepley         qc          += Nc;
19321555c271SMatthew G. Knepley       }
19339566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
1934f8527842SMatthew G. Knepley       for (fc = 0; fc < numComponents; ++fc) {
1935f8527842SMatthew G. Knepley         for (d = 0; d < coordDim; ++d) {
1936f8527842SMatthew G. Knepley           gradsum[fc*coordDim+d] += grad[fc*coordDim+d];
1937f8527842SMatthew G. Knepley         }
1938f8527842SMatthew G. Knepley       }
1939f8527842SMatthew G. Knepley       volsum += vol;
1940db1066baSMatthew G. Knepley       if (debug) {
19419566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " gradient: [", v, cell));
19421555c271SMatthew G. Knepley         for (fc = 0; fc < numComponents; ++fc) {
19431555c271SMatthew G. Knepley           for (d = 0; d < coordDim; ++d) {
19449566063dSJacob Faibussowitsch             if (fc || d > 0) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
19459566063dSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(grad[fc*coordDim+d])));
19461555c271SMatthew G. Knepley           }
19471555c271SMatthew G. Knepley         }
19489566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
1949db1066baSMatthew G. Knepley       }
19501555c271SMatthew G. Knepley     }
19511555c271SMatthew G. Knepley     for (fc = 0; fc < numComponents; ++fc) {
19521555c271SMatthew G. Knepley       for (d = 0; d < coordDim; ++d) gradsum[fc*coordDim+d] /= volsum;
19531555c271SMatthew G. Knepley     }
19549566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
19559566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmC, NULL, locC, v, gradsum, INSERT_VALUES));
19561555c271SMatthew G. Knepley   }
19579566063dSJacob Faibussowitsch   PetscCall(PetscFree6(gradsum,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
19581555c271SMatthew G. Knepley   PetscFunctionReturn(0);
19591555c271SMatthew G. Knepley }
19601555c271SMatthew G. Knepley 
1961338f77d5SMatthew G. Knepley static PetscErrorCode DMPlexComputeIntegral_Internal(DM dm, Vec X, PetscInt cStart, PetscInt cEnd, PetscScalar *cintegral, void *user)
196273d901b8SMatthew G. Knepley {
1963338f77d5SMatthew G. Knepley   DM                 dmAux = NULL;
196461aaff12SToby Isaac   PetscDS            prob,    probAux = NULL;
196573d901b8SMatthew G. Knepley   PetscSection       section, sectionAux;
1966338f77d5SMatthew G. Knepley   Vec                locX,    locA;
1967c330f8ffSToby Isaac   PetscInt           dim, numCells = cEnd - cStart, c, f;
1968c330f8ffSToby Isaac   PetscBool          useFVM = PETSC_FALSE;
1969338f77d5SMatthew G. Knepley   /* DS */
1970338f77d5SMatthew G. Knepley   PetscInt           Nf,    totDim,    *uOff, *uOff_x, numConstants;
1971338f77d5SMatthew G. Knepley   PetscInt           NfAux, totDimAux, *aOff;
1972338f77d5SMatthew G. Knepley   PetscScalar       *u, *a;
1973338f77d5SMatthew G. Knepley   const PetscScalar *constants;
1974338f77d5SMatthew G. Knepley   /* Geometry */
1975c330f8ffSToby Isaac   PetscFEGeom       *cgeomFEM;
1976338f77d5SMatthew G. Knepley   DM                 dmGrad;
1977c330f8ffSToby Isaac   PetscQuadrature    affineQuad = NULL;
1978338f77d5SMatthew G. Knepley   Vec                cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL;
1979b5a3613cSMatthew G. Knepley   PetscFVCellGeom   *cgeomFVM;
1980338f77d5SMatthew G. Knepley   const PetscScalar *lgrad;
1981b7260050SToby Isaac   PetscInt           maxDegree;
1982c330f8ffSToby Isaac   DMField            coordField;
1983c330f8ffSToby Isaac   IS                 cellIS;
198473d901b8SMatthew G. Knepley 
198573d901b8SMatthew G. Knepley   PetscFunctionBegin;
19869566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
19879566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
19889566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
19899566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
1990338f77d5SMatthew G. Knepley   /* Determine which discretizations we have */
1991b5a3613cSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
1992b5a3613cSMatthew G. Knepley     PetscObject  obj;
1993b5a3613cSMatthew G. Knepley     PetscClassId id;
1994b5a3613cSMatthew G. Knepley 
19959566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
19969566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1997338f77d5SMatthew G. Knepley     if (id == PETSCFV_CLASSID) useFVM = PETSC_TRUE;
1998338f77d5SMatthew G. Knepley   }
1999338f77d5SMatthew G. Knepley   /* Get local solution with boundary values */
20009566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &locX));
20019566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
20029566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
20039566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
2004338f77d5SMatthew G. Knepley   /* Read DS information */
20059566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
20069566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
20079566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
20089566063dSJacob Faibussowitsch   PetscCall(ISCreateStride(PETSC_COMM_SELF,numCells,cStart,1,&cellIS));
20099566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
2010338f77d5SMatthew G. Knepley   /* Read Auxiliary DS information */
20119566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
20129a2a23afSMatthew G. Knepley   if (locA) {
20139566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
20149566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
20159566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
20169566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
20179566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
20189566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
2019338f77d5SMatthew G. Knepley   }
2020338f77d5SMatthew G. Knepley   /* Allocate data  arrays */
20219566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(numCells*totDim, &u));
20229566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells*totDimAux, &a));
2023338f77d5SMatthew G. Knepley   /* Read out geometry */
20249566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm,&coordField));
20259566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
2026b7260050SToby Isaac   if (maxDegree <= 1) {
20279566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad));
2028c330f8ffSToby Isaac     if (affineQuad) {
20299566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&cgeomFEM));
2030338f77d5SMatthew G. Knepley     }
2031b5a3613cSMatthew G. Knepley   }
2032b5a3613cSMatthew G. Knepley   if (useFVM) {
2033338f77d5SMatthew G. Knepley     PetscFV   fv = NULL;
2034b5a3613cSMatthew G. Knepley     Vec       grad;
2035b5a3613cSMatthew G. Knepley     PetscInt  fStart, fEnd;
2036b5a3613cSMatthew G. Knepley     PetscBool compGrad;
2037b5a3613cSMatthew G. Knepley 
2038338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
2039338f77d5SMatthew G. Knepley       PetscObject  obj;
2040338f77d5SMatthew G. Knepley       PetscClassId id;
2041338f77d5SMatthew G. Knepley 
20429566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
20439566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
2044338f77d5SMatthew G. Knepley       if (id == PETSCFV_CLASSID) {fv = (PetscFV) obj; break;}
2045338f77d5SMatthew G. Knepley     }
20469566063dSJacob Faibussowitsch     PetscCall(PetscFVGetComputeGradients(fv, &compGrad));
20479566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, PETSC_TRUE));
20489566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGeometryFVM(dm, &cellGeometryFVM, &faceGeometryFVM));
20499566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGradientFVM(dm, fv, faceGeometryFVM, cellGeometryFVM, &dmGrad));
20509566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, compGrad));
20519566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM));
2052b5a3613cSMatthew G. Knepley     /* Reconstruct and limit cell gradients */
20539566063dSJacob Faibussowitsch     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
20549566063dSJacob Faibussowitsch     PetscCall(DMGetGlobalVector(dmGrad, &grad));
20559566063dSJacob Faibussowitsch     PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
2056b5a3613cSMatthew G. Knepley     /* Communicate gradient values */
20579566063dSJacob Faibussowitsch     PetscCall(DMGetLocalVector(dmGrad, &locGrad));
20589566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
20599566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
20609566063dSJacob Faibussowitsch     PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
2061b5a3613cSMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
20629566063dSJacob Faibussowitsch     PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, 0.0, faceGeometryFVM, cellGeometryFVM, locGrad));
20639566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
2064b5a3613cSMatthew G. Knepley   }
2065338f77d5SMatthew G. Knepley   /* Read out data from inputs */
206673d901b8SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
206773d901b8SMatthew G. Knepley     PetscScalar *x = NULL;
206873d901b8SMatthew G. Knepley     PetscInt     i;
206973d901b8SMatthew G. Knepley 
20709566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, section, locX, c, NULL, &x));
20710f2d7e86SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[c*totDim+i] = x[i];
20729566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, section, locX, c, NULL, &x));
207373d901b8SMatthew G. Knepley     if (dmAux) {
20749566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, locA, c, NULL, &x));
20750f2d7e86SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[c*totDimAux+i] = x[i];
20769566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, locA, c, NULL, &x));
207773d901b8SMatthew G. Knepley     }
207873d901b8SMatthew G. Knepley   }
2079338f77d5SMatthew G. Knepley   /* Do integration for each field */
208073d901b8SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
2081c1f031eeSMatthew G. Knepley     PetscObject  obj;
2082c1f031eeSMatthew G. Knepley     PetscClassId id;
2083c1f031eeSMatthew G. Knepley     PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
208473d901b8SMatthew G. Knepley 
20859566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
20869566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
2087c1f031eeSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
2088c1f031eeSMatthew G. Knepley       PetscFE         fe = (PetscFE) obj;
2089c1f031eeSMatthew G. Knepley       PetscQuadrature q;
2090c330f8ffSToby Isaac       PetscFEGeom     *chunkGeom = NULL;
2091c1f031eeSMatthew G. Knepley       PetscInt        Nq, Nb;
2092c1f031eeSMatthew G. Knepley 
20939566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
20949566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &q));
20959566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
20969566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
2097c1f031eeSMatthew G. Knepley       blockSize = Nb*Nq;
209873d901b8SMatthew G. Knepley       batchSize = numBlocks * blockSize;
20999566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
210073d901b8SMatthew G. Knepley       numChunks = numCells / (numBatches*batchSize);
210173d901b8SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
210273d901b8SMatthew G. Knepley       Nr        = numCells % (numBatches*batchSize);
210373d901b8SMatthew G. Knepley       offset    = numCells - Nr;
2104c330f8ffSToby Isaac       if (!affineQuad) {
21059566063dSJacob Faibussowitsch         PetscCall(DMFieldCreateFEGeom(coordField,cellIS,q,PETSC_FALSE,&cgeomFEM));
2106c330f8ffSToby Isaac       }
21079566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom));
21089566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrate(prob, f, Ne, chunkGeom, u, probAux, a, cintegral));
21099566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&chunkGeom));
21109566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrate(prob, f, Nr, chunkGeom, &u[offset*totDim], probAux, &a[offset*totDimAux], &cintegral[offset*Nf]));
21119566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&chunkGeom));
2112c330f8ffSToby Isaac       if (!affineQuad) {
21139566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomDestroy(&cgeomFEM));
2114c330f8ffSToby Isaac       }
2115c1f031eeSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
2116c1f031eeSMatthew G. Knepley       PetscInt       foff;
2117420e96edSMatthew G. Knepley       PetscPointFunc obj_func;
2118b69edc29SMatthew G. Knepley       PetscScalar    lint;
2119c1f031eeSMatthew G. Knepley 
21209566063dSJacob Faibussowitsch       PetscCall(PetscDSGetObjective(prob, f, &obj_func));
21219566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, f, &foff));
2122c1f031eeSMatthew G. Knepley       if (obj_func) {
2123c1f031eeSMatthew G. Knepley         for (c = 0; c < numCells; ++c) {
2124b5a3613cSMatthew G. Knepley           PetscScalar *u_x;
2125b5a3613cSMatthew G. Knepley 
21269566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, c, lgrad, &u_x));
2127338f77d5SMatthew G. Knepley           obj_func(dim, Nf, NfAux, uOff, uOff_x, &u[totDim*c+foff], NULL, u_x, aOff, NULL, &a[totDimAux*c], NULL, NULL, 0.0, cgeomFVM[c].centroid, numConstants, constants, &lint);
2128338f77d5SMatthew G. Knepley           cintegral[c*Nf+f] += PetscRealPart(lint)*cgeomFVM[c].volume;
212973d901b8SMatthew G. Knepley         }
2130c1f031eeSMatthew G. Knepley       }
213163a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
2132c1f031eeSMatthew G. Knepley   }
2133338f77d5SMatthew G. Knepley   /* Cleanup data arrays */
2134b5a3613cSMatthew G. Knepley   if (useFVM) {
21359566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
21369566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM));
21379566063dSJacob Faibussowitsch     PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
21389566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&faceGeometryFVM));
21399566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&cellGeometryFVM));
21409566063dSJacob Faibussowitsch     PetscCall(DMDestroy(&dmGrad));
2141b5a3613cSMatthew G. Knepley   }
21429566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscFree(a));
21439566063dSJacob Faibussowitsch   PetscCall(PetscFree(u));
2144338f77d5SMatthew G. Knepley   /* Cleanup */
2145f99c8401SToby Isaac   if (affineQuad) {
21469566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomDestroy(&cgeomFEM));
2147f99c8401SToby Isaac   }
21489566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&affineQuad));
21499566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cellIS));
21509566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &locX));
2151338f77d5SMatthew G. Knepley   PetscFunctionReturn(0);
2152338f77d5SMatthew G. Knepley }
2153338f77d5SMatthew G. Knepley 
2154338f77d5SMatthew G. Knepley /*@
2155338f77d5SMatthew G. Knepley   DMPlexComputeIntegralFEM - Form the integral over the domain from the global input X using pointwise functions specified by the user
2156338f77d5SMatthew G. Knepley 
2157338f77d5SMatthew G. Knepley   Input Parameters:
2158338f77d5SMatthew G. Knepley + dm - The mesh
2159338f77d5SMatthew G. Knepley . X  - Global input vector
2160338f77d5SMatthew G. Knepley - user - The user context
2161338f77d5SMatthew G. Knepley 
2162338f77d5SMatthew G. Knepley   Output Parameter:
2163338f77d5SMatthew G. Knepley . integral - Integral for each field
2164338f77d5SMatthew G. Knepley 
2165338f77d5SMatthew G. Knepley   Level: developer
2166338f77d5SMatthew G. Knepley 
2167db781477SPatrick Sanan .seealso: `DMPlexSNESComputeResidualFEM()`
2168338f77d5SMatthew G. Knepley @*/
2169b8feb594SMatthew G. Knepley PetscErrorCode DMPlexComputeIntegralFEM(DM dm, Vec X, PetscScalar *integral, void *user)
2170338f77d5SMatthew G. Knepley {
2171338f77d5SMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dm->data;
2172b8feb594SMatthew G. Knepley   PetscScalar   *cintegral, *lintegral;
2173412e9a14SMatthew G. Knepley   PetscInt       Nf, f, cellHeight, cStart, cEnd, cell;
2174338f77d5SMatthew G. Knepley 
2175338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2176338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2177338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2178dadcf809SJacob Faibussowitsch   PetscValidScalarPointer(integral, 3);
21799566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0));
21809566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
21819566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
21829566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2183338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
21849566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &lintegral, (cEnd-cStart)*Nf, &cintegral));
21859566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user));
2186338f77d5SMatthew G. Knepley   /* Sum up values */
2187338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2188338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2189338f77d5SMatthew G. Knepley 
21909566063dSJacob Faibussowitsch     if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c*Nf]));
2191b8feb594SMatthew G. Knepley     for (f = 0; f < Nf; ++f) lintegral[f] += cintegral[c*Nf+f];
2192338f77d5SMatthew G. Knepley   }
21931c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(lintegral, integral, Nf, MPIU_SCALAR, MPIU_SUM, PetscObjectComm((PetscObject) dm)));
219473d901b8SMatthew G. Knepley   if (mesh->printFEM) {
21959566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject) dm), "Integral:"));
21969566063dSJacob Faibussowitsch     for (f = 0; f < Nf; ++f) PetscCall(PetscPrintf(PetscObjectComm((PetscObject) dm), " %g", (double) PetscRealPart(integral[f])));
21979566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject) dm), "\n"));
219873d901b8SMatthew G. Knepley   }
21999566063dSJacob Faibussowitsch   PetscCall(PetscFree2(lintegral, cintegral));
22009566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0));
2201338f77d5SMatthew G. Knepley   PetscFunctionReturn(0);
2202338f77d5SMatthew G. Knepley }
2203338f77d5SMatthew G. Knepley 
2204338f77d5SMatthew G. Knepley /*@
2205338f77d5SMatthew G. Knepley   DMPlexComputeCellwiseIntegralFEM - Form the vector of cellwise integrals F from the global input X using pointwise functions specified by the user
2206338f77d5SMatthew G. Knepley 
2207338f77d5SMatthew G. Knepley   Input Parameters:
2208338f77d5SMatthew G. Knepley + dm - The mesh
2209338f77d5SMatthew G. Knepley . X  - Global input vector
2210338f77d5SMatthew G. Knepley - user - The user context
2211338f77d5SMatthew G. Knepley 
2212338f77d5SMatthew G. Knepley   Output Parameter:
2213338f77d5SMatthew G. Knepley . integral - Cellwise integrals for each field
2214338f77d5SMatthew G. Knepley 
2215338f77d5SMatthew G. Knepley   Level: developer
2216338f77d5SMatthew G. Knepley 
2217db781477SPatrick Sanan .seealso: `DMPlexSNESComputeResidualFEM()`
2218338f77d5SMatthew G. Knepley @*/
2219338f77d5SMatthew G. Knepley PetscErrorCode DMPlexComputeCellwiseIntegralFEM(DM dm, Vec X, Vec F, void *user)
2220338f77d5SMatthew G. Knepley {
2221338f77d5SMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dm->data;
2222338f77d5SMatthew G. Knepley   DM             dmF;
2223338f77d5SMatthew G. Knepley   PetscSection   sectionF;
2224338f77d5SMatthew G. Knepley   PetscScalar   *cintegral, *af;
2225412e9a14SMatthew G. Knepley   PetscInt       Nf, f, cellHeight, cStart, cEnd, cell;
2226338f77d5SMatthew G. Knepley 
2227338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2228338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2229338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2230338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(F, VEC_CLASSID, 3);
22319566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0));
22329566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
22339566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
22349566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2235338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
22369566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1((cEnd-cStart)*Nf, &cintegral));
22379566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user));
2238338f77d5SMatthew G. Knepley   /* Put values in F*/
22399566063dSJacob Faibussowitsch   PetscCall(VecGetDM(F, &dmF));
22409566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmF, &sectionF));
22419566063dSJacob Faibussowitsch   PetscCall(VecGetArray(F, &af));
2242338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2243338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2244338f77d5SMatthew G. Knepley     PetscInt       dof, off;
2245338f77d5SMatthew G. Knepley 
22469566063dSJacob Faibussowitsch     if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c*Nf]));
22479566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetDof(sectionF, cell, &dof));
22489566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetOffset(sectionF, cell, &off));
224963a3b9bcSJacob Faibussowitsch     PetscCheck(dof == Nf,PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of cell dofs %" PetscInt_FMT " != %" PetscInt_FMT, dof, Nf);
2250338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) af[off+f] = cintegral[c*Nf+f];
2251338f77d5SMatthew G. Knepley   }
22529566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(F, &af));
22539566063dSJacob Faibussowitsch   PetscCall(PetscFree(cintegral));
22549566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0));
225573d901b8SMatthew G. Knepley   PetscFunctionReturn(0);
225673d901b8SMatthew G. Knepley }
225773d901b8SMatthew G. Knepley 
22589b6f715bSMatthew G. Knepley static PetscErrorCode DMPlexComputeBdIntegral_Internal(DM dm, Vec locX, IS pointIS,
22599b6f715bSMatthew G. Knepley                                                        void (*func)(PetscInt, PetscInt, PetscInt,
226064c72086SMatthew G. Knepley                                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
226164c72086SMatthew G. Knepley                                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
226264c72086SMatthew G. Knepley                                                                     PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]),
226364c72086SMatthew G. Knepley                                                        PetscScalar *fintegral, void *user)
226464c72086SMatthew G. Knepley {
22659b6f715bSMatthew G. Knepley   DM                 plex = NULL, plexA = NULL;
2266a6e0b375SMatthew G. Knepley   DMEnclosureType    encAux;
22679b6f715bSMatthew G. Knepley   PetscDS            prob, probAux = NULL;
22689b6f715bSMatthew G. Knepley   PetscSection       section, sectionAux = NULL;
22699b6f715bSMatthew G. Knepley   Vec                locA = NULL;
22709b6f715bSMatthew G. Knepley   DMField            coordField;
22719b6f715bSMatthew G. Knepley   PetscInt           Nf,        totDim,        *uOff, *uOff_x;
22729b6f715bSMatthew G. Knepley   PetscInt           NfAux = 0, totDimAux = 0, *aOff = NULL;
22739b6f715bSMatthew G. Knepley   PetscScalar       *u, *a = NULL;
227464c72086SMatthew G. Knepley   const PetscScalar *constants;
22759b6f715bSMatthew G. Knepley   PetscInt           numConstants, f;
227664c72086SMatthew G. Knepley 
227764c72086SMatthew G. Knepley   PetscFunctionBegin;
22789566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
22799566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
22809566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
22819566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
22829566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
228364c72086SMatthew G. Knepley   /* Determine which discretizations we have */
22849b6f715bSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
228564c72086SMatthew G. Knepley     PetscObject  obj;
228664c72086SMatthew G. Knepley     PetscClassId id;
228764c72086SMatthew G. Knepley 
22889566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
22899566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
229063a3b9bcSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Not supported for FVM (field %" PetscInt_FMT ")", f);
229164c72086SMatthew G. Knepley   }
229264c72086SMatthew G. Knepley   /* Read DS information */
22939566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
22949566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
22959566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
22969566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
229764c72086SMatthew G. Knepley   /* Read Auxiliary DS information */
22989566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
22999b6f715bSMatthew G. Knepley   if (locA) {
23009b6f715bSMatthew G. Knepley     DM dmAux;
23019b6f715bSMatthew G. Knepley 
23029566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
23039566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
23049566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
23059566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
23069566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
23079566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
23089566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
23099566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
231064c72086SMatthew G. Knepley   }
23119b6f715bSMatthew G. Knepley   /* Integrate over points */
23129b6f715bSMatthew G. Knepley   {
23139b6f715bSMatthew G. Knepley     PetscFEGeom    *fgeom, *chunkGeom = NULL;
2314b7260050SToby Isaac     PetscInt        maxDegree;
23159b6f715bSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
23169b6f715bSMatthew G. Knepley     const PetscInt *points;
23179b6f715bSMatthew G. Knepley     PetscInt        numFaces, face, Nq, field;
23189b6f715bSMatthew G. Knepley     PetscInt        numChunks, chunkSize, chunk, Nr, offset;
231964c72086SMatthew G. Knepley 
23209566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
23219566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
23229566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(numFaces*totDim, &u, locA ? numFaces*totDimAux : 0, &a));
23239566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
232464c72086SMatthew G. Knepley     for (field = 0; field < Nf; ++field) {
232564c72086SMatthew G. Knepley       PetscFE fe;
232664c72086SMatthew G. Knepley 
23279566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *) &fe));
23289566063dSJacob Faibussowitsch       if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
23299b6f715bSMatthew G. Knepley       if (!qGeom) {
23309566063dSJacob Faibussowitsch         PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
23319566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) qGeom));
23329b6f715bSMatthew G. Knepley       }
23339566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
23349566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
23359b6f715bSMatthew G. Knepley       for (face = 0; face < numFaces; ++face) {
2336f15274beSMatthew Knepley         const PetscInt point = points[face], *support;
23379b6f715bSMatthew G. Knepley         PetscScalar    *x    = NULL;
2338f15274beSMatthew Knepley         PetscInt       i;
23399b6f715bSMatthew G. Knepley 
23409566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, point, &support));
23419566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
23429b6f715bSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
23439566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
23449b6f715bSMatthew G. Knepley         if (locA) {
23459b6f715bSMatthew G. Knepley           PetscInt subp;
23469566063dSJacob Faibussowitsch           PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
23479566063dSJacob Faibussowitsch           PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
23489b6f715bSMatthew G. Knepley           for (i = 0; i < totDimAux; ++i) a[f*totDimAux+i] = x[i];
23499566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
23509b6f715bSMatthew G. Knepley         }
23519b6f715bSMatthew G. Knepley       }
23529b6f715bSMatthew G. Knepley       /* Get blocking */
23539b6f715bSMatthew G. Knepley       {
23549b6f715bSMatthew G. Knepley         PetscQuadrature q;
23559b6f715bSMatthew G. Knepley         PetscInt        numBatches, batchSize, numBlocks, blockSize;
23569b6f715bSMatthew G. Knepley         PetscInt        Nq, Nb;
23579b6f715bSMatthew G. Knepley 
23589566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
23599566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &q));
23609566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
23619566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
236264c72086SMatthew G. Knepley         blockSize = Nb*Nq;
236364c72086SMatthew G. Knepley         batchSize = numBlocks * blockSize;
23649b6f715bSMatthew G. Knepley         chunkSize = numBatches*batchSize;
23659566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
23669b6f715bSMatthew G. Knepley         numChunks = numFaces / chunkSize;
23679b6f715bSMatthew G. Knepley         Nr        = numFaces % chunkSize;
236864c72086SMatthew G. Knepley         offset    = numFaces - Nr;
236964c72086SMatthew G. Knepley       }
23709b6f715bSMatthew G. Knepley       /* Do integration for each field */
23719b6f715bSMatthew G. Knepley       for (chunk = 0; chunk < numChunks; ++chunk) {
23729566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(fgeom, chunk*chunkSize, (chunk+1)*chunkSize, &chunkGeom));
23739566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateBd(prob, field, func, chunkSize, chunkGeom, u, probAux, a, fintegral));
23749566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
237564c72086SMatthew G. Knepley       }
23769566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
23779566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBd(prob, field, func, Nr, chunkGeom, &u[offset*totDim], probAux, a ? &a[offset*totDimAux] : NULL, &fintegral[offset*Nf]));
23789566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
237964c72086SMatthew G. Knepley       /* Cleanup data arrays */
23809566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
23819566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&qGeom));
23829566063dSJacob Faibussowitsch       PetscCall(PetscFree2(u, a));
23839566063dSJacob Faibussowitsch       PetscCall(ISRestoreIndices(pointIS, &points));
238464c72086SMatthew G. Knepley     }
238564c72086SMatthew G. Knepley   }
23869566063dSJacob Faibussowitsch   if (plex)  PetscCall(DMDestroy(&plex));
23879566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
23889b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
23899b6f715bSMatthew G. Knepley }
23909b6f715bSMatthew G. Knepley 
23919b6f715bSMatthew G. Knepley /*@
23929b6f715bSMatthew G. Knepley   DMPlexComputeBdIntegral - Form the integral over the specified boundary from the global input X using pointwise functions specified by the user
23939b6f715bSMatthew G. Knepley 
23949b6f715bSMatthew G. Knepley   Input Parameters:
23959b6f715bSMatthew G. Knepley + dm      - The mesh
23969b6f715bSMatthew G. Knepley . X       - Global input vector
23979b6f715bSMatthew G. Knepley . label   - The boundary DMLabel
23989b6f715bSMatthew G. Knepley . numVals - The number of label values to use, or PETSC_DETERMINE for all values
23999b6f715bSMatthew G. Knepley . vals    - The label values to use, or PETSC_NULL for all values
240067b8a455SSatish Balay . func    - The function to integrate along the boundary
24019b6f715bSMatthew G. Knepley - user    - The user context
24029b6f715bSMatthew G. Knepley 
24039b6f715bSMatthew G. Knepley   Output Parameter:
24049b6f715bSMatthew G. Knepley . integral - Integral for each field
24059b6f715bSMatthew G. Knepley 
24069b6f715bSMatthew G. Knepley   Level: developer
24079b6f715bSMatthew G. Knepley 
2408db781477SPatrick Sanan .seealso: `DMPlexComputeIntegralFEM()`, `DMPlexComputeBdResidualFEM()`
24099b6f715bSMatthew G. Knepley @*/
24109b6f715bSMatthew G. Knepley PetscErrorCode DMPlexComputeBdIntegral(DM dm, Vec X, DMLabel label, PetscInt numVals, const PetscInt vals[],
24119b6f715bSMatthew G. Knepley                                        void (*func)(PetscInt, PetscInt, PetscInt,
24129b6f715bSMatthew G. Knepley                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
24139b6f715bSMatthew G. Knepley                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
24149b6f715bSMatthew G. Knepley                                                     PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]),
24159b6f715bSMatthew G. Knepley                                        PetscScalar *integral, void *user)
24169b6f715bSMatthew G. Knepley {
24179b6f715bSMatthew G. Knepley   Vec            locX;
24189b6f715bSMatthew G. Knepley   PetscSection   section;
24199b6f715bSMatthew G. Knepley   DMLabel        depthLabel;
24209b6f715bSMatthew G. Knepley   IS             facetIS;
24219b6f715bSMatthew G. Knepley   PetscInt       dim, Nf, f, v;
24229b6f715bSMatthew G. Knepley 
24239b6f715bSMatthew G. Knepley   PetscFunctionBegin;
24249b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
24259b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
24269b6f715bSMatthew G. Knepley   PetscValidPointer(label, 3);
2427dadcf809SJacob Faibussowitsch   if (vals) PetscValidIntPointer(vals, 5);
2428dadcf809SJacob Faibussowitsch   PetscValidScalarPointer(integral, 7);
24299566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0));
24309566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
24319566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
24329566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim-1, &facetIS));
24339566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
24349566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
24359b6f715bSMatthew G. Knepley   /* Get local solution with boundary values */
24369566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &locX));
24379566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
24389566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
24399566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
24409b6f715bSMatthew G. Knepley   /* Loop over label values */
24419566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(integral, Nf));
24429b6f715bSMatthew G. Knepley   for (v = 0; v < numVals; ++v) {
24439b6f715bSMatthew G. Knepley     IS           pointIS;
24449b6f715bSMatthew G. Knepley     PetscInt     numFaces, face;
24459b6f715bSMatthew G. Knepley     PetscScalar *fintegral;
24469b6f715bSMatthew G. Knepley 
24479566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, vals[v], &pointIS));
24489b6f715bSMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
24499b6f715bSMatthew G. Knepley     {
24509b6f715bSMatthew G. Knepley       IS isectIS;
24519b6f715bSMatthew G. Knepley 
24529b6f715bSMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
24539566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
24549566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
24559b6f715bSMatthew G. Knepley       pointIS = isectIS;
24569b6f715bSMatthew G. Knepley     }
24579566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
24589566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(numFaces*Nf, &fintegral));
24599566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdIntegral_Internal(dm, locX, pointIS, func, fintegral, user));
24609b6f715bSMatthew G. Knepley     /* Sum point contributions into integral */
24619b6f715bSMatthew G. Knepley     for (f = 0; f < Nf; ++f) for (face = 0; face < numFaces; ++face) integral[f] += fintegral[face*Nf+f];
24629566063dSJacob Faibussowitsch     PetscCall(PetscFree(fintegral));
24639566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
24649b6f715bSMatthew G. Knepley   }
24659566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &locX));
24669566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
24679566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0));
246864c72086SMatthew G. Knepley   PetscFunctionReturn(0);
246964c72086SMatthew G. Knepley }
247064c72086SMatthew G. Knepley 
2471d69c5d34SMatthew G. Knepley /*@
2472cf51de39SMatthew G. Knepley   DMPlexComputeInterpolatorNested - Form the local portion of the interpolation matrix I from the coarse DM to a uniformly refined DM.
2473d69c5d34SMatthew G. Knepley 
2474d69c5d34SMatthew G. Knepley   Input Parameters:
2475cf51de39SMatthew G. Knepley + dmc  - The coarse mesh
2476cf51de39SMatthew G. Knepley . dmf  - The fine mesh
2477cf51de39SMatthew G. Knepley . isRefined - Flag indicating regular refinement, rather than the same topology
2478d69c5d34SMatthew G. Knepley - user - The user context
2479d69c5d34SMatthew G. Knepley 
2480d69c5d34SMatthew G. Knepley   Output Parameter:
2481934789fcSMatthew G. Knepley . In  - The interpolation matrix
2482d69c5d34SMatthew G. Knepley 
2483d69c5d34SMatthew G. Knepley   Level: developer
2484d69c5d34SMatthew G. Knepley 
2485db781477SPatrick Sanan .seealso: `DMPlexComputeInterpolatorGeneral()`, `DMPlexComputeJacobianFEM()`
2486d69c5d34SMatthew G. Knepley @*/
2487cf51de39SMatthew G. Knepley PetscErrorCode DMPlexComputeInterpolatorNested(DM dmc, DM dmf, PetscBool isRefined, Mat In, void *user)
2488d69c5d34SMatthew G. Knepley {
2489d69c5d34SMatthew G. Knepley   DM_Plex          *mesh  = (DM_Plex *) dmc->data;
2490d69c5d34SMatthew G. Knepley   const char       *name  = "Interpolator";
2491d69c5d34SMatthew G. Knepley   PetscFE          *feRef;
249297c42addSMatthew G. Knepley   PetscFV          *fvRef;
2493d69c5d34SMatthew G. Knepley   PetscSection      fsection, fglobalSection;
2494d69c5d34SMatthew G. Knepley   PetscSection      csection, cglobalSection;
2495d69c5d34SMatthew G. Knepley   PetscScalar      *elemMat;
2496485ad865SMatthew G. Knepley   PetscInt          dim, Nf, f, fieldI, fieldJ, offsetI, offsetJ, cStart, cEnd, c;
24972ea9c922SToby Isaac   PetscInt          cTotDim=0, rTotDim = 0;
2498d69c5d34SMatthew G. Knepley 
2499d69c5d34SMatthew G. Knepley   PetscFunctionBegin;
25009566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM,dmc,dmf,0,0));
25019566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
25029566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
25039566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
25049566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
25059566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
25069566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
25079566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
25089566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &feRef, Nf, &fvRef));
2509d69c5d34SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
25102ea9c922SToby Isaac     PetscObject  obj, objc;
25112ea9c922SToby Isaac     PetscClassId id, idc;
25122ea9c922SToby Isaac     PetscInt     rNb = 0, Nc = 0, cNb = 0;
2513d69c5d34SMatthew G. Knepley 
25149566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmf, f, NULL, &obj));
25159566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
251697c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
251797c42addSMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
251897c42addSMatthew G. Knepley 
2519cf51de39SMatthew G. Knepley       if (isRefined) {
25209566063dSJacob Faibussowitsch         PetscCall(PetscFERefine(fe, &feRef[f]));
2521cf51de39SMatthew G. Knepley       } else {
25229566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) fe));
2523cf51de39SMatthew G. Knepley         feRef[f] = fe;
2524cf51de39SMatthew G. Knepley       }
25259566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &rNb));
25269566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
252797c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
252897c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV) obj;
252997c42addSMatthew G. Knepley       PetscDualSpace Q;
253097c42addSMatthew G. Knepley 
2531cf51de39SMatthew G. Knepley       if (isRefined) {
25329566063dSJacob Faibussowitsch         PetscCall(PetscFVRefine(fv, &fvRef[f]));
2533cf51de39SMatthew G. Knepley       } else {
25349566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) fv));
2535cf51de39SMatthew G. Knepley         fvRef[f] = fv;
2536cf51de39SMatthew G. Knepley       }
25379566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
25389566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &rNb));
25399566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
25409566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
254197c42addSMatthew G. Knepley     }
25429566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmc, f, NULL, &objc));
25439566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(objc, &idc));
25442ea9c922SToby Isaac     if (idc == PETSCFE_CLASSID) {
25452ea9c922SToby Isaac       PetscFE fe = (PetscFE) objc;
25462ea9c922SToby Isaac 
25479566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &cNb));
25482ea9c922SToby Isaac     } else if (id == PETSCFV_CLASSID) {
25492ea9c922SToby Isaac       PetscFV        fv = (PetscFV) obj;
25502ea9c922SToby Isaac       PetscDualSpace Q;
25512ea9c922SToby Isaac 
25529566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
25539566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &cNb));
2554d69c5d34SMatthew G. Knepley     }
25552ea9c922SToby Isaac     rTotDim += rNb;
25562ea9c922SToby Isaac     cTotDim += cNb;
25572ea9c922SToby Isaac   }
25589566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(rTotDim*cTotDim,&elemMat));
25599566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, rTotDim*cTotDim));
2560d69c5d34SMatthew G. Knepley   for (fieldI = 0, offsetI = 0; fieldI < Nf; ++fieldI) {
2561d69c5d34SMatthew G. Knepley     PetscDualSpace   Qref;
2562d69c5d34SMatthew G. Knepley     PetscQuadrature  f;
2563d69c5d34SMatthew G. Knepley     const PetscReal *qpoints, *qweights;
2564d69c5d34SMatthew G. Knepley     PetscReal       *points;
2565d69c5d34SMatthew G. Knepley     PetscInt         npoints = 0, Nc, Np, fpdim, i, k, p, d;
2566d69c5d34SMatthew G. Knepley 
2567d69c5d34SMatthew G. Knepley     /* Compose points from all dual basis functionals */
256897c42addSMatthew G. Knepley     if (feRef[fieldI]) {
25699566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[fieldI], &Qref));
25709566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[fieldI], &Nc));
257197c42addSMatthew G. Knepley     } else {
25729566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[fieldI], &Qref));
25739566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[fieldI], &Nc));
257497c42addSMatthew G. Knepley     }
25759566063dSJacob Faibussowitsch     PetscCall(PetscDualSpaceGetDimension(Qref, &fpdim));
2576d69c5d34SMatthew G. Knepley     for (i = 0; i < fpdim; ++i) {
25779566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
25789566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, NULL, NULL));
2579d69c5d34SMatthew G. Knepley       npoints += Np;
2580d69c5d34SMatthew G. Knepley     }
25819566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(npoints*dim,&points));
2582d69c5d34SMatthew G. Knepley     for (i = 0, k = 0; i < fpdim; ++i) {
25839566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
25849566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL));
2585d69c5d34SMatthew G. Knepley       for (p = 0; p < Np; ++p, ++k) for (d = 0; d < dim; ++d) points[k*dim+d] = qpoints[p*dim+d];
2586d69c5d34SMatthew G. Knepley     }
2587d69c5d34SMatthew G. Knepley 
2588d69c5d34SMatthew G. Knepley     for (fieldJ = 0, offsetJ = 0; fieldJ < Nf; ++fieldJ) {
258997c42addSMatthew G. Knepley       PetscObject  obj;
259097c42addSMatthew G. Knepley       PetscClassId id;
25919c3cf19fSMatthew G. Knepley       PetscInt     NcJ = 0, cpdim = 0, j, qNc;
2592d69c5d34SMatthew G. Knepley 
25939566063dSJacob Faibussowitsch       PetscCall(DMGetField(dmc, fieldJ, NULL, &obj));
25949566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
259597c42addSMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
259697c42addSMatthew G. Knepley         PetscFE           fe = (PetscFE) obj;
2597ef0bb6c7SMatthew G. Knepley         PetscTabulation T  = NULL;
2598d69c5d34SMatthew G. Knepley 
2599d69c5d34SMatthew G. Knepley         /* Evaluate basis at points */
26009566063dSJacob Faibussowitsch         PetscCall(PetscFEGetNumComponents(fe, &NcJ));
26019566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &cpdim));
2602ffe73a53SMatthew G. Knepley         /* For now, fields only interpolate themselves */
2603ffe73a53SMatthew G. Knepley         if (fieldI == fieldJ) {
260463a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
26059566063dSJacob Faibussowitsch           PetscCall(PetscFECreateTabulation(fe, 1, npoints, points, 0, &T));
2606d69c5d34SMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
26079566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
26089566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
260963a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
2610d69c5d34SMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
261136a6d9c0SMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
2612d172c84bSMatthew G. Knepley                 /*
2613d172c84bSMatthew G. Knepley                    cTotDim:            Total columns in element interpolation matrix, sum of number of dual basis functionals in each field
2614d172c84bSMatthew G. Knepley                    offsetI, offsetJ:   Offsets into the larger element interpolation matrix for different fields
2615d172c84bSMatthew G. Knepley                    fpdim, i, cpdim, j: Dofs for fine and coarse grids, correspond to dual space basis functionals
2616d172c84bSMatthew G. Knepley                    qNC, Nc, Ncj, c:    Number of components in this field
2617d172c84bSMatthew G. Knepley                    Np, p:              Number of quad points in the fine grid functional i
2618d172c84bSMatthew G. Knepley                    k:                  i*Np + p, overall point number for the interpolation
2619d172c84bSMatthew G. Knepley                 */
2620ef0bb6c7SMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i)*cTotDim + offsetJ + j] += T->T[0][k*cpdim*NcJ+j*Nc+c]*qweights[p*qNc+c];
262136a6d9c0SMatthew G. Knepley               }
2622d69c5d34SMatthew G. Knepley             }
2623d69c5d34SMatthew G. Knepley           }
26249566063dSJacob Faibussowitsch           PetscCall(PetscTabulationDestroy(&T));
2625ffe73a53SMatthew G. Knepley         }
262697c42addSMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
262797c42addSMatthew G. Knepley         PetscFV        fv = (PetscFV) obj;
262897c42addSMatthew G. Knepley 
262997c42addSMatthew G. Knepley         /* Evaluate constant function at points */
26309566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &NcJ));
263197c42addSMatthew G. Knepley         cpdim = 1;
263297c42addSMatthew G. Knepley         /* For now, fields only interpolate themselves */
263397c42addSMatthew G. Knepley         if (fieldI == fieldJ) {
263463a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
263597c42addSMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
26369566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
26379566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
263863a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
263997c42addSMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
264097c42addSMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
2641458eb97cSMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i)*cTotDim + offsetJ + j] += 1.0*qweights[p*qNc+c];
264297c42addSMatthew G. Knepley               }
264397c42addSMatthew G. Knepley             }
264497c42addSMatthew G. Knepley           }
264597c42addSMatthew G. Knepley         }
264697c42addSMatthew G. Knepley       }
2647d172c84bSMatthew G. Knepley       offsetJ += cpdim;
2648d69c5d34SMatthew G. Knepley     }
2649d172c84bSMatthew G. Knepley     offsetI += fpdim;
26509566063dSJacob Faibussowitsch     PetscCall(PetscFree(points));
2651d69c5d34SMatthew G. Knepley   }
26529566063dSJacob Faibussowitsch   if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(0, name, rTotDim, cTotDim, elemMat));
26537f5b169aSMatthew G. Knepley   /* Preallocate matrix */
26547f5b169aSMatthew G. Knepley   {
2655c094ef40SMatthew G. Knepley     Mat          preallocator;
2656c094ef40SMatthew G. Knepley     PetscScalar *vals;
2657c094ef40SMatthew G. Knepley     PetscInt    *cellCIndices, *cellFIndices;
2658c094ef40SMatthew G. Knepley     PetscInt     locRows, locCols, cell;
26597f5b169aSMatthew G. Knepley 
26609566063dSJacob Faibussowitsch     PetscCall(MatGetLocalSize(In, &locRows, &locCols));
26619566063dSJacob Faibussowitsch     PetscCall(MatCreate(PetscObjectComm((PetscObject) In), &preallocator));
26629566063dSJacob Faibussowitsch     PetscCall(MatSetType(preallocator, MATPREALLOCATOR));
26639566063dSJacob Faibussowitsch     PetscCall(MatSetSizes(preallocator, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE));
26649566063dSJacob Faibussowitsch     PetscCall(MatSetUp(preallocator));
26659566063dSJacob Faibussowitsch     PetscCall(PetscCalloc3(rTotDim*cTotDim, &vals,cTotDim,&cellCIndices,rTotDim,&cellFIndices));
26667f5b169aSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
2667cf51de39SMatthew G. Knepley       if (isRefined) {
26689566063dSJacob Faibussowitsch         PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, cell, cellCIndices, cellFIndices));
26699566063dSJacob Faibussowitsch         PetscCall(MatSetValues(preallocator, rTotDim, cellFIndices, cTotDim, cellCIndices, vals, INSERT_VALUES));
2670cf51de39SMatthew G. Knepley       } else {
26719566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, preallocator, cell, vals, INSERT_VALUES));
2672cf51de39SMatthew G. Knepley       }
26737f5b169aSMatthew G. Knepley     }
26749566063dSJacob Faibussowitsch     PetscCall(PetscFree3(vals,cellCIndices,cellFIndices));
26759566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(preallocator, MAT_FINAL_ASSEMBLY));
26769566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(preallocator, MAT_FINAL_ASSEMBLY));
26779566063dSJacob Faibussowitsch     PetscCall(MatPreallocatorPreallocate(preallocator, PETSC_TRUE, In));
26789566063dSJacob Faibussowitsch     PetscCall(MatDestroy(&preallocator));
26797f5b169aSMatthew G. Knepley   }
26807f5b169aSMatthew G. Knepley   /* Fill matrix */
26819566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(In));
2682d69c5d34SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
2683cf51de39SMatthew G. Knepley     if (isRefined) {
26849566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosureRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES));
2685cf51de39SMatthew G. Knepley     } else {
26869566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES));
2687cf51de39SMatthew G. Knepley     }
2688d69c5d34SMatthew G. Knepley   }
26899566063dSJacob Faibussowitsch   for (f = 0; f < Nf; ++f) PetscCall(PetscFEDestroy(&feRef[f]));
26909566063dSJacob Faibussowitsch   PetscCall(PetscFree2(feRef,fvRef));
26919566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
26929566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
26939566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
26949318fe57SMatthew G. Knepley   if (mesh->printFEM > 1) {
26959566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)In), "%s:\n", name));
26969566063dSJacob Faibussowitsch     PetscCall(MatChop(In, 1.0e-10));
26979566063dSJacob Faibussowitsch     PetscCall(MatView(In, NULL));
2698d69c5d34SMatthew G. Knepley   }
26999566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM,dmc,dmf,0,0));
2700d69c5d34SMatthew G. Knepley   PetscFunctionReturn(0);
2701d69c5d34SMatthew G. Knepley }
27026c73c22cSMatthew G. Knepley 
2703bd041c0cSMatthew G. Knepley PetscErrorCode DMPlexComputeMassMatrixNested(DM dmc, DM dmf, Mat mass, void *user)
2704bd041c0cSMatthew G. Knepley {
2705bd041c0cSMatthew G. Knepley   SETERRQ(PetscObjectComm((PetscObject) dmc), PETSC_ERR_SUP, "Laziness");
2706bd041c0cSMatthew G. Knepley }
2707bd041c0cSMatthew G. Knepley 
270868132eb9SMatthew G. Knepley /*@
270968132eb9SMatthew G. Knepley   DMPlexComputeInterpolatorGeneral - Form the local portion of the interpolation matrix I from the coarse DM to a non-nested fine DM.
271068132eb9SMatthew G. Knepley 
271168132eb9SMatthew G. Knepley   Input Parameters:
271268132eb9SMatthew G. Knepley + dmf  - The fine mesh
271368132eb9SMatthew G. Knepley . dmc  - The coarse mesh
271468132eb9SMatthew G. Knepley - user - The user context
271568132eb9SMatthew G. Knepley 
271668132eb9SMatthew G. Knepley   Output Parameter:
271768132eb9SMatthew G. Knepley . In  - The interpolation matrix
271868132eb9SMatthew G. Knepley 
271968132eb9SMatthew G. Knepley   Level: developer
272068132eb9SMatthew G. Knepley 
2721db781477SPatrick Sanan .seealso: `DMPlexComputeInterpolatorNested()`, `DMPlexComputeJacobianFEM()`
272268132eb9SMatthew G. Knepley @*/
272368132eb9SMatthew G. Knepley PetscErrorCode DMPlexComputeInterpolatorGeneral(DM dmc, DM dmf, Mat In, void *user)
27244ef9d792SMatthew G. Knepley {
272564e98e1dSMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dmf->data;
272664e98e1dSMatthew G. Knepley   const char    *name = "Interpolator";
27274ef9d792SMatthew G. Knepley   PetscDS        prob;
27284ef9d792SMatthew G. Knepley   PetscSection   fsection, csection, globalFSection, globalCSection;
2729e8f14785SLisandro Dalcin   PetscHSetIJ    ht;
27304ef9d792SMatthew G. Knepley   PetscLayout    rLayout;
27314ef9d792SMatthew G. Knepley   PetscInt      *dnz, *onz;
27324ef9d792SMatthew G. Knepley   PetscInt       locRows, rStart, rEnd;
27334ef9d792SMatthew G. Knepley   PetscReal     *x, *v0, *J, *invJ, detJ;
27344ef9d792SMatthew G. Knepley   PetscReal     *v0c, *Jc, *invJc, detJc;
27354ef9d792SMatthew G. Knepley   PetscScalar   *elemMat;
27364ef9d792SMatthew G. Knepley   PetscInt       dim, Nf, field, totDim, cStart, cEnd, cell, ccell;
27374ef9d792SMatthew G. Knepley 
27384ef9d792SMatthew G. Knepley   PetscFunctionBegin;
27399566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM,dmc,dmf,0,0));
27409566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
27419566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
27429566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
27439566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
27449566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim,&v0,dim*dim,&J,dim*dim,&invJ));
27459566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim,&v0c,dim*dim,&Jc,dim*dim,&invJc));
27469566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
27479566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
27489566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
27499566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
27509566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd));
27519566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
27529566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
27534ef9d792SMatthew G. Knepley 
27549566063dSJacob Faibussowitsch   PetscCall(MatGetLocalSize(In, &locRows, NULL));
27559566063dSJacob Faibussowitsch   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject) In), &rLayout));
27569566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetLocalSize(rLayout, locRows));
27579566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetBlockSize(rLayout, 1));
27589566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetUp(rLayout));
27599566063dSJacob Faibussowitsch   PetscCall(PetscLayoutGetRange(rLayout, &rStart, &rEnd));
27609566063dSJacob Faibussowitsch   PetscCall(PetscLayoutDestroy(&rLayout));
27619566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(locRows,&dnz,locRows,&onz));
27629566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJCreate(&ht));
27634ef9d792SMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
27644ef9d792SMatthew G. Knepley     PetscObject      obj;
27654ef9d792SMatthew G. Knepley     PetscClassId     id;
2766c0d7054bSMatthew G. Knepley     PetscDualSpace   Q = NULL;
27674ef9d792SMatthew G. Knepley     PetscQuadrature  f;
276817f047d8SMatthew G. Knepley     const PetscReal *qpoints;
2769d0f6233fSMatthew G. Knepley     PetscInt         Nc, Np, fpdim, off, i, d;
27704ef9d792SMatthew G. Knepley 
2771d0f6233fSMatthew G. Knepley     PetscCall(PetscDSGetFieldOffset(prob, field, &off));
27729566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
27739566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
27744ef9d792SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
27754ef9d792SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
27764ef9d792SMatthew G. Knepley 
27779566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(fe, &Q));
27789566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
27794ef9d792SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
27804ef9d792SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
27814ef9d792SMatthew G. Knepley 
27829566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
27834ef9d792SMatthew G. Knepley       Nc   = 1;
27844ef9d792SMatthew G. Knepley     }
27859566063dSJacob Faibussowitsch     PetscCall(PetscDualSpaceGetDimension(Q, &fpdim));
27864ef9d792SMatthew G. Knepley     /* For each fine grid cell */
27874ef9d792SMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
27884ef9d792SMatthew G. Knepley       PetscInt *findices,   *cindices;
27894ef9d792SMatthew G. Knepley       PetscInt  numFIndices, numCIndices;
27904ef9d792SMatthew G. Knepley 
27919566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
27929566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
2793d0f6233fSMatthew G. Knepley       PetscCheck(numFIndices == totDim,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %" PetscInt_FMT " != %" PetscInt_FMT " dual basis vecs", numFIndices, totDim);
27944ef9d792SMatthew G. Knepley       for (i = 0; i < fpdim; ++i) {
27954ef9d792SMatthew G. Knepley         Vec             pointVec;
27964ef9d792SMatthew G. Knepley         PetscScalar    *pV;
27973a93e3b7SToby Isaac         PetscSF         coarseCellSF = NULL;
27983a93e3b7SToby Isaac         const PetscSFNode *coarseCells;
27999c3cf19fSMatthew G. Knepley         PetscInt        numCoarseCells, q, c;
28004ef9d792SMatthew G. Knepley 
28014ef9d792SMatthew G. Knepley         /* Get points from the dual basis functional quadrature */
28029566063dSJacob Faibussowitsch         PetscCall(PetscDualSpaceGetFunctional(Q, i, &f));
28039566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL));
28049566063dSJacob Faibussowitsch         PetscCall(VecCreateSeq(PETSC_COMM_SELF, Np*dim, &pointVec));
28059566063dSJacob Faibussowitsch         PetscCall(VecSetBlockSize(pointVec, dim));
28069566063dSJacob Faibussowitsch         PetscCall(VecGetArray(pointVec, &pV));
28074ef9d792SMatthew G. Knepley         for (q = 0; q < Np; ++q) {
2808c330f8ffSToby Isaac           const PetscReal xi0[3] = {-1., -1., -1.};
2809c330f8ffSToby Isaac 
28104ef9d792SMatthew G. Knepley           /* Transform point to real space */
2811c330f8ffSToby Isaac           CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
28124ef9d792SMatthew G. Knepley           for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
28134ef9d792SMatthew G. Knepley         }
28149566063dSJacob Faibussowitsch         PetscCall(VecRestoreArray(pointVec, &pV));
28154ef9d792SMatthew G. Knepley         /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
28161555c271SMatthew G. Knepley         /* OPT: Pack all quad points from fine cell */
28179566063dSJacob Faibussowitsch         PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
28189566063dSJacob Faibussowitsch         PetscCall(PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view"));
28194ef9d792SMatthew G. Knepley         /* Update preallocation info */
28209566063dSJacob Faibussowitsch         PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
28215f80ce2aSJacob Faibussowitsch         PetscCheck(numCoarseCells == Np,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
28229c3cf19fSMatthew G. Knepley         {
2823e8f14785SLisandro Dalcin           PetscHashIJKey key;
2824e8f14785SLisandro Dalcin           PetscBool      missing;
28254ef9d792SMatthew G. Knepley 
2826d0f6233fSMatthew G. Knepley           key.i = findices[i+off];
2827e8f14785SLisandro Dalcin           if (key.i >= 0) {
28284ef9d792SMatthew G. Knepley             /* Get indices for coarse elements */
28294ef9d792SMatthew G. Knepley             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
28309566063dSJacob Faibussowitsch               PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
28314ef9d792SMatthew G. Knepley               for (c = 0; c < numCIndices; ++c) {
2832e8f14785SLisandro Dalcin                 key.j = cindices[c];
2833e8f14785SLisandro Dalcin                 if (key.j < 0) continue;
28349566063dSJacob Faibussowitsch                 PetscCall(PetscHSetIJQueryAdd(ht, key, &missing));
28354ef9d792SMatthew G. Knepley                 if (missing) {
2836e8f14785SLisandro Dalcin                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i-rStart];
2837e8f14785SLisandro Dalcin                   else                                     ++onz[key.i-rStart];
28384ef9d792SMatthew G. Knepley                 }
28394ef9d792SMatthew G. Knepley               }
28409566063dSJacob Faibussowitsch               PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
28414ef9d792SMatthew G. Knepley             }
28424ef9d792SMatthew G. Knepley           }
28438c543595SMatthew G. Knepley         }
28449566063dSJacob Faibussowitsch         PetscCall(PetscSFDestroy(&coarseCellSF));
28459566063dSJacob Faibussowitsch         PetscCall(VecDestroy(&pointVec));
28464ef9d792SMatthew G. Knepley       }
28479566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
28484ef9d792SMatthew G. Knepley     }
28494ef9d792SMatthew G. Knepley   }
28509566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJDestroy(&ht));
28519566063dSJacob Faibussowitsch   PetscCall(MatXAIJSetPreallocation(In, 1, dnz, onz, NULL, NULL));
28529566063dSJacob Faibussowitsch   PetscCall(MatSetOption(In, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE));
28539566063dSJacob Faibussowitsch   PetscCall(PetscFree2(dnz,onz));
28544ef9d792SMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
28554ef9d792SMatthew G. Knepley     PetscObject       obj;
28564ef9d792SMatthew G. Knepley     PetscClassId      id;
2857c0d7054bSMatthew G. Knepley     PetscDualSpace    Q = NULL;
2858ef0bb6c7SMatthew G. Knepley     PetscTabulation T = NULL;
28594ef9d792SMatthew G. Knepley     PetscQuadrature   f;
28604ef9d792SMatthew G. Knepley     const PetscReal  *qpoints, *qweights;
2861d0f6233fSMatthew G. Knepley     PetscInt          Nc, qNc, Np, fpdim, off, i, d;
28624ef9d792SMatthew G. Knepley 
2863d0f6233fSMatthew G. Knepley     PetscCall(PetscDSGetFieldOffset(prob, field, &off));
28649566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
28659566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
28664ef9d792SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
28674ef9d792SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
28684ef9d792SMatthew G. Knepley 
28699566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(fe, &Q));
28709566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
28719566063dSJacob Faibussowitsch       PetscCall(PetscFECreateTabulation(fe, 1, 1, x, 0, &T));
28724ef9d792SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
28734ef9d792SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
28744ef9d792SMatthew G. Knepley 
28759566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
28764ef9d792SMatthew G. Knepley       Nc   = 1;
287763a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dmc),PETSC_ERR_ARG_WRONG,"Unknown discretization type for field %" PetscInt_FMT,field);
28789566063dSJacob Faibussowitsch     PetscCall(PetscDualSpaceGetDimension(Q, &fpdim));
28794ef9d792SMatthew G. Knepley     /* For each fine grid cell */
28804ef9d792SMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
28814ef9d792SMatthew G. Knepley       PetscInt *findices,   *cindices;
28824ef9d792SMatthew G. Knepley       PetscInt  numFIndices, numCIndices;
28834ef9d792SMatthew G. Knepley 
28849566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
28859566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
2886d0f6233fSMatthew G. Knepley       PetscCheck(numFIndices == totDim,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %" PetscInt_FMT " != %" PetscInt_FMT " dual basis vecs", numFIndices, totDim);
28874ef9d792SMatthew G. Knepley       for (i = 0; i < fpdim; ++i) {
28884ef9d792SMatthew G. Knepley         Vec             pointVec;
28894ef9d792SMatthew G. Knepley         PetscScalar    *pV;
289012111d7cSToby Isaac         PetscSF         coarseCellSF = NULL;
28913a93e3b7SToby Isaac         const PetscSFNode *coarseCells;
2892d0f6233fSMatthew G. Knepley         PetscInt        numCoarseCells, cpdim, row = findices[i+off], q, c, j;
28934ef9d792SMatthew G. Knepley 
28944ef9d792SMatthew G. Knepley         /* Get points from the dual basis functional quadrature */
28959566063dSJacob Faibussowitsch         PetscCall(PetscDualSpaceGetFunctional(Q, i, &f));
28969566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, &qpoints, &qweights));
289763a3b9bcSJacob Faibussowitsch         PetscCheck(qNc == Nc,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, Nc);
28989566063dSJacob Faibussowitsch         PetscCall(VecCreateSeq(PETSC_COMM_SELF, Np*dim, &pointVec));
28999566063dSJacob Faibussowitsch         PetscCall(VecSetBlockSize(pointVec, dim));
29009566063dSJacob Faibussowitsch         PetscCall(VecGetArray(pointVec, &pV));
29014ef9d792SMatthew G. Knepley         for (q = 0; q < Np; ++q) {
2902c330f8ffSToby Isaac           const PetscReal xi0[3] = {-1., -1., -1.};
2903c330f8ffSToby Isaac 
29044ef9d792SMatthew G. Knepley           /* Transform point to real space */
2905c330f8ffSToby Isaac           CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
29064ef9d792SMatthew G. Knepley           for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
29074ef9d792SMatthew G. Knepley         }
29089566063dSJacob Faibussowitsch         PetscCall(VecRestoreArray(pointVec, &pV));
29094ef9d792SMatthew G. Knepley         /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
29101555c271SMatthew G. Knepley         /* OPT: Read this out from preallocation information */
29119566063dSJacob Faibussowitsch         PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
29124ef9d792SMatthew G. Knepley         /* Update preallocation info */
29139566063dSJacob Faibussowitsch         PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
29145f80ce2aSJacob Faibussowitsch         PetscCheck(numCoarseCells == Np,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
29159566063dSJacob Faibussowitsch         PetscCall(VecGetArray(pointVec, &pV));
29164ef9d792SMatthew G. Knepley         for (ccell = 0; ccell < numCoarseCells; ++ccell) {
2917826eb36dSMatthew G. Knepley           PetscReal pVReal[3];
2918c330f8ffSToby Isaac           const PetscReal xi0[3] = {-1., -1., -1.};
2919826eb36dSMatthew G. Knepley 
29209566063dSJacob Faibussowitsch           PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
29214ef9d792SMatthew G. Knepley           /* Transform points from real space to coarse reference space */
29229566063dSJacob Faibussowitsch           PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
2923e2d86523SMatthew G. Knepley           for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell*dim+d]);
2924c330f8ffSToby Isaac           CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
29254ef9d792SMatthew G. Knepley 
29264ef9d792SMatthew G. Knepley           if (id == PETSCFE_CLASSID) {
29274ef9d792SMatthew G. Knepley             PetscFE fe = (PetscFE) obj;
29284ef9d792SMatthew G. Knepley 
29294ef9d792SMatthew G. Knepley             /* Evaluate coarse basis on contained point */
29309566063dSJacob Faibussowitsch             PetscCall(PetscFEGetDimension(fe, &cpdim));
29319566063dSJacob Faibussowitsch             PetscCall(PetscFEComputeTabulation(fe, 1, x, 0, T));
29329566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
29334ef9d792SMatthew G. Knepley             /* Get elemMat entries by multiplying by weight */
29344ef9d792SMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
2935ef0bb6c7SMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j*Nc + c]*qweights[ccell*qNc + c];
29364ef9d792SMatthew G. Knepley             }
29374ef9d792SMatthew G. Knepley           } else {
29384ef9d792SMatthew G. Knepley             cpdim = 1;
29394ef9d792SMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
29409c3cf19fSMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0*qweights[ccell*qNc + c];
29414ef9d792SMatthew G. Knepley             }
29424ef9d792SMatthew G. Knepley           }
29434ef9d792SMatthew G. Knepley           /* Update interpolator */
29449566063dSJacob Faibussowitsch           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
2945d0f6233fSMatthew G. Knepley           PetscCheck(numCIndices == totDim,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, totDim);
2946d0f6233fSMatthew G. Knepley           PetscCall(MatSetValues(In, 1, &row, cpdim, &cindices[off], elemMat, INSERT_VALUES));
29479566063dSJacob Faibussowitsch           PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
29484ef9d792SMatthew G. Knepley         }
29499566063dSJacob Faibussowitsch         PetscCall(VecRestoreArray(pointVec, &pV));
29509566063dSJacob Faibussowitsch         PetscCall(PetscSFDestroy(&coarseCellSF));
29519566063dSJacob Faibussowitsch         PetscCall(VecDestroy(&pointVec));
29524ef9d792SMatthew G. Knepley       }
29539566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
29544ef9d792SMatthew G. Knepley     }
29559566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
29564ef9d792SMatthew G. Knepley   }
29579566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0,J,invJ));
29589566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c,Jc,invJc));
29599566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
29609566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
29619566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
29629566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM,dmc,dmf,0,0));
29634ef9d792SMatthew G. Knepley   PetscFunctionReturn(0);
29644ef9d792SMatthew G. Knepley }
29654ef9d792SMatthew G. Knepley 
296646fa42a0SMatthew G. Knepley /*@
2967bd041c0cSMatthew G. Knepley   DMPlexComputeMassMatrixGeneral - Form the local portion of the mass matrix M from the coarse DM to a non-nested fine DM.
2968bd041c0cSMatthew G. Knepley 
2969bd041c0cSMatthew G. Knepley   Input Parameters:
2970bd041c0cSMatthew G. Knepley + dmf  - The fine mesh
2971bd041c0cSMatthew G. Knepley . dmc  - The coarse mesh
2972bd041c0cSMatthew G. Knepley - user - The user context
2973bd041c0cSMatthew G. Knepley 
2974bd041c0cSMatthew G. Knepley   Output Parameter:
2975bd041c0cSMatthew G. Knepley . mass  - The mass matrix
2976bd041c0cSMatthew G. Knepley 
2977bd041c0cSMatthew G. Knepley   Level: developer
2978bd041c0cSMatthew G. Knepley 
2979db781477SPatrick Sanan .seealso: `DMPlexComputeMassMatrixNested()`, `DMPlexComputeInterpolatorNested()`, `DMPlexComputeInterpolatorGeneral()`, `DMPlexComputeJacobianFEM()`
2980bd041c0cSMatthew G. Knepley @*/
2981bd041c0cSMatthew G. Knepley PetscErrorCode DMPlexComputeMassMatrixGeneral(DM dmc, DM dmf, Mat mass, void *user)
2982bd041c0cSMatthew G. Knepley {
2983bd041c0cSMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dmf->data;
2984bd041c0cSMatthew G. Knepley   const char    *name = "Mass Matrix";
2985bd041c0cSMatthew G. Knepley   PetscDS        prob;
2986bd041c0cSMatthew G. Knepley   PetscSection   fsection, csection, globalFSection, globalCSection;
2987e8f14785SLisandro Dalcin   PetscHSetIJ    ht;
2988bd041c0cSMatthew G. Knepley   PetscLayout    rLayout;
2989bd041c0cSMatthew G. Knepley   PetscInt      *dnz, *onz;
2990bd041c0cSMatthew G. Knepley   PetscInt       locRows, rStart, rEnd;
2991bd041c0cSMatthew G. Knepley   PetscReal     *x, *v0, *J, *invJ, detJ;
2992bd041c0cSMatthew G. Knepley   PetscReal     *v0c, *Jc, *invJc, detJc;
2993bd041c0cSMatthew G. Knepley   PetscScalar   *elemMat;
2994bd041c0cSMatthew G. Knepley   PetscInt       dim, Nf, field, totDim, cStart, cEnd, cell, ccell;
2995bd041c0cSMatthew G. Knepley 
2996bd041c0cSMatthew G. Knepley   PetscFunctionBegin;
29979566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
29989566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
29999566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
30009566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
30019566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim,&v0,dim*dim,&J,dim*dim,&invJ));
30029566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim,&v0c,dim*dim,&Jc,dim*dim,&invJc));
30039566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
30049566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
30059566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
30069566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
30079566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd));
30089566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
30099566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
3010bd041c0cSMatthew G. Knepley 
30119566063dSJacob Faibussowitsch   PetscCall(MatGetLocalSize(mass, &locRows, NULL));
30129566063dSJacob Faibussowitsch   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject) mass), &rLayout));
30139566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetLocalSize(rLayout, locRows));
30149566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetBlockSize(rLayout, 1));
30159566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetUp(rLayout));
30169566063dSJacob Faibussowitsch   PetscCall(PetscLayoutGetRange(rLayout, &rStart, &rEnd));
30179566063dSJacob Faibussowitsch   PetscCall(PetscLayoutDestroy(&rLayout));
30189566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(locRows,&dnz,locRows,&onz));
30199566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJCreate(&ht));
3020bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
3021bd041c0cSMatthew G. Knepley     PetscObject      obj;
3022bd041c0cSMatthew G. Knepley     PetscClassId     id;
3023bd041c0cSMatthew G. Knepley     PetscQuadrature  quad;
3024bd041c0cSMatthew G. Knepley     const PetscReal *qpoints;
3025bd041c0cSMatthew G. Knepley     PetscInt         Nq, Nc, i, d;
3026bd041c0cSMatthew G. Knepley 
30279566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
30289566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
30299566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetQuadrature((PetscFE) obj, &quad));
30309566063dSJacob Faibussowitsch     else                       PetscCall(PetscFVGetQuadrature((PetscFV) obj, &quad));
30319566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, NULL));
3032bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
3033bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3034bd041c0cSMatthew G. Knepley       Vec                pointVec;
3035bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
3036bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
3037bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
3038bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, q, c;
3039bd041c0cSMatthew G. Knepley       PetscInt          *findices,   *cindices;
3040bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
3041bd041c0cSMatthew G. Knepley 
30429566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
30439566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3044bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
30459566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq*dim, &pointVec));
30469566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
30479566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3048bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
3049c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3050c330f8ffSToby Isaac 
3051bd041c0cSMatthew G. Knepley         /* Transform point to real space */
3052c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
3053bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
3054bd041c0cSMatthew G. Knepley       }
30559566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
3056bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
30579566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
30589566063dSJacob Faibussowitsch       PetscCall(PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view"));
3059bd041c0cSMatthew G. Knepley       /* Update preallocation info */
30609566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
30615f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
3062bd041c0cSMatthew G. Knepley       {
3063e8f14785SLisandro Dalcin         PetscHashIJKey key;
3064e8f14785SLisandro Dalcin         PetscBool      missing;
3065bd041c0cSMatthew G. Knepley 
3066bd041c0cSMatthew G. Knepley         for (i = 0; i < numFIndices; ++i) {
3067e8f14785SLisandro Dalcin           key.i = findices[i];
3068e8f14785SLisandro Dalcin           if (key.i >= 0) {
3069bd041c0cSMatthew G. Knepley             /* Get indices for coarse elements */
3070bd041c0cSMatthew G. Knepley             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
30719566063dSJacob Faibussowitsch               PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3072bd041c0cSMatthew G. Knepley               for (c = 0; c < numCIndices; ++c) {
3073e8f14785SLisandro Dalcin                 key.j = cindices[c];
3074e8f14785SLisandro Dalcin                 if (key.j < 0) continue;
30759566063dSJacob Faibussowitsch                 PetscCall(PetscHSetIJQueryAdd(ht, key, &missing));
3076bd041c0cSMatthew G. Knepley                 if (missing) {
3077e8f14785SLisandro Dalcin                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i-rStart];
3078e8f14785SLisandro Dalcin                   else                                     ++onz[key.i-rStart];
3079bd041c0cSMatthew G. Knepley                 }
3080bd041c0cSMatthew G. Knepley               }
30819566063dSJacob Faibussowitsch               PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3082bd041c0cSMatthew G. Knepley             }
3083bd041c0cSMatthew G. Knepley           }
3084bd041c0cSMatthew G. Knepley         }
3085bd041c0cSMatthew G. Knepley       }
30869566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
30879566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
30889566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
3089bd041c0cSMatthew G. Knepley     }
3090bd041c0cSMatthew G. Knepley   }
30919566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJDestroy(&ht));
30929566063dSJacob Faibussowitsch   PetscCall(MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL));
30939566063dSJacob Faibussowitsch   PetscCall(MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE));
30949566063dSJacob Faibussowitsch   PetscCall(PetscFree2(dnz,onz));
3095bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
3096bd041c0cSMatthew G. Knepley     PetscObject       obj;
3097bd041c0cSMatthew G. Knepley     PetscClassId      id;
3098ef0bb6c7SMatthew G. Knepley     PetscTabulation T, Tfine;
3099bd041c0cSMatthew G. Knepley     PetscQuadrature   quad;
3100bd041c0cSMatthew G. Knepley     const PetscReal  *qpoints, *qweights;
3101bd041c0cSMatthew G. Knepley     PetscInt          Nq, Nc, i, d;
3102bd041c0cSMatthew G. Knepley 
31039566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
31049566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
3105ef0bb6c7SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
31069566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature((PetscFE) obj, &quad));
31079566063dSJacob Faibussowitsch       PetscCall(PetscFEGetCellTabulation((PetscFE) obj, 1, &Tfine));
31089566063dSJacob Faibussowitsch       PetscCall(PetscFECreateTabulation((PetscFE) obj, 1, 1, x, 0, &T));
3109ef0bb6c7SMatthew G. Knepley     } else {
31109566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature((PetscFV) obj, &quad));
3111ef0bb6c7SMatthew G. Knepley     }
31129566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, &qweights));
3113bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
3114bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3115bd041c0cSMatthew G. Knepley       Vec                pointVec;
3116bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
3117bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
3118bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
3119bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, cpdim, q, c, j;
3120bd041c0cSMatthew G. Knepley       PetscInt          *findices,   *cindices;
3121bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
3122bd041c0cSMatthew G. Knepley 
31239566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
31249566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3125bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
31269566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq*dim, &pointVec));
31279566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
31289566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3129bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
3130c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3131c330f8ffSToby Isaac 
3132bd041c0cSMatthew G. Knepley         /* Transform point to real space */
3133c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
3134bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
3135bd041c0cSMatthew G. Knepley       }
31369566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
3137bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
31389566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
3139bd041c0cSMatthew G. Knepley       /* Update matrix */
31409566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
31415f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
31429566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3143bd041c0cSMatthew G. Knepley       for (ccell = 0; ccell < numCoarseCells; ++ccell) {
3144bd041c0cSMatthew G. Knepley         PetscReal pVReal[3];
3145c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3146c330f8ffSToby Isaac 
31479566063dSJacob Faibussowitsch         PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3148bd041c0cSMatthew G. Knepley         /* Transform points from real space to coarse reference space */
31499566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
3150bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell*dim+d]);
3151c330f8ffSToby Isaac         CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
3152bd041c0cSMatthew G. Knepley 
3153bd041c0cSMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
3154bd041c0cSMatthew G. Knepley           PetscFE fe = (PetscFE) obj;
3155bd041c0cSMatthew G. Knepley 
3156bd041c0cSMatthew G. Knepley           /* Evaluate coarse basis on contained point */
31579566063dSJacob Faibussowitsch           PetscCall(PetscFEGetDimension(fe, &cpdim));
31589566063dSJacob Faibussowitsch           PetscCall(PetscFEComputeTabulation(fe, 1, x, 0, T));
3159bd041c0cSMatthew G. Knepley           /* Get elemMat entries by multiplying by weight */
3160bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
31619566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3162bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3163ef0bb6c7SMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j*Nc + c]*Tfine->T[0][(ccell*numFIndices + i)*Nc + c]*qweights[ccell*Nc + c]*detJ;
3164bd041c0cSMatthew G. Knepley             }
3165bd041c0cSMatthew G. Knepley             /* Update interpolator */
31669566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
316763a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
31689566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3169bd041c0cSMatthew G. Knepley           }
3170bd041c0cSMatthew G. Knepley         } else {
3171bd041c0cSMatthew G. Knepley           cpdim = 1;
3172bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
31739566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3174bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3175bd041c0cSMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0*1.0*qweights[ccell*Nc + c]*detJ;
3176bd041c0cSMatthew G. Knepley             }
3177bd041c0cSMatthew G. Knepley             /* Update interpolator */
31789566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
317963a3b9bcSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "Nq: %" PetscInt_FMT " %" PetscInt_FMT " Nf: %" PetscInt_FMT " %" PetscInt_FMT " Nc: %" PetscInt_FMT " %" PetscInt_FMT "\n", ccell, Nq, i, numFIndices, j, numCIndices));
318063a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
31819566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3182bd041c0cSMatthew G. Knepley           }
3183bd041c0cSMatthew G. Knepley         }
31849566063dSJacob Faibussowitsch         PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3185bd041c0cSMatthew G. Knepley       }
31869566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
31879566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
31889566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
31899566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
3190bd041c0cSMatthew G. Knepley     }
31919566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
3192bd041c0cSMatthew G. Knepley   }
31939566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0,J,invJ));
31949566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c,Jc,invJc));
31959566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
31969566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY));
31979566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY));
3198bd041c0cSMatthew G. Knepley   PetscFunctionReturn(0);
3199bd041c0cSMatthew G. Knepley }
3200bd041c0cSMatthew G. Knepley 
3201bd041c0cSMatthew G. Knepley /*@
320246fa42a0SMatthew G. Knepley   DMPlexComputeInjectorFEM - Compute a mapping from coarse unknowns to fine unknowns
320346fa42a0SMatthew G. Knepley 
320446fa42a0SMatthew G. Knepley   Input Parameters:
320546fa42a0SMatthew G. Knepley + dmc  - The coarse mesh
320646fa42a0SMatthew G. Knepley - dmf  - The fine mesh
320746fa42a0SMatthew G. Knepley - user - The user context
320846fa42a0SMatthew G. Knepley 
320946fa42a0SMatthew G. Knepley   Output Parameter:
321046fa42a0SMatthew G. Knepley . sc   - The mapping
321146fa42a0SMatthew G. Knepley 
321246fa42a0SMatthew G. Knepley   Level: developer
321346fa42a0SMatthew G. Knepley 
3214db781477SPatrick Sanan .seealso: `DMPlexComputeInterpolatorNested()`, `DMPlexComputeJacobianFEM()`
321546fa42a0SMatthew G. Knepley @*/
32167c927364SMatthew G. Knepley PetscErrorCode DMPlexComputeInjectorFEM(DM dmc, DM dmf, VecScatter *sc, void *user)
32177c927364SMatthew G. Knepley {
3218e9d4ef1bSMatthew G. Knepley   PetscDS        prob;
32197c927364SMatthew G. Knepley   PetscFE       *feRef;
322097c42addSMatthew G. Knepley   PetscFV       *fvRef;
32217c927364SMatthew G. Knepley   Vec            fv, cv;
32227c927364SMatthew G. Knepley   IS             fis, cis;
32237c927364SMatthew G. Knepley   PetscSection   fsection, fglobalSection, csection, cglobalSection;
32247c927364SMatthew G. Knepley   PetscInt      *cmap, *cellCIndices, *cellFIndices, *cindices, *findices;
3225485ad865SMatthew G. Knepley   PetscInt       cTotDim, fTotDim = 0, Nf, f, field, cStart, cEnd, c, dim, d, startC, endC, offsetC, offsetF, m;
32266f3d3cbcSMatthew G. Knepley   PetscBool     *needAvg;
32277c927364SMatthew G. Knepley 
32287c927364SMatthew G. Knepley   PetscFunctionBegin;
32299566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InjectorFEM,dmc,dmf,0,0));
32309566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
32319566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
32329566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
32339566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
32349566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
32359566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
32369566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
32379566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
32389566063dSJacob Faibussowitsch   PetscCall(PetscCalloc3(Nf,&feRef,Nf,&fvRef,Nf,&needAvg));
32397c927364SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
324097c42addSMatthew G. Knepley     PetscObject  obj;
324197c42addSMatthew G. Knepley     PetscClassId id;
3242aa7890ccSMatthew G. Knepley     PetscInt     fNb = 0, Nc = 0;
32437c927364SMatthew G. Knepley 
32449566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
32459566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
324697c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
324797c42addSMatthew G. Knepley       PetscFE    fe = (PetscFE) obj;
32486f3d3cbcSMatthew G. Knepley       PetscSpace sp;
32499b2fc754SMatthew G. Knepley       PetscInt   maxDegree;
325097c42addSMatthew G. Knepley 
32519566063dSJacob Faibussowitsch       PetscCall(PetscFERefine(fe, &feRef[f]));
32529566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &fNb));
32539566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
32549566063dSJacob Faibussowitsch       PetscCall(PetscFEGetBasisSpace(fe, &sp));
32559566063dSJacob Faibussowitsch       PetscCall(PetscSpaceGetDegree(sp, NULL, &maxDegree));
32569b2fc754SMatthew G. Knepley       if (!maxDegree) needAvg[f] = PETSC_TRUE;
325797c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
325897c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV) obj;
325997c42addSMatthew G. Knepley       PetscDualSpace Q;
326097c42addSMatthew G. Knepley 
32619566063dSJacob Faibussowitsch       PetscCall(PetscFVRefine(fv, &fvRef[f]));
32629566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
32639566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &fNb));
32649566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
32656f3d3cbcSMatthew G. Knepley       needAvg[f] = PETSC_TRUE;
326697c42addSMatthew G. Knepley     }
3267d172c84bSMatthew G. Knepley     fTotDim += fNb;
32687c927364SMatthew G. Knepley   }
32699566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &cTotDim));
32709566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(cTotDim,&cmap));
32717c927364SMatthew G. Knepley   for (field = 0, offsetC = 0, offsetF = 0; field < Nf; ++field) {
32727c927364SMatthew G. Knepley     PetscFE        feC;
327397c42addSMatthew G. Knepley     PetscFV        fvC;
32747c927364SMatthew G. Knepley     PetscDualSpace QF, QC;
3275d172c84bSMatthew G. Knepley     PetscInt       order = -1, NcF, NcC, fpdim, cpdim;
32767c927364SMatthew G. Knepley 
327797c42addSMatthew G. Knepley     if (feRef[field]) {
32789566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *) &feC));
32799566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feC, &NcC));
32809566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[field], &NcF));
32819566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[field], &QF));
32829566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetOrder(QF, &order));
32839566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
32849566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feC, &QC));
32859566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
328697c42addSMatthew G. Knepley     } else {
32879566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *) &fvC));
32889566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvC, &NcC));
32899566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[field], &NcF));
32909566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[field], &QF));
32919566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
32929566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvC, &QC));
32939566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
329497c42addSMatthew G. Knepley     }
329563a3b9bcSJacob Faibussowitsch     PetscCheck(NcF == NcC,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, NcF, NcC);
32967c927364SMatthew G. Knepley     for (c = 0; c < cpdim; ++c) {
32977c927364SMatthew G. Knepley       PetscQuadrature  cfunc;
3298d172c84bSMatthew G. Knepley       const PetscReal *cqpoints, *cqweights;
3299d172c84bSMatthew G. Knepley       PetscInt         NqcC, NpC;
330097c42addSMatthew G. Knepley       PetscBool        found = PETSC_FALSE;
33017c927364SMatthew G. Knepley 
33029566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(QC, c, &cfunc));
33039566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(cfunc, NULL, &NqcC, &NpC, &cqpoints, &cqweights));
330463a3b9bcSJacob Faibussowitsch       PetscCheck(NqcC == NcC,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcC, NcC);
33051dca8a05SBarry Smith       PetscCheck(NpC == 1 || !feRef[field],PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Do not know how to do injection for moments");
33067c927364SMatthew G. Knepley       for (f = 0; f < fpdim; ++f) {
33077c927364SMatthew G. Knepley         PetscQuadrature  ffunc;
3308d172c84bSMatthew G. Knepley         const PetscReal *fqpoints, *fqweights;
33097c927364SMatthew G. Knepley         PetscReal        sum = 0.0;
3310d172c84bSMatthew G. Knepley         PetscInt         NqcF, NpF;
33117c927364SMatthew G. Knepley 
33129566063dSJacob Faibussowitsch         PetscCall(PetscDualSpaceGetFunctional(QF, f, &ffunc));
33139566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(ffunc, NULL, &NqcF, &NpF, &fqpoints, &fqweights));
331463a3b9bcSJacob Faibussowitsch         PetscCheck(NqcF == NcF,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcF, NcF);
33157c927364SMatthew G. Knepley         if (NpC != NpF) continue;
33167c927364SMatthew G. Knepley         for (d = 0; d < dim; ++d) sum += PetscAbsReal(cqpoints[d] - fqpoints[d]);
33177c927364SMatthew G. Knepley         if (sum > 1.0e-9) continue;
3318d172c84bSMatthew G. Knepley         for (d = 0; d < NcC; ++d) sum += PetscAbsReal(cqweights[d]*fqweights[d]);
3319d172c84bSMatthew G. Knepley         if (sum < 1.0e-9) continue;
3320d172c84bSMatthew G. Knepley         cmap[offsetC+c] = offsetF+f;
332197c42addSMatthew G. Knepley         found = PETSC_TRUE;
33227c927364SMatthew G. Knepley         break;
33237c927364SMatthew G. Knepley       }
332497c42addSMatthew G. Knepley       if (!found) {
332597c42addSMatthew G. Knepley         /* TODO We really want the average here, but some asshole put VecScatter in the interface */
3326d172c84bSMatthew G. Knepley         if (fvRef[field] || (feRef[field] && order == 0)) {
3327d172c84bSMatthew G. Knepley           cmap[offsetC+c] = offsetF+0;
332897c42addSMatthew G. Knepley         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate matching functional for injection");
332997c42addSMatthew G. Knepley       }
33307c927364SMatthew G. Knepley     }
3331d172c84bSMatthew G. Knepley     offsetC += cpdim;
3332d172c84bSMatthew G. Knepley     offsetF += fpdim;
33337c927364SMatthew G. Knepley   }
33349566063dSJacob Faibussowitsch   for (f = 0; f < Nf; ++f) {PetscCall(PetscFEDestroy(&feRef[f]));PetscCall(PetscFVDestroy(&fvRef[f]));}
33359566063dSJacob Faibussowitsch   PetscCall(PetscFree3(feRef,fvRef,needAvg));
33367c927364SMatthew G. Knepley 
33379566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmf, &fv));
33389566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmc, &cv));
33399566063dSJacob Faibussowitsch   PetscCall(VecGetOwnershipRange(cv, &startC, &endC));
33409566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(cglobalSection, &m));
33419566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(cTotDim,&cellCIndices,fTotDim,&cellFIndices));
33429566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m,&cindices));
33439566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m,&findices));
33447c927364SMatthew G. Knepley   for (d = 0; d < m; ++d) cindices[d] = findices[d] = -1;
33457c927364SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
33469566063dSJacob Faibussowitsch     PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, c, cellCIndices, cellFIndices));
33477c927364SMatthew G. Knepley     for (d = 0; d < cTotDim; ++d) {
33480bd915a7SMatthew G. Knepley       if ((cellCIndices[d] < startC) || (cellCIndices[d] >= endC)) continue;
334963a3b9bcSJacob Faibussowitsch       PetscCheck(!(findices[cellCIndices[d]-startC] >= 0) || !(findices[cellCIndices[d]-startC] != cellFIndices[cmap[d]]),PETSC_COMM_SELF, PETSC_ERR_PLIB, "Coarse dof %" PetscInt_FMT " maps to both %" PetscInt_FMT " and %" PetscInt_FMT, cindices[cellCIndices[d]-startC], findices[cellCIndices[d]-startC], cellFIndices[cmap[d]]);
33507c927364SMatthew G. Knepley       cindices[cellCIndices[d]-startC] = cellCIndices[d];
33517c927364SMatthew G. Knepley       findices[cellCIndices[d]-startC] = cellFIndices[cmap[d]];
33527c927364SMatthew G. Knepley     }
33537c927364SMatthew G. Knepley   }
33549566063dSJacob Faibussowitsch   PetscCall(PetscFree(cmap));
33559566063dSJacob Faibussowitsch   PetscCall(PetscFree2(cellCIndices,cellFIndices));
33567c927364SMatthew G. Knepley 
33579566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis));
33589566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis));
33599566063dSJacob Faibussowitsch   PetscCall(VecScatterCreate(cv, cis, fv, fis, sc));
33609566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cis));
33619566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&fis));
33629566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmf, &fv));
33639566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmc, &cv));
33649566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InjectorFEM,dmc,dmf,0,0));
3365cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
3366cb1e1211SMatthew G Knepley }
3367a1cf66bbSMatthew G. Knepley 
33682f856554SMatthew G. Knepley /*@C
33692f856554SMatthew G. Knepley   DMPlexGetCellFields - Retrieve the field values values for a chunk of cells
33702f856554SMatthew G. Knepley 
33712f856554SMatthew G. Knepley   Input Parameters:
33722f856554SMatthew G. Knepley + dm     - The DM
33732f856554SMatthew G. Knepley . cellIS - The cells to include
33742f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
33752f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
33762f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
33772f856554SMatthew G. Knepley 
33782f856554SMatthew G. Knepley   Output Parameters:
33792f856554SMatthew G. Knepley + u   - The field coefficients
33802f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
33812f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
33822f856554SMatthew G. Knepley 
33832f856554SMatthew G. Knepley   Level: developer
33842f856554SMatthew G. Knepley 
3385db781477SPatrick Sanan .seealso: `DMPlexGetFaceFields()`
33862f856554SMatthew G. Knepley @*/
33872f856554SMatthew G. Knepley PetscErrorCode DMPlexGetCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
33882f856554SMatthew G. Knepley {
33892f856554SMatthew G. Knepley   DM              plex, plexA = NULL;
3390a6e0b375SMatthew G. Knepley   DMEnclosureType encAux;
33912f856554SMatthew G. Knepley   PetscSection    section, sectionAux;
33922f856554SMatthew G. Knepley   PetscDS         prob;
33932f856554SMatthew G. Knepley   const PetscInt *cells;
33942f856554SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, totDim, totDimAux, c;
33952f856554SMatthew G. Knepley 
33962f856554SMatthew G. Knepley   PetscFunctionBegin;
33972f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3398064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
3399064a246eSJacob Faibussowitsch   if (locX_t) {PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4);}
3400064a246eSJacob Faibussowitsch   if (locA)   {PetscValidHeaderSpecific(locA, VEC_CLASSID, 5);}
3401064a246eSJacob Faibussowitsch   PetscValidPointer(u, 6);
3402064a246eSJacob Faibussowitsch   PetscValidPointer(u_t, 7);
3403064a246eSJacob Faibussowitsch   PetscValidPointer(a, 8);
34049566063dSJacob Faibussowitsch   PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE));
34059566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
34069566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
34079566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob));
34089566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
34092f856554SMatthew G. Knepley   if (locA) {
34102f856554SMatthew G. Knepley     DM      dmAux;
34112f856554SMatthew G. Knepley     PetscDS probAux;
34122f856554SMatthew G. Knepley 
34139566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
34149566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
34159566063dSJacob Faibussowitsch     PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE));
34169566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
34179566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
34189566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
34192f856554SMatthew G. Knepley   }
34202f856554SMatthew G. Knepley   numCells = cEnd - cStart;
34219566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, u));
34229566063dSJacob Faibussowitsch   if (locX_t) PetscCall(DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, u_t)); else {*u_t = NULL;}
34239566063dSJacob Faibussowitsch   if (locA)   PetscCall(DMGetWorkArray(dm, numCells*totDimAux, MPIU_SCALAR, a)); else {*a = NULL;}
34242f856554SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
34252f856554SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
34262f856554SMatthew G. Knepley     const PetscInt cind = c - cStart;
34272f856554SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL, *ul = *u, *ul_t = *u_t, *al = *a;
34282f856554SMatthew G. Knepley     PetscInt       i;
34292f856554SMatthew G. Knepley 
34309566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x));
34312f856554SMatthew G. Knepley     for (i = 0; i < totDim; ++i) ul[cind*totDim+i] = x[i];
34329566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x));
34332f856554SMatthew G. Knepley     if (locX_t) {
34349566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t));
34352f856554SMatthew G. Knepley       for (i = 0; i < totDim; ++i) ul_t[cind*totDim+i] = x_t[i];
34369566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t));
34372f856554SMatthew G. Knepley     }
34382f856554SMatthew G. Knepley     if (locA) {
34392f856554SMatthew G. Knepley       PetscInt subcell;
34409566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell));
34419566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, NULL, &x));
34422f856554SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) al[cind*totDimAux+i] = x[i];
34439566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, NULL, &x));
34442f856554SMatthew G. Knepley     }
34452f856554SMatthew G. Knepley   }
34469566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
34479566063dSJacob Faibussowitsch   if (locA) PetscCall(DMDestroy(&plexA));
34489566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
34492f856554SMatthew G. Knepley   PetscFunctionReturn(0);
34502f856554SMatthew G. Knepley }
34512f856554SMatthew G. Knepley 
34522f856554SMatthew G. Knepley /*@C
34532f856554SMatthew G. Knepley   DMPlexRestoreCellFields - Restore the field values values for a chunk of cells
34542f856554SMatthew G. Knepley 
34552f856554SMatthew G. Knepley   Input Parameters:
34562f856554SMatthew G. Knepley + dm     - The DM
34572f856554SMatthew G. Knepley . cellIS - The cells to include
34582f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
34592f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
34602f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
34612f856554SMatthew G. Knepley 
34622f856554SMatthew G. Knepley   Output Parameters:
34632f856554SMatthew G. Knepley + u   - The field coefficients
34642f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
34652f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
34662f856554SMatthew G. Knepley 
34672f856554SMatthew G. Knepley   Level: developer
34682f856554SMatthew G. Knepley 
3469db781477SPatrick Sanan .seealso: `DMPlexGetFaceFields()`
34702f856554SMatthew G. Knepley @*/
34712f856554SMatthew G. Knepley PetscErrorCode DMPlexRestoreCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
34722f856554SMatthew G. Knepley {
34732f856554SMatthew G. Knepley   PetscFunctionBegin;
34749566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u));
34759566063dSJacob Faibussowitsch   if (locX_t) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u_t));
34769566063dSJacob Faibussowitsch   if (locA)   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, a));
34772f856554SMatthew G. Knepley   PetscFunctionReturn(0);
34782f856554SMatthew G. Knepley }
34792f856554SMatthew G. Knepley 
3480faff4f7fSMatthew G. Knepley /*
3481faff4f7fSMatthew G. Knepley   Get the auxiliary field vectors for the negative side (s = 0) and positive side (s = 1) of the interfaace
3482faff4f7fSMatthew G. Knepley */
3483148442b3SMatthew G. Knepley static PetscErrorCode DMPlexGetHybridAuxFields(DM dm, DM dmAux[], PetscDS dsAux[], IS cellIS, Vec locA[], PetscScalar *a[])
34846528b96dSMatthew G. Knepley {
348504c51a94SMatthew G. Knepley   DM              plexA[2];
3486148442b3SMatthew G. Knepley   DMEnclosureType encAux[2];
348704c51a94SMatthew G. Knepley   PetscSection    sectionAux[2];
34886528b96dSMatthew G. Knepley   const PetscInt *cells;
3489148442b3SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, c, s, totDimAux[2];
34906528b96dSMatthew G. Knepley 
34916528b96dSMatthew G. Knepley   PetscFunctionBegin;
3492148442b3SMatthew G. Knepley   PetscValidPointer(locA, 5);
349304c51a94SMatthew G. Knepley   if (!locA[0] || !locA[1]) PetscFunctionReturn(0);
3494148442b3SMatthew G. Knepley   PetscValidPointer(dmAux, 2);
3495148442b3SMatthew G. Knepley   PetscValidPointer(dsAux, 3);
3496148442b3SMatthew G. Knepley   PetscValidPointer(a, 6);
34979566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
34986528b96dSMatthew G. Knepley   numCells = cEnd - cStart;
3499148442b3SMatthew G. Knepley   for (s = 0; s < 2; ++s) {
3500148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(dmAux[s], DM_CLASSID, 2);
3501148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(dsAux[s], PETSCDS_CLASSID, 3);
3502148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(locA[s], VEC_CLASSID, 5);
35039566063dSJacob Faibussowitsch     PetscCall(DMPlexConvertPlex(dmAux[s], &plexA[s], PETSC_FALSE));
35049566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux[s], dm, &encAux[s]));
35059566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux[s], &sectionAux[s]));
35069566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[s], &totDimAux[s]));
35079566063dSJacob Faibussowitsch     PetscCall(DMGetWorkArray(dmAux[s], numCells*totDimAux[s], MPIU_SCALAR, &a[s]));
350804c51a94SMatthew G. Knepley   }
3509148442b3SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
35106528b96dSMatthew G. Knepley     const PetscInt  cell = cells ? cells[c] : c;
35116528b96dSMatthew G. Knepley     const PetscInt  cind = c - cStart;
35126528b96dSMatthew G. Knepley     const PetscInt *cone, *ornt;
35136528b96dSMatthew G. Knepley 
35149566063dSJacob Faibussowitsch     PetscCall(DMPlexGetCone(dm, cell, &cone));
35159566063dSJacob Faibussowitsch     PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
35161dca8a05SBarry Smith     PetscCheck(!ornt[0],PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", cone[0], cell, ornt[0]);
3517148442b3SMatthew G. Knepley     for (s = 0; s < 2; ++s) {
35185fedec97SMatthew G. Knepley       const PetscInt *support;
3519148442b3SMatthew G. Knepley       PetscScalar    *x = NULL, *al = a[s];
3520148442b3SMatthew G. Knepley       const PetscInt  tdA = totDimAux[s];
35215fedec97SMatthew G. Knepley       PetscInt        ssize, scell;
3522148442b3SMatthew G. Knepley       PetscInt        subface, Na, i;
35236528b96dSMatthew G. Knepley 
35249566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, cone[s], &support));
35259566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupportSize(dm, cone[s], &ssize));
352663a3b9bcSJacob Faibussowitsch       PetscCheck(ssize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[s], cell, ssize);
35275fedec97SMatthew G. Knepley       if      (support[0] == cell) scell = support[1];
35285fedec97SMatthew G. Knepley       else if (support[1] == cell) scell = support[0];
352963a3b9bcSJacob Faibussowitsch       else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[s], cell);
35305fedec97SMatthew G. Knepley 
35319566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(plexA[s], dm, encAux[s], scell, &subface));
35329566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexA[s], sectionAux[s], locA[s], subface, &Na, &x));
35336528b96dSMatthew G. Knepley       for (i = 0; i < Na; ++i) al[cind*tdA+i] = x[i];
35349566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexA[s], sectionAux[s], locA[s], subface, &Na, &x));
35356528b96dSMatthew G. Knepley     }
35366528b96dSMatthew G. Knepley   }
35379566063dSJacob Faibussowitsch   for (s = 0; s < 2; ++s) PetscCall(DMDestroy(&plexA[s]));
35389566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
35396528b96dSMatthew G. Knepley   PetscFunctionReturn(0);
35406528b96dSMatthew G. Knepley }
35416528b96dSMatthew G. Knepley 
354204c51a94SMatthew G. Knepley static PetscErrorCode DMPlexRestoreHybridAuxFields(DM dmAux[], PetscDS dsAux[], IS cellIS, Vec locA[], PetscScalar *a[])
35436528b96dSMatthew G. Knepley {
35446528b96dSMatthew G. Knepley   PetscFunctionBegin;
354504c51a94SMatthew G. Knepley   if (!locA[0] || !locA[1]) PetscFunctionReturn(0);
35469566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dmAux[0], 0, MPIU_SCALAR, &a[0]));
35479566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dmAux[1], 0, MPIU_SCALAR, &a[1]));
35486528b96dSMatthew G. Knepley   PetscFunctionReturn(0);
35496528b96dSMatthew G. Knepley }
35506528b96dSMatthew G. Knepley 
35512f856554SMatthew G. Knepley /*@C
35522f856554SMatthew G. Knepley   DMPlexGetFaceFields - Retrieve the field values values for a chunk of faces
35532f856554SMatthew G. Knepley 
35542f856554SMatthew G. Knepley   Input Parameters:
35552f856554SMatthew G. Knepley + dm     - The DM
35562f856554SMatthew G. Knepley . fStart - The first face to include
35572f856554SMatthew G. Knepley . fEnd   - The first face to exclude
35582f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
35592f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
35602f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
35612f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
35622f856554SMatthew G. Knepley - locaGrad - A local vector with field gradients, or NULL
35632f856554SMatthew G. Knepley 
35642f856554SMatthew G. Knepley   Output Parameters:
35652f856554SMatthew G. Knepley + Nface - The number of faces with field values
35662f856554SMatthew G. Knepley . uL - The field values at the left side of the face
35672f856554SMatthew G. Knepley - uR - The field values at the right side of the face
35682f856554SMatthew G. Knepley 
35692f856554SMatthew G. Knepley   Level: developer
35702f856554SMatthew G. Knepley 
3571db781477SPatrick Sanan .seealso: `DMPlexGetCellFields()`
35722f856554SMatthew G. Knepley @*/
35732f856554SMatthew G. Knepley PetscErrorCode DMPlexGetFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
35742f856554SMatthew G. Knepley {
35752f856554SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad = NULL;
35762f856554SMatthew G. Knepley   PetscSection       section;
35772f856554SMatthew G. Knepley   PetscDS            prob;
35782f856554SMatthew G. Knepley   DMLabel            ghostLabel;
35792f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom, *x, *lgrad;
35802f856554SMatthew G. Knepley   PetscBool         *isFE;
35812f856554SMatthew G. Knepley   PetscInt           dim, Nf, f, Nc, numFaces = fEnd - fStart, iface, face;
35822f856554SMatthew G. Knepley 
35832f856554SMatthew G. Knepley   PetscFunctionBegin;
35842f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
35852f856554SMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 4);
35862f856554SMatthew G. Knepley   if (locX_t) {PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 5);}
35872f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 6);
35882f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 7);
35892f856554SMatthew G. Knepley   if (locGrad) {PetscValidHeaderSpecific(locGrad, VEC_CLASSID, 8);}
3590064a246eSJacob Faibussowitsch   PetscValidPointer(uL, 10);
3591064a246eSJacob Faibussowitsch   PetscValidPointer(uR, 11);
35929566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
35939566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
35949566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
35959566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
35969566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalComponents(prob, &Nc));
35979566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(Nf, &isFE));
35982f856554SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
35992f856554SMatthew G. Knepley     PetscObject  obj;
36002f856554SMatthew G. Knepley     PetscClassId id;
36012f856554SMatthew G. Knepley 
36029566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
36039566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
36042f856554SMatthew G. Knepley     if (id == PETSCFE_CLASSID)      {isFE[f] = PETSC_TRUE;}
36052f856554SMatthew G. Knepley     else if (id == PETSCFV_CLASSID) {isFE[f] = PETSC_FALSE;}
36062f856554SMatthew G. Knepley     else                            {isFE[f] = PETSC_FALSE;}
36072f856554SMatthew G. Knepley   }
36089566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
36099566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(locX, &x));
36109566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
36119566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
36129566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
36139566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
36142f856554SMatthew G. Knepley   if (locGrad) {
36159566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locGrad, &dmGrad));
36169566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
36172f856554SMatthew G. Knepley   }
36189566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces*Nc, MPIU_SCALAR, uL));
36199566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces*Nc, MPIU_SCALAR, uR));
36202f856554SMatthew G. Knepley   /* Right now just eat the extra work for FE (could make a cell loop) */
36212f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
36222f856554SMatthew G. Knepley     const PetscInt        *cells;
36232f856554SMatthew G. Knepley     PetscFVFaceGeom       *fg;
36242f856554SMatthew G. Knepley     PetscFVCellGeom       *cgL, *cgR;
36252f856554SMatthew G. Knepley     PetscScalar           *xL, *xR, *gL, *gR;
36262f856554SMatthew G. Knepley     PetscScalar           *uLl = *uL, *uRl = *uR;
36272f856554SMatthew G. Knepley     PetscInt               ghost, nsupp, nchild;
36282f856554SMatthew G. Knepley 
36299566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
36309566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
36319566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
36322f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
36339566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
36349566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
36359566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
36369566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
36372f856554SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
36382f856554SMatthew G. Knepley       PetscInt off;
36392f856554SMatthew G. Knepley 
36409566063dSJacob Faibussowitsch       PetscCall(PetscDSGetComponentOffset(prob, f, &off));
36412f856554SMatthew G. Knepley       if (isFE[f]) {
36422f856554SMatthew G. Knepley         const PetscInt *cone;
36432f856554SMatthew G. Knepley         PetscInt        comp, coneSizeL, coneSizeR, faceLocL, faceLocR, ldof, rdof, d;
36442f856554SMatthew G. Knepley 
36452f856554SMatthew G. Knepley         xL = xR = NULL;
36469566063dSJacob Faibussowitsch         PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
36479566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **) &xL));
36489566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **) &xR));
36499566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[0], &cone));
36509566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[0], &coneSizeL));
36512f856554SMatthew G. Knepley         for (faceLocL = 0; faceLocL < coneSizeL; ++faceLocL) if (cone[faceLocL] == face) break;
36529566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[1], &cone));
36539566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[1], &coneSizeR));
36542f856554SMatthew G. Knepley         for (faceLocR = 0; faceLocR < coneSizeR; ++faceLocR) if (cone[faceLocR] == face) break;
36551dca8a05SBarry Smith         PetscCheck(faceLocL != coneSizeL || faceLocR != coneSizeR,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %" PetscInt_FMT " in cone of cell %" PetscInt_FMT " or cell %" PetscInt_FMT, face, cells[0], cells[1]);
36562f856554SMatthew G. Knepley         /* Check that FEM field has values in the right cell (sometimes its an FV ghost cell) */
36572f856554SMatthew G. Knepley         /* TODO: this is a hack that might not be right for nonconforming */
36582f856554SMatthew G. Knepley         if (faceLocL < coneSizeL) {
36599566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocL, xL, &uLl[iface*Nc+off]));
36609566063dSJacob Faibussowitsch           if (rdof == ldof && faceLocR < coneSizeR) PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface*Nc+off]));
36612f856554SMatthew G. Knepley           else              {for (d = 0; d < comp; ++d) uRl[iface*Nc+off+d] = uLl[iface*Nc+off+d];}
36622f856554SMatthew G. Knepley         }
36632f856554SMatthew G. Knepley         else {
36649566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface*Nc+off]));
36659566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
36662f856554SMatthew G. Knepley           for (d = 0; d < comp; ++d) uLl[iface*Nc+off+d] = uRl[iface*Nc+off+d];
36672f856554SMatthew G. Knepley         }
36689566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **) &xL));
36699566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **) &xR));
36702f856554SMatthew G. Knepley       } else {
36712f856554SMatthew G. Knepley         PetscFV  fv;
36722f856554SMatthew G. Knepley         PetscInt numComp, c;
36732f856554SMatthew G. Knepley 
36749566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, (PetscObject *) &fv));
36759566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &numComp));
36769566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[0], f, x, &xL));
36779566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[1], f, x, &xR));
36782f856554SMatthew G. Knepley         if (dmGrad) {
36792f856554SMatthew G. Knepley           PetscReal dxL[3], dxR[3];
36802f856554SMatthew G. Knepley 
36819566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], lgrad, &gL));
36829566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[1], lgrad, &gR));
36832f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgL->centroid, fg->centroid, dxL);
36842f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgR->centroid, fg->centroid, dxR);
36852f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
36862f856554SMatthew G. Knepley             uLl[iface*Nc+off+c] = xL[c] + DMPlex_DotD_Internal(dim, &gL[c*dim], dxL);
36872f856554SMatthew G. Knepley             uRl[iface*Nc+off+c] = xR[c] + DMPlex_DotD_Internal(dim, &gR[c*dim], dxR);
36882f856554SMatthew G. Knepley           }
36892f856554SMatthew G. Knepley         } else {
36902f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
36912f856554SMatthew G. Knepley             uLl[iface*Nc+off+c] = xL[c];
36922f856554SMatthew G. Knepley             uRl[iface*Nc+off+c] = xR[c];
36932f856554SMatthew G. Knepley           }
36942f856554SMatthew G. Knepley         }
36952f856554SMatthew G. Knepley       }
36962f856554SMatthew G. Knepley     }
36972f856554SMatthew G. Knepley     ++iface;
36982f856554SMatthew G. Knepley   }
36992f856554SMatthew G. Knepley   *Nface = iface;
37009566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(locX, &x));
37019566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
37029566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
37032f856554SMatthew G. Knepley   if (locGrad) {
37049566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
37052f856554SMatthew G. Knepley   }
37069566063dSJacob Faibussowitsch   PetscCall(PetscFree(isFE));
37072f856554SMatthew G. Knepley   PetscFunctionReturn(0);
37082f856554SMatthew G. Knepley }
37092f856554SMatthew G. Knepley 
37102f856554SMatthew G. Knepley /*@C
37112f856554SMatthew G. Knepley   DMPlexRestoreFaceFields - Restore the field values values for a chunk of faces
37122f856554SMatthew G. Knepley 
37132f856554SMatthew G. Knepley   Input Parameters:
37142f856554SMatthew G. Knepley + dm     - The DM
37152f856554SMatthew G. Knepley . fStart - The first face to include
37162f856554SMatthew G. Knepley . fEnd   - The first face to exclude
37172f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
37182f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
37192f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
37202f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
37212f856554SMatthew G. Knepley - locaGrad - A local vector with field gradients, or NULL
37222f856554SMatthew G. Knepley 
37232f856554SMatthew G. Knepley   Output Parameters:
37242f856554SMatthew G. Knepley + Nface - The number of faces with field values
37252f856554SMatthew G. Knepley . uL - The field values at the left side of the face
37262f856554SMatthew G. Knepley - uR - The field values at the right side of the face
37272f856554SMatthew G. Knepley 
37282f856554SMatthew G. Knepley   Level: developer
37292f856554SMatthew G. Knepley 
3730db781477SPatrick Sanan .seealso: `DMPlexGetFaceFields()`
37312f856554SMatthew G. Knepley @*/
37322f856554SMatthew G. Knepley PetscErrorCode DMPlexRestoreFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
37332f856554SMatthew G. Knepley {
37342f856554SMatthew G. Knepley   PetscFunctionBegin;
37359566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uL));
37369566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uR));
37372f856554SMatthew G. Knepley   PetscFunctionReturn(0);
37382f856554SMatthew G. Knepley }
37392f856554SMatthew G. Knepley 
37402f856554SMatthew G. Knepley /*@C
37412f856554SMatthew G. Knepley   DMPlexGetFaceGeometry - Retrieve the geometric values for a chunk of faces
37422f856554SMatthew G. Knepley 
37432f856554SMatthew G. Knepley   Input Parameters:
37442f856554SMatthew G. Knepley + dm     - The DM
37452f856554SMatthew G. Knepley . fStart - The first face to include
37462f856554SMatthew G. Knepley . fEnd   - The first face to exclude
37472f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
37482f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
37492f856554SMatthew G. Knepley 
37502f856554SMatthew G. Knepley   Output Parameters:
37512f856554SMatthew G. Knepley + Nface - The number of faces with field values
37522f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
37532f856554SMatthew G. Knepley - vol   - The cell volume
37542f856554SMatthew G. Knepley 
37552f856554SMatthew G. Knepley   Level: developer
37562f856554SMatthew G. Knepley 
3757db781477SPatrick Sanan .seealso: `DMPlexGetCellFields()`
37582f856554SMatthew G. Knepley @*/
37592f856554SMatthew G. Knepley PetscErrorCode DMPlexGetFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
37602f856554SMatthew G. Knepley {
37612f856554SMatthew G. Knepley   DM                 dmFace, dmCell;
37622f856554SMatthew G. Knepley   DMLabel            ghostLabel;
37632f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom;
37642f856554SMatthew G. Knepley   PetscInt           dim, numFaces = fEnd - fStart, iface, face;
37652f856554SMatthew G. Knepley 
37662f856554SMatthew G. Knepley   PetscFunctionBegin;
37672f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
37682f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 4);
37692f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 5);
3770064a246eSJacob Faibussowitsch   PetscValidPointer(fgeom, 7);
3771064a246eSJacob Faibussowitsch   PetscValidPointer(vol, 8);
37729566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
37739566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
37749566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
37759566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
37769566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
37779566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
37789566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(numFaces, fgeom));
37799566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces*2, MPIU_SCALAR, vol));
37802f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
37812f856554SMatthew G. Knepley     const PetscInt        *cells;
37822f856554SMatthew G. Knepley     PetscFVFaceGeom       *fg;
37832f856554SMatthew G. Knepley     PetscFVCellGeom       *cgL, *cgR;
37842f856554SMatthew G. Knepley     PetscFVFaceGeom       *fgeoml = *fgeom;
37852f856554SMatthew G. Knepley     PetscReal             *voll   = *vol;
37862f856554SMatthew G. Knepley     PetscInt               ghost, d, nchild, nsupp;
37872f856554SMatthew G. Knepley 
37889566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
37899566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
37909566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
37912f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
37929566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
37939566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
37949566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
37959566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
37962f856554SMatthew G. Knepley     for (d = 0; d < dim; ++d) {
37972f856554SMatthew G. Knepley       fgeoml[iface].centroid[d] = fg->centroid[d];
37982f856554SMatthew G. Knepley       fgeoml[iface].normal[d]   = fg->normal[d];
37992f856554SMatthew G. Knepley     }
38002f856554SMatthew G. Knepley     voll[iface*2+0] = cgL->volume;
38012f856554SMatthew G. Knepley     voll[iface*2+1] = cgR->volume;
38022f856554SMatthew G. Knepley     ++iface;
38032f856554SMatthew G. Knepley   }
38042f856554SMatthew G. Knepley   *Nface = iface;
38059566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
38069566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
38072f856554SMatthew G. Knepley   PetscFunctionReturn(0);
38082f856554SMatthew G. Knepley }
38092f856554SMatthew G. Knepley 
38102f856554SMatthew G. Knepley /*@C
38112f856554SMatthew G. Knepley   DMPlexRestoreFaceGeometry - Restore the field values values for a chunk of faces
38122f856554SMatthew G. Knepley 
38132f856554SMatthew G. Knepley   Input Parameters:
38142f856554SMatthew G. Knepley + dm     - The DM
38152f856554SMatthew G. Knepley . fStart - The first face to include
38162f856554SMatthew G. Knepley . fEnd   - The first face to exclude
38172f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
38182f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
38192f856554SMatthew G. Knepley 
38202f856554SMatthew G. Knepley   Output Parameters:
38212f856554SMatthew G. Knepley + Nface - The number of faces with field values
38222f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
38232f856554SMatthew G. Knepley - vol   - The cell volume
38242f856554SMatthew G. Knepley 
38252f856554SMatthew G. Knepley   Level: developer
38262f856554SMatthew G. Knepley 
3827db781477SPatrick Sanan .seealso: `DMPlexGetFaceFields()`
38282f856554SMatthew G. Knepley @*/
38292f856554SMatthew G. Knepley PetscErrorCode DMPlexRestoreFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
38302f856554SMatthew G. Knepley {
38312f856554SMatthew G. Knepley   PetscFunctionBegin;
38329566063dSJacob Faibussowitsch   PetscCall(PetscFree(*fgeom));
38339566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_REAL, vol));
38342f856554SMatthew G. Knepley   PetscFunctionReturn(0);
38352f856554SMatthew G. Knepley }
38362f856554SMatthew G. Knepley 
3837a1cf66bbSMatthew G. Knepley PetscErrorCode DMSNESGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
3838a1cf66bbSMatthew G. Knepley {
3839a1cf66bbSMatthew G. Knepley   char            composeStr[33] = {0};
3840a1cf66bbSMatthew G. Knepley   PetscObjectId   id;
3841a1cf66bbSMatthew G. Knepley   PetscContainer  container;
3842a1cf66bbSMatthew G. Knepley 
3843a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
38449566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad,&id));
384563a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMSNESGetFEGeom_%" PetscInt64_FMT "\n", id));
38469566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject) pointIS, composeStr, (PetscObject *) &container));
3847a1cf66bbSMatthew G. Knepley   if (container) {
38489566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **) geom));
3849a1cf66bbSMatthew G. Knepley   } else {
38509566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom));
38519566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF,&container));
38529566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *) *geom));
38539566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom));
38549566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject) pointIS, composeStr, (PetscObject) container));
38559566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
3856a1cf66bbSMatthew G. Knepley   }
3857a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
3858a1cf66bbSMatthew G. Knepley }
3859a1cf66bbSMatthew G. Knepley 
3860a1cf66bbSMatthew G. Knepley PetscErrorCode DMSNESRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
3861a1cf66bbSMatthew G. Knepley {
3862a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
3863a1cf66bbSMatthew G. Knepley   *geom = NULL;
3864a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
3865a1cf66bbSMatthew G. Knepley }
3866a1cf66bbSMatthew G. Knepley 
386792d50984SMatthew G. Knepley PetscErrorCode DMPlexComputeResidual_Patch_Internal(DM dm, PetscSection section, IS cellIS, PetscReal t, Vec locX, Vec locX_t, Vec locF, void *user)
386892d50984SMatthew G. Knepley {
386992d50984SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *) dm->data;
387092d50984SMatthew G. Knepley   const char      *name       = "Residual";
387192d50984SMatthew G. Knepley   DM               dmAux      = NULL;
387292d50984SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
387392d50984SMatthew G. Knepley   PetscDS          prob       = NULL;
387492d50984SMatthew G. Knepley   PetscDS          probAux    = NULL;
387592d50984SMatthew G. Knepley   PetscBool        useFEM     = PETSC_FALSE;
387692d50984SMatthew G. Knepley   PetscBool        isImplicit = (locX_t || t == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
387792d50984SMatthew G. Knepley   DMField          coordField = NULL;
3878c0006e53SPatrick Farrell   Vec              locA;
3879c0006e53SPatrick Farrell   PetscScalar     *u = NULL, *u_t, *a, *uL = NULL, *uR = NULL;
388092d50984SMatthew G. Knepley   IS               chunkIS;
388192d50984SMatthew G. Knepley   const PetscInt  *cells;
388292d50984SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
3883364207b6SKarl Rupp   PetscInt         Nf, f, totDim, totDimAux, numChunks, cellChunkSize, chunk, fStart, fEnd;
388492d50984SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
388506ad1575SMatthew G. Knepley   PetscFormKey key;
388692d50984SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
388792d50984SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
388892d50984SMatthew G. Knepley 
388992d50984SMatthew G. Knepley   PetscFunctionBegin;
38909566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0));
389192d50984SMatthew G. Knepley   /* FEM+FVM */
389292d50984SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
38939566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
38949566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
38959566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
38969566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
38979566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
389892d50984SMatthew G. Knepley   if (locA) {
38999566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
39009566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
39019566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
390292d50984SMatthew G. Knepley   }
390392d50984SMatthew G. Knepley   /* 2: Get geometric data */
390492d50984SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
390592d50984SMatthew G. Knepley     PetscObject  obj;
390692d50984SMatthew G. Knepley     PetscClassId id;
390792d50984SMatthew G. Knepley     PetscBool    fimp;
390892d50984SMatthew G. Knepley 
39099566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(prob, f, &fimp));
391092d50984SMatthew G. Knepley     if (isImplicit != fimp) continue;
39119566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
39129566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
391392d50984SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {useFEM = PETSC_TRUE;}
39145f80ce2aSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Use of FVM with PCPATCH not yet implemented");
391592d50984SMatthew G. Knepley   }
391692d50984SMatthew G. Knepley   if (useFEM) {
39179566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
39189566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
391992d50984SMatthew G. Knepley     if (maxDegree <= 1) {
39209566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad));
392192d50984SMatthew G. Knepley       if (affineQuad) {
39229566063dSJacob Faibussowitsch         PetscCall(DMSNESGetFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
392392d50984SMatthew G. Knepley       }
392492d50984SMatthew G. Knepley     } else {
39259566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf,&quads,Nf,&geoms));
392692d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
392792d50984SMatthew G. Knepley         PetscObject  obj;
392892d50984SMatthew G. Knepley         PetscClassId id;
392992d50984SMatthew G. Knepley         PetscBool    fimp;
393092d50984SMatthew G. Knepley 
39319566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(prob, f, &fimp));
393292d50984SMatthew G. Knepley         if (isImplicit != fimp) continue;
39339566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
39349566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
393592d50984SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
393692d50984SMatthew G. Knepley           PetscFE fe = (PetscFE) obj;
393792d50984SMatthew G. Knepley 
39389566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
39399566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
39409566063dSJacob Faibussowitsch           PetscCall(DMSNESGetFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
394192d50984SMatthew G. Knepley         }
394292d50984SMatthew G. Knepley       }
394392d50984SMatthew G. Knepley     }
394492d50984SMatthew G. Knepley   }
394592d50984SMatthew G. Knepley   /* Loop over chunks */
39469566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
39479566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
39489566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
394992d50984SMatthew G. Knepley   numCells      = cEnd - cStart;
395092d50984SMatthew G. Knepley   numChunks     = 1;
395192d50984SMatthew G. Knepley   cellChunkSize = numCells/numChunks;
395292d50984SMatthew G. Knepley   numChunks     = PetscMin(1,numCells);
39536528b96dSMatthew G. Knepley   key.label     = NULL;
39546528b96dSMatthew G. Knepley   key.value     = 0;
395506ad1575SMatthew G. Knepley   key.part      = 0;
395692d50984SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
3957c0006e53SPatrick Farrell     PetscScalar     *elemVec, *fluxL = NULL, *fluxR = NULL;
3958c0006e53SPatrick Farrell     PetscReal       *vol = NULL;
3959c0006e53SPatrick Farrell     PetscFVFaceGeom *fgeom = NULL;
396092d50984SMatthew G. Knepley     PetscInt         cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
3961c0006e53SPatrick Farrell     PetscInt         numFaces = 0;
396292d50984SMatthew G. Knepley 
396392d50984SMatthew G. Knepley     /* Extract field coefficients */
396492d50984SMatthew G. Knepley     if (useFEM) {
39659566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
39669566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
39679566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
39689566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells*totDim));
396992d50984SMatthew G. Knepley     }
397092d50984SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
397192d50984SMatthew G. Knepley     /* Loop over fields */
397292d50984SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
397392d50984SMatthew G. Knepley       PetscObject  obj;
397492d50984SMatthew G. Knepley       PetscClassId id;
397592d50984SMatthew G. Knepley       PetscBool    fimp;
397692d50984SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
397792d50984SMatthew G. Knepley 
39786528b96dSMatthew G. Knepley       key.field = f;
39799566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(prob, f, &fimp));
398092d50984SMatthew G. Knepley       if (isImplicit != fimp) continue;
39819566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
39829566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
398392d50984SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
398492d50984SMatthew G. Knepley         PetscFE         fe = (PetscFE) obj;
398592d50984SMatthew G. Knepley         PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
398692d50984SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
398792d50984SMatthew G. Knepley         PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
398892d50984SMatthew G. Knepley         PetscInt        Nq, Nb;
398992d50984SMatthew G. Knepley 
39909566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
39919566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
39929566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
399392d50984SMatthew G. Knepley         blockSize = Nb;
399492d50984SMatthew G. Knepley         batchSize = numBlocks * blockSize;
39959566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
399692d50984SMatthew G. Knepley         numChunks = numCells / (numBatches*batchSize);
399792d50984SMatthew G. Knepley         Ne        = numChunks*numBatches*batchSize;
399892d50984SMatthew G. Knepley         Nr        = numCells % (numBatches*batchSize);
399992d50984SMatthew G. Knepley         offset    = numCells - Nr;
400092d50984SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
400192d50984SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
40029566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom,0,offset,&chunkGeom));
40039566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(prob, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
40049566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom,offset,numCells,&chunkGeom));
40059566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(prob, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, &elemVec[offset*totDim]));
40069566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom,offset,numCells,&chunkGeom));
400792d50984SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
400892d50984SMatthew G. Knepley         PetscFV fv = (PetscFV) obj;
400992d50984SMatthew G. Knepley 
401092d50984SMatthew G. Knepley         Ne = numFaces;
401192d50984SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
401292d50984SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
40139566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
401463a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
401592d50984SMatthew G. Knepley     }
401692d50984SMatthew G. Knepley     /* Loop over domain */
401792d50984SMatthew G. Knepley     if (useFEM) {
401892d50984SMatthew G. Knepley       /* Add elemVec to locX */
401992d50984SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
402092d50984SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
402192d50984SMatthew G. Knepley         const PetscInt cind = c - cStart;
402292d50984SMatthew G. Knepley 
40239566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]));
402492d50984SMatthew G. Knepley         if (ghostLabel) {
402592d50984SMatthew G. Knepley           PetscInt ghostVal;
402692d50984SMatthew G. Knepley 
40279566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel,cell,&ghostVal));
402892d50984SMatthew G. Knepley           if (ghostVal > 0) continue;
402992d50984SMatthew G. Knepley         }
40309566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES));
403192d50984SMatthew G. Knepley       }
403292d50984SMatthew G. Knepley     }
403392d50984SMatthew G. Knepley     /* Handle time derivative */
403492d50984SMatthew G. Knepley     if (locX_t) {
403592d50984SMatthew G. Knepley       PetscScalar *x_t, *fa;
403692d50984SMatthew G. Knepley 
40379566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
40389566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
403992d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
404092d50984SMatthew G. Knepley         PetscFV      fv;
404192d50984SMatthew G. Knepley         PetscObject  obj;
404292d50984SMatthew G. Knepley         PetscClassId id;
404392d50984SMatthew G. Knepley         PetscInt     pdim, d;
404492d50984SMatthew G. Knepley 
40459566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
40469566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
404792d50984SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
404892d50984SMatthew G. Knepley         fv   = (PetscFV) obj;
40499566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
405092d50984SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
405192d50984SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
405292d50984SMatthew G. Knepley           PetscScalar   *u_t, *r;
405392d50984SMatthew G. Knepley 
405492d50984SMatthew G. Knepley           if (ghostLabel) {
405592d50984SMatthew G. Knepley             PetscInt ghostVal;
405692d50984SMatthew G. Knepley 
40579566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
405892d50984SMatthew G. Knepley             if (ghostVal > 0) continue;
405992d50984SMatthew G. Knepley           }
40609566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
40619566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
406292d50984SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
406392d50984SMatthew G. Knepley         }
406492d50984SMatthew G. Knepley       }
40659566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
40669566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
406792d50984SMatthew G. Knepley     }
406892d50984SMatthew G. Knepley     if (useFEM) {
40699566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
40709566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
407192d50984SMatthew G. Knepley     }
407292d50984SMatthew G. Knepley   }
40739566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
40749566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
407592d50984SMatthew G. Knepley   /* TODO Could include boundary residual here (see DMPlexComputeResidual_Internal) */
407692d50984SMatthew G. Knepley   if (useFEM) {
407792d50984SMatthew G. Knepley     if (maxDegree <= 1) {
40789566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
40799566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
408092d50984SMatthew G. Knepley     } else {
408192d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
40829566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
40839566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
408492d50984SMatthew G. Knepley       }
40859566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads,geoms));
408692d50984SMatthew G. Knepley     }
408792d50984SMatthew G. Knepley   }
40889566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0));
408992d50984SMatthew G. Knepley   PetscFunctionReturn(0);
409092d50984SMatthew G. Knepley }
409192d50984SMatthew G. Knepley 
4092a1cf66bbSMatthew G. Knepley /*
4093a1cf66bbSMatthew G. Knepley   We always assemble JacP, and if the matrix is different from Jac and two different sets of point functions are provided, we also assemble Jac
4094a1cf66bbSMatthew G. Knepley 
4095a1cf66bbSMatthew G. Knepley   X   - The local solution vector
4096a5b23f4aSJose E. Roman   X_t - The local solution time derivative vector, or NULL
4097a1cf66bbSMatthew G. Knepley */
4098a1cf66bbSMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Patch_Internal(DM dm, PetscSection section, PetscSection globalSection, IS cellIS,
4099a1cf66bbSMatthew G. Knepley                                                     PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *ctx)
4100a1cf66bbSMatthew G. Knepley {
4101a1cf66bbSMatthew G. Knepley   DM_Plex         *mesh  = (DM_Plex *) dm->data;
4102a1cf66bbSMatthew G. Knepley   const char      *name = "Jacobian", *nameP = "JacobianPre";
4103a1cf66bbSMatthew G. Knepley   DM               dmAux = NULL;
4104a1cf66bbSMatthew G. Knepley   PetscDS          prob,   probAux = NULL;
4105a1cf66bbSMatthew G. Knepley   PetscSection     sectionAux = NULL;
4106a1cf66bbSMatthew G. Knepley   Vec              A;
4107a1cf66bbSMatthew G. Knepley   DMField          coordField;
4108a1cf66bbSMatthew G. Knepley   PetscFEGeom     *cgeomFEM;
4109a1cf66bbSMatthew G. Knepley   PetscQuadrature  qGeom = NULL;
4110a1cf66bbSMatthew G. Knepley   Mat              J = Jac, JP = JacP;
4111a1cf66bbSMatthew G. Knepley   PetscScalar     *work, *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL, *elemMatD = NULL;
4112e432b41dSStefano Zampini   PetscBool        hasJac, hasPrec, hasDyn, assembleJac, *isFE, hasFV = PETSC_FALSE;
4113a1cf66bbSMatthew G. Knepley   const PetscInt  *cells;
411406ad1575SMatthew G. Knepley   PetscFormKey key;
41159b2fc754SMatthew G. Knepley   PetscInt         Nf, fieldI, fieldJ, maxDegree, numCells, cStart, cEnd, numChunks, chunkSize, chunk, totDim, totDimAux = 0, sz, wsz, off = 0, offCell = 0;
4116a1cf66bbSMatthew G. Knepley 
4117a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
41189566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
41199566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
41209566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0));
41219566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
41229566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &A));
41239a2a23afSMatthew G. Knepley   if (A) {
41249566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
41259566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
41269566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
4127a1cf66bbSMatthew G. Knepley   }
4128a1cf66bbSMatthew G. Knepley   /* Get flags */
41299566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
41309566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, Nf, MPIU_BOOL, &isFE));
4131a1cf66bbSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
4132a1cf66bbSMatthew G. Knepley     PetscObject  disc;
4133a1cf66bbSMatthew G. Knepley     PetscClassId id;
41349566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &disc));
41359566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(disc, &id));
4136a1cf66bbSMatthew G. Knepley     if (id == PETSCFE_CLASSID)      {isFE[fieldI] = PETSC_TRUE;}
4137a1cf66bbSMatthew G. Knepley     else if (id == PETSCFV_CLASSID) {hasFV = PETSC_TRUE; isFE[fieldI] = PETSC_FALSE;}
4138a1cf66bbSMatthew G. Knepley   }
41399566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
41409566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
41419566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
4142a1cf66bbSMatthew G. Knepley   assembleJac = hasJac && hasPrec && (Jac != JacP) ? PETSC_TRUE : PETSC_FALSE;
4143a1cf66bbSMatthew G. Knepley   hasDyn      = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
41449566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE)); /* No allocated space for FV stuff, so ignore the zero entries */
41459566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
41469566063dSJacob Faibussowitsch   if (probAux) PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
4147a1cf66bbSMatthew G. Knepley   /* Compute batch sizes */
4148a1cf66bbSMatthew G. Knepley   if (isFE[0]) {
4149a1cf66bbSMatthew G. Knepley     PetscFE         fe;
4150a1cf66bbSMatthew G. Knepley     PetscQuadrature q;
4151a1cf66bbSMatthew G. Knepley     PetscInt        numQuadPoints, numBatches, batchSize, numBlocks, blockSize, Nb;
4152a1cf66bbSMatthew G. Knepley 
41539566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *) &fe));
41549566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &q));
41559566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(q, NULL, NULL, &numQuadPoints, NULL, NULL));
41569566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
41579566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
4158a1cf66bbSMatthew G. Knepley     blockSize = Nb*numQuadPoints;
4159a1cf66bbSMatthew G. Knepley     batchSize = numBlocks  * blockSize;
4160a1cf66bbSMatthew G. Knepley     chunkSize = numBatches * batchSize;
4161a1cf66bbSMatthew G. Knepley     numChunks = numCells / chunkSize + numCells % chunkSize;
41629566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
4163a1cf66bbSMatthew G. Knepley   } else {
4164a1cf66bbSMatthew G. Knepley     chunkSize = numCells;
4165a1cf66bbSMatthew G. Knepley     numChunks = 1;
4166a1cf66bbSMatthew G. Knepley   }
4167a1cf66bbSMatthew G. Knepley   /* Get work space */
4168a1cf66bbSMatthew G. Knepley   wsz  = (((X?1:0) + (X_t?1:0) + (dmAux?1:0))*totDim + ((hasJac?1:0) + (hasPrec?1:0) + (hasDyn?1:0))*totDim*totDim)*chunkSize;
41699566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, wsz, MPIU_SCALAR, &work));
41709566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(work, wsz));
4171a1cf66bbSMatthew G. Knepley   off      = 0;
4172a1cf66bbSMatthew G. Knepley   u        = X       ? (sz = chunkSize*totDim,        off += sz, work+off-sz) : NULL;
4173a1cf66bbSMatthew G. Knepley   u_t      = X_t     ? (sz = chunkSize*totDim,        off += sz, work+off-sz) : NULL;
4174a1cf66bbSMatthew G. Knepley   a        = dmAux   ? (sz = chunkSize*totDimAux,     off += sz, work+off-sz) : NULL;
4175a1cf66bbSMatthew G. Knepley   elemMat  = hasJac  ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
4176a1cf66bbSMatthew G. Knepley   elemMatP = hasPrec ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
4177a1cf66bbSMatthew G. Knepley   elemMatD = hasDyn  ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
417863a3b9bcSJacob Faibussowitsch   PetscCheck(off == wsz,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error is workspace size %" PetscInt_FMT " should be %" PetscInt_FMT, off, wsz);
4179a1cf66bbSMatthew G. Knepley   /* Setup geometry */
41809566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
41819566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
41829566063dSJacob Faibussowitsch   if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
4183a1cf66bbSMatthew G. Knepley   if (!qGeom) {
4184a1cf66bbSMatthew G. Knepley     PetscFE fe;
4185a1cf66bbSMatthew G. Knepley 
41869566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *) &fe));
41879566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &qGeom));
41889566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject) qGeom));
4189a1cf66bbSMatthew G. Knepley   }
41909566063dSJacob Faibussowitsch   PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
4191a1cf66bbSMatthew G. Knepley   /* Compute volume integrals */
41929566063dSJacob Faibussowitsch   if (assembleJac) PetscCall(MatZeroEntries(J));
41939566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(JP));
41946528b96dSMatthew G. Knepley   key.label = NULL;
41956528b96dSMatthew G. Knepley   key.value = 0;
419606ad1575SMatthew G. Knepley   key.part  = 0;
4197a1cf66bbSMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk, offCell += chunkSize) {
4198a1cf66bbSMatthew G. Knepley     const PetscInt   Ncell = PetscMin(chunkSize, numCells - offCell);
4199a1cf66bbSMatthew G. Knepley     PetscInt         c;
4200a1cf66bbSMatthew G. Knepley 
4201a1cf66bbSMatthew G. Knepley     /* Extract values */
4202a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4203a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c+offCell] : c+offCell;
4204a1cf66bbSMatthew G. Knepley       PetscScalar   *x = NULL,  *x_t = NULL;
4205a1cf66bbSMatthew G. Knepley       PetscInt       i;
4206a1cf66bbSMatthew G. Knepley 
4207a1cf66bbSMatthew G. Knepley       if (X) {
42089566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x));
4209a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[c*totDim+i] = x[i];
42109566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x));
4211a1cf66bbSMatthew G. Knepley       }
4212a1cf66bbSMatthew G. Knepley       if (X_t) {
42139566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t));
4214a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[c*totDim+i] = x_t[i];
42159566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t));
4216a1cf66bbSMatthew G. Knepley       }
4217a1cf66bbSMatthew G. Knepley       if (dmAux) {
42189566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, A, cell, NULL, &x));
4219a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[c*totDimAux+i] = x[i];
42209566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, A, cell, NULL, &x));
4221a1cf66bbSMatthew G. Knepley       }
4222a1cf66bbSMatthew G. Knepley     }
4223a1cf66bbSMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
4224a1cf66bbSMatthew G. Knepley       PetscFE fe;
42259566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
4226a1cf66bbSMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
42276528b96dSMatthew G. Knepley         key.field = fieldI*Nf + fieldJ;
42289566063dSJacob Faibussowitsch         if (hasJac)  PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN,     key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMat));
42299566063dSJacob Faibussowitsch         if (hasPrec) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatP));
42309566063dSJacob Faibussowitsch         if (hasDyn)  PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatD));
4231a1cf66bbSMatthew G. Knepley       }
4232a1cf66bbSMatthew G. Knepley       /* For finite volume, add the identity */
4233a1cf66bbSMatthew G. Knepley       if (!isFE[fieldI]) {
4234a1cf66bbSMatthew G. Knepley         PetscFV  fv;
4235a1cf66bbSMatthew G. Knepley         PetscInt eOffset = 0, Nc, fc, foff;
4236a1cf66bbSMatthew G. Knepley 
42379566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(prob, fieldI, &foff));
42389566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fv));
42399566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &Nc));
4240a1cf66bbSMatthew G. Knepley         for (c = 0; c < chunkSize; ++c, eOffset += totDim*totDim) {
4241a1cf66bbSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
4242a1cf66bbSMatthew G. Knepley             const PetscInt i = foff + fc;
4243a1cf66bbSMatthew G. Knepley             if (hasJac)  {elemMat [eOffset+i*totDim+i] = 1.0;}
4244a1cf66bbSMatthew G. Knepley             if (hasPrec) {elemMatP[eOffset+i*totDim+i] = 1.0;}
4245a1cf66bbSMatthew G. Knepley           }
4246a1cf66bbSMatthew G. Knepley         }
4247a1cf66bbSMatthew G. Knepley       }
4248a1cf66bbSMatthew G. Knepley     }
4249a1cf66bbSMatthew G. Knepley     /*   Add contribution from X_t */
4250a1cf66bbSMatthew G. Knepley     if (hasDyn) {for (c = 0; c < chunkSize*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];}
4251a1cf66bbSMatthew G. Knepley     /* Insert values into matrix */
4252a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4253a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c+offCell] : c+offCell;
4254a1cf66bbSMatthew G. Knepley       if (mesh->printFEM > 1) {
42559566063dSJacob Faibussowitsch         if (hasJac)  PetscCall(DMPrintCellMatrix(cell, name,  totDim, totDim, &elemMat[(c-cStart)*totDim*totDim]));
42569566063dSJacob Faibussowitsch         if (hasPrec) PetscCall(DMPrintCellMatrix(cell, nameP, totDim, totDim, &elemMatP[(c-cStart)*totDim*totDim]));
4257a1cf66bbSMatthew G. Knepley       }
42589566063dSJacob Faibussowitsch       if (assembleJac) PetscCall(DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[(c-cStart)*totDim*totDim], ADD_VALUES));
42599566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JP, cell, &elemMat[(c-cStart)*totDim*totDim], ADD_VALUES));
4260a1cf66bbSMatthew G. Knepley     }
4261a1cf66bbSMatthew G. Knepley   }
4262a1cf66bbSMatthew G. Knepley   /* Cleanup */
42639566063dSJacob Faibussowitsch   PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
42649566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&qGeom));
42659566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
42669566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, Nf, MPIU_BOOL, &isFE));
42679566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, ((1 + (X_t?1:0) + (dmAux?1:0))*totDim + ((hasJac?1:0) + (hasPrec?1:0) + (hasDyn?1:0))*totDim*totDim)*chunkSize, MPIU_SCALAR, &work));
4268a1cf66bbSMatthew G. Knepley   /* Compute boundary integrals */
42699566063dSJacob Faibussowitsch   /* PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, ctx)); */
4270a1cf66bbSMatthew G. Knepley   /* Assemble matrix */
42719566063dSJacob Faibussowitsch   if (assembleJac) {PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));}
42729566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
42739566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0));
4274a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
4275a1cf66bbSMatthew G. Knepley }
42763e9753d6SMatthew G. Knepley 
42773e9753d6SMatthew G. Knepley /******** FEM Assembly Function ********/
42783e9753d6SMatthew G. Knepley 
42793e9753d6SMatthew G. Knepley static PetscErrorCode DMConvertPlex_Internal(DM dm, DM *plex, PetscBool copy)
42803e9753d6SMatthew G. Knepley {
42813e9753d6SMatthew G. Knepley   PetscBool      isPlex;
42823e9753d6SMatthew G. Knepley 
42833e9753d6SMatthew G. Knepley   PetscFunctionBegin;
42849566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject) dm, DMPLEX, &isPlex));
42853e9753d6SMatthew G. Knepley   if (isPlex) {
42863e9753d6SMatthew G. Knepley     *plex = dm;
42879566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject) dm));
42883e9753d6SMatthew G. Knepley   } else {
42899566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject) dm, "dm_plex", (PetscObject *) plex));
42903e9753d6SMatthew G. Knepley     if (!*plex) {
42919566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm,DMPLEX,plex));
42929566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject) dm, "dm_plex", (PetscObject) *plex));
42933e9753d6SMatthew G. Knepley       if (copy) {
42949566063dSJacob Faibussowitsch         PetscCall(DMCopyAuxiliaryVec(dm, *plex));
42953e9753d6SMatthew G. Knepley       }
42963e9753d6SMatthew G. Knepley     } else {
42979566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject) *plex));
42983e9753d6SMatthew G. Knepley     }
42993e9753d6SMatthew G. Knepley   }
43003e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
43013e9753d6SMatthew G. Knepley }
43023e9753d6SMatthew G. Knepley 
43033e9753d6SMatthew G. Knepley /*@
43043e9753d6SMatthew G. Knepley   DMPlexGetGeometryFVM - Return precomputed geometric data
43053e9753d6SMatthew G. Knepley 
43063e9753d6SMatthew G. Knepley   Collective on DM
43073e9753d6SMatthew G. Knepley 
43083e9753d6SMatthew G. Knepley   Input Parameter:
43093e9753d6SMatthew G. Knepley . dm - The DM
43103e9753d6SMatthew G. Knepley 
43113e9753d6SMatthew G. Knepley   Output Parameters:
43123e9753d6SMatthew G. Knepley + facegeom - The values precomputed from face geometry
43133e9753d6SMatthew G. Knepley . cellgeom - The values precomputed from cell geometry
43143e9753d6SMatthew G. Knepley - minRadius - The minimum radius over the mesh of an inscribed sphere in a cell
43153e9753d6SMatthew G. Knepley 
43163e9753d6SMatthew G. Knepley   Level: developer
43173e9753d6SMatthew G. Knepley 
4318db781477SPatrick Sanan .seealso: `DMTSSetRHSFunctionLocal()`
43193e9753d6SMatthew G. Knepley @*/
43203e9753d6SMatthew G. Knepley PetscErrorCode DMPlexGetGeometryFVM(DM dm, Vec *facegeom, Vec *cellgeom, PetscReal *minRadius)
43213e9753d6SMatthew G. Knepley {
43223e9753d6SMatthew G. Knepley   DM             plex;
43233e9753d6SMatthew G. Knepley 
43243e9753d6SMatthew G. Knepley   PetscFunctionBegin;
43253e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
43269566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm,&plex,PETSC_TRUE));
43279566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, NULL, cellgeom, facegeom, NULL));
43289566063dSJacob Faibussowitsch   if (minRadius) PetscCall(DMPlexGetMinRadius(plex, minRadius));
43299566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
43303e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
43313e9753d6SMatthew G. Knepley }
43323e9753d6SMatthew G. Knepley 
43333e9753d6SMatthew G. Knepley /*@
43343e9753d6SMatthew G. Knepley   DMPlexGetGradientDM - Return gradient data layout
43353e9753d6SMatthew G. Knepley 
43363e9753d6SMatthew G. Knepley   Collective on DM
43373e9753d6SMatthew G. Knepley 
43383e9753d6SMatthew G. Knepley   Input Parameters:
43393e9753d6SMatthew G. Knepley + dm - The DM
43403e9753d6SMatthew G. Knepley - fv - The PetscFV
43413e9753d6SMatthew G. Knepley 
43423e9753d6SMatthew G. Knepley   Output Parameter:
43433e9753d6SMatthew G. Knepley . dmGrad - The layout for gradient values
43443e9753d6SMatthew G. Knepley 
43453e9753d6SMatthew G. Knepley   Level: developer
43463e9753d6SMatthew G. Knepley 
4347db781477SPatrick Sanan .seealso: `DMPlexGetGeometryFVM()`
43483e9753d6SMatthew G. Knepley @*/
43493e9753d6SMatthew G. Knepley PetscErrorCode DMPlexGetGradientDM(DM dm, PetscFV fv, DM *dmGrad)
43503e9753d6SMatthew G. Knepley {
43513e9753d6SMatthew G. Knepley   DM             plex;
43523e9753d6SMatthew G. Knepley   PetscBool      computeGradients;
43533e9753d6SMatthew G. Knepley 
43543e9753d6SMatthew G. Knepley   PetscFunctionBegin;
43553e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
43563e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(fv,PETSCFV_CLASSID,2);
43573e9753d6SMatthew G. Knepley   PetscValidPointer(dmGrad,3);
43589566063dSJacob Faibussowitsch   PetscCall(PetscFVGetComputeGradients(fv, &computeGradients));
43593e9753d6SMatthew G. Knepley   if (!computeGradients) {*dmGrad = NULL; PetscFunctionReturn(0);}
43609566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm,&plex,PETSC_TRUE));
43619566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, fv, NULL, NULL, dmGrad));
43629566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
43633e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
43643e9753d6SMatthew G. Knepley }
43653e9753d6SMatthew G. Knepley 
43660c290148SMatthew G. Knepley static PetscErrorCode DMPlexComputeBdResidual_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF, DMField coordField, IS facetIS)
43673e9753d6SMatthew G. Knepley {
43683e9753d6SMatthew G. Knepley   DM_Plex         *mesh = (DM_Plex *) dm->data;
43693e9753d6SMatthew G. Knepley   DM               plex = NULL, plexA = NULL;
43703e9753d6SMatthew G. Knepley   DMEnclosureType  encAux;
43713e9753d6SMatthew G. Knepley   PetscDS          prob, probAux = NULL;
43723e9753d6SMatthew G. Knepley   PetscSection     section, sectionAux = NULL;
43733e9753d6SMatthew G. Knepley   Vec              locA = NULL;
43743e9753d6SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t = NULL, *a = NULL, *elemVec = NULL;
43753e9753d6SMatthew G. Knepley   PetscInt         totDim, totDimAux = 0;
43763e9753d6SMatthew G. Knepley 
43773e9753d6SMatthew G. Knepley   PetscFunctionBegin;
43789566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
43799566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
43809566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
43819566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
43829566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
43833e9753d6SMatthew G. Knepley   if (locA) {
43843e9753d6SMatthew G. Knepley     DM dmAux;
43853e9753d6SMatthew G. Knepley 
43869566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
43879566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
43889566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
43899566063dSJacob Faibussowitsch     PetscCall(DMGetDS(plexA, &probAux));
43909566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
43919566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
43923e9753d6SMatthew G. Knepley   }
43930c290148SMatthew G. Knepley   {
43943e9753d6SMatthew G. Knepley     PetscFEGeom     *fgeom;
43953e9753d6SMatthew G. Knepley     PetscInt         maxDegree;
43963e9753d6SMatthew G. Knepley     PetscQuadrature  qGeom = NULL;
43973e9753d6SMatthew G. Knepley     IS               pointIS;
43983e9753d6SMatthew G. Knepley     const PetscInt  *points;
43993e9753d6SMatthew G. Knepley     PetscInt         numFaces, face, Nq;
44003e9753d6SMatthew G. Knepley 
44019566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(key.label, key.value, &pointIS));
44020c290148SMatthew G. Knepley     if (!pointIS) goto end; /* No points with that id on this process */
44033e9753d6SMatthew G. Knepley     {
44043e9753d6SMatthew G. Knepley       IS isectIS;
44053e9753d6SMatthew G. Knepley 
44063e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
44079566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS,pointIS,&isectIS));
44089566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
44093e9753d6SMatthew G. Knepley       pointIS = isectIS;
44103e9753d6SMatthew G. Knepley     }
44119566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS,&numFaces));
44129566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS,&points));
44139566063dSJacob Faibussowitsch     PetscCall(PetscMalloc4(numFaces*totDim, &u, locX_t ? numFaces*totDim : 0, &u_t, numFaces*totDim, &elemVec, locA ? numFaces*totDimAux : 0, &a));
44149566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,pointIS,NULL,&maxDegree));
44153e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
44169566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,pointIS,&qGeom));
44173e9753d6SMatthew G. Knepley     }
44183e9753d6SMatthew G. Knepley     if (!qGeom) {
44193e9753d6SMatthew G. Knepley       PetscFE fe;
44203e9753d6SMatthew G. Knepley 
44219566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *) &fe));
44229566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
44239566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
44243e9753d6SMatthew G. Knepley     }
44259566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
44269566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom));
44273e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
4428f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
44293e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
4430f15274beSMatthew Knepley       PetscInt       i;
44313e9753d6SMatthew G. Knepley 
44329566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
44339566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
44343e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
44359566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
44363e9753d6SMatthew G. Knepley       if (locX_t) {
44379566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
44383e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face*totDim+i] = x[i];
44399566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
44403e9753d6SMatthew G. Knepley       }
44413e9753d6SMatthew G. Knepley       if (locA) {
44423e9753d6SMatthew G. Knepley         PetscInt subp;
44433e9753d6SMatthew G. Knepley 
44449566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
44459566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
44463e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face*totDimAux+i] = x[i];
44479566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
44483e9753d6SMatthew G. Knepley       }
44493e9753d6SMatthew G. Knepley     }
44509566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(elemVec, numFaces*totDim));
44513e9753d6SMatthew G. Knepley     {
44523e9753d6SMatthew G. Knepley       PetscFE         fe;
44533e9753d6SMatthew G. Knepley       PetscInt        Nb;
44543e9753d6SMatthew G. Knepley       PetscFEGeom     *chunkGeom = NULL;
44553e9753d6SMatthew G. Knepley       /* Conforming batches */
44563e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
44573e9753d6SMatthew G. Knepley       /* Remainder */
44583e9753d6SMatthew G. Knepley       PetscInt        Nr, offset;
44593e9753d6SMatthew G. Knepley 
44609566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *) &fe));
44619566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
44629566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
44633e9753d6SMatthew G. Knepley       /* TODO: documentation is unclear about what is going on with these numbers: how should Nb / Nq factor in ? */
44643e9753d6SMatthew G. Knepley       blockSize = Nb;
44653e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
44669566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
44673e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches*batchSize);
44683e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
44693e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches*batchSize);
44703e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
44719566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom,0,offset,&chunkGeom));
44729566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
44739566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
44749566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom,offset,numFaces,&chunkGeom));
44759566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, a ? &a[offset*totDimAux] : NULL, t, &elemVec[offset*totDim]));
44769566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom,offset,numFaces,&chunkGeom));
44773e9753d6SMatthew G. Knepley     }
44783e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
44793e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
44803e9753d6SMatthew G. Knepley 
44819566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(point, "BdResidual", totDim, &elemVec[face*totDim]));
44829566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
44839566063dSJacob Faibussowitsch       PetscCall(DMPlexVecSetClosure(plex, NULL, locF, support[0], &elemVec[face*totDim], ADD_ALL_VALUES));
44843e9753d6SMatthew G. Knepley     }
44859566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom));
44869566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
44879566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
44889566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
44899566063dSJacob Faibussowitsch     PetscCall(PetscFree4(u, u_t, elemVec, a));
44903e9753d6SMatthew G. Knepley   }
44910c290148SMatthew G. Knepley   end:
44929566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
44939566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexA));
44943e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
44953e9753d6SMatthew G. Knepley }
44963e9753d6SMatthew G. Knepley 
44970c290148SMatthew G. Knepley PetscErrorCode DMPlexComputeBdResidualSingle(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF)
44983e9753d6SMatthew G. Knepley {
44993e9753d6SMatthew G. Knepley   DMField        coordField;
45003e9753d6SMatthew G. Knepley   DMLabel        depthLabel;
45013e9753d6SMatthew G. Knepley   IS             facetIS;
45023e9753d6SMatthew G. Knepley   PetscInt       dim;
45033e9753d6SMatthew G. Knepley 
45043e9753d6SMatthew G. Knepley   PetscFunctionBegin;
45059566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
45069566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
45079566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim-1, &facetIS));
45089566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
45099566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS));
45109566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
45113e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
45123e9753d6SMatthew G. Knepley }
45133e9753d6SMatthew G. Knepley 
45143e9753d6SMatthew G. Knepley PetscErrorCode DMPlexComputeBdResidual_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
45153e9753d6SMatthew G. Knepley {
45163e9753d6SMatthew G. Knepley   PetscDS        prob;
45173e9753d6SMatthew G. Knepley   PetscInt       numBd, bd;
45183e9753d6SMatthew G. Knepley   DMField        coordField = NULL;
45193e9753d6SMatthew G. Knepley   IS             facetIS    = NULL;
45203e9753d6SMatthew G. Knepley   DMLabel        depthLabel;
45213e9753d6SMatthew G. Knepley   PetscInt       dim;
45223e9753d6SMatthew G. Knepley 
45233e9753d6SMatthew G. Knepley   PetscFunctionBegin;
45249566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
45259566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
45269566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
45279566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel,dim - 1,&facetIS));
45289566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
45293e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
453045480ffeSMatthew G. Knepley     PetscWeakForm           wf;
45313e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
45323e9753d6SMatthew G. Knepley     DMLabel                 label;
45333e9753d6SMatthew G. Knepley     const PetscInt         *values;
45340c290148SMatthew G. Knepley     PetscInt                field, numValues, v;
45353e9753d6SMatthew G. Knepley     PetscObject             obj;
45363e9753d6SMatthew G. Knepley     PetscClassId            id;
45370c290148SMatthew G. Knepley     PetscFormKey            key;
45383e9753d6SMatthew G. Knepley 
45399566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &field, NULL, NULL, NULL, NULL, NULL));
45409566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
45419566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
45423e9753d6SMatthew G. Knepley     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
45433e9753d6SMatthew G. Knepley     if (!facetIS) {
45443e9753d6SMatthew G. Knepley       DMLabel  depthLabel;
45453e9753d6SMatthew G. Knepley       PetscInt dim;
45463e9753d6SMatthew G. Knepley 
45479566063dSJacob Faibussowitsch       PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
45489566063dSJacob Faibussowitsch       PetscCall(DMGetDimension(dm, &dim));
45499566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
45503e9753d6SMatthew G. Knepley     }
45519566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
45520c290148SMatthew G. Knepley     for (v = 0; v < numValues; ++v) {
45530c290148SMatthew G. Knepley       key.label = label;
45540c290148SMatthew G. Knepley       key.value = values[v];
45550c290148SMatthew G. Knepley       key.field = field;
45560c290148SMatthew G. Knepley       key.part  = 0;
45579566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS));
45580c290148SMatthew G. Knepley     }
45593e9753d6SMatthew G. Knepley   }
45609566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
45613e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
45623e9753d6SMatthew G. Knepley }
45633e9753d6SMatthew G. Knepley 
456406ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeResidual_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
45653e9753d6SMatthew G. Knepley {
45663e9753d6SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *) dm->data;
45673e9753d6SMatthew G. Knepley   const char      *name       = "Residual";
45683e9753d6SMatthew G. Knepley   DM               dmAux      = NULL;
45693e9753d6SMatthew G. Knepley   DM               dmGrad     = NULL;
45703e9753d6SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
45716528b96dSMatthew G. Knepley   PetscDS          ds         = NULL;
45726528b96dSMatthew G. Knepley   PetscDS          dsAux      = NULL;
45733e9753d6SMatthew G. Knepley   PetscSection     section    = NULL;
45743e9753d6SMatthew G. Knepley   PetscBool        useFEM     = PETSC_FALSE;
45753e9753d6SMatthew G. Knepley   PetscBool        useFVM     = PETSC_FALSE;
45763e9753d6SMatthew G. Knepley   PetscBool        isImplicit = (locX_t || time == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
45773e9753d6SMatthew G. Knepley   PetscFV          fvm        = NULL;
45783e9753d6SMatthew G. Knepley   PetscFVCellGeom *cgeomFVM   = NULL;
45793e9753d6SMatthew G. Knepley   PetscFVFaceGeom *fgeomFVM   = NULL;
45803e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
45813e9753d6SMatthew G. Knepley   Vec              locA, cellGeometryFVM = NULL, faceGeometryFVM = NULL, grad, locGrad = NULL;
45823e9753d6SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a, *uL, *uR;
45833e9753d6SMatthew G. Knepley   IS               chunkIS;
45843e9753d6SMatthew G. Knepley   const PetscInt  *cells;
45853e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
45863e9753d6SMatthew G. Knepley   PetscInt         Nf, f, totDim, totDimAux, numChunks, cellChunkSize, faceChunkSize, chunk, fStart, fEnd;
45873e9753d6SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
45883e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
45893e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
45903e9753d6SMatthew G. Knepley 
45913e9753d6SMatthew G. Knepley   PetscFunctionBegin;
45929566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0));
45933e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
45943e9753d6SMatthew G. Knepley   /* TODO The FVM geometry is over-manipulated. Make the precalc functions return exactly what we need */
45953e9753d6SMatthew G. Knepley   /* FEM+FVM */
45969566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
45979566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
45983e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
45999566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
46009566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
46019566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds));
46029566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
46039566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
46049566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
46053e9753d6SMatthew G. Knepley   if (locA) {
46063e9753d6SMatthew G. Knepley     PetscInt subcell;
46079566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
4608*1059d808SMatthew G. Knepley     PetscCall(DMGetEnclosurePoint(dmAux, dm, DM_ENC_UNKNOWN, cells ? cells[cStart] : cStart, &subcell));
46099566063dSJacob Faibussowitsch     PetscCall(DMGetCellDS(dmAux, subcell, &dsAux));
46109566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux));
46113e9753d6SMatthew G. Knepley   }
46123e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
46133e9753d6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
46143e9753d6SMatthew G. Knepley     PetscObject  obj;
46153e9753d6SMatthew G. Knepley     PetscClassId id;
46163e9753d6SMatthew G. Knepley     PetscBool    fimp;
46173e9753d6SMatthew G. Knepley 
46189566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(ds, f, &fimp));
46193e9753d6SMatthew G. Knepley     if (isImplicit != fimp) continue;
46209566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(ds, f, &obj));
46219566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
46223e9753d6SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {useFEM = PETSC_TRUE;}
46233e9753d6SMatthew G. Knepley     if (id == PETSCFV_CLASSID) {useFVM = PETSC_TRUE; fvm = (PetscFV) obj;}
46243e9753d6SMatthew G. Knepley   }
46253e9753d6SMatthew G. Knepley   if (useFEM) {
46269566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
46279566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
46283e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
46299566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad));
46303e9753d6SMatthew G. Knepley       if (affineQuad) {
46319566063dSJacob Faibussowitsch         PetscCall(DMSNESGetFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
46323e9753d6SMatthew G. Knepley       }
46333e9753d6SMatthew G. Knepley     } else {
46349566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf,&quads,Nf,&geoms));
46353e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
46363e9753d6SMatthew G. Knepley         PetscObject  obj;
46373e9753d6SMatthew G. Knepley         PetscClassId id;
46383e9753d6SMatthew G. Knepley         PetscBool    fimp;
46393e9753d6SMatthew G. Knepley 
46409566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(ds, f, &fimp));
46413e9753d6SMatthew G. Knepley         if (isImplicit != fimp) continue;
46429566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
46439566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
46443e9753d6SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
46453e9753d6SMatthew G. Knepley           PetscFE fe = (PetscFE) obj;
46463e9753d6SMatthew G. Knepley 
46479566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
46489566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
46499566063dSJacob Faibussowitsch           PetscCall(DMSNESGetFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
46503e9753d6SMatthew G. Knepley         }
46513e9753d6SMatthew G. Knepley       }
46523e9753d6SMatthew G. Knepley     }
46533e9753d6SMatthew G. Knepley   }
46543e9753d6SMatthew G. Knepley   if (useFVM) {
46559566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL));
46569566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(faceGeometryFVM, (const PetscScalar **) &fgeomFVM));
46579566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM));
46583e9753d6SMatthew G. Knepley     /* Reconstruct and limit cell gradients */
46599566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGradientDM(dm, fvm, &dmGrad));
46603e9753d6SMatthew G. Knepley     if (dmGrad) {
46619566063dSJacob Faibussowitsch       PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
46629566063dSJacob Faibussowitsch       PetscCall(DMGetGlobalVector(dmGrad, &grad));
46639566063dSJacob Faibussowitsch       PetscCall(DMPlexReconstructGradients_Internal(dm, fvm, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
46643e9753d6SMatthew G. Knepley       /* Communicate gradient values */
46659566063dSJacob Faibussowitsch       PetscCall(DMGetLocalVector(dmGrad, &locGrad));
46669566063dSJacob Faibussowitsch       PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
46679566063dSJacob Faibussowitsch       PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
46689566063dSJacob Faibussowitsch       PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
46693e9753d6SMatthew G. Knepley     }
46703e9753d6SMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
46719566063dSJacob Faibussowitsch     PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, time, faceGeometryFVM, cellGeometryFVM, locGrad));
46723e9753d6SMatthew G. Knepley   }
46733e9753d6SMatthew G. Knepley   /* Loop over chunks */
46749566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
46753e9753d6SMatthew G. Knepley   numCells      = cEnd - cStart;
46763e9753d6SMatthew G. Knepley   numChunks     = 1;
46773e9753d6SMatthew G. Knepley   cellChunkSize = numCells/numChunks;
46783e9753d6SMatthew G. Knepley   faceChunkSize = (fEnd - fStart)/numChunks;
46793e9753d6SMatthew G. Knepley   numChunks     = PetscMin(1,numCells);
46803e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
46813e9753d6SMatthew G. Knepley     PetscScalar     *elemVec, *fluxL, *fluxR;
46823e9753d6SMatthew G. Knepley     PetscReal       *vol;
46833e9753d6SMatthew G. Knepley     PetscFVFaceGeom *fgeom;
46843e9753d6SMatthew G. Knepley     PetscInt         cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
46853e9753d6SMatthew G. Knepley     PetscInt         fS = fStart+chunk*faceChunkSize, fE = PetscMin(fS+faceChunkSize, fEnd), numFaces = 0, face;
46863e9753d6SMatthew G. Knepley 
46873e9753d6SMatthew G. Knepley     /* Extract field coefficients */
46883e9753d6SMatthew G. Knepley     if (useFEM) {
46899566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
46909566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
46919566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
46929566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells*totDim));
46933e9753d6SMatthew G. Knepley     }
46943e9753d6SMatthew G. Knepley     if (useFVM) {
46959566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
46969566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
46979566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxL));
46989566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxR));
46999566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxL, numFaces*totDim));
47009566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxR, numFaces*totDim));
47013e9753d6SMatthew G. Knepley     }
47023e9753d6SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
47033e9753d6SMatthew G. Knepley     /* Loop over fields */
47043e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
47053e9753d6SMatthew G. Knepley       PetscObject  obj;
47063e9753d6SMatthew G. Knepley       PetscClassId id;
47073e9753d6SMatthew G. Knepley       PetscBool    fimp;
47083e9753d6SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
47093e9753d6SMatthew G. Knepley 
47106528b96dSMatthew G. Knepley       key.field = f;
47119566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(ds, f, &fimp));
47123e9753d6SMatthew G. Knepley       if (isImplicit != fimp) continue;
47139566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, &obj));
47149566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
47153e9753d6SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
47163e9753d6SMatthew G. Knepley         PetscFE         fe = (PetscFE) obj;
47173e9753d6SMatthew G. Knepley         PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
47183e9753d6SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
47193e9753d6SMatthew G. Knepley         PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
47203e9753d6SMatthew G. Knepley         PetscInt        Nq, Nb;
47213e9753d6SMatthew G. Knepley 
47229566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
47239566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
47249566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
47253e9753d6SMatthew G. Knepley         blockSize = Nb;
47263e9753d6SMatthew G. Knepley         batchSize = numBlocks * blockSize;
47279566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
47283e9753d6SMatthew G. Knepley         numChunks = numCells / (numBatches*batchSize);
47293e9753d6SMatthew G. Knepley         Ne        = numChunks*numBatches*batchSize;
47303e9753d6SMatthew G. Knepley         Nr        = numCells % (numBatches*batchSize);
47313e9753d6SMatthew G. Knepley         offset    = numCells - Nr;
47323e9753d6SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
47333e9753d6SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
47349566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom,0,offset,&chunkGeom));
47359566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(ds, key, Ne, chunkGeom, u, u_t, dsAux, a, t, elemVec));
47369566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom,offset,numCells,&chunkGeom));
47379566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(ds, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux, &a[offset*totDimAux], t, &elemVec[offset*totDim]));
47389566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom,offset,numCells,&chunkGeom));
47393e9753d6SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
47403e9753d6SMatthew G. Knepley         PetscFV fv = (PetscFV) obj;
47413e9753d6SMatthew G. Knepley 
47423e9753d6SMatthew G. Knepley         Ne = numFaces;
47433e9753d6SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
47443e9753d6SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
47459566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, ds, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
474663a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
47473e9753d6SMatthew G. Knepley     }
47483e9753d6SMatthew G. Knepley     /* Loop over domain */
47493e9753d6SMatthew G. Knepley     if (useFEM) {
47503e9753d6SMatthew G. Knepley       /* Add elemVec to locX */
47513e9753d6SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
47523e9753d6SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
47533e9753d6SMatthew G. Knepley         const PetscInt cind = c - cStart;
47543e9753d6SMatthew G. Knepley 
47559566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]));
47563e9753d6SMatthew G. Knepley         if (ghostLabel) {
47573e9753d6SMatthew G. Knepley           PetscInt ghostVal;
47583e9753d6SMatthew G. Knepley 
47599566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel,cell,&ghostVal));
47603e9753d6SMatthew G. Knepley           if (ghostVal > 0) continue;
47613e9753d6SMatthew G. Knepley         }
47629566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES));
47633e9753d6SMatthew G. Knepley       }
47643e9753d6SMatthew G. Knepley     }
47653e9753d6SMatthew G. Knepley     if (useFVM) {
47663e9753d6SMatthew G. Knepley       PetscScalar *fa;
47673e9753d6SMatthew G. Knepley       PetscInt     iface;
47683e9753d6SMatthew G. Knepley 
47699566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
47703e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
47713e9753d6SMatthew G. Knepley         PetscFV      fv;
47723e9753d6SMatthew G. Knepley         PetscObject  obj;
47733e9753d6SMatthew G. Knepley         PetscClassId id;
47743e9753d6SMatthew G. Knepley         PetscInt     foff, pdim;
47753e9753d6SMatthew G. Knepley 
47769566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
47779566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(ds, f, &foff));
47789566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
47793e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
47803e9753d6SMatthew G. Knepley         fv   = (PetscFV) obj;
47819566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
47823e9753d6SMatthew G. Knepley         /* Accumulate fluxes to cells */
47833e9753d6SMatthew G. Knepley         for (face = fS, iface = 0; face < fE; ++face) {
47843e9753d6SMatthew G. Knepley           const PetscInt *scells;
47853e9753d6SMatthew G. Knepley           PetscScalar    *fL = NULL, *fR = NULL;
47863e9753d6SMatthew G. Knepley           PetscInt        ghost, d, nsupp, nchild;
47873e9753d6SMatthew G. Knepley 
47889566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
47899566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
47909566063dSJacob Faibussowitsch           PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
47913e9753d6SMatthew G. Knepley           if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
47929566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupport(dm, face, &scells));
47939566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel,scells[0],&ghost));
47949566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[0], f, fa, &fL));
47959566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel,scells[1],&ghost));
47969566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[1], f, fa, &fR));
47973e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) {
47983e9753d6SMatthew G. Knepley             if (fL) fL[d] -= fluxL[iface*totDim+foff+d];
47993e9753d6SMatthew G. Knepley             if (fR) fR[d] += fluxR[iface*totDim+foff+d];
48003e9753d6SMatthew G. Knepley           }
48013e9753d6SMatthew G. Knepley           ++iface;
48023e9753d6SMatthew G. Knepley         }
48033e9753d6SMatthew G. Knepley       }
48049566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
48053e9753d6SMatthew G. Knepley     }
48063e9753d6SMatthew G. Knepley     /* Handle time derivative */
48073e9753d6SMatthew G. Knepley     if (locX_t) {
48083e9753d6SMatthew G. Knepley       PetscScalar *x_t, *fa;
48093e9753d6SMatthew G. Knepley 
48109566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
48119566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
48123e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
48133e9753d6SMatthew G. Knepley         PetscFV      fv;
48143e9753d6SMatthew G. Knepley         PetscObject  obj;
48153e9753d6SMatthew G. Knepley         PetscClassId id;
48163e9753d6SMatthew G. Knepley         PetscInt     pdim, d;
48173e9753d6SMatthew G. Knepley 
48189566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
48199566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
48203e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
48213e9753d6SMatthew G. Knepley         fv   = (PetscFV) obj;
48229566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
48233e9753d6SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
48243e9753d6SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
48253e9753d6SMatthew G. Knepley           PetscScalar   *u_t, *r;
48263e9753d6SMatthew G. Knepley 
48273e9753d6SMatthew G. Knepley           if (ghostLabel) {
48283e9753d6SMatthew G. Knepley             PetscInt ghostVal;
48293e9753d6SMatthew G. Knepley 
48309566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
48313e9753d6SMatthew G. Knepley             if (ghostVal > 0) continue;
48323e9753d6SMatthew G. Knepley           }
48339566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
48349566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
48353e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
48363e9753d6SMatthew G. Knepley         }
48373e9753d6SMatthew G. Knepley       }
48389566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
48399566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
48403e9753d6SMatthew G. Knepley     }
48413e9753d6SMatthew G. Knepley     if (useFEM) {
48429566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
48439566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
48443e9753d6SMatthew G. Knepley     }
48453e9753d6SMatthew G. Knepley     if (useFVM) {
48469566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
48479566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
48489566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxL));
48499566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxR));
48509566063dSJacob Faibussowitsch       if (dmGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
48513e9753d6SMatthew G. Knepley     }
48523e9753d6SMatthew G. Knepley   }
48539566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
48549566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
48553e9753d6SMatthew G. Knepley 
48563e9753d6SMatthew G. Knepley   if (useFEM) {
48579566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdResidual_Internal(dm, locX, locX_t, t, locF, user));
48583e9753d6SMatthew G. Knepley 
48593e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
48609566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
48619566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
48623e9753d6SMatthew G. Knepley     } else {
48633e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
48649566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
48659566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
48663e9753d6SMatthew G. Knepley       }
48679566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads,geoms));
48683e9753d6SMatthew G. Knepley     }
48693e9753d6SMatthew G. Knepley   }
48703e9753d6SMatthew G. Knepley 
48713e9753d6SMatthew G. Knepley   /* FEM */
48723e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
48733e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
48743e9753d6SMatthew G. Knepley   /* 3: Handle boundary values */
48753e9753d6SMatthew G. Knepley   /* 4: Loop over domain */
48763e9753d6SMatthew G. Knepley   /*   Extract coefficients */
48773e9753d6SMatthew G. Knepley   /* Loop over fields */
48783e9753d6SMatthew G. Knepley   /*   Set tiling for FE*/
48793e9753d6SMatthew G. Knepley   /*   Integrate FE residual to get elemVec */
48803e9753d6SMatthew G. Knepley   /*     Loop over subdomain */
48813e9753d6SMatthew G. Knepley   /*       Loop over quad points */
48823e9753d6SMatthew G. Knepley   /*         Transform coords to real space */
48833e9753d6SMatthew G. Knepley   /*         Evaluate field and aux fields at point */
48843e9753d6SMatthew G. Knepley   /*         Evaluate residual at point */
48853e9753d6SMatthew G. Knepley   /*         Transform residual to real space */
48863e9753d6SMatthew G. Knepley   /*       Add residual to elemVec */
48873e9753d6SMatthew G. Knepley   /* Loop over domain */
48883e9753d6SMatthew G. Knepley   /*   Add elemVec to locX */
48893e9753d6SMatthew G. Knepley 
48903e9753d6SMatthew G. Knepley   /* FVM */
48913e9753d6SMatthew G. Knepley   /* Get geometric data */
48923e9753d6SMatthew G. Knepley   /* If using gradients */
48933e9753d6SMatthew G. Knepley   /*   Compute gradient data */
48943e9753d6SMatthew G. Knepley   /*   Loop over domain faces */
48953e9753d6SMatthew G. Knepley   /*     Count computational faces */
48963e9753d6SMatthew G. Knepley   /*     Reconstruct cell gradient */
48973e9753d6SMatthew G. Knepley   /*   Loop over domain cells */
48983e9753d6SMatthew G. Knepley   /*     Limit cell gradients */
48993e9753d6SMatthew G. Knepley   /* Handle boundary values */
49003e9753d6SMatthew G. Knepley   /* Loop over domain faces */
49013e9753d6SMatthew G. Knepley   /*   Read out field, centroid, normal, volume for each side of face */
49023e9753d6SMatthew G. Knepley   /* Riemann solve over faces */
49033e9753d6SMatthew G. Knepley   /* Loop over domain faces */
49043e9753d6SMatthew G. Knepley   /*   Accumulate fluxes to cells */
49053e9753d6SMatthew G. Knepley   /* TODO Change printFEM to printDisc here */
49063e9753d6SMatthew G. Knepley   if (mesh->printFEM) {
49073e9753d6SMatthew G. Knepley     Vec         locFbc;
49083e9753d6SMatthew G. Knepley     PetscInt    pStart, pEnd, p, maxDof;
49093e9753d6SMatthew G. Knepley     PetscScalar *zeroes;
49103e9753d6SMatthew G. Knepley 
49119566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(locF,&locFbc));
49129566063dSJacob Faibussowitsch     PetscCall(VecCopy(locF,locFbc));
49139566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetChart(section,&pStart,&pEnd));
49149566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetMaxDof(section,&maxDof));
49159566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(maxDof,&zeroes));
49163e9753d6SMatthew G. Knepley     for (p = pStart; p < pEnd; p++) {
49179566063dSJacob Faibussowitsch       PetscCall(VecSetValuesSection(locFbc,section,p,zeroes,INSERT_BC_VALUES));
49183e9753d6SMatthew G. Knepley     }
49199566063dSJacob Faibussowitsch     PetscCall(PetscFree(zeroes));
49209566063dSJacob Faibussowitsch     PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc));
49219566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&locFbc));
49223e9753d6SMatthew G. Knepley   }
49239566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0));
49243e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
49253e9753d6SMatthew G. Knepley }
49263e9753d6SMatthew G. Knepley 
49276528b96dSMatthew G. Knepley /*
49286528b96dSMatthew G. Knepley   1) Allow multiple kernels for BdResidual for hybrid DS
49296528b96dSMatthew G. Knepley 
49306528b96dSMatthew G. Knepley   DONE 2) Get out dsAux for either side at the same time as cohesive cell dsAux
49316528b96dSMatthew G. Knepley 
49326528b96dSMatthew G. Knepley   DONE 3) Change DMGetCellFields() to get different aux data a[] for each side
49336528b96dSMatthew G. Knepley      - I think I just need to replace a[] with the closure from each face
49346528b96dSMatthew G. Knepley 
49356528b96dSMatthew G. Knepley   4) Run both kernels for each non-hybrid field with correct dsAux, and then hybrid field as before
49366528b96dSMatthew G. Knepley */
493706ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeResidual_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
49383e9753d6SMatthew G. Knepley {
49393e9753d6SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *) dm->data;
49403e9753d6SMatthew G. Knepley   const char      *name       = "Hybrid Residual";
494104c51a94SMatthew G. Knepley   DM               dmAux[3]   = {NULL, NULL, NULL};
49423e9753d6SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
49436528b96dSMatthew G. Knepley   PetscDS          ds         = NULL;
49446528b96dSMatthew G. Knepley   PetscDS          dsAux[3]   = {NULL, NULL, NULL};
494504c51a94SMatthew G. Knepley   Vec              locA[3]    = {NULL, NULL, NULL};
49463e9753d6SMatthew G. Knepley   PetscSection     section    = NULL;
49473e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
49486528b96dSMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a[3];
49493e9753d6SMatthew G. Knepley   PetscScalar     *elemVec;
49503e9753d6SMatthew G. Knepley   IS               chunkIS;
49513e9753d6SMatthew G. Knepley   const PetscInt  *cells;
49523e9753d6SMatthew G. Knepley   PetscInt        *faces;
49533e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
49546528b96dSMatthew G. Knepley   PetscInt         Nf, f, totDim, totDimAux[3], numChunks, cellChunkSize, chunk;
49553e9753d6SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
49563e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
49573e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
49583e9753d6SMatthew G. Knepley 
49593e9753d6SMatthew G. Knepley   PetscFunctionBegin;
49605fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
49615fedec97SMatthew G. Knepley     const char *name;
49629566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject) key[0].label, &name));
496363a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
49645fedec97SMatthew G. Knepley   }
49659566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0));
49663e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
49673e9753d6SMatthew G. Knepley   /* FEM */
49689566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
49699566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
49703e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
49719566063dSJacob Faibussowitsch   PetscCall(DMGetSection(dm, &section));
49729566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
4973*1059d808SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds));
49749566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
49759566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
49769566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
497704c51a94SMatthew G. Knepley   if (locA[2]) {
4978*1059d808SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
4979*1059d808SMatthew G. Knepley 
49809566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
4981*1059d808SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2]));
49829566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
49836528b96dSMatthew G. Knepley     {
49846528b96dSMatthew G. Knepley       const PetscInt *cone;
49856528b96dSMatthew G. Knepley       PetscInt        c;
49866528b96dSMatthew G. Knepley 
4987*1059d808SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
49886528b96dSMatthew G. Knepley       for (c = 0; c < 2; ++c) {
49896528b96dSMatthew G. Knepley         const PetscInt *support;
49906528b96dSMatthew G. Knepley         PetscInt ssize, s;
49916528b96dSMatthew G. Knepley 
49929566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
49939566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
4994*1059d808SMatthew G. Knepley         PetscCheck(ssize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
4995*1059d808SMatthew G. Knepley         if      (support[0] == cellStart) s = 1;
4996*1059d808SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
4997*1059d808SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
49989566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
49999566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
500004c51a94SMatthew G. Knepley         else         {dmAux[c] = dmAux[2];}
50019566063dSJacob Faibussowitsch         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c]));
50029566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
50036528b96dSMatthew G. Knepley       }
50046528b96dSMatthew G. Knepley     }
50053e9753d6SMatthew G. Knepley   }
50063e9753d6SMatthew G. Knepley   /* 2: Setup geometric data */
50079566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
50089566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
50093e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
50109566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
50113e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
50123e9753d6SMatthew G. Knepley       PetscFE fe;
50133e9753d6SMatthew G. Knepley 
50149566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *) &fe));
50153e9753d6SMatthew G. Knepley       if (fe) {
50169566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
50179566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) quads[f]));
50183e9753d6SMatthew G. Knepley       }
50193e9753d6SMatthew G. Knepley     }
50203e9753d6SMatthew G. Knepley   }
50213e9753d6SMatthew G. Knepley   /* Loop over chunks */
50223e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
50233e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal) numCells)/cellChunkSize);
5024a4158a15SMatthew G. Knepley   PetscCall(PetscCalloc1(1*cellChunkSize, &faces));
5025a4158a15SMatthew G. Knepley   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 1*cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS));
50263e9753d6SMatthew G. Knepley   /* Extract field coefficients */
50273e9753d6SMatthew G. Knepley   /* NOTE This needs the end cap faces to have identical orientations */
50289566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
50299566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHybridAuxFields(dm, dmAux, dsAux, cellIS, locA, a));
50309566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, cellChunkSize*totDim, MPIU_SCALAR, &elemVec));
50313e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
50323e9753d6SMatthew G. Knepley     PetscInt cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
50333e9753d6SMatthew G. Knepley 
50349566063dSJacob Faibussowitsch     PetscCall(PetscMemzero(elemVec, cellChunkSize*totDim * sizeof(PetscScalar)));
50353e9753d6SMatthew G. Knepley     /* Get faces */
50363e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
50373e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
50383e9753d6SMatthew G. Knepley       const PetscInt *cone;
50399566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(dm, cell, &cone));
5040a4158a15SMatthew G. Knepley       faces[0*cellChunkSize+(c-cS)] = cone[0];
5041a4158a15SMatthew G. Knepley       /*faces[1*cellChunkSize+(c-cS)] = cone[1];*/
50423e9753d6SMatthew G. Knepley     }
5043a4158a15SMatthew G. Knepley     PetscCall(ISGeneralSetIndices(chunkIS, 1*cellChunkSize, faces, PETSC_USE_POINTER));
50443e9753d6SMatthew G. Knepley     /* Get geometric data */
50453e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
50469566063dSJacob Faibussowitsch       if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad));
50479566063dSJacob Faibussowitsch       if (affineQuad)  PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom));
50483e9753d6SMatthew G. Knepley     } else {
50493e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
50509566063dSJacob Faibussowitsch         if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]));
50513e9753d6SMatthew G. Knepley       }
50523e9753d6SMatthew G. Knepley     }
50533e9753d6SMatthew G. Knepley     /* Loop over fields */
50543e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
50553e9753d6SMatthew G. Knepley       PetscFE         fe;
50563e9753d6SMatthew G. Knepley       PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
5057148442b3SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
50583e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
50593e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
50605fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
50613e9753d6SMatthew G. Knepley 
50629566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *) &fe));
50633e9753d6SMatthew G. Knepley       if (!fe) continue;
50649566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
50659566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
50669566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
50673e9753d6SMatthew G. Knepley       blockSize = Nb;
50683e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
50699566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
50703e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches*batchSize);
50713e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
50723e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches*batchSize);
50733e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
50749566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom,0,offset,&chunkGeom));
50759566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom,offset,numCells,&remGeom));
50769566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, f, &isCohesiveField));
50775fedec97SMatthew G. Knepley       chunkGeom->isCohesive = remGeom->isCohesive = PETSC_TRUE;
50786528b96dSMatthew G. Knepley       key[0].field = f;
50796528b96dSMatthew G. Knepley       key[1].field = f;
50805fedec97SMatthew G. Knepley       key[2].field = f;
50819566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, elemVec));
50829566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[0], 0, Nr, remGeom,  &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[0], &a[0][offset*totDimAux[0]], t, &elemVec[offset*totDim]));
50839566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, elemVec));
50849566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[1], 1, Nr, remGeom,  &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[1], &a[1][offset*totDimAux[1]], t, &elemVec[offset*totDim]));
50859566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, elemVec));
50869566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[2], 2, Nr, remGeom,  &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[2], &a[2][offset*totDimAux[2]], t, &elemVec[offset*totDim]));
50879566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom,offset,numCells,&remGeom));
50889566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom,0,offset,&chunkGeom));
50893e9753d6SMatthew G. Knepley     }
50903e9753d6SMatthew G. Knepley     /* Add elemVec to locX */
50913e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
50923e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
50933e9753d6SMatthew G. Knepley       const PetscInt cind = c - cStart;
50943e9753d6SMatthew G. Knepley 
50959566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]));
50963e9753d6SMatthew G. Knepley       if (ghostLabel) {
50973e9753d6SMatthew G. Knepley         PetscInt ghostVal;
50983e9753d6SMatthew G. Knepley 
50999566063dSJacob Faibussowitsch         PetscCall(DMLabelGetValue(ghostLabel,cell,&ghostVal));
51003e9753d6SMatthew G. Knepley         if (ghostVal > 0) continue;
51013e9753d6SMatthew G. Knepley       }
51029566063dSJacob Faibussowitsch       PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES));
51033e9753d6SMatthew G. Knepley     }
51043e9753d6SMatthew G. Knepley   }
51059566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
51069566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreHybridAuxFields(dmAux, dsAux, cellIS, locA, a));
51079566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
51089566063dSJacob Faibussowitsch   PetscCall(PetscFree(faces));
51099566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&chunkIS));
51109566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
51113e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
51129566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
51139566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&affineQuad));
51143e9753d6SMatthew G. Knepley   } else {
51153e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
51169566063dSJacob Faibussowitsch       if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
51179566063dSJacob Faibussowitsch       if (quads) PetscCall(PetscQuadratureDestroy(&quads[f]));
51183e9753d6SMatthew G. Knepley     }
51199566063dSJacob Faibussowitsch     PetscCall(PetscFree2(quads,geoms));
51203e9753d6SMatthew G. Knepley   }
51219566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0));
51223e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
51233e9753d6SMatthew G. Knepley }
51243e9753d6SMatthew G. Knepley 
512545480ffeSMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobian_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP, DMField coordField, IS facetIS)
51263e9753d6SMatthew G. Knepley {
51273e9753d6SMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *) dm->data;
51283e9753d6SMatthew G. Knepley   DM              plex = NULL, plexA = NULL, tdm;
51293e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
51303e9753d6SMatthew G. Knepley   PetscDS         prob, probAux = NULL;
51313e9753d6SMatthew G. Knepley   PetscSection    section, sectionAux = NULL;
5132e432b41dSStefano Zampini   PetscSection    globalSection;
51333e9753d6SMatthew G. Knepley   Vec             locA = NULL, tv;
51343e9753d6SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL;
51353e9753d6SMatthew G. Knepley   PetscInt        v;
51363e9753d6SMatthew G. Knepley   PetscInt        Nf, totDim, totDimAux = 0;
5137e432b41dSStefano Zampini   PetscBool       transform;
51383e9753d6SMatthew G. Knepley 
51393e9753d6SMatthew G. Knepley   PetscFunctionBegin;
51409566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
51419566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
51429566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
51439566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
51449566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
51459566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
51469566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
51479566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
51489566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, label, values[0], 0, &locA));
51493e9753d6SMatthew G. Knepley   if (locA) {
51503e9753d6SMatthew G. Knepley     DM dmAux;
51513e9753d6SMatthew G. Knepley 
51529566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
51539566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
51549566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
51559566063dSJacob Faibussowitsch     PetscCall(DMGetDS(plexA, &probAux));
51569566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
51579566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
51583e9753d6SMatthew G. Knepley   }
51593e9753d6SMatthew G. Knepley 
51609566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
51613e9753d6SMatthew G. Knepley   for (v = 0; v < numValues; ++v) {
51623e9753d6SMatthew G. Knepley     PetscFEGeom     *fgeom;
51633e9753d6SMatthew G. Knepley     PetscInt         maxDegree;
51643e9753d6SMatthew G. Knepley     PetscQuadrature  qGeom = NULL;
51653e9753d6SMatthew G. Knepley     IS               pointIS;
51663e9753d6SMatthew G. Knepley     const PetscInt  *points;
516706ad1575SMatthew G. Knepley     PetscFormKey key;
51683e9753d6SMatthew G. Knepley     PetscInt         numFaces, face, Nq;
51693e9753d6SMatthew G. Knepley 
517045480ffeSMatthew G. Knepley     key.label = label;
517145480ffeSMatthew G. Knepley     key.value = values[v];
517206ad1575SMatthew G. Knepley     key.part  = 0;
51739566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
51743e9753d6SMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
51753e9753d6SMatthew G. Knepley     {
51763e9753d6SMatthew G. Knepley       IS isectIS;
51773e9753d6SMatthew G. Knepley 
51783e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a prior if one is a superset of the other */
51799566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS,pointIS,&isectIS));
51809566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
51813e9753d6SMatthew G. Knepley       pointIS = isectIS;
51823e9753d6SMatthew G. Knepley     }
51839566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
51849566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
51859566063dSJacob Faibussowitsch     PetscCall(PetscMalloc4(numFaces*totDim, &u, locX_t ? numFaces*totDim : 0, &u_t, numFaces*totDim*totDim, &elemMat, locA ? numFaces*totDimAux : 0, &a));
51869566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,pointIS,NULL,&maxDegree));
51873e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
51889566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,pointIS,&qGeom));
51893e9753d6SMatthew G. Knepley     }
51903e9753d6SMatthew G. Knepley     if (!qGeom) {
51913e9753d6SMatthew G. Knepley       PetscFE fe;
51923e9753d6SMatthew G. Knepley 
51939566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
51949566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
51959566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
51963e9753d6SMatthew G. Knepley     }
51979566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
51989566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom));
51993e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
5200f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
52013e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
5202f15274beSMatthew Knepley       PetscInt       i;
52033e9753d6SMatthew G. Knepley 
52049566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
52059566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
52063e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
52079566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
52083e9753d6SMatthew G. Knepley       if (locX_t) {
52099566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
52103e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face*totDim+i] = x[i];
52119566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
52123e9753d6SMatthew G. Knepley       }
52133e9753d6SMatthew G. Knepley       if (locA) {
52143e9753d6SMatthew G. Knepley         PetscInt subp;
52159566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
52169566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
52173e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face*totDimAux+i] = x[i];
52189566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
52193e9753d6SMatthew G. Knepley       }
52203e9753d6SMatthew G. Knepley     }
52219566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(elemMat, numFaces*totDim*totDim));
52223e9753d6SMatthew G. Knepley     {
52233e9753d6SMatthew G. Knepley       PetscFE         fe;
52243e9753d6SMatthew G. Knepley       PetscInt        Nb;
52253e9753d6SMatthew G. Knepley       /* Conforming batches */
52263e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
52273e9753d6SMatthew G. Knepley       /* Remainder */
52283e9753d6SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL;
52293e9753d6SMatthew G. Knepley       PetscInt        fieldJ, Nr, offset;
52303e9753d6SMatthew G. Knepley 
52319566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
52329566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
52339566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
52343e9753d6SMatthew G. Knepley       blockSize = Nb;
52353e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
52369566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
52373e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches*batchSize);
52383e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
52393e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches*batchSize);
52403e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
52419566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom,0,offset,&chunkGeom));
52423e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
524345480ffeSMatthew G. Knepley         key.field = fieldI*Nf+fieldJ;
52449566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateBdJacobian(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
52453e9753d6SMatthew G. Knepley       }
52469566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom,offset,numFaces,&chunkGeom));
52473e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
524845480ffeSMatthew G. Knepley         key.field = fieldI*Nf+fieldJ;
52499566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateBdJacobian(prob, wf, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, a ? &a[offset*totDimAux] : NULL, t, X_tShift, &elemMat[offset*totDim*totDim]));
52503e9753d6SMatthew G. Knepley       }
52519566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom,offset,numFaces,&chunkGeom));
52523e9753d6SMatthew G. Knepley     }
52533e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
52543e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
52553e9753d6SMatthew G. Knepley 
52563e9753d6SMatthew G. Knepley       /* Transform to global basis before insertion in Jacobian */
52579566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
52589566063dSJacob Faibussowitsch       if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMat[face*totDim*totDim]));
52599566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face*totDim*totDim]));
52609566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, support[0], &elemMat[face*totDim*totDim], ADD_VALUES));
52613e9753d6SMatthew G. Knepley     }
52629566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom));
52639566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
52649566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
52659566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
52669566063dSJacob Faibussowitsch     PetscCall(PetscFree4(u, u_t, elemMat, a));
52673e9753d6SMatthew G. Knepley   }
52689566063dSJacob Faibussowitsch   if (plex)  PetscCall(DMDestroy(&plex));
52699566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
52703e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
52713e9753d6SMatthew G. Knepley }
52723e9753d6SMatthew G. Knepley 
527345480ffeSMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobianSingle(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP)
52743e9753d6SMatthew G. Knepley {
52753e9753d6SMatthew G. Knepley   DMField        coordField;
52763e9753d6SMatthew G. Knepley   DMLabel        depthLabel;
52773e9753d6SMatthew G. Knepley   IS             facetIS;
52783e9753d6SMatthew G. Knepley   PetscInt       dim;
52793e9753d6SMatthew G. Knepley 
52803e9753d6SMatthew G. Knepley   PetscFunctionBegin;
52819566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
52829566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
52839566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim-1, &facetIS));
52849566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
52859566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, field, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS));
52869566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
52873e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
52883e9753d6SMatthew G. Knepley }
52893e9753d6SMatthew G. Knepley 
52903e9753d6SMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobian_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP, void *user)
52913e9753d6SMatthew G. Knepley {
52923e9753d6SMatthew G. Knepley   PetscDS          prob;
52933e9753d6SMatthew G. Knepley   PetscInt         dim, numBd, bd;
52943e9753d6SMatthew G. Knepley   DMLabel          depthLabel;
52953e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
52963e9753d6SMatthew G. Knepley   IS               facetIS;
52973e9753d6SMatthew G. Knepley 
52983e9753d6SMatthew G. Knepley   PetscFunctionBegin;
52999566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
53009566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
53019566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
53029566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim-1, &facetIS));
53039566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
53049566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
53053e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
530645480ffeSMatthew G. Knepley     PetscWeakForm           wf;
53073e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
53083e9753d6SMatthew G. Knepley     DMLabel                 label;
53093e9753d6SMatthew G. Knepley     const PetscInt         *values;
53103e9753d6SMatthew G. Knepley     PetscInt                fieldI, numValues;
53113e9753d6SMatthew G. Knepley     PetscObject             obj;
53123e9753d6SMatthew G. Knepley     PetscClassId            id;
53133e9753d6SMatthew G. Knepley 
53149566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &fieldI, NULL, NULL, NULL, NULL, NULL));
53159566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &obj));
53169566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
53173e9753d6SMatthew G. Knepley     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
53189566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, fieldI, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS));
53193e9753d6SMatthew G. Knepley   }
53209566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
53213e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
53223e9753d6SMatthew G. Knepley }
53233e9753d6SMatthew G. Knepley 
532406ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP,void *user)
53253e9753d6SMatthew G. Knepley {
53263e9753d6SMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *) dm->data;
53273e9753d6SMatthew G. Knepley   const char     *name  = "Jacobian";
53289a2a23afSMatthew G. Knepley   DM              dmAux = NULL, plex, tdm;
53293e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
53303e9753d6SMatthew G. Knepley   Vec             A, tv;
53313e9753d6SMatthew G. Knepley   DMField         coordField;
53323e9753d6SMatthew G. Knepley   PetscDS         prob, probAux = NULL;
5333e432b41dSStefano Zampini   PetscSection    section, globalSection, sectionAux;
53343e9753d6SMatthew G. Knepley   PetscScalar    *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL;
53353e9753d6SMatthew G. Knepley   const PetscInt *cells;
53363e9753d6SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
53373e9753d6SMatthew G. Knepley   PetscInt        totDim, totDimAux, cStart, cEnd, numCells, c;
5338e432b41dSStefano Zampini   PetscBool       hasJac, hasPrec, hasDyn, hasFV = PETSC_FALSE, transform;
53393e9753d6SMatthew G. Knepley 
53403e9753d6SMatthew G. Knepley   PetscFunctionBegin;
53419566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0));
53429566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
53439566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
53449566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
53459566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
53469566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
53479566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
53489566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
53499566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob));
53509566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
53519566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
53529566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
53539566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
53543e9753d6SMatthew G. Knepley   /* user passed in the same matrix, avoid double contributions and
53553e9753d6SMatthew G. Knepley      only assemble the Jacobian */
53563e9753d6SMatthew G. Knepley   if (hasJac && Jac == JacP) hasPrec = PETSC_FALSE;
53579566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
53583e9753d6SMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
53599566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
53609a2a23afSMatthew G. Knepley   if (A) {
53619566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
53629566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
53639566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plex));
53649566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plex, &sectionAux));
53659566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
53669566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
53673e9753d6SMatthew G. Knepley   }
53689566063dSJacob Faibussowitsch   PetscCall(PetscMalloc5(numCells*totDim,&u,X_t ? numCells*totDim : 0,&u_t,hasJac ? numCells*totDim*totDim : 0,&elemMat,hasPrec ? numCells*totDim*totDim : 0, &elemMatP,hasDyn ? numCells*totDim*totDim : 0, &elemMatD));
53699566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells*totDimAux, &a));
53709566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
53713e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
53723e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
53733e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
53743e9753d6SMatthew G. Knepley     PetscScalar   *x = NULL,  *x_t = NULL;
53753e9753d6SMatthew G. Knepley     PetscInt       i;
53763e9753d6SMatthew G. Knepley 
53779566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x));
53783e9753d6SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind*totDim+i] = x[i];
53799566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x));
53803e9753d6SMatthew G. Knepley     if (X_t) {
53819566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t));
53823e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind*totDim+i] = x_t[i];
53839566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t));
53843e9753d6SMatthew G. Knepley     }
53853e9753d6SMatthew G. Knepley     if (dmAux) {
53863e9753d6SMatthew G. Knepley       PetscInt subcell;
53879566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
53889566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x));
53893e9753d6SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind*totDimAux+i] = x[i];
53909566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x));
53913e9753d6SMatthew G. Knepley     }
53923e9753d6SMatthew G. Knepley   }
53939566063dSJacob Faibussowitsch   if (hasJac)  PetscCall(PetscArrayzero(elemMat,  numCells*totDim*totDim));
53949566063dSJacob Faibussowitsch   if (hasPrec) PetscCall(PetscArrayzero(elemMatP, numCells*totDim*totDim));
53959566063dSJacob Faibussowitsch   if (hasDyn)  PetscCall(PetscArrayzero(elemMatD, numCells*totDim*totDim));
53963e9753d6SMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
53973e9753d6SMatthew G. Knepley     PetscClassId    id;
53983e9753d6SMatthew G. Knepley     PetscFE         fe;
53993e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
54003e9753d6SMatthew G. Knepley     PetscInt        Nb;
54013e9753d6SMatthew G. Knepley     /* Conforming batches */
54023e9753d6SMatthew G. Knepley     PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
54033e9753d6SMatthew G. Knepley     /* Remainder */
54043e9753d6SMatthew G. Knepley     PetscInt        Nr, offset, Nq;
54053e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
54063e9753d6SMatthew G. Knepley     PetscFEGeom     *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
54073e9753d6SMatthew G. Knepley 
54089566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
54099566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId((PetscObject) fe, &id));
54103e9753d6SMatthew G. Knepley     if (id == PETSCFV_CLASSID) {hasFV = PETSC_TRUE; continue;}
54119566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
54129566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
54139566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
54143e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
54159566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&qGeom));
54163e9753d6SMatthew G. Knepley     }
54173e9753d6SMatthew G. Knepley     if (!qGeom) {
54189566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe,&qGeom));
54199566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
54203e9753d6SMatthew G. Knepley     }
54219566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
54229566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM));
54233e9753d6SMatthew G. Knepley     blockSize = Nb;
54243e9753d6SMatthew G. Knepley     batchSize = numBlocks * blockSize;
54259566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
54263e9753d6SMatthew G. Knepley     numChunks = numCells / (numBatches*batchSize);
54273e9753d6SMatthew G. Knepley     Ne        = numChunks*numBatches*batchSize;
54283e9753d6SMatthew G. Knepley     Nr        = numCells % (numBatches*batchSize);
54293e9753d6SMatthew G. Knepley     offset    = numCells - Nr;
54309566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom));
54319566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&remGeom));
54323e9753d6SMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
54336528b96dSMatthew G. Knepley       key.field = fieldI*Nf+fieldJ;
54343e9753d6SMatthew G. Knepley       if (hasJac) {
54359566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
54369566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMat[offset*totDim*totDim]));
54373e9753d6SMatthew G. Knepley       }
54383e9753d6SMatthew G. Knepley       if (hasPrec) {
54399566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatP));
54409566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatP[offset*totDim*totDim]));
54413e9753d6SMatthew G. Knepley       }
54423e9753d6SMatthew G. Knepley       if (hasDyn) {
54439566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
54449566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatD[offset*totDim*totDim]));
54453e9753d6SMatthew G. Knepley       }
54463e9753d6SMatthew G. Knepley     }
54479566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&remGeom));
54489566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,0,offset,&chunkGeom));
54499566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM));
54509566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
54513e9753d6SMatthew G. Knepley   }
54523e9753d6SMatthew G. Knepley   /*   Add contribution from X_t */
54533e9753d6SMatthew G. Knepley   if (hasDyn) {for (c = 0; c < numCells*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];}
54543e9753d6SMatthew G. Knepley   if (hasFV) {
54553e9753d6SMatthew G. Knepley     PetscClassId id;
54563e9753d6SMatthew G. Knepley     PetscFV      fv;
54573e9753d6SMatthew G. Knepley     PetscInt     offsetI, NcI, NbI = 1, fc, f;
54583e9753d6SMatthew G. Knepley 
54593e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
54609566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fv));
54619566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, fieldI, &offsetI));
54629566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId((PetscObject) fv, &id));
54633e9753d6SMatthew G. Knepley       if (id != PETSCFV_CLASSID) continue;
54643e9753d6SMatthew G. Knepley       /* Put in the identity */
54659566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &NcI));
54663e9753d6SMatthew G. Knepley       for (c = cStart; c < cEnd; ++c) {
54673e9753d6SMatthew G. Knepley         const PetscInt cind    = c - cStart;
54683e9753d6SMatthew G. Knepley         const PetscInt eOffset = cind*totDim*totDim;
54693e9753d6SMatthew G. Knepley         for (fc = 0; fc < NcI; ++fc) {
54703e9753d6SMatthew G. Knepley           for (f = 0; f < NbI; ++f) {
54713e9753d6SMatthew G. Knepley             const PetscInt i = offsetI + f*NcI+fc;
54723e9753d6SMatthew G. Knepley             if (hasPrec) {
54733e9753d6SMatthew G. Knepley               if (hasJac) {elemMat[eOffset+i*totDim+i] = 1.0;}
54743e9753d6SMatthew G. Knepley               elemMatP[eOffset+i*totDim+i] = 1.0;
54753e9753d6SMatthew G. Knepley             } else {elemMat[eOffset+i*totDim+i] = 1.0;}
54763e9753d6SMatthew G. Knepley           }
54773e9753d6SMatthew G. Knepley         }
54783e9753d6SMatthew G. Knepley       }
54793e9753d6SMatthew G. Knepley     }
54803e9753d6SMatthew G. Knepley     /* No allocated space for FV stuff, so ignore the zero entries */
54819566063dSJacob Faibussowitsch     PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE));
54823e9753d6SMatthew G. Knepley   }
54833e9753d6SMatthew G. Knepley   /* Insert values into matrix */
54843e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
54853e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
54863e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
54873e9753d6SMatthew G. Knepley 
54883e9753d6SMatthew G. Knepley     /* Transform to global basis before insertion in Jacobian */
54899566063dSJacob Faibussowitsch     if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, cell, PETSC_TRUE, totDim, &elemMat[cind*totDim*totDim]));
54903e9753d6SMatthew G. Knepley     if (hasPrec) {
54913e9753d6SMatthew G. Knepley       if (hasJac) {
54929566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
54939566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[cind*totDim*totDim], ADD_VALUES));
54943e9753d6SMatthew G. Knepley       }
54959566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind*totDim*totDim]));
54969566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES));
54973e9753d6SMatthew G. Knepley     } else {
54983e9753d6SMatthew G. Knepley       if (hasJac) {
54999566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
55009566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMat[cind*totDim*totDim], ADD_VALUES));
55013e9753d6SMatthew G. Knepley       }
55023e9753d6SMatthew G. Knepley     }
55033e9753d6SMatthew G. Knepley   }
55049566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
55059566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
55069566063dSJacob Faibussowitsch   PetscCall(PetscFree5(u,u_t,elemMat,elemMatP,elemMatD));
55073e9753d6SMatthew G. Knepley   if (dmAux) {
55089566063dSJacob Faibussowitsch     PetscCall(PetscFree(a));
55099566063dSJacob Faibussowitsch     PetscCall(DMDestroy(&plex));
55103e9753d6SMatthew G. Knepley   }
55113e9753d6SMatthew G. Knepley   /* Compute boundary integrals */
55129566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, user));
55133e9753d6SMatthew G. Knepley   /* Assemble matrix */
55143e9753d6SMatthew G. Knepley   if (hasJac && hasPrec) {
55159566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
55169566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
55173e9753d6SMatthew G. Knepley   }
55189566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
55199566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
55209566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0));
55213e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
55223e9753d6SMatthew G. Knepley }
55233e9753d6SMatthew G. Knepley 
552406ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user)
55253e9753d6SMatthew G. Knepley {
55263e9753d6SMatthew G. Knepley   DM_Plex         *mesh          = (DM_Plex *) dm->data;
55273e9753d6SMatthew G. Knepley   const char      *name          = "Hybrid Jacobian";
5528148442b3SMatthew G. Knepley   DM               dmAux[3]      = {NULL, NULL, NULL};
5529148442b3SMatthew G. Knepley   DMLabel          ghostLabel    = NULL;
55303e9753d6SMatthew G. Knepley   DM               plex          = NULL;
55313e9753d6SMatthew G. Knepley   DM               plexA         = NULL;
5532148442b3SMatthew G. Knepley   PetscDS          ds            = NULL;
5533148442b3SMatthew G. Knepley   PetscDS          dsAux[3]      = {NULL, NULL, NULL};
5534148442b3SMatthew G. Knepley   Vec              locA[3]       = {NULL, NULL, NULL};
55353e9753d6SMatthew G. Knepley   PetscSection     section       = NULL;
5536148442b3SMatthew G. Knepley   PetscSection     sectionAux[3] = {NULL, NULL, NULL};
55373e9753d6SMatthew G. Knepley   DMField          coordField    = NULL;
5538148442b3SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a[3];
55393e9753d6SMatthew G. Knepley   PetscScalar     *elemMat, *elemMatP;
5540e432b41dSStefano Zampini   PetscSection     globalSection;
55413e9753d6SMatthew G. Knepley   IS               chunkIS;
55423e9753d6SMatthew G. Knepley   const PetscInt  *cells;
55433e9753d6SMatthew G. Knepley   PetscInt        *faces;
55443e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
5545148442b3SMatthew G. Knepley   PetscInt         Nf, fieldI, fieldJ, totDim, totDimAux[3], numChunks, cellChunkSize, chunk;
55463e9753d6SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
55473e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
55483e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
5549e432b41dSStefano Zampini   PetscBool        hasBdJac, hasBdPrec;
55503e9753d6SMatthew G. Knepley 
55513e9753d6SMatthew G. Knepley   PetscFunctionBegin;
55525fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
55535fedec97SMatthew G. Knepley     const char *name;
55549566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject) key[0].label, &name));
555563a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
55565fedec97SMatthew G. Knepley   }
55579566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0));
55589566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
55599566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
55609566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
55619566063dSJacob Faibussowitsch   PetscCall(DMGetSection(dm, &section));
55629566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
55639566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
5564*1059d808SMatthew G. Knepley   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds));
55659566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
55669566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
55679566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobian(ds, &hasBdJac));
55689566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobianPreconditioner(ds, &hasBdPrec));
55699566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
5570148442b3SMatthew G. Knepley   if (locA[2]) {
5571*1059d808SMatthew G. Knepley     const PetscInt cellStart = cells ? cells[cStart] : cStart;
5572*1059d808SMatthew G. Knepley 
55739566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
55749566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux[2], DMPLEX, &plexA));
55759566063dSJacob Faibussowitsch     PetscCall(DMGetSection(dmAux[2], &sectionAux[2]));
5576*1059d808SMatthew G. Knepley     PetscCall(DMGetCellDS(dmAux[2], cellStart, &dsAux[2]));
55779566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
5578148442b3SMatthew G. Knepley     {
5579148442b3SMatthew G. Knepley       const PetscInt *cone;
5580148442b3SMatthew G. Knepley       PetscInt        c;
5581148442b3SMatthew G. Knepley 
5582*1059d808SMatthew G. Knepley       PetscCall(DMPlexGetCone(dm, cellStart, &cone));
5583148442b3SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
5584148442b3SMatthew G. Knepley         const PetscInt *support;
5585148442b3SMatthew G. Knepley         PetscInt ssize, s;
5586148442b3SMatthew G. Knepley 
55879566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
55889566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
5589*1059d808SMatthew G. Knepley         PetscCheck(ssize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cellStart, ssize);
5590*1059d808SMatthew G. Knepley         if      (support[0] == cellStart) s = 1;
5591*1059d808SMatthew G. Knepley         else if (support[1] == cellStart) s = 0;
5592*1059d808SMatthew G. Knepley         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cellStart);
55939566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
55949566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
5595148442b3SMatthew G. Knepley         else         {dmAux[c] = dmAux[2];}
55969566063dSJacob Faibussowitsch         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c]));
55979566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
5598148442b3SMatthew G. Knepley       }
5599148442b3SMatthew G. Knepley     }
56003e9753d6SMatthew G. Knepley   }
56019566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
56029566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
56033e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
56043e9753d6SMatthew G. Knepley     PetscInt f;
56059566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
56063e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
56073e9753d6SMatthew G. Knepley       PetscFE fe;
56083e9753d6SMatthew G. Knepley 
56099566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *) &fe));
56103e9753d6SMatthew G. Knepley       if (fe) {
56119566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
56129566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) quads[f]));
56133e9753d6SMatthew G. Knepley       }
56143e9753d6SMatthew G. Knepley     }
56153e9753d6SMatthew G. Knepley   }
56163e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
56173e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal) numCells)/cellChunkSize);
5618a4158a15SMatthew G. Knepley   PetscCall(PetscCalloc1(1*cellChunkSize, &faces));
5619a4158a15SMatthew G. Knepley   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 1*cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS));
56209566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
56219566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHybridAuxFields(dm, dmAux, dsAux, cellIS, locA, a));
56229566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, hasBdJac  ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMat));
56239566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMatP));
56243e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
56253e9753d6SMatthew G. Knepley     PetscInt cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
56263e9753d6SMatthew G. Knepley 
56279566063dSJacob Faibussowitsch     if (hasBdJac)  PetscCall(PetscMemzero(elemMat,  numCells*totDim*totDim * sizeof(PetscScalar)));
56289566063dSJacob Faibussowitsch     if (hasBdPrec) PetscCall(PetscMemzero(elemMatP, numCells*totDim*totDim * sizeof(PetscScalar)));
56293e9753d6SMatthew G. Knepley     /* Get faces */
56303e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
56313e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
56323e9753d6SMatthew G. Knepley       const PetscInt *cone;
56339566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(plex, cell, &cone));
5634a4158a15SMatthew G. Knepley       faces[0*cellChunkSize+(c-cS)] = cone[0];
5635a4158a15SMatthew G. Knepley       /*faces[2*cellChunkSize+(c-cS)] = cone[1];*/
56363e9753d6SMatthew G. Knepley     }
5637a4158a15SMatthew G. Knepley     PetscCall(ISGeneralSetIndices(chunkIS, 1*cellChunkSize, faces, PETSC_USE_POINTER));
56383e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
56399566063dSJacob Faibussowitsch       if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad));
56409566063dSJacob Faibussowitsch       if (affineQuad)  PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom));
56413e9753d6SMatthew G. Knepley     } else {
56423e9753d6SMatthew G. Knepley       PetscInt f;
56433e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
56449566063dSJacob Faibussowitsch         if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]));
56453e9753d6SMatthew G. Knepley       }
56463e9753d6SMatthew G. Knepley     }
56473e9753d6SMatthew G. Knepley 
56483e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
56493e9753d6SMatthew G. Knepley       PetscFE         feI;
56503e9753d6SMatthew G. Knepley       PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[fieldI];
56513e9753d6SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
56523e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[fieldI];
56533e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
56545fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
56553e9753d6SMatthew G. Knepley 
56569566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *) &feI));
56573e9753d6SMatthew G. Knepley       if (!feI) continue;
56589566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(feI, NULL, &numBlocks, NULL, &numBatches));
56599566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
56609566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feI, &Nb));
56613e9753d6SMatthew G. Knepley       blockSize = Nb;
56623e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
56639566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(feI, blockSize, numBlocks, batchSize, numBatches));
56643e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches*batchSize);
56653e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
56663e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches*batchSize);
56673e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
56689566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom,0,offset,&chunkGeom));
56699566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom,offset,numCells,&remGeom));
56709566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, fieldI, &isCohesiveField));
56713e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
56723e9753d6SMatthew G. Knepley         PetscFE feJ;
56733e9753d6SMatthew G. Knepley 
56749566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, fieldJ, (PetscObject *) &feJ));
56753e9753d6SMatthew G. Knepley         if (!feJ) continue;
5676148442b3SMatthew G. Knepley         key[0].field = fieldI*Nf+fieldJ;
5677148442b3SMatthew G. Knepley         key[1].field = fieldI*Nf+fieldJ;
56785fedec97SMatthew G. Knepley         key[2].field = fieldI*Nf+fieldJ;
5679148442b3SMatthew G. Knepley         if (hasBdJac) {
56809566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMat));
56819566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[0], 0, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[0], &a[0][offset*totDimAux[0]], t, X_tShift, &elemMat[offset*totDim*totDim]));
56829566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMat));
56839566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[1], 1, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[1], &a[1][offset*totDimAux[1]], t, X_tShift, &elemMat[offset*totDim*totDim]));
5684148442b3SMatthew G. Knepley         }
5685148442b3SMatthew G. Knepley         if (hasBdPrec) {
56869566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatP));
56879566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[0], 0, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[0], &a[0][offset*totDimAux[0]], t, X_tShift, &elemMatP[offset*totDim*totDim]));
56889566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatP));
56899566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[1], 1, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[1], &a[1][offset*totDimAux[1]], t, X_tShift, &elemMatP[offset*totDim*totDim]));
5690148442b3SMatthew G. Knepley         }
56915fedec97SMatthew G. Knepley         if (hasBdJac) {
56929566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMat));
56939566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[2], 2, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[2], &a[2][offset*totDimAux[2]], t, X_tShift, &elemMat[offset*totDim*totDim]));
5694148442b3SMatthew G. Knepley         }
56955fedec97SMatthew G. Knepley         if (hasBdPrec) {
56969566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatP));
56979566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[2], 2, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[2], &a[2][offset*totDimAux[2]], t, X_tShift, &elemMatP[offset*totDim*totDim]));
56983e9753d6SMatthew G. Knepley         }
56993e9753d6SMatthew G. Knepley       }
57009566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom,offset,numCells,&remGeom));
57019566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom,0,offset,&chunkGeom));
57023e9753d6SMatthew G. Knepley     }
57033e9753d6SMatthew G. Knepley     /* Insert values into matrix */
57043e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
57053e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
57063e9753d6SMatthew G. Knepley       const PetscInt cind = c - cS;
57073e9753d6SMatthew G. Knepley 
57083e9753d6SMatthew G. Knepley       if (hasBdPrec) {
57093e9753d6SMatthew G. Knepley         if (hasBdJac) {
57109566063dSJacob Faibussowitsch           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
57119566063dSJacob Faibussowitsch           PetscCall(DMPlexMatSetClosure(plex, section, globalSection, Jac, cell, &elemMat[cind*totDim*totDim], ADD_VALUES));
57123e9753d6SMatthew G. Knepley         }
57139566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind*totDim*totDim]));
57149566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES));
57153e9753d6SMatthew G. Knepley       } else if (hasBdJac) {
57169566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
57179566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMat[cind*totDim*totDim], ADD_VALUES));
57183e9753d6SMatthew G. Knepley       }
57193e9753d6SMatthew G. Knepley     }
57203e9753d6SMatthew G. Knepley   }
57219566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
57229566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreHybridAuxFields(dmAux, dsAux, cellIS, locA, a));
57239566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, hasBdJac  ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMat));
57249566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMatP));
57259566063dSJacob Faibussowitsch   PetscCall(PetscFree(faces));
57269566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&chunkIS));
57279566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
57283e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
57299566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
57309566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&affineQuad));
57313e9753d6SMatthew G. Knepley   } else {
57323e9753d6SMatthew G. Knepley     PetscInt f;
57333e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
57349566063dSJacob Faibussowitsch       if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE, &geoms[f]));
57359566063dSJacob Faibussowitsch       if (quads) PetscCall(PetscQuadratureDestroy(&quads[f]));
57363e9753d6SMatthew G. Knepley     }
57379566063dSJacob Faibussowitsch     PetscCall(PetscFree2(quads,geoms));
57383e9753d6SMatthew G. Knepley   }
57399566063dSJacob Faibussowitsch   if (dmAux[2]) PetscCall(DMDestroy(&plexA));
57409566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
57413e9753d6SMatthew G. Knepley   /* Assemble matrix */
57423e9753d6SMatthew G. Knepley   if (hasBdJac && hasBdPrec) {
57439566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
57449566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
57453e9753d6SMatthew G. Knepley   }
57469566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
57479566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
57489566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0));
57493e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
57503e9753d6SMatthew G. Knepley }
57518e3a2eefSMatthew G. Knepley 
57528e3a2eefSMatthew G. Knepley /*
57538e3a2eefSMatthew G. Knepley   DMPlexComputeJacobian_Action_Internal - Form the local portion of the Jacobian action Z = J(X) Y at the local solution X using pointwise functions specified by the user.
57548e3a2eefSMatthew G. Knepley 
57558e3a2eefSMatthew G. Knepley   Input Parameters:
57568e3a2eefSMatthew G. Knepley + dm     - The mesh
57578e3a2eefSMatthew G. Knepley . key    - The PetscWeakFormKey indcating where integration should happen
575806ad1575SMatthew G. Knepley . cellIS - The cells to integrate over
57598e3a2eefSMatthew G. Knepley . t      - The time
57608e3a2eefSMatthew G. Knepley . X_tShift - The multiplier for the Jacobian with repsect to X_t
57618e3a2eefSMatthew G. Knepley . X      - Local solution vector
57628e3a2eefSMatthew G. Knepley . X_t    - Time-derivative of the local solution vector
57638e3a2eefSMatthew G. Knepley . Y      - Local input vector
576406ad1575SMatthew G. Knepley - user   - the user context
57658e3a2eefSMatthew G. Knepley 
57668e3a2eefSMatthew G. Knepley   Output Parameter:
57678e3a2eefSMatthew G. Knepley . Z - Local output vector
57688e3a2eefSMatthew G. Knepley 
57698e3a2eefSMatthew G. Knepley   Note:
57708e3a2eefSMatthew G. Knepley   We form the residual one batch of elements at a time. This allows us to offload work onto an accelerator,
57718e3a2eefSMatthew G. Knepley   like a GPU, or vectorize on a multicore machine.
57728e3a2eefSMatthew G. Knepley */
577306ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Action_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Vec Y, Vec Z, void *user)
57748e3a2eefSMatthew G. Knepley {
57758e3a2eefSMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *) dm->data;
57768e3a2eefSMatthew G. Knepley   const char     *name  = "Jacobian";
57778e3a2eefSMatthew G. Knepley   DM              dmAux = NULL, plex, plexAux = NULL;
57788e3a2eefSMatthew G. Knepley   DMEnclosureType encAux;
57798e3a2eefSMatthew G. Knepley   Vec             A;
57808e3a2eefSMatthew G. Knepley   DMField         coordField;
57818e3a2eefSMatthew G. Knepley   PetscDS         prob, probAux = NULL;
57828e3a2eefSMatthew G. Knepley   PetscQuadrature quad;
57838e3a2eefSMatthew G. Knepley   PetscSection    section, globalSection, sectionAux;
57848e3a2eefSMatthew G. Knepley   PetscScalar    *elemMat, *elemMatD, *u, *u_t, *a = NULL, *y, *z;
57858e3a2eefSMatthew G. Knepley   const PetscInt *cells;
57868e3a2eefSMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
57878e3a2eefSMatthew G. Knepley   PetscInt        totDim, totDimAux = 0, cStart, cEnd, numCells, c;
57888e3a2eefSMatthew G. Knepley   PetscBool       hasDyn;
57898e3a2eefSMatthew G. Knepley 
57908e3a2eefSMatthew G. Knepley   PetscFunctionBegin;
57919566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0));
57929566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
57938e3a2eefSMatthew G. Knepley   if (!cellIS) {
57948e3a2eefSMatthew G. Knepley     PetscInt depth;
57958e3a2eefSMatthew G. Knepley 
57969566063dSJacob Faibussowitsch     PetscCall(DMPlexGetDepth(plex, &depth));
57979566063dSJacob Faibussowitsch     PetscCall(DMGetStratumIS(plex, "dim", depth, &cellIS));
57989566063dSJacob Faibussowitsch     if (!cellIS) PetscCall(DMGetStratumIS(plex, "depth", depth, &cellIS));
57998e3a2eefSMatthew G. Knepley   } else {
58009566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject) cellIS));
58018e3a2eefSMatthew G. Knepley   }
58029566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
58039566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
58049566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
58059566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
58069566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob));
58079566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
58089566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
58099566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
58108e3a2eefSMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
58119566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
58128e3a2eefSMatthew G. Knepley   if (A) {
58139566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
58149566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
58159566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexAux));
58169566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexAux, &sectionAux));
58179566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
58189566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
58198e3a2eefSMatthew G. Knepley   }
58209566063dSJacob Faibussowitsch   PetscCall(VecSet(Z, 0.0));
58219566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numCells*totDim,&u,X_t ? numCells*totDim : 0,&u_t,numCells*totDim*totDim,&elemMat,hasDyn ? numCells*totDim*totDim : 0, &elemMatD,numCells*totDim,&y,totDim,&z));
58229566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells*totDimAux, &a));
58239566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
58248e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
58258e3a2eefSMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
58268e3a2eefSMatthew G. Knepley     const PetscInt cind = c - cStart;
58278e3a2eefSMatthew G. Knepley     PetscScalar   *x = NULL,  *x_t = NULL;
58288e3a2eefSMatthew G. Knepley     PetscInt       i;
58298e3a2eefSMatthew G. Knepley 
58309566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, X, cell, NULL, &x));
58318e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind*totDim+i] = x[i];
58329566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, X, cell, NULL, &x));
58338e3a2eefSMatthew G. Knepley     if (X_t) {
58349566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, X_t, cell, NULL, &x_t));
58358e3a2eefSMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind*totDim+i] = x_t[i];
58369566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, X_t, cell, NULL, &x_t));
58378e3a2eefSMatthew G. Knepley     }
58388e3a2eefSMatthew G. Knepley     if (dmAux) {
58398e3a2eefSMatthew G. Knepley       PetscInt subcell;
58409566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
58419566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexAux, sectionAux, A, subcell, NULL, &x));
58428e3a2eefSMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind*totDimAux+i] = x[i];
58439566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexAux, sectionAux, A, subcell, NULL, &x));
58448e3a2eefSMatthew G. Knepley     }
58459566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, Y, cell, NULL, &x));
58468e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) y[cind*totDim+i] = x[i];
58479566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, Y, cell, NULL, &x));
58488e3a2eefSMatthew G. Knepley   }
58499566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, numCells*totDim*totDim));
58509566063dSJacob Faibussowitsch   if (hasDyn)  PetscCall(PetscArrayzero(elemMatD, numCells*totDim*totDim));
58518e3a2eefSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
58528e3a2eefSMatthew G. Knepley     PetscFE  fe;
58538e3a2eefSMatthew G. Knepley     PetscInt Nb;
58548e3a2eefSMatthew G. Knepley     /* Conforming batches */
58558e3a2eefSMatthew G. Knepley     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
58568e3a2eefSMatthew G. Knepley     /* Remainder */
58578e3a2eefSMatthew G. Knepley     PetscInt Nr, offset, Nq;
58588e3a2eefSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
58598e3a2eefSMatthew G. Knepley     PetscInt    maxDegree;
58608e3a2eefSMatthew G. Knepley     PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
58618e3a2eefSMatthew G. Knepley 
58629566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
58639566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
58649566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
58659566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
58669566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
58679566063dSJacob Faibussowitsch     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&qGeom));
58688e3a2eefSMatthew G. Knepley     if (!qGeom) {
58699566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe,&qGeom));
58709566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
58718e3a2eefSMatthew G. Knepley     }
58729566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
58739566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM));
58748e3a2eefSMatthew G. Knepley     blockSize = Nb;
58758e3a2eefSMatthew G. Knepley     batchSize = numBlocks * blockSize;
58769566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
58778e3a2eefSMatthew G. Knepley     numChunks = numCells / (numBatches*batchSize);
58788e3a2eefSMatthew G. Knepley     Ne        = numChunks*numBatches*batchSize;
58798e3a2eefSMatthew G. Knepley     Nr        = numCells % (numBatches*batchSize);
58808e3a2eefSMatthew G. Knepley     offset    = numCells - Nr;
58819566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom));
58829566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&remGeom));
58838e3a2eefSMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
58848e3a2eefSMatthew G. Knepley       key.field = fieldI*Nf + fieldJ;
58859566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
58869566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMat[offset*totDim*totDim]));
58878e3a2eefSMatthew G. Knepley       if (hasDyn) {
58889566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
58899566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatD[offset*totDim*totDim]));
58908e3a2eefSMatthew G. Knepley       }
58918e3a2eefSMatthew G. Knepley     }
58929566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&remGeom));
58939566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,0,offset,&chunkGeom));
58949566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM));
58959566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
58968e3a2eefSMatthew G. Knepley   }
58978e3a2eefSMatthew G. Knepley   if (hasDyn) {
58988e3a2eefSMatthew G. Knepley     for (c = 0; c < numCells*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];
58998e3a2eefSMatthew G. Knepley   }
59008e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
59018e3a2eefSMatthew G. Knepley     const PetscInt     cell = cells ? cells[c] : c;
59028e3a2eefSMatthew G. Knepley     const PetscInt     cind = c - cStart;
59038e3a2eefSMatthew G. Knepley     const PetscBLASInt M = totDim, one = 1;
59048e3a2eefSMatthew G. Knepley     const PetscScalar  a = 1.0, b = 0.0;
59058e3a2eefSMatthew G. Knepley 
59068e3a2eefSMatthew G. Knepley     PetscStackCallBLAS("BLASgemv", BLASgemv_("N", &M, &M, &a, &elemMat[cind*totDim*totDim], &M, &y[cind*totDim], &one, &b, z, &one));
59078e3a2eefSMatthew G. Knepley     if (mesh->printFEM > 1) {
59089566063dSJacob Faibussowitsch       PetscCall(DMPrintCellMatrix(c, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
59099566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Y",  totDim, &y[cind*totDim]));
59109566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Z",  totDim, z));
59118e3a2eefSMatthew G. Knepley     }
59129566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dm, section, Z, cell, z, ADD_VALUES));
59138e3a2eefSMatthew G. Knepley   }
59149566063dSJacob Faibussowitsch   PetscCall(PetscFree6(u,u_t,elemMat,elemMatD,y,z));
59158e3a2eefSMatthew G. Knepley   if (mesh->printFEM) {
59169566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)Z), "Z:\n"));
59179566063dSJacob Faibussowitsch     PetscCall(VecView(Z, NULL));
59188e3a2eefSMatthew G. Knepley   }
5919*1059d808SMatthew G. Knepley   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
59209566063dSJacob Faibussowitsch   PetscCall(PetscFree(a));
59219566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cellIS));
59229566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexAux));
59239566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
59249566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0));
59258e3a2eefSMatthew G. Knepley   PetscFunctionReturn(0);
59268e3a2eefSMatthew G. Knepley }
5927