xref: /petsc/src/dm/impls/plex/plexfem.c (revision 1dca8a0504492127e77eac64bc165d7372dd6d63)
1af0996ceSBarry Smith #include <petsc/private/dmpleximpl.h>   /*I      "petscdmplex.h"   I*/
2da97024aSMatthew G. Knepley #include <petscsf.h>
3cb1e1211SMatthew G Knepley 
48e3a2eefSMatthew G. Knepley #include <petscblaslapack.h>
5e8f14785SLisandro Dalcin #include <petsc/private/hashsetij.h>
6af0996ceSBarry Smith #include <petsc/private/petscfeimpl.h>
7af0996ceSBarry Smith #include <petsc/private/petscfvimpl.h>
8a0845e3aSMatthew G. Knepley 
95f0b18bfSMatthew G. Knepley PetscBool Clementcite = PETSC_FALSE;
105f0b18bfSMatthew G. Knepley const char ClementCitation[] = "@article{clement1975approximation,\n"
115f0b18bfSMatthew G. Knepley                                "  title   = {Approximation by finite element functions using local regularization},\n"
125f0b18bfSMatthew G. Knepley                                "  author  = {Philippe Cl{\\'e}ment},\n"
135f0b18bfSMatthew G. Knepley                                "  journal = {Revue fran{\\c{c}}aise d'automatique, informatique, recherche op{\\'e}rationnelle. Analyse num{\\'e}rique},\n"
145f0b18bfSMatthew G. Knepley                                "  volume  = {9},\n"
155f0b18bfSMatthew G. Knepley                                "  number  = {R2},\n"
165f0b18bfSMatthew G. Knepley                                "  pages   = {77--84},\n"
175f0b18bfSMatthew G. Knepley                                "  year    = {1975}\n}\n";
185f0b18bfSMatthew G. Knepley 
192f856554SMatthew G. Knepley static PetscErrorCode DMPlexConvertPlex(DM dm, DM *plex, PetscBool copy)
202f856554SMatthew G. Knepley {
212f856554SMatthew G. Knepley   PetscBool      isPlex;
222f856554SMatthew G. Knepley 
232f856554SMatthew G. Knepley   PetscFunctionBegin;
249566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject) dm, DMPLEX, &isPlex));
252f856554SMatthew G. Knepley   if (isPlex) {
262f856554SMatthew G. Knepley     *plex = dm;
279566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject) dm));
282f856554SMatthew G. Knepley   } else {
299566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject) dm, "dm_plex", (PetscObject *) plex));
302f856554SMatthew G. Knepley     if (!*plex) {
319566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm, DMPLEX, plex));
329566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject) dm, "dm_plex", (PetscObject) *plex));
332f856554SMatthew G. Knepley       if (copy) {
342f856554SMatthew G. Knepley         DMSubDomainHookLink link;
359a2a23afSMatthew G. Knepley 
369566063dSJacob Faibussowitsch         PetscCall(DMCopyAuxiliaryVec(dm, *plex));
379a2a23afSMatthew G. Knepley         /* Run the subdomain hook (this will copy the DMSNES/DMTS) */
382f856554SMatthew G. Knepley         for (link = dm->subdomainhook; link; link = link->next) {
399566063dSJacob Faibussowitsch           if (link->ddhook) PetscCall((*link->ddhook)(dm, *plex, link->ctx));
402f856554SMatthew G. Knepley         }
412f856554SMatthew G. Knepley       }
422f856554SMatthew G. Knepley     } else {
439566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject) *plex));
442f856554SMatthew G. Knepley     }
452f856554SMatthew G. Knepley   }
462f856554SMatthew G. Knepley   PetscFunctionReturn(0);
472f856554SMatthew G. Knepley }
482f856554SMatthew G. Knepley 
499b6f715bSMatthew G. Knepley static PetscErrorCode PetscContainerUserDestroy_PetscFEGeom (void *ctx)
509b6f715bSMatthew G. Knepley {
519b6f715bSMatthew G. Knepley   PetscFEGeom *geom = (PetscFEGeom *) ctx;
529b6f715bSMatthew G. Knepley 
539b6f715bSMatthew G. Knepley   PetscFunctionBegin;
549566063dSJacob Faibussowitsch   PetscCall(PetscFEGeomDestroy(&geom));
559b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
569b6f715bSMatthew G. Knepley }
579b6f715bSMatthew G. Knepley 
589b6f715bSMatthew G. Knepley static PetscErrorCode DMPlexGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
599b6f715bSMatthew G. Knepley {
609b6f715bSMatthew G. Knepley   char            composeStr[33] = {0};
619b6f715bSMatthew G. Knepley   PetscObjectId   id;
629b6f715bSMatthew G. Knepley   PetscContainer  container;
639b6f715bSMatthew G. Knepley 
649b6f715bSMatthew G. Knepley   PetscFunctionBegin;
659566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad,&id));
6663a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMPlexGetFEGeom_%" PetscInt64_FMT "\n", id));
679566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject) pointIS, composeStr, (PetscObject *) &container));
689b6f715bSMatthew G. Knepley   if (container) {
699566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **) geom));
709b6f715bSMatthew G. Knepley   } else {
719566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom));
729566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF,&container));
739566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *) *geom));
749566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom));
759566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject) pointIS, composeStr, (PetscObject) container));
769566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
779b6f715bSMatthew G. Knepley   }
789b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
799b6f715bSMatthew G. Knepley }
809b6f715bSMatthew G. Knepley 
819b6f715bSMatthew G. Knepley static PetscErrorCode DMPlexRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
829b6f715bSMatthew G. Knepley {
839b6f715bSMatthew G. Knepley   PetscFunctionBegin;
849b6f715bSMatthew G. Knepley   *geom = NULL;
859b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
869b6f715bSMatthew G. Knepley }
879b6f715bSMatthew G. Knepley 
8846fa42a0SMatthew G. Knepley /*@
8946fa42a0SMatthew G. Knepley   DMPlexGetScale - Get the scale for the specified fundamental unit
9046fa42a0SMatthew G. Knepley 
9146fa42a0SMatthew G. Knepley   Not collective
9246fa42a0SMatthew G. Knepley 
934165533cSJose E. Roman   Input Parameters:
9446fa42a0SMatthew G. Knepley + dm   - the DM
9546fa42a0SMatthew G. Knepley - unit - The SI unit
9646fa42a0SMatthew G. Knepley 
974165533cSJose E. Roman   Output Parameter:
9846fa42a0SMatthew G. Knepley . scale - The value used to scale all quantities with this unit
9946fa42a0SMatthew G. Knepley 
10046fa42a0SMatthew G. Knepley   Level: advanced
10146fa42a0SMatthew G. Knepley 
10246fa42a0SMatthew G. Knepley .seealso: DMPlexSetScale(), PetscUnit
10346fa42a0SMatthew G. Knepley @*/
104cb1e1211SMatthew G Knepley PetscErrorCode DMPlexGetScale(DM dm, PetscUnit unit, PetscReal *scale)
105cb1e1211SMatthew G Knepley {
106cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex*) dm->data;
107cb1e1211SMatthew G Knepley 
108cb1e1211SMatthew G Knepley   PetscFunctionBegin;
109cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
110dadcf809SJacob Faibussowitsch   PetscValidRealPointer(scale, 3);
111cb1e1211SMatthew G Knepley   *scale = mesh->scale[unit];
112cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
113cb1e1211SMatthew G Knepley }
114cb1e1211SMatthew G Knepley 
11546fa42a0SMatthew G. Knepley /*@
11646fa42a0SMatthew G. Knepley   DMPlexSetScale - Set the scale for the specified fundamental unit
11746fa42a0SMatthew G. Knepley 
11846fa42a0SMatthew G. Knepley   Not collective
11946fa42a0SMatthew G. Knepley 
1204165533cSJose E. Roman   Input Parameters:
12146fa42a0SMatthew G. Knepley + dm   - the DM
12246fa42a0SMatthew G. Knepley . unit - The SI unit
12346fa42a0SMatthew G. Knepley - scale - The value used to scale all quantities with this unit
12446fa42a0SMatthew G. Knepley 
12546fa42a0SMatthew G. Knepley   Level: advanced
12646fa42a0SMatthew G. Knepley 
12746fa42a0SMatthew G. Knepley .seealso: DMPlexGetScale(), PetscUnit
12846fa42a0SMatthew G. Knepley @*/
129cb1e1211SMatthew G Knepley PetscErrorCode DMPlexSetScale(DM dm, PetscUnit unit, PetscReal scale)
130cb1e1211SMatthew G Knepley {
131cb1e1211SMatthew G Knepley   DM_Plex *mesh = (DM_Plex*) dm->data;
132cb1e1211SMatthew G Knepley 
133cb1e1211SMatthew G Knepley   PetscFunctionBegin;
134cb1e1211SMatthew G Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
135cb1e1211SMatthew G Knepley   mesh->scale[unit] = scale;
136cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
137cb1e1211SMatthew G Knepley }
138cb1e1211SMatthew G Knepley 
139327779f2SMatthew Knepley static PetscErrorCode DMPlexProjectRigidBody_Private(PetscInt dim, PetscReal t, const PetscReal X[], PetscInt Nc, PetscScalar *mode, void *ctx)
140026175e5SToby Isaac {
14112adca46SMatthew G. Knepley   const PetscInt eps[3][3][3] = {{{0, 0, 0}, {0, 0, 1}, {0, -1, 0}}, {{0, 0, -1}, {0, 0, 0}, {1, 0, 0}}, {{0, 1, 0}, {-1, 0, 0}, {0, 0, 0}}};
142026175e5SToby Isaac   PetscInt *ctxInt  = (PetscInt *) ctx;
143ad917190SMatthew G. Knepley   PetscInt  dim2    = ctxInt[0];
144026175e5SToby Isaac   PetscInt  d       = ctxInt[1];
145026175e5SToby Isaac   PetscInt  i, j, k = dim > 2 ? d - dim : d;
146026175e5SToby Isaac 
147ad917190SMatthew G. Knepley   PetscFunctionBegin;
14863a3b9bcSJacob Faibussowitsch   PetscCheck(dim == dim2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Input dimension %" PetscInt_FMT " does not match context dimension %" PetscInt_FMT, dim, dim2);
149026175e5SToby Isaac   for (i = 0; i < dim; i++) mode[i] = 0.;
150026175e5SToby Isaac   if (d < dim) {
15112adca46SMatthew G. Knepley     mode[d] = 1.; /* Translation along axis d */
152ad917190SMatthew G. Knepley   } else {
153026175e5SToby Isaac     for (i = 0; i < dim; i++) {
154026175e5SToby Isaac       for (j = 0; j < dim; j++) {
15512adca46SMatthew G. Knepley         mode[j] += eps[i][j][k]*X[i]; /* Rotation about axis d */
156026175e5SToby Isaac       }
157026175e5SToby Isaac     }
158026175e5SToby Isaac   }
159ad917190SMatthew G. Knepley   PetscFunctionReturn(0);
160026175e5SToby Isaac }
161026175e5SToby Isaac 
162cc4e42d9SMartin Diehl /*@
16312adca46SMatthew G. Knepley   DMPlexCreateRigidBody - For the default global section, create rigid body modes by function space interpolation
164cb1e1211SMatthew G Knepley 
165d083f849SBarry Smith   Collective on dm
166cb1e1211SMatthew G Knepley 
1674165533cSJose E. Roman   Input Parameters:
16856cf3b9cSMatthew G. Knepley + dm - the DM
16956cf3b9cSMatthew G. Knepley - field - The field number for the rigid body space, or 0 for the default
170cb1e1211SMatthew G Knepley 
1714165533cSJose E. Roman   Output Parameter:
172cb1e1211SMatthew G Knepley . sp - the null space
173cb1e1211SMatthew G Knepley 
17412adca46SMatthew G. Knepley   Note: This is necessary to provide a suitable coarse space for algebraic multigrid
175cb1e1211SMatthew G Knepley 
176cb1e1211SMatthew G Knepley   Level: advanced
177cb1e1211SMatthew G Knepley 
17812adca46SMatthew G. Knepley .seealso: MatNullSpaceCreate(), PCGAMG
179cb1e1211SMatthew G Knepley @*/
18056cf3b9cSMatthew G. Knepley PetscErrorCode DMPlexCreateRigidBody(DM dm, PetscInt field, MatNullSpace *sp)
181cb1e1211SMatthew G Knepley {
18256cf3b9cSMatthew G. Knepley   PetscErrorCode (**func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *);
183cb1e1211SMatthew G Knepley   MPI_Comm          comm;
184026175e5SToby Isaac   Vec               mode[6];
185026175e5SToby Isaac   PetscSection      section, globalSection;
18656cf3b9cSMatthew G. Knepley   PetscInt          dim, dimEmbed, Nf, n, m, mmin, d, i, j;
187cb1e1211SMatthew G Knepley 
188cb1e1211SMatthew G Knepley   PetscFunctionBegin;
1899566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject) dm, &comm));
1909566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
1919566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
1929566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
19363a3b9bcSJacob Faibussowitsch   PetscCheck(!Nf || !(field < 0 || field >= Nf),comm, PETSC_ERR_ARG_OUTOFRANGE, "Field %" PetscInt_FMT " is not in [0, %" PetscInt_FMT ")", field, Nf);
19456cf3b9cSMatthew G. Knepley   if (dim == 1 && Nf < 2) {
1959566063dSJacob Faibussowitsch     PetscCall(MatNullSpaceCreate(comm, PETSC_TRUE, 0, NULL, sp));
196cb1e1211SMatthew G Knepley     PetscFunctionReturn(0);
197cb1e1211SMatthew G Knepley   }
1989566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
1999566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
2009566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
2019566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(Nf, &func));
202b247467aSMatthew G. Knepley   m    = (dim*(dim+1))/2;
2039566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
2049566063dSJacob Faibussowitsch   PetscCall(VecSetType(mode[0], dm->vectype));
2059566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
2069566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
2079566063dSJacob Faibussowitsch   PetscCall(VecGetSize(mode[0], &n));
208b247467aSMatthew G. Knepley   mmin = PetscMin(m, n);
20956cf3b9cSMatthew G. Knepley   func[field] = DMPlexProjectRigidBody_Private;
2109566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
211026175e5SToby Isaac   for (d = 0; d < m; d++) {
212330485fdSToby Isaac     PetscInt ctx[2];
213026175e5SToby Isaac     void    *voidctx = (void *) (&ctx[0]);
214cb1e1211SMatthew G Knepley 
2159d8fbdeaSMatthew G. Knepley     ctx[0] = dimEmbed;
216330485fdSToby Isaac     ctx[1] = d;
2179566063dSJacob Faibussowitsch     PetscCall(DMProjectFunction(dm, 0.0, func, &voidctx, INSERT_VALUES, mode[d]));
218cb1e1211SMatthew G Knepley   }
2193b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
220b50a2c0aSJacob Faibussowitsch   for (i = 0; i < mmin; ++i) {
221b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
222b50a2c0aSJacob Faibussowitsch 
2239566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
2249566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], mmin-i-1, mode+i+1, dots+i+1));
225b50a2c0aSJacob Faibussowitsch     for (j = i+1; j < mmin; ++j) {
226b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
2279566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
228b50a2c0aSJacob Faibussowitsch     }
229cb1e1211SMatthew G Knepley   }
2309566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, mmin, mode, sp));
2319566063dSJacob Faibussowitsch   for (i = 0; i < m; ++i) PetscCall(VecDestroy(&mode[i]));
2329566063dSJacob Faibussowitsch   PetscCall(PetscFree(func));
233cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
234cb1e1211SMatthew G Knepley }
235cb1e1211SMatthew G Knepley 
236cc4e42d9SMartin Diehl /*@
23712adca46SMatthew G. Knepley   DMPlexCreateRigidBodies - For the default global section, create rigid body modes by function space interpolation
23812adca46SMatthew G. Knepley 
239d083f849SBarry Smith   Collective on dm
24012adca46SMatthew G. Knepley 
2414165533cSJose E. Roman   Input Parameters:
24212adca46SMatthew G. Knepley + dm    - the DM
24312adca46SMatthew G. Knepley . nb    - The number of bodies
24412adca46SMatthew G. Knepley . label - The DMLabel marking each domain
24512adca46SMatthew G. Knepley . nids  - The number of ids per body
24612adca46SMatthew G. Knepley - ids   - An array of the label ids in sequence for each domain
24712adca46SMatthew G. Knepley 
2484165533cSJose E. Roman   Output Parameter:
24912adca46SMatthew G. Knepley . sp - the null space
25012adca46SMatthew G. Knepley 
25112adca46SMatthew G. Knepley   Note: This is necessary to provide a suitable coarse space for algebraic multigrid
25212adca46SMatthew G. Knepley 
25312adca46SMatthew G. Knepley   Level: advanced
25412adca46SMatthew G. Knepley 
25512adca46SMatthew G. Knepley .seealso: MatNullSpaceCreate()
25612adca46SMatthew G. Knepley @*/
25712adca46SMatthew G. Knepley PetscErrorCode DMPlexCreateRigidBodies(DM dm, PetscInt nb, DMLabel label, const PetscInt nids[], const PetscInt ids[], MatNullSpace *sp)
25812adca46SMatthew G. Knepley {
25912adca46SMatthew G. Knepley   MPI_Comm       comm;
26012adca46SMatthew G. Knepley   PetscSection   section, globalSection;
26112adca46SMatthew G. Knepley   Vec           *mode;
26212adca46SMatthew G. Knepley   PetscScalar   *dots;
26312adca46SMatthew G. Knepley   PetscInt       dim, dimEmbed, n, m, b, d, i, j, off;
26412adca46SMatthew G. Knepley 
26512adca46SMatthew G. Knepley   PetscFunctionBegin;
2669566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject)dm,&comm));
2679566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
2689566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dimEmbed));
2699566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
2709566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
2719566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(globalSection, &n));
27212adca46SMatthew G. Knepley   m    = nb * (dim*(dim+1))/2;
2739566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(m, &mode, m, &dots));
2749566063dSJacob Faibussowitsch   PetscCall(VecCreate(comm, &mode[0]));
2759566063dSJacob Faibussowitsch   PetscCall(VecSetSizes(mode[0], n, PETSC_DETERMINE));
2769566063dSJacob Faibussowitsch   PetscCall(VecSetUp(mode[0]));
2779566063dSJacob Faibussowitsch   for (i = 1; i < m; ++i) PetscCall(VecDuplicate(mode[0], &mode[i]));
27812adca46SMatthew G. Knepley   for (b = 0, off = 0; b < nb; ++b) {
27912adca46SMatthew G. Knepley     for (d = 0; d < m/nb; ++d) {
28012adca46SMatthew G. Knepley       PetscInt         ctx[2];
28112adca46SMatthew G. Knepley       PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal *, PetscInt, PetscScalar *, void *) = DMPlexProjectRigidBody_Private;
28212adca46SMatthew G. Knepley       void            *voidctx = (void *) (&ctx[0]);
28312adca46SMatthew G. Knepley 
28412adca46SMatthew G. Knepley       ctx[0] = dimEmbed;
28512adca46SMatthew G. Knepley       ctx[1] = d;
2869566063dSJacob Faibussowitsch       PetscCall(DMProjectFunctionLabel(dm, 0.0, label, nids[b], &ids[off], 0, NULL, &func, &voidctx, INSERT_VALUES, mode[d]));
28712adca46SMatthew G. Knepley       off   += nids[b];
28812adca46SMatthew G. Knepley     }
28912adca46SMatthew G. Knepley   }
2903b2202bfSJacob Faibussowitsch   /* Orthonormalize system */
291606c1a1cSJacob Faibussowitsch   for (i = 0; i < m; ++i) {
292b77f2eeeSJacob Faibussowitsch     PetscScalar dots[6];
2935a0e29b9SJacob Faibussowitsch 
2949566063dSJacob Faibussowitsch     PetscCall(VecNormalize(mode[i], NULL));
2959566063dSJacob Faibussowitsch     PetscCall(VecMDot(mode[i], m-i-1, mode+i+1, dots+i+1));
2965a0e29b9SJacob Faibussowitsch     for (j = i+1; j < m; ++j) {
297b77f2eeeSJacob Faibussowitsch       dots[j] *= -1.0;
2989566063dSJacob Faibussowitsch       PetscCall(VecAXPY(mode[j], dots[j], mode[i]));
2995a0e29b9SJacob Faibussowitsch     }
30012adca46SMatthew G. Knepley   }
3019566063dSJacob Faibussowitsch   PetscCall(MatNullSpaceCreate(comm, PETSC_FALSE, m, mode, sp));
3029566063dSJacob Faibussowitsch   for (i = 0; i< m; ++i) PetscCall(VecDestroy(&mode[i]));
3039566063dSJacob Faibussowitsch   PetscCall(PetscFree2(mode, dots));
30412adca46SMatthew G. Knepley   PetscFunctionReturn(0);
30512adca46SMatthew G. Knepley }
30612adca46SMatthew G. Knepley 
307b29cfa1cSToby Isaac /*@
308b29cfa1cSToby Isaac   DMPlexSetMaxProjectionHeight - In DMPlexProjectXXXLocal() functions, the projected values of a basis function's dofs
309b29cfa1cSToby Isaac   are computed by associating the basis function with one of the mesh points in its transitively-closed support, and
310b29cfa1cSToby Isaac   evaluating the dual space basis of that point.  A basis function is associated with the point in its
311b29cfa1cSToby Isaac   transitively-closed support whose mesh height is highest (w.r.t. DAG height), but not greater than the maximum
312b29cfa1cSToby Isaac   projection height, which is set with this function.  By default, the maximum projection height is zero, which means
313b29cfa1cSToby Isaac   that only mesh cells are used to project basis functions.  A height of one, for example, evaluates a cell-interior
314b29cfa1cSToby Isaac   basis functions using its cells dual space basis, but all other basis functions with the dual space basis of a face.
315b29cfa1cSToby Isaac 
316b29cfa1cSToby Isaac   Input Parameters:
317b29cfa1cSToby Isaac + dm - the DMPlex object
318b29cfa1cSToby Isaac - height - the maximum projection height >= 0
319b29cfa1cSToby Isaac 
320b29cfa1cSToby Isaac   Level: advanced
321b29cfa1cSToby Isaac 
3224d6f44ffSToby Isaac .seealso: DMPlexGetMaxProjectionHeight(), DMProjectFunctionLocal(), DMProjectFunctionLabelLocal()
323b29cfa1cSToby Isaac @*/
324b29cfa1cSToby Isaac PetscErrorCode DMPlexSetMaxProjectionHeight(DM dm, PetscInt height)
325b29cfa1cSToby Isaac {
326b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *) dm->data;
327b29cfa1cSToby Isaac 
328b29cfa1cSToby Isaac   PetscFunctionBegin;
329b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
330b29cfa1cSToby Isaac   plex->maxProjectionHeight = height;
331b29cfa1cSToby Isaac   PetscFunctionReturn(0);
332b29cfa1cSToby Isaac }
333b29cfa1cSToby Isaac 
334b29cfa1cSToby Isaac /*@
335b29cfa1cSToby Isaac   DMPlexGetMaxProjectionHeight - Get the maximum height (w.r.t. DAG) of mesh points used to evaluate dual bases in
336b29cfa1cSToby Isaac   DMPlexProjectXXXLocal() functions.
337b29cfa1cSToby Isaac 
338b29cfa1cSToby Isaac   Input Parameters:
339b29cfa1cSToby Isaac . dm - the DMPlex object
340b29cfa1cSToby Isaac 
341b29cfa1cSToby Isaac   Output Parameters:
342b29cfa1cSToby Isaac . height - the maximum projection height
343b29cfa1cSToby Isaac 
344b29cfa1cSToby Isaac   Level: intermediate
345b29cfa1cSToby Isaac 
3464d6f44ffSToby Isaac .seealso: DMPlexSetMaxProjectionHeight(), DMProjectFunctionLocal(), DMProjectFunctionLabelLocal()
347b29cfa1cSToby Isaac @*/
348b29cfa1cSToby Isaac PetscErrorCode DMPlexGetMaxProjectionHeight(DM dm, PetscInt *height)
349b29cfa1cSToby Isaac {
350b29cfa1cSToby Isaac   DM_Plex *plex = (DM_Plex *) dm->data;
351b29cfa1cSToby Isaac 
352b29cfa1cSToby Isaac   PetscFunctionBegin;
353b29cfa1cSToby Isaac   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
354b29cfa1cSToby Isaac   *height = plex->maxProjectionHeight;
355b29cfa1cSToby Isaac   PetscFunctionReturn(0);
356b29cfa1cSToby Isaac }
357b29cfa1cSToby Isaac 
358ca3d3a14SMatthew G. Knepley typedef struct {
359ca3d3a14SMatthew G. Knepley   PetscReal    alpha; /* The first Euler angle, and in 2D the only one */
360ca3d3a14SMatthew G. Knepley   PetscReal    beta;  /* The second Euler angle */
361ca3d3a14SMatthew G. Knepley   PetscReal    gamma; /* The third Euler angle */
362ca3d3a14SMatthew G. Knepley   PetscInt     dim;   /* The dimension of R */
363ca3d3a14SMatthew G. Knepley   PetscScalar *R;     /* The rotation matrix, transforming a vector in the local basis to the global basis */
364ca3d3a14SMatthew G. Knepley   PetscScalar *RT;    /* The transposed rotation matrix, transforming a vector in the global basis to the local basis */
365ca3d3a14SMatthew G. Knepley } RotCtx;
366ca3d3a14SMatthew G. Knepley 
367ca3d3a14SMatthew G. Knepley /*
368ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
369ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
370ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
371ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
372ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
373ca3d3a14SMatthew G. Knepley */
374ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformSetUp_Rotation_Internal(DM dm, void *ctx)
375ca3d3a14SMatthew G. Knepley {
376ca3d3a14SMatthew G. Knepley   RotCtx        *rc  = (RotCtx *) ctx;
377ca3d3a14SMatthew G. Knepley   PetscInt       dim = rc->dim;
378ca3d3a14SMatthew G. Knepley   PetscReal      c1, s1, c2, s2, c3, s3;
379ca3d3a14SMatthew G. Knepley 
380ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
3819566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(PetscSqr(dim), &rc->R, PetscSqr(dim), &rc->RT));
382ca3d3a14SMatthew G. Knepley   switch (dim) {
383ca3d3a14SMatthew G. Knepley   case 2:
384ca3d3a14SMatthew G. Knepley     c1 = PetscCosReal(rc->alpha);s1 = PetscSinReal(rc->alpha);
385ca3d3a14SMatthew G. Knepley     rc->R[0] =  c1;rc->R[1] = s1;
386ca3d3a14SMatthew G. Knepley     rc->R[2] = -s1;rc->R[3] = c1;
3879566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
388b458e8f1SJose E. Roman     DMPlex_Transpose2D_Internal(rc->RT);
389ca3d3a14SMatthew G. Knepley     break;
390ca3d3a14SMatthew G. Knepley   case 3:
391ca3d3a14SMatthew G. Knepley     c1 = PetscCosReal(rc->alpha);s1 = PetscSinReal(rc->alpha);
392ca3d3a14SMatthew G. Knepley     c2 = PetscCosReal(rc->beta); s2 = PetscSinReal(rc->beta);
393ca3d3a14SMatthew G. Knepley     c3 = PetscCosReal(rc->gamma);s3 = PetscSinReal(rc->gamma);
394ca3d3a14SMatthew G. Knepley     rc->R[0] =  c1*c3 - c2*s1*s3;rc->R[1] =  c3*s1    + c1*c2*s3;rc->R[2] = s2*s3;
395ca3d3a14SMatthew G. Knepley     rc->R[3] = -c1*s3 - c2*c3*s1;rc->R[4] =  c1*c2*c3 - s1*s3;   rc->R[5] = c3*s2;
396ca3d3a14SMatthew G. Knepley     rc->R[6] =  s1*s2;           rc->R[7] = -c1*s2;              rc->R[8] = c2;
3979566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(rc->RT, rc->R, PetscSqr(dim)));
398b458e8f1SJose E. Roman     DMPlex_Transpose3D_Internal(rc->RT);
399ca3d3a14SMatthew G. Knepley     break;
40063a3b9bcSJacob Faibussowitsch   default: SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_OUTOFRANGE, "Dimension %" PetscInt_FMT " not supported", dim);
401ca3d3a14SMatthew G. Knepley   }
402ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
403ca3d3a14SMatthew G. Knepley }
404ca3d3a14SMatthew G. Knepley 
405ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformDestroy_Rotation_Internal(DM dm, void *ctx)
406ca3d3a14SMatthew G. Knepley {
407ca3d3a14SMatthew G. Knepley   RotCtx        *rc = (RotCtx *) ctx;
408ca3d3a14SMatthew G. Knepley 
409ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
4109566063dSJacob Faibussowitsch   PetscCall(PetscFree2(rc->R, rc->RT));
4119566063dSJacob Faibussowitsch   PetscCall(PetscFree(rc));
412ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
413ca3d3a14SMatthew G. Knepley }
414ca3d3a14SMatthew G. Knepley 
415ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformGetMatrix_Rotation_Internal(DM dm, const PetscReal x[], PetscBool l2g, const PetscScalar **A, void *ctx)
416ca3d3a14SMatthew G. Knepley {
417ca3d3a14SMatthew G. Knepley   RotCtx *rc = (RotCtx *) ctx;
418ca3d3a14SMatthew G. Knepley 
419ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
420ca3d3a14SMatthew G. Knepley   PetscValidPointer(ctx, 5);
421ca3d3a14SMatthew G. Knepley   if (l2g) {*A = rc->R;}
422ca3d3a14SMatthew G. Knepley   else     {*A = rc->RT;}
423ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
424ca3d3a14SMatthew G. Knepley }
425ca3d3a14SMatthew G. Knepley 
426ab6a9622SMatthew G. Knepley PetscErrorCode DMPlexBasisTransformApplyReal_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscReal *y, PetscReal *z, void *ctx)
427ab6a9622SMatthew G. Knepley {
428ec277c0fSMatthew G. Knepley   PetscFunctionBegin;
429ab6a9622SMatthew G. Knepley   #if defined(PETSC_USE_COMPLEX)
430ab6a9622SMatthew G. Knepley   switch (dim) {
431ab6a9622SMatthew G. Knepley     case 2:
432ab6a9622SMatthew G. Knepley     {
43327104ee2SJacob Faibussowitsch       PetscScalar yt[2] = {y[0], y[1]}, zt[2] = {0.0,0.0};
434ab6a9622SMatthew G. Knepley 
4359566063dSJacob Faibussowitsch       PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
436ab6a9622SMatthew G. Knepley       z[0] = PetscRealPart(zt[0]); z[1] = PetscRealPart(zt[1]);
437ab6a9622SMatthew G. Knepley     }
4389b4815dcSMatthew G. Knepley     break;
439ab6a9622SMatthew G. Knepley     case 3:
440ab6a9622SMatthew G. Knepley     {
44127104ee2SJacob Faibussowitsch       PetscScalar yt[3] = {y[0], y[1], y[2]}, zt[3] = {0.0,0.0,0.0};
442ab6a9622SMatthew G. Knepley 
4439566063dSJacob Faibussowitsch       PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, yt, zt, ctx));
444ab6a9622SMatthew G. Knepley       z[0] = PetscRealPart(zt[0]); z[1] = PetscRealPart(zt[1]); z[2] = PetscRealPart(zt[2]);
445ab6a9622SMatthew G. Knepley     }
4469b4815dcSMatthew G. Knepley     break;
447ab6a9622SMatthew G. Knepley   }
448ab6a9622SMatthew G. Knepley   #else
4499566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransformApply_Internal(dm, x, l2g, dim, y, z, ctx));
450ab6a9622SMatthew G. Knepley   #endif
451ec277c0fSMatthew G. Knepley   PetscFunctionReturn(0);
452ab6a9622SMatthew G. Knepley }
453ab6a9622SMatthew G. Knepley 
454ec277c0fSMatthew G. Knepley PetscErrorCode DMPlexBasisTransformApply_Internal(DM dm, const PetscReal x[], PetscBool l2g, PetscInt dim, const PetscScalar *y, PetscScalar *z, void *ctx)
455ca3d3a14SMatthew G. Knepley {
456ca3d3a14SMatthew G. Knepley   const PetscScalar *A;
457ca3d3a14SMatthew G. Knepley 
458ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
4599566063dSJacob Faibussowitsch   PetscCall((*dm->transformGetMatrix)(dm, x, l2g, &A, ctx));
460ca3d3a14SMatthew G. Knepley   switch (dim) {
4611fba962aSMatthew G. Knepley   case 2: DMPlex_Mult2D_Internal(A, 1, y, z);break;
4621fba962aSMatthew G. Knepley   case 3: DMPlex_Mult3D_Internal(A, 1, y, z);break;
463ca3d3a14SMatthew G. Knepley   }
464ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
465ca3d3a14SMatthew G. Knepley }
466ca3d3a14SMatthew G. Knepley 
467ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformField_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscInt f, PetscBool l2g, PetscScalar *a)
468ca3d3a14SMatthew G. Knepley {
469ca3d3a14SMatthew G. Knepley   PetscSection       ts;
470ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tva;
471ca3d3a14SMatthew G. Knepley   PetscInt           dof;
472ca3d3a14SMatthew G. Knepley 
473ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
4749566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
4759566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, p, f, &dof));
4769566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
4779566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, p, f, ta, &tva));
478ca3d3a14SMatthew G. Knepley   if (l2g) {
479ca3d3a14SMatthew G. Knepley     switch (dof) {
4801fba962aSMatthew G. Knepley     case 4: DMPlex_Mult2D_Internal(tva, 1, a, a);break;
4811fba962aSMatthew G. Knepley     case 9: DMPlex_Mult3D_Internal(tva, 1, a, a);break;
482ca3d3a14SMatthew G. Knepley     }
483ca3d3a14SMatthew G. Knepley   } else {
484ca3d3a14SMatthew G. Knepley     switch (dof) {
4851fba962aSMatthew G. Knepley     case 4: DMPlex_MultTranspose2D_Internal(tva, 1, a, a);break;
4861fba962aSMatthew G. Knepley     case 9: DMPlex_MultTranspose3D_Internal(tva, 1, a, a);break;
487ca3d3a14SMatthew G. Knepley     }
488ca3d3a14SMatthew G. Knepley   }
4899566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
490ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
491ca3d3a14SMatthew G. Knepley }
492ca3d3a14SMatthew G. Knepley 
493ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransformFieldTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt pf, PetscInt f, PetscInt pg, PetscInt g, PetscBool l2g, PetscInt lda, PetscScalar *a)
494ca3d3a14SMatthew G. Knepley {
495ca3d3a14SMatthew G. Knepley   PetscSection       s, ts;
496ca3d3a14SMatthew G. Knepley   const PetscScalar *ta, *tvaf, *tvag;
497ca3d3a14SMatthew G. Knepley   PetscInt           fdof, gdof, fpdof, gpdof;
498ca3d3a14SMatthew G. Knepley 
499ca3d3a14SMatthew G. Knepley   PetscFunctionBeginHot;
5009566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
5019566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
5029566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pf, f, &fpdof));
5039566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(s, pg, g, &gpdof));
5049566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pf, f, &fdof));
5059566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetFieldDof(ts, pg, g, &gdof));
5069566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
5079566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pf, f, ta, &tvaf));
5089566063dSJacob Faibussowitsch   PetscCall(DMPlexPointLocalFieldRead(tdm, pg, g, ta, &tvag));
509ca3d3a14SMatthew G. Knepley   if (l2g) {
510ca3d3a14SMatthew G. Knepley     switch (fdof) {
511ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMult2D_Internal(tvaf, gpdof, lda, a, a);break;
512ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMult3D_Internal(tvaf, gpdof, lda, a, a);break;
513ca3d3a14SMatthew G. Knepley     }
514ca3d3a14SMatthew G. Knepley     switch (gdof) {
515ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultTransposeLeft2D_Internal(tvag, fpdof, lda, a, a);break;
516ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultTransposeLeft3D_Internal(tvag, fpdof, lda, a, a);break;
517ca3d3a14SMatthew G. Knepley     }
518ca3d3a14SMatthew G. Knepley   } else {
519ca3d3a14SMatthew G. Knepley     switch (fdof) {
520ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultTranspose2D_Internal(tvaf, gpdof, lda, a, a);break;
521ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultTranspose3D_Internal(tvaf, gpdof, lda, a, a);break;
522ca3d3a14SMatthew G. Knepley     }
523ca3d3a14SMatthew G. Knepley     switch (gdof) {
524ca3d3a14SMatthew G. Knepley     case 4: DMPlex_MatMultLeft2D_Internal(tvag, fpdof, lda, a, a);break;
525ca3d3a14SMatthew G. Knepley     case 9: DMPlex_MatMultLeft3D_Internal(tvag, fpdof, lda, a, a);break;
526ca3d3a14SMatthew G. Knepley     }
527ca3d3a14SMatthew G. Knepley   }
5289566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
529ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
530ca3d3a14SMatthew G. Knepley }
531ca3d3a14SMatthew G. Knepley 
532ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexBasisTransformPoint_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool fieldActive[], PetscBool l2g, PetscScalar *a)
533ca3d3a14SMatthew G. Knepley {
534ca3d3a14SMatthew G. Knepley   PetscSection    s;
535ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
536ca3d3a14SMatthew G. Knepley   IS              clPoints;
537ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
538ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
539ca3d3a14SMatthew G. Knepley   PetscInt        Nf, f, Np, cp, dof, d = 0;
540ca3d3a14SMatthew G. Knepley 
541ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5429566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
5439566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
5449566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
545ca3d3a14SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
546ca3d3a14SMatthew G. Knepley     for (cp = 0; cp < Np*2; cp += 2) {
5479566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cp], f, &dof));
548ca3d3a14SMatthew G. Knepley       if (!dof) continue;
5499566063dSJacob Faibussowitsch       if (fieldActive[f]) PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, points[cp], f, l2g, &a[d]));
550ca3d3a14SMatthew G. Knepley       d += dof;
551ca3d3a14SMatthew G. Knepley     }
552ca3d3a14SMatthew G. Knepley   }
5539566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
554ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
555ca3d3a14SMatthew G. Knepley }
556ca3d3a14SMatthew G. Knepley 
557ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexBasisTransformPointTensor_Internal(DM dm, DM tdm, Vec tv, PetscInt p, PetscBool l2g, PetscInt lda, PetscScalar *a)
558ca3d3a14SMatthew G. Knepley {
559ca3d3a14SMatthew G. Knepley   PetscSection    s;
560ca3d3a14SMatthew G. Knepley   PetscSection    clSection;
561ca3d3a14SMatthew G. Knepley   IS              clPoints;
562ca3d3a14SMatthew G. Knepley   const PetscInt *clp;
563ca3d3a14SMatthew G. Knepley   PetscInt       *points = NULL;
5648bdb3c71SMatthew G. Knepley   PetscInt        Nf, f, g, Np, cpf, cpg, fdof, gdof, r, c = 0;
565ca3d3a14SMatthew G. Knepley 
566ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5679566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
5689566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
5699566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
570ca3d3a14SMatthew G. Knepley   for (f = 0, r = 0; f < Nf; ++f) {
571ca3d3a14SMatthew G. Knepley     for (cpf = 0; cpf < Np*2; cpf += 2) {
5729566063dSJacob Faibussowitsch       PetscCall(PetscSectionGetFieldDof(s, points[cpf], f, &fdof));
573ca3d3a14SMatthew G. Knepley       for (g = 0, c = 0; g < Nf; ++g) {
574ca3d3a14SMatthew G. Knepley         for (cpg = 0; cpg < Np*2; cpg += 2) {
5759566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldDof(s, points[cpg], g, &gdof));
5769566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformFieldTensor_Internal(dm, tdm, tv, points[cpf], f, points[cpg], g, l2g, lda, &a[r*lda+c]));
577ca3d3a14SMatthew G. Knepley           c += gdof;
578ca3d3a14SMatthew G. Knepley         }
579ca3d3a14SMatthew G. Knepley       }
58063a3b9bcSJacob Faibussowitsch       PetscCheck(c == lda,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of columns %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
581ca3d3a14SMatthew G. Knepley       r += fdof;
582ca3d3a14SMatthew G. Knepley     }
583ca3d3a14SMatthew G. Knepley   }
58463a3b9bcSJacob Faibussowitsch   PetscCheck(r == lda,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Invalid number of rows %" PetscInt_FMT " should be %" PetscInt_FMT, c, lda);
5859566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCompressedClosure(dm, s, p, &Np, &points, &clSection, &clPoints, &clp));
586ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
587ca3d3a14SMatthew G. Knepley }
588ca3d3a14SMatthew G. Knepley 
589ca3d3a14SMatthew G. Knepley static PetscErrorCode DMPlexBasisTransform_Internal(DM dm, Vec lv, PetscBool l2g)
590ca3d3a14SMatthew G. Knepley {
591ca3d3a14SMatthew G. Knepley   DM                 tdm;
592ca3d3a14SMatthew G. Knepley   Vec                tv;
593ca3d3a14SMatthew G. Knepley   PetscSection       ts, s;
594ca3d3a14SMatthew G. Knepley   const PetscScalar *ta;
595ca3d3a14SMatthew G. Knepley   PetscScalar       *a, *va;
596ca3d3a14SMatthew G. Knepley   PetscInt           pStart, pEnd, p, Nf, f;
597ca3d3a14SMatthew G. Knepley 
598ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
5999566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
6009566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
6019566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(tdm, &ts));
6029566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &s));
6039566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetChart(s, &pStart, &pEnd));
6049566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(s, &Nf));
6059566063dSJacob Faibussowitsch   PetscCall(VecGetArray(lv, &a));
6069566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(tv, &ta));
607ca3d3a14SMatthew G. Knepley   for (p = pStart; p < pEnd; ++p) {
608ca3d3a14SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
6099566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalFieldRef(dm, p, f, a, &va));
6109566063dSJacob Faibussowitsch       PetscCall(DMPlexBasisTransformField_Internal(dm, tdm, tv, p, f, l2g, va));
611ca3d3a14SMatthew G. Knepley     }
612ca3d3a14SMatthew G. Knepley   }
6139566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(lv, &a));
6149566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(tv, &ta));
615ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
616ca3d3a14SMatthew G. Knepley }
617ca3d3a14SMatthew G. Knepley 
618ca3d3a14SMatthew G. Knepley /*@
619ca3d3a14SMatthew G. Knepley   DMPlexGlobalToLocalBasis - Transform the values in the given local vector from the global basis to the local basis
620ca3d3a14SMatthew G. Knepley 
621ca3d3a14SMatthew G. Knepley   Input Parameters:
622ca3d3a14SMatthew G. Knepley + dm - The DM
623ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the global basis
624ca3d3a14SMatthew G. Knepley 
625ca3d3a14SMatthew G. Knepley   Output Parameters:
626ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the local basis
627ca3d3a14SMatthew G. Knepley 
628c0f8e1fdSMatthew G. Knepley   Note: This method is only intended to be called inside DMGlobalToLocal(). It is unlikely that a user will have a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
629c0f8e1fdSMatthew G. Knepley 
630ca3d3a14SMatthew G. Knepley   Level: developer
631ca3d3a14SMatthew G. Knepley 
632436bc73aSJed Brown .seealso: DMPlexLocalToGlobalBasis(), DMGetLocalSection(), DMPlexCreateBasisRotation()
633ca3d3a14SMatthew G. Knepley @*/
634ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexGlobalToLocalBasis(DM dm, Vec lv)
635ca3d3a14SMatthew G. Knepley {
636ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
637ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
638ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
6399566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_FALSE));
640ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
641ca3d3a14SMatthew G. Knepley }
642ca3d3a14SMatthew G. Knepley 
643ca3d3a14SMatthew G. Knepley /*@
644ca3d3a14SMatthew G. Knepley   DMPlexLocalToGlobalBasis - Transform the values in the given local vector from the local basis to the global basis
645ca3d3a14SMatthew G. Knepley 
646ca3d3a14SMatthew G. Knepley   Input Parameters:
647ca3d3a14SMatthew G. Knepley + dm - The DM
648ca3d3a14SMatthew G. Knepley - lv - A local vector with values in the local basis
649ca3d3a14SMatthew G. Knepley 
650ca3d3a14SMatthew G. Knepley   Output Parameters:
651ca3d3a14SMatthew G. Knepley . lv - A local vector with values in the global basis
652ca3d3a14SMatthew G. Knepley 
653c0f8e1fdSMatthew G. Knepley   Note: This method is only intended to be called inside DMGlobalToLocal(). It is unlikely that a user would want a local vector full of coefficients for the global basis unless they are reimplementing GlobalToLocal.
654c0f8e1fdSMatthew G. Knepley 
655ca3d3a14SMatthew G. Knepley   Level: developer
656ca3d3a14SMatthew G. Knepley 
657436bc73aSJed Brown .seealso: DMPlexGlobalToLocalBasis(), DMGetLocalSection(), DMPlexCreateBasisRotation()
658ca3d3a14SMatthew G. Knepley @*/
659ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexLocalToGlobalBasis(DM dm, Vec lv)
660ca3d3a14SMatthew G. Knepley {
661ca3d3a14SMatthew G. Knepley   PetscFunctionBegin;
662ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
663ca3d3a14SMatthew G. Knepley   PetscValidHeaderSpecific(lv, VEC_CLASSID, 2);
6649566063dSJacob Faibussowitsch   PetscCall(DMPlexBasisTransform_Internal(dm, lv, PETSC_TRUE));
665ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
666ca3d3a14SMatthew G. Knepley }
667ca3d3a14SMatthew G. Knepley 
668ca3d3a14SMatthew G. Knepley /*@
669ca3d3a14SMatthew G. Knepley   DMPlexCreateBasisRotation - Create an internal transformation from the global basis, used to specify boundary conditions
670ca3d3a14SMatthew G. Knepley     and global solutions, to a local basis, appropriate for discretization integrals and assembly.
671ca3d3a14SMatthew G. Knepley 
672ca3d3a14SMatthew G. Knepley   Input Parameters:
673ca3d3a14SMatthew G. Knepley + dm    - The DM
674ca3d3a14SMatthew G. Knepley . alpha - The first Euler angle, and in 2D the only one
675ca3d3a14SMatthew G. Knepley . beta  - The second Euler angle
676f0fc11ceSJed Brown - gamma - The third Euler angle
677ca3d3a14SMatthew G. Knepley 
678ca3d3a14SMatthew G. Knepley   Note: Following https://en.wikipedia.org/wiki/Euler_angles, we will specify Euler angles by extrinsic rotations, meaning that
679ca3d3a14SMatthew G. Knepley   we rotate with respect to a fixed initial coordinate system, the local basis (x-y-z). The global basis (X-Y-Z) is reached as follows:
680ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates about the z axis by alpha. The X axis is now at angle alpha with respect to the x axis.
681ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates again about the x axis by beta. The Z axis is now at angle beta with respect to the z axis.
682ca3d3a14SMatthew G. Knepley   $ The XYZ system rotates a third time about the z axis by gamma.
683ca3d3a14SMatthew G. Knepley 
684ca3d3a14SMatthew G. Knepley   Level: developer
685ca3d3a14SMatthew G. Knepley 
686ca3d3a14SMatthew G. Knepley .seealso: DMPlexGlobalToLocalBasis(), DMPlexLocalToGlobalBasis()
687ca3d3a14SMatthew G. Knepley @*/
688ca3d3a14SMatthew G. Knepley PetscErrorCode DMPlexCreateBasisRotation(DM dm, PetscReal alpha, PetscReal beta, PetscReal gamma)
689ca3d3a14SMatthew G. Knepley {
690ca3d3a14SMatthew G. Knepley   RotCtx        *rc;
691ca3d3a14SMatthew G. Knepley   PetscInt       cdim;
692ca3d3a14SMatthew G. Knepley 
693362febeeSStefano Zampini   PetscFunctionBegin;
6949566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
6959566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(1, &rc));
696ca3d3a14SMatthew G. Knepley   dm->transformCtx       = rc;
697ca3d3a14SMatthew G. Knepley   dm->transformSetUp     = DMPlexBasisTransformSetUp_Rotation_Internal;
698ca3d3a14SMatthew G. Knepley   dm->transformDestroy   = DMPlexBasisTransformDestroy_Rotation_Internal;
699ca3d3a14SMatthew G. Knepley   dm->transformGetMatrix = DMPlexBasisTransformGetMatrix_Rotation_Internal;
700ca3d3a14SMatthew G. Knepley   rc->dim   = cdim;
701ca3d3a14SMatthew G. Knepley   rc->alpha = alpha;
702ca3d3a14SMatthew G. Knepley   rc->beta  = beta;
703ca3d3a14SMatthew G. Knepley   rc->gamma = gamma;
7049566063dSJacob Faibussowitsch   PetscCall((*dm->transformSetUp)(dm, dm->transformCtx));
7059566063dSJacob Faibussowitsch   PetscCall(DMConstructBasisTransform_Internal(dm));
706ca3d3a14SMatthew G. Knepley   PetscFunctionReturn(0);
707ca3d3a14SMatthew G. Knepley }
708ca3d3a14SMatthew G. Knepley 
709b278463cSMatthew G. Knepley /*@C
710ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssential - Insert boundary values into a local vector using a function of the coordinates
711b278463cSMatthew G. Knepley 
712b278463cSMatthew G. Knepley   Input Parameters:
713b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
714b278463cSMatthew G. Knepley . time   - The time
715b278463cSMatthew G. Knepley . field  - The field to constrain
7161c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
7171c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
718b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
719b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
720b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
721b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
722b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
723b278463cSMatthew G. Knepley 
724b278463cSMatthew G. Knepley   Output Parameter:
725b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
726b278463cSMatthew G. Knepley 
727b278463cSMatthew G. Knepley   Level: developer
728b278463cSMatthew G. Knepley 
729ece3a9fcSMatthew G. Knepley .seealso: DMPlexInsertBoundaryValuesEssentialField(), DMPlexInsertBoundaryValuesEssentialBdField(), DMAddBoundary()
730b278463cSMatthew G. Knepley @*/
7311c531cf8SMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesEssential(DM dm, PetscReal time, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[], PetscErrorCode (*func)(PetscInt, PetscReal, const PetscReal[], PetscInt, PetscScalar *, void *), void *ctx, Vec locX)
73255f2e967SMatthew G. Knepley {
7330163fd50SMatthew G. Knepley   PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal x[], PetscInt, PetscScalar *u, void *ctx);
73455f2e967SMatthew G. Knepley   void            **ctxs;
735d7ddef95SMatthew G. Knepley   PetscInt          numFields;
736d7ddef95SMatthew G. Knepley 
737d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
7389566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
7399566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields,&funcs,numFields,&ctxs));
740d7ddef95SMatthew G. Knepley   funcs[field] = func;
741d7ddef95SMatthew G. Knepley   ctxs[field]  = ctx;
7429566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLabelLocal(dm, time, label, numids, ids, Nc, comps, funcs, ctxs, INSERT_BC_VALUES, locX));
7439566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs,ctxs));
744d7ddef95SMatthew G. Knepley   PetscFunctionReturn(0);
745d7ddef95SMatthew G. Knepley }
746d7ddef95SMatthew G. Knepley 
747b278463cSMatthew G. Knepley /*@C
748ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialField - Insert boundary values into a local vector using a function of the coordinates and field data
749b278463cSMatthew G. Knepley 
750b278463cSMatthew G. Knepley   Input Parameters:
751b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
752b278463cSMatthew G. Knepley . time   - The time
753b278463cSMatthew G. Knepley . locU   - A local vector with the input solution values
754b278463cSMatthew G. Knepley . field  - The field to constrain
7551c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
7561c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
757b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
758b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
759b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
760b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
761b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
762b278463cSMatthew G. Knepley 
763b278463cSMatthew G. Knepley   Output Parameter:
764b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
765b278463cSMatthew G. Knepley 
766b278463cSMatthew G. Knepley   Level: developer
767b278463cSMatthew G. Knepley 
768ece3a9fcSMatthew G. Knepley .seealso: DMPlexInsertBoundaryValuesEssential(), DMPlexInsertBoundaryValuesEssentialBdField(), DMAddBoundary()
769b278463cSMatthew G. Knepley @*/
7701c531cf8SMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesEssentialField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
771c60e475cSMatthew G. Knepley                                                         void (*func)(PetscInt, PetscInt, PetscInt,
772c60e475cSMatthew G. Knepley                                                                      const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
773c60e475cSMatthew G. Knepley                                                                      const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
77497b6e6e8SMatthew G. Knepley                                                                      PetscReal, const PetscReal[], PetscInt, const PetscScalar[],
775b278463cSMatthew G. Knepley                                                                      PetscScalar[]),
776b278463cSMatthew G. Knepley                                                         void *ctx, Vec locX)
777c60e475cSMatthew G. Knepley {
778c60e475cSMatthew G. Knepley   void (**funcs)(PetscInt, PetscInt, PetscInt,
779c60e475cSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
780c60e475cSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
78197b6e6e8SMatthew G. Knepley                  PetscReal, const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
782c60e475cSMatthew G. Knepley   void            **ctxs;
783c60e475cSMatthew G. Knepley   PetscInt          numFields;
784c60e475cSMatthew G. Knepley 
785c60e475cSMatthew G. Knepley   PetscFunctionBegin;
7869566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
7879566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields,&funcs,numFields,&ctxs));
788c60e475cSMatthew G. Knepley   funcs[field] = func;
789c60e475cSMatthew G. Knepley   ctxs[field]  = ctx;
7909566063dSJacob Faibussowitsch   PetscCall(DMProjectFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
7919566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs,ctxs));
792c60e475cSMatthew G. Knepley   PetscFunctionReturn(0);
793c60e475cSMatthew G. Knepley }
794c60e475cSMatthew G. Knepley 
795b278463cSMatthew G. Knepley /*@C
796ece3a9fcSMatthew G. Knepley   DMPlexInsertBoundaryValuesEssentialBdField - Insert boundary values into a local vector using a function of the coodinates and boundary field data
797ece3a9fcSMatthew G. Knepley 
798ece3a9fcSMatthew G. Knepley   Collective on dm
799ece3a9fcSMatthew G. Knepley 
800ece3a9fcSMatthew G. Knepley   Input Parameters:
801ece3a9fcSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
802ece3a9fcSMatthew G. Knepley . time   - The time
803ece3a9fcSMatthew G. Knepley . locU   - A local vector with the input solution values
804ece3a9fcSMatthew G. Knepley . field  - The field to constrain
805ece3a9fcSMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
806ece3a9fcSMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
807ece3a9fcSMatthew G. Knepley . label  - The DMLabel defining constrained points
808ece3a9fcSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
809ece3a9fcSMatthew G. Knepley . ids    - An array of ids for constrained points
810ece3a9fcSMatthew G. Knepley . func   - A pointwise function giving boundary values, the calling sequence is given in DMProjectBdFieldLabelLocal()
811ece3a9fcSMatthew G. Knepley - ctx    - An optional user context for bcFunc
812ece3a9fcSMatthew G. Knepley 
813ece3a9fcSMatthew G. Knepley   Output Parameter:
814ece3a9fcSMatthew G. Knepley . locX   - A local vector to receive the boundary values
815ece3a9fcSMatthew G. Knepley 
816ece3a9fcSMatthew G. Knepley   Level: developer
817ece3a9fcSMatthew G. Knepley 
818ece3a9fcSMatthew G. Knepley .seealso: DMProjectBdFieldLabelLocal(), DMPlexInsertBoundaryValuesEssential(), DMPlexInsertBoundaryValuesEssentialField(), DMAddBoundary()
819ece3a9fcSMatthew G. Knepley @*/
820ece3a9fcSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesEssentialBdField(DM dm, PetscReal time, Vec locU, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
821ece3a9fcSMatthew G. Knepley                                                           void (*func)(PetscInt, PetscInt, PetscInt,
822ece3a9fcSMatthew G. Knepley                                                                        const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
823ece3a9fcSMatthew G. Knepley                                                                        const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
824ece3a9fcSMatthew G. Knepley                                                                        PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[],
825ece3a9fcSMatthew G. Knepley                                                                        PetscScalar[]),
826ece3a9fcSMatthew G. Knepley                                                           void *ctx, Vec locX)
827ece3a9fcSMatthew G. Knepley {
828ece3a9fcSMatthew G. Knepley   void (**funcs)(PetscInt, PetscInt, PetscInt,
829ece3a9fcSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
830ece3a9fcSMatthew G. Knepley                  const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
831ece3a9fcSMatthew G. Knepley                  PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]);
832ece3a9fcSMatthew G. Knepley   void            **ctxs;
833ece3a9fcSMatthew G. Knepley   PetscInt          numFields;
834ece3a9fcSMatthew G. Knepley 
835ece3a9fcSMatthew G. Knepley   PetscFunctionBegin;
8369566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
8379566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(numFields,&funcs,numFields,&ctxs));
838ece3a9fcSMatthew G. Knepley   funcs[field] = func;
839ece3a9fcSMatthew G. Knepley   ctxs[field]  = ctx;
8409566063dSJacob Faibussowitsch   PetscCall(DMProjectBdFieldLabelLocal(dm, time, label, numids, ids, Nc, comps, locU, funcs, INSERT_BC_VALUES, locX));
8419566063dSJacob Faibussowitsch   PetscCall(PetscFree2(funcs,ctxs));
842ece3a9fcSMatthew G. Knepley   PetscFunctionReturn(0);
843ece3a9fcSMatthew G. Knepley }
844ece3a9fcSMatthew G. Knepley 
845ece3a9fcSMatthew G. Knepley /*@C
846b278463cSMatthew G. Knepley   DMPlexInsertBoundaryValuesRiemann - Insert boundary values into a local vector
847b278463cSMatthew G. Knepley 
848b278463cSMatthew G. Knepley   Input Parameters:
849b278463cSMatthew G. Knepley + dm     - The DM, with a PetscDS that matches the problem being constrained
850b278463cSMatthew G. Knepley . time   - The time
851b278463cSMatthew G. Knepley . faceGeometry - A vector with the FVM face geometry information
852b278463cSMatthew G. Knepley . cellGeometry - A vector with the FVM cell geometry information
853b278463cSMatthew G. Knepley . Grad         - A vector with the FVM cell gradient information
854b278463cSMatthew G. Knepley . field  - The field to constrain
8551c531cf8SMatthew G. Knepley . Nc     - The number of constrained field components, or 0 for all components
8561c531cf8SMatthew G. Knepley . comps  - An array of constrained component numbers, or NULL for all components
857b278463cSMatthew G. Knepley . label  - The DMLabel defining constrained points
858b278463cSMatthew G. Knepley . numids - The number of DMLabel ids for constrained points
859b278463cSMatthew G. Knepley . ids    - An array of ids for constrained points
860b278463cSMatthew G. Knepley . func   - A pointwise function giving boundary values
861b278463cSMatthew G. Knepley - ctx    - An optional user context for bcFunc
862b278463cSMatthew G. Knepley 
863b278463cSMatthew G. Knepley   Output Parameter:
864b278463cSMatthew G. Knepley . locX   - A local vector to receives the boundary values
865b278463cSMatthew G. Knepley 
866b278463cSMatthew G. Knepley   Note: This implementation currently ignores the numcomps/comps argument from DMAddBoundary()
867b278463cSMatthew G. Knepley 
868b278463cSMatthew G. Knepley   Level: developer
869b278463cSMatthew G. Knepley 
870b278463cSMatthew G. Knepley .seealso: DMPlexInsertBoundaryValuesEssential(), DMPlexInsertBoundaryValuesEssentialField(), DMAddBoundary()
871b278463cSMatthew G. Knepley @*/
8721c531cf8SMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValuesRiemann(DM dm, PetscReal time, Vec faceGeometry, Vec cellGeometry, Vec Grad, PetscInt field, PetscInt Nc, const PetscInt comps[], DMLabel label, PetscInt numids, const PetscInt ids[],
873b278463cSMatthew G. Knepley                                                  PetscErrorCode (*func)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*), void *ctx, Vec locX)
874d7ddef95SMatthew G. Knepley {
87561f58d28SMatthew G. Knepley   PetscDS            prob;
876da97024aSMatthew G. Knepley   PetscSF            sf;
877d7ddef95SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad;
87820369375SToby Isaac   const PetscScalar *facegeom, *cellgeom = NULL, *grad;
879da97024aSMatthew G. Knepley   const PetscInt    *leaves;
880d7ddef95SMatthew G. Knepley   PetscScalar       *x, *fx;
881520b3818SMatthew G. Knepley   PetscInt           dim, nleaves, loc, fStart, fEnd, pdim, i;
8825f80ce2aSJacob Faibussowitsch   PetscErrorCode     ierru = 0;
883d7ddef95SMatthew G. Knepley 
884d7ddef95SMatthew G. Knepley   PetscFunctionBegin;
8859566063dSJacob Faibussowitsch   PetscCall(DMGetPointSF(dm, &sf));
8869566063dSJacob Faibussowitsch   PetscCall(PetscSFGetGraph(sf, NULL, &nleaves, &leaves, NULL));
887da97024aSMatthew G. Knepley   nleaves = PetscMax(0, nleaves);
8889566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
8899566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
8909566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
8919566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
8929566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
89320369375SToby Isaac   if (cellGeometry) {
8949566063dSJacob Faibussowitsch     PetscCall(VecGetDM(cellGeometry, &dmCell));
8959566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
89620369375SToby Isaac   }
897d7ddef95SMatthew G. Knepley   if (Grad) {
898c0a6632aSMatthew G. Knepley     PetscFV fv;
899c0a6632aSMatthew G. Knepley 
9009566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *) &fv));
9019566063dSJacob Faibussowitsch     PetscCall(VecGetDM(Grad, &dmGrad));
9029566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(Grad, &grad));
9039566063dSJacob Faibussowitsch     PetscCall(PetscFVGetNumComponents(fv, &pdim));
9049566063dSJacob Faibussowitsch     PetscCall(DMGetWorkArray(dm, pdim, MPIU_SCALAR, &fx));
905d7ddef95SMatthew G. Knepley   }
9069566063dSJacob Faibussowitsch   PetscCall(VecGetArray(locX, &x));
907d7ddef95SMatthew G. Knepley   for (i = 0; i < numids; ++i) {
908d7ddef95SMatthew G. Knepley     IS              faceIS;
909d7ddef95SMatthew G. Knepley     const PetscInt *faces;
910d7ddef95SMatthew G. Knepley     PetscInt        numFaces, f;
911d7ddef95SMatthew G. Knepley 
9129566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, ids[i], &faceIS));
913d7ddef95SMatthew G. Knepley     if (!faceIS) continue; /* No points with that id on this process */
9149566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(faceIS, &numFaces));
9159566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(faceIS, &faces));
916d7ddef95SMatthew G. Knepley     for (f = 0; f < numFaces; ++f) {
917d7ddef95SMatthew G. Knepley       const PetscInt         face = faces[f], *cells;
918640bce14SSatish Balay       PetscFVFaceGeom        *fg;
919d7ddef95SMatthew G. Knepley 
920d7ddef95SMatthew G. Knepley       if ((face < fStart) || (face >= fEnd)) continue; /* Refinement adds non-faces to labels */
9219566063dSJacob Faibussowitsch       PetscCall(PetscFindInt(face, nleaves, (PetscInt *) leaves, &loc));
922da97024aSMatthew G. Knepley       if (loc >= 0) continue;
9239566063dSJacob Faibussowitsch       PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
9249566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, face, &cells));
925d7ddef95SMatthew G. Knepley       if (Grad) {
926640bce14SSatish Balay         PetscFVCellGeom       *cg;
927640bce14SSatish Balay         PetscScalar           *cx, *cgrad;
928d7ddef95SMatthew G. Knepley         PetscScalar           *xG;
929d7ddef95SMatthew G. Knepley         PetscReal              dx[3];
930d7ddef95SMatthew G. Knepley         PetscInt               d;
931d7ddef95SMatthew G. Knepley 
9329566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cg));
9339566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &cx));
9349566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], grad, &cgrad));
9359566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
936d7ddef95SMatthew G. Knepley         DMPlex_WaxpyD_Internal(dim, -1, cg->centroid, fg->centroid, dx);
937d7ddef95SMatthew G. Knepley         for (d = 0; d < pdim; ++d) fx[d] = cx[d] + DMPlex_DotD_Internal(dim, &cgrad[d*dim], dx);
9389566063dSJacob Faibussowitsch         PetscCall((*func)(time, fg->centroid, fg->normal, fx, xG, ctx));
939d7ddef95SMatthew G. Knepley       } else {
940640bce14SSatish Balay         PetscScalar *xI;
941d7ddef95SMatthew G. Knepley         PetscScalar *xG;
942d7ddef95SMatthew G. Knepley 
9439566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalRead(dm, cells[0], x, &xI));
9449566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRef(dm, cells[1], field, x, &xG));
945e735a8a9SMatthew G. Knepley         ierru = (*func)(time, fg->centroid, fg->normal, xI, xG, ctx);
946e735a8a9SMatthew G. Knepley         if (ierru) {
9479566063dSJacob Faibussowitsch           PetscCall(ISRestoreIndices(faceIS, &faces));
9489566063dSJacob Faibussowitsch           PetscCall(ISDestroy(&faceIS));
949e735a8a9SMatthew G. Knepley           goto cleanup;
950e735a8a9SMatthew G. Knepley         }
951d7ddef95SMatthew G. Knepley       }
952d7ddef95SMatthew G. Knepley     }
9539566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(faceIS, &faces));
9549566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&faceIS));
955d7ddef95SMatthew G. Knepley   }
956e735a8a9SMatthew G. Knepley   cleanup:
9579566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(locX, &x));
958d7ddef95SMatthew G. Knepley   if (Grad) {
9599566063dSJacob Faibussowitsch     PetscCall(DMRestoreWorkArray(dm, pdim, MPIU_SCALAR, &fx));
9609566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(Grad, &grad));
961d7ddef95SMatthew G. Knepley   }
9629566063dSJacob Faibussowitsch   if (cellGeometry) PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
9639566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
9649566063dSJacob Faibussowitsch   PetscCall(ierru);
965d7ddef95SMatthew G. Knepley   PetscFunctionReturn(0);
966d7ddef95SMatthew G. Knepley }
967d7ddef95SMatthew G. Knepley 
9680c364540SMatthew G. Knepley static PetscErrorCode zero(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nc, PetscScalar *u, void *ctx)
9690c364540SMatthew G. Knepley {
9700c364540SMatthew G. Knepley   PetscInt c;
9710c364540SMatthew G. Knepley   for (c = 0; c < Nc; ++c) u[c] = 0.0;
9720c364540SMatthew G. Knepley   return 0;
9730c364540SMatthew G. Knepley }
9740c364540SMatthew G. Knepley 
975f1d73a7aSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
976d7ddef95SMatthew G. Knepley {
9770c364540SMatthew G. Knepley   PetscObject    isZero;
978e5e52638SMatthew G. Knepley   PetscDS        prob;
979d7ddef95SMatthew G. Knepley   PetscInt       numBd, b;
98055f2e967SMatthew G. Knepley 
98155f2e967SMatthew G. Knepley   PetscFunctionBegin;
9829566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
9839566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
9849566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject) locX, "__Vec_bc_zero__", &isZero));
98555f2e967SMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
98645480ffeSMatthew G. Knepley     PetscWeakForm           wf;
987f971fd6bSMatthew G. Knepley     DMBoundaryConditionType type;
98845480ffeSMatthew G. Knepley     const char             *name;
989d7ddef95SMatthew G. Knepley     DMLabel                 label;
9901c531cf8SMatthew G. Knepley     PetscInt                field, Nc;
9911c531cf8SMatthew G. Knepley     const PetscInt         *comps;
992d7ddef95SMatthew G. Knepley     PetscObject             obj;
993d7ddef95SMatthew G. Knepley     PetscClassId            id;
99445480ffeSMatthew G. Knepley     void                  (*bvfunc)(void);
995d7ddef95SMatthew G. Knepley     PetscInt                numids;
996d7ddef95SMatthew G. Knepley     const PetscInt         *ids;
99755f2e967SMatthew G. Knepley     void                   *ctx;
99855f2e967SMatthew G. Knepley 
9999566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, &bvfunc, NULL, &ctx));
1000f971fd6bSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
10019566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
10029566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1003d7ddef95SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1004c60e475cSMatthew G. Knepley       switch (type) {
1005c60e475cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
1006c60e475cSMatthew G. Knepley       case DM_BC_ESSENTIAL:
100745480ffeSMatthew G. Knepley         {
100845480ffeSMatthew G. Knepley           PetscSimplePointFunc func = (PetscSimplePointFunc) bvfunc;
100945480ffeSMatthew G. Knepley 
101045480ffeSMatthew G. Knepley           if (isZero) func = zero;
10119566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelAddCells(dm,label));
10129566063dSJacob Faibussowitsch           PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func, ctx, locX));
10139566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelClearCells(dm,label));
101445480ffeSMatthew G. Knepley         }
1015c60e475cSMatthew G. Knepley         break;
1016c60e475cSMatthew G. Knepley       case DM_BC_ESSENTIAL_FIELD:
101745480ffeSMatthew G. Knepley         {
101845480ffeSMatthew G. Knepley           PetscPointFunc func = (PetscPointFunc) bvfunc;
101945480ffeSMatthew G. Knepley 
10209566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelAddCells(dm,label));
10219566063dSJacob Faibussowitsch           PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func, ctx, locX));
10229566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelClearCells(dm,label));
102345480ffeSMatthew G. Knepley         }
1024c60e475cSMatthew G. Knepley         break;
1025c60e475cSMatthew G. Knepley       default: break;
1026c60e475cSMatthew G. Knepley       }
1027d7ddef95SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
102845480ffeSMatthew G. Knepley       {
102945480ffeSMatthew G. Knepley         PetscErrorCode (*func)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*) = (PetscErrorCode (*)(PetscReal,const PetscReal*,const PetscReal*,const PetscScalar*,PetscScalar*,void*)) bvfunc;
103045480ffeSMatthew G. Knepley 
103143ea7facSMatthew G. Knepley         if (!faceGeomFVM) continue;
10329566063dSJacob Faibussowitsch         PetscCall(DMPlexInsertBoundaryValuesRiemann(dm, time, faceGeomFVM, cellGeomFVM, gradFVM, field, Nc, comps, label, numids, ids, func, ctx, locX));
103345480ffeSMatthew G. Knepley       }
103463a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
103555f2e967SMatthew G. Knepley   }
103655f2e967SMatthew G. Knepley   PetscFunctionReturn(0);
103755f2e967SMatthew G. Knepley }
103855f2e967SMatthew G. Knepley 
103956cf3b9cSMatthew G. Knepley PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues_Plex(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
104056cf3b9cSMatthew G. Knepley {
104156cf3b9cSMatthew G. Knepley   PetscObject    isZero;
104256cf3b9cSMatthew G. Knepley   PetscDS        prob;
104356cf3b9cSMatthew G. Knepley   PetscInt       numBd, b;
104456cf3b9cSMatthew G. Knepley 
104556cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
104656cf3b9cSMatthew G. Knepley   if (!locX) PetscFunctionReturn(0);
10479566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
10489566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
10499566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject) locX, "__Vec_bc_zero__", &isZero));
105056cf3b9cSMatthew G. Knepley   for (b = 0; b < numBd; ++b) {
105145480ffeSMatthew G. Knepley     PetscWeakForm           wf;
105256cf3b9cSMatthew G. Knepley     DMBoundaryConditionType type;
105345480ffeSMatthew G. Knepley     const char             *name;
105456cf3b9cSMatthew G. Knepley     DMLabel                 label;
105556cf3b9cSMatthew G. Knepley     PetscInt                field, Nc;
105656cf3b9cSMatthew G. Knepley     const PetscInt         *comps;
105756cf3b9cSMatthew G. Knepley     PetscObject             obj;
105856cf3b9cSMatthew G. Knepley     PetscClassId            id;
105956cf3b9cSMatthew G. Knepley     PetscInt                numids;
106056cf3b9cSMatthew G. Knepley     const PetscInt         *ids;
106145480ffeSMatthew G. Knepley     void                  (*bvfunc)(void);
106256cf3b9cSMatthew G. Knepley     void                   *ctx;
106356cf3b9cSMatthew G. Knepley 
10649566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, b, &wf, &type, &name, &label, &numids, &ids, &field, &Nc, &comps, NULL, &bvfunc, &ctx));
106556cf3b9cSMatthew G. Knepley     if (insertEssential != (type & DM_BC_ESSENTIAL)) continue;
10669566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
10679566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
106856cf3b9cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
106956cf3b9cSMatthew G. Knepley       switch (type) {
107056cf3b9cSMatthew G. Knepley         /* for FEM, there is no insertion to be done for non-essential boundary conditions */
107156cf3b9cSMatthew G. Knepley       case DM_BC_ESSENTIAL:
107245480ffeSMatthew G. Knepley         {
107345480ffeSMatthew G. Knepley           PetscSimplePointFunc func_t = (PetscSimplePointFunc) bvfunc;
107445480ffeSMatthew G. Knepley 
107545480ffeSMatthew G. Knepley           if (isZero) func_t = zero;
10769566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelAddCells(dm,label));
10779566063dSJacob Faibussowitsch           PetscCall(DMPlexInsertBoundaryValuesEssential(dm, time, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
10789566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelClearCells(dm,label));
107945480ffeSMatthew G. Knepley         }
108056cf3b9cSMatthew G. Knepley         break;
108156cf3b9cSMatthew G. Knepley       case DM_BC_ESSENTIAL_FIELD:
108245480ffeSMatthew G. Knepley         {
108345480ffeSMatthew G. Knepley           PetscPointFunc func_t = (PetscPointFunc) bvfunc;
108445480ffeSMatthew G. Knepley 
10859566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelAddCells(dm,label));
10869566063dSJacob Faibussowitsch           PetscCall(DMPlexInsertBoundaryValuesEssentialField(dm, time, locX, field, Nc, comps, label, numids, ids, func_t, ctx, locX));
10879566063dSJacob Faibussowitsch           PetscCall(DMPlexLabelClearCells(dm,label));
108845480ffeSMatthew G. Knepley         }
108956cf3b9cSMatthew G. Knepley         break;
109056cf3b9cSMatthew G. Knepley       default: break;
109156cf3b9cSMatthew G. Knepley       }
109263a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
109356cf3b9cSMatthew G. Knepley   }
109456cf3b9cSMatthew G. Knepley   PetscFunctionReturn(0);
109556cf3b9cSMatthew G. Knepley }
109656cf3b9cSMatthew G. Knepley 
1097f1d73a7aSMatthew G. Knepley /*@
1098f1d73a7aSMatthew G. Knepley   DMPlexInsertBoundaryValues - Puts coefficients which represent boundary values into the local solution vector
1099f1d73a7aSMatthew G. Knepley 
1100ed808b8fSJed Brown   Not Collective
1101ed808b8fSJed Brown 
1102f1d73a7aSMatthew G. Knepley   Input Parameters:
1103f1d73a7aSMatthew G. Knepley + dm - The DM
1104f1d73a7aSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
1105f1d73a7aSMatthew G. Knepley . time - The time
1106f1d73a7aSMatthew G. Knepley . faceGeomFVM - Face geometry data for FV discretizations
1107f1d73a7aSMatthew G. Knepley . cellGeomFVM - Cell geometry data for FV discretizations
1108f1d73a7aSMatthew G. Knepley - gradFVM - Gradient reconstruction data for FV discretizations
1109f1d73a7aSMatthew G. Knepley 
1110f1d73a7aSMatthew G. Knepley   Output Parameters:
1111f1d73a7aSMatthew G. Knepley . locX - Solution updated with boundary values
1112f1d73a7aSMatthew G. Knepley 
1113ed808b8fSJed Brown   Level: intermediate
1114f1d73a7aSMatthew G. Knepley 
1115ed808b8fSJed Brown .seealso: DMProjectFunctionLabelLocal(), DMAddBoundary()
1116f1d73a7aSMatthew G. Knepley @*/
1117f1d73a7aSMatthew G. Knepley PetscErrorCode DMPlexInsertBoundaryValues(DM dm, PetscBool insertEssential, Vec locX, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
1118f1d73a7aSMatthew G. Knepley {
1119f1d73a7aSMatthew G. Knepley   PetscFunctionBegin;
1120f1d73a7aSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1121064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
1122064a246eSJacob Faibussowitsch   if (faceGeomFVM) {PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);}
1123064a246eSJacob Faibussowitsch   if (cellGeomFVM) {PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);}
1124064a246eSJacob Faibussowitsch   if (gradFVM)     {PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);}
1125cac4c232SBarry Smith   PetscTryMethod(dm,"DMPlexInsertBoundaryValues_C",(DM,PetscBool,Vec,PetscReal,Vec,Vec,Vec),(dm,insertEssential,locX,time,faceGeomFVM,cellGeomFVM,gradFVM));
1126f1d73a7aSMatthew G. Knepley   PetscFunctionReturn(0);
1127f1d73a7aSMatthew G. Knepley }
1128f1d73a7aSMatthew G. Knepley 
112956cf3b9cSMatthew G. Knepley /*@
1130a5b23f4aSJose E. Roman   DMPlexInsertTimeDerivativeBoundaryValues - Puts coefficients which represent boundary values of the time derivative into the local solution vector
113156cf3b9cSMatthew G. Knepley 
113256cf3b9cSMatthew G. Knepley   Input Parameters:
113356cf3b9cSMatthew G. Knepley + dm - The DM
113456cf3b9cSMatthew G. Knepley . insertEssential - Should I insert essential (e.g. Dirichlet) or inessential (e.g. Neumann) boundary conditions
113556cf3b9cSMatthew G. Knepley . time - The time
113656cf3b9cSMatthew G. Knepley . faceGeomFVM - Face geometry data for FV discretizations
113756cf3b9cSMatthew G. Knepley . cellGeomFVM - Cell geometry data for FV discretizations
113856cf3b9cSMatthew G. Knepley - gradFVM - Gradient reconstruction data for FV discretizations
113956cf3b9cSMatthew G. Knepley 
114056cf3b9cSMatthew G. Knepley   Output Parameters:
114156cf3b9cSMatthew G. Knepley . locX_t - Solution updated with boundary values
114256cf3b9cSMatthew G. Knepley 
114356cf3b9cSMatthew G. Knepley   Level: developer
114456cf3b9cSMatthew G. Knepley 
114556cf3b9cSMatthew G. Knepley .seealso: DMProjectFunctionLabelLocal()
114656cf3b9cSMatthew G. Knepley @*/
114756cf3b9cSMatthew G. Knepley PetscErrorCode DMPlexInsertTimeDerivativeBoundaryValues(DM dm, PetscBool insertEssential, Vec locX_t, PetscReal time, Vec faceGeomFVM, Vec cellGeomFVM, Vec gradFVM)
114856cf3b9cSMatthew G. Knepley {
114956cf3b9cSMatthew G. Knepley   PetscFunctionBegin;
115056cf3b9cSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
1151064a246eSJacob Faibussowitsch   if (locX_t)      {PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 3);}
1152064a246eSJacob Faibussowitsch   if (faceGeomFVM) {PetscValidHeaderSpecific(faceGeomFVM, VEC_CLASSID, 5);}
1153064a246eSJacob Faibussowitsch   if (cellGeomFVM) {PetscValidHeaderSpecific(cellGeomFVM, VEC_CLASSID, 6);}
1154064a246eSJacob Faibussowitsch   if (gradFVM)     {PetscValidHeaderSpecific(gradFVM, VEC_CLASSID, 7);}
1155cac4c232SBarry Smith   PetscTryMethod(dm,"DMPlexInsertTimeDerviativeBoundaryValues_C",(DM,PetscBool,Vec,PetscReal,Vec,Vec,Vec),(dm,insertEssential,locX_t,time,faceGeomFVM,cellGeomFVM,gradFVM));
115656cf3b9cSMatthew G. Knepley   PetscFunctionReturn(0);
115756cf3b9cSMatthew G. Knepley }
115856cf3b9cSMatthew G. Knepley 
11590709b2feSToby Isaac PetscErrorCode DMComputeL2Diff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
1160cb1e1211SMatthew G Knepley {
1161574a98acSMatthew G. Knepley   Vec              localX;
1162574a98acSMatthew G. Knepley 
1163574a98acSMatthew G. Knepley   PetscFunctionBegin;
11649566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
11659566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, localX, time, NULL, NULL, NULL));
11669566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
11679566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
11689566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeL2DiffLocal(dm, time, funcs, ctxs, localX, diff));
11699566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
1170574a98acSMatthew G. Knepley   PetscFunctionReturn(0);
1171574a98acSMatthew G. Knepley }
1172574a98acSMatthew G. Knepley 
1173574a98acSMatthew G. Knepley /*@C
1174ca3d3a14SMatthew G. Knepley   DMComputeL2DiffLocal - This function computes the L_2 difference between a function u and an FEM interpolant solution u_h.
1175574a98acSMatthew G. Knepley 
1176d083f849SBarry Smith   Collective on dm
1177c0f8e1fdSMatthew G. Knepley 
1178574a98acSMatthew G. Knepley   Input Parameters:
1179574a98acSMatthew G. Knepley + dm     - The DM
1180574a98acSMatthew G. Knepley . time   - The time
1181574a98acSMatthew G. Knepley . funcs  - The functions to evaluate for each field component
1182574a98acSMatthew G. Knepley . ctxs   - Optional array of contexts to pass to each function, or NULL.
1183574a98acSMatthew G. Knepley - localX - The coefficient vector u_h, a local vector
1184574a98acSMatthew G. Knepley 
1185574a98acSMatthew G. Knepley   Output Parameter:
1186574a98acSMatthew G. Knepley . diff - The diff ||u - u_h||_2
1187574a98acSMatthew G. Knepley 
1188574a98acSMatthew G. Knepley   Level: developer
1189574a98acSMatthew G. Knepley 
1190574a98acSMatthew G. Knepley .seealso: DMProjectFunction(), DMComputeL2FieldDiff(), DMComputeL2GradientDiff()
1191574a98acSMatthew G. Knepley @*/
1192574a98acSMatthew G. Knepley PetscErrorCode DMPlexComputeL2DiffLocal(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec localX, PetscReal *diff)
1193574a98acSMatthew G. Knepley {
11940f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
1195ca3d3a14SMatthew G. Knepley   DM               tdm;
1196ca3d3a14SMatthew G. Knepley   Vec              tv;
1197cb1e1211SMatthew G Knepley   PetscSection     section;
1198c5bbbd5bSMatthew G. Knepley   PetscQuadrature  quad;
11994bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
120015496722SMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
12014bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
1202cb1e1211SMatthew G Knepley   PetscReal        localDiff = 0.0;
12037318780aSToby Isaac   const PetscReal *quadWeights;
1204412e9a14SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cellHeight, cStart, cEnd, c, field, fieldOffset;
1205ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1206cb1e1211SMatthew G Knepley 
1207cb1e1211SMatthew G Knepley   PetscFunctionBegin;
12089566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
12099566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
12102a4e142eSMatthew G. Knepley   fegeom.dimEmbed = coordDim;
12119566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
12129566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
12139566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
12149566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
12159566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1216cb1e1211SMatthew G Knepley   for (field = 0; field < numFields; ++field) {
121715496722SMatthew G. Knepley     PetscObject  obj;
121815496722SMatthew G. Knepley     PetscClassId id;
1219c5bbbd5bSMatthew G. Knepley     PetscInt     Nc;
1220c5bbbd5bSMatthew G. Knepley 
12219566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
12229566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
122315496722SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
122415496722SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
122515496722SMatthew G. Knepley 
12269566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
12279566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
122815496722SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
122915496722SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
123015496722SMatthew G. Knepley 
12319566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
12329566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
123363a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1234c5bbbd5bSMatthew G. Knepley     numComponents += Nc;
1235cb1e1211SMatthew G Knepley   }
12369566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
123763a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents),PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
12389566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents,&funcVal,numComponents,&interpolant,coordDim*Nq,&coords,Nq,&fegeom.detJ,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ));
12399566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
12409566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
1241cb1e1211SMatthew G Knepley   for (c = cStart; c < cEnd; ++c) {
1242a1e44745SMatthew G. Knepley     PetscScalar *x = NULL;
1243cb1e1211SMatthew G Knepley     PetscReal    elemDiff = 0.0;
12449c3cf19fSMatthew G. Knepley     PetscInt     qc = 0;
1245cb1e1211SMatthew G Knepley 
12469566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
12479566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x));
1248cb1e1211SMatthew G Knepley 
124915496722SMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
125015496722SMatthew G. Knepley       PetscObject  obj;
125115496722SMatthew G. Knepley       PetscClassId id;
1252c110b1eeSGeoffrey Irving       void * const ctx = ctxs ? ctxs[field] : NULL;
125315496722SMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1254cb1e1211SMatthew G Knepley 
12559566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
12569566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
12579566063dSJacob Faibussowitsch       if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &Nc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
12589566063dSJacob Faibussowitsch       else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &Nc));Nb = 1;}
125963a3b9bcSJacob Faibussowitsch       else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1260cb1e1211SMatthew G Knepley       if (debug) {
1261cb1e1211SMatthew G Knepley         char title[1024];
126263a3b9bcSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
12639566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1264cb1e1211SMatthew G Knepley       }
12657318780aSToby Isaac       for (q = 0; q < Nq; ++q) {
12662a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1267d0609cedSBarry Smith         PetscErrorCode ierr;
12682a4e142eSMatthew G. Knepley 
12692a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
12702a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q*coordDim*coordDim];
12712a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
12722a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
127363a3b9bcSJacob Faibussowitsch         PetscCheck(fegeom.detJ[q] > 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", point %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1274d3a7d86cSMatthew G. Knepley         if (transform) {
1275d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*Nq];
12769566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim*q], PETSC_TRUE, coordDim, &coords[coordDim*q], gcoords, dm->transformCtx));
1277d3a7d86cSMatthew G. Knepley         } else {
1278d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*q];
1279d3a7d86cSMatthew G. Knepley         }
12809566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal,Nc));
1281ca3d3a14SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, Nc, funcVal, ctx);
1282e735a8a9SMatthew G. Knepley         if (ierr) {
12839566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
12849566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
12859566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
1286e735a8a9SMatthew G. Knepley         }
12879566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim*q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
12889566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID)      PetscCall(PetscFEInterpolate_Static((PetscFE) obj, &x[fieldOffset], &qgeom, q, interpolant));
12899566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV) obj, &x[fieldOffset], q, interpolant));
12902df84da0SMatthew G. Knepley         else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
129115496722SMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1292beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
129363a3b9bcSJacob Faibussowitsch           if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g (%g, %g)\n", c, field, fc, (double)(coordDim > 0 ? coords[coordDim*q] : 0.), (double)(coordDim > 1 ? coords[coordDim*q+1] : 0.),(double)(coordDim > 2 ? coords[coordDim*q+2] : 0.), (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q]), (double)PetscRealPart(interpolant[fc]), (double)PetscRealPart(funcVal[fc])));
12944bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
1295cb1e1211SMatthew G Knepley         }
1296cb1e1211SMatthew G Knepley       }
12979c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
1298beaa55a6SMatthew G. Knepley       qc += Nc;
1299cb1e1211SMatthew G Knepley     }
13009566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
13019566063dSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
1302cb1e1211SMatthew G Knepley     localDiff += elemDiff;
1303cb1e1211SMatthew G Knepley   }
13049566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
13051c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
1306cb1e1211SMatthew G Knepley   *diff = PetscSqrtReal(*diff);
1307cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
1308cb1e1211SMatthew G Knepley }
1309cb1e1211SMatthew G Knepley 
1310b698f381SToby Isaac PetscErrorCode DMComputeL2GradientDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, const PetscReal n[], PetscReal *diff)
1311cb1e1211SMatthew G Knepley {
13120f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
1313ca3d3a14SMatthew G. Knepley   DM               tdm;
1314cb1e1211SMatthew G Knepley   PetscSection     section;
131540e14135SMatthew G. Knepley   PetscQuadrature  quad;
1316ca3d3a14SMatthew G. Knepley   Vec              localX, tv;
13179c3cf19fSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
13182a4e142eSMatthew G. Knepley   const PetscReal *quadWeights;
13194bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
13204bee2e38SMatthew G. Knepley   PetscReal       *coords, *gcoords;
132140e14135SMatthew G. Knepley   PetscReal        localDiff = 0.0;
1322485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, qNc = 0, Nq = 0, numFields, numComponents = 0, cStart, cEnd, c, field, fieldOffset;
1323ca3d3a14SMatthew G. Knepley   PetscBool        transform;
1324cb1e1211SMatthew G Knepley 
1325cb1e1211SMatthew G Knepley   PetscFunctionBegin;
13269566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
13279566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
13284bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
13299566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
13309566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
13319566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
13329566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
13339566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
13349566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
13359566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
13369566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
1337652b88e8SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
13380f2d7e86SMatthew G. Knepley     PetscFE  fe;
133940e14135SMatthew G. Knepley     PetscInt Nc;
1340652b88e8SMatthew G. Knepley 
13419566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, (PetscObject *) &fe));
13429566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
13439566063dSJacob Faibussowitsch     PetscCall(PetscFEGetNumComponents(fe, &Nc));
134440e14135SMatthew G. Knepley     numComponents += Nc;
1345652b88e8SMatthew G. Knepley   }
13469566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, NULL, &quadWeights));
134763a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents),PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
13489566063dSJacob Faibussowitsch   /* PetscCall(DMProjectFunctionLocal(dm, fe, funcs, INSERT_BC_VALUES, localX)); */
13499566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents,&funcVal,coordDim*Nq,&coords,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ,numComponents*coordDim,&interpolant,Nq,&fegeom.detJ));
13509566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
135140e14135SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
135240e14135SMatthew G. Knepley     PetscScalar *x = NULL;
135340e14135SMatthew G. Knepley     PetscReal    elemDiff = 0.0;
13549c3cf19fSMatthew G. Knepley     PetscInt     qc = 0;
1355652b88e8SMatthew G. Knepley 
13569566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
13579566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x));
135840e14135SMatthew G. Knepley 
13599c3cf19fSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
13600f2d7e86SMatthew G. Knepley       PetscFE          fe;
136151259fa3SMatthew G. Knepley       void * const     ctx = ctxs ? ctxs[field] : NULL;
13629c3cf19fSMatthew G. Knepley       PetscInt         Nb, Nc, q, fc;
136340e14135SMatthew G. Knepley 
13649566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, (PetscObject *) &fe));
13659566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
13669566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
136740e14135SMatthew G. Knepley       if (debug) {
136840e14135SMatthew G. Knepley         char title[1024];
13699566063dSJacob Faibussowitsch         PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, field));
13709566063dSJacob Faibussowitsch         PetscCall(DMPrintCellVector(c, title, Nb, &x[fieldOffset]));
1371652b88e8SMatthew G. Knepley       }
13729c3cf19fSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
13732a4e142eSMatthew G. Knepley         PetscFEGeom    qgeom;
1374d0609cedSBarry Smith         PetscErrorCode ierr;
13752a4e142eSMatthew G. Knepley 
13762a4e142eSMatthew G. Knepley         qgeom.dimEmbed = fegeom.dimEmbed;
13772a4e142eSMatthew G. Knepley         qgeom.J        = &fegeom.J[q*coordDim*coordDim];
13782a4e142eSMatthew G. Knepley         qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
13792a4e142eSMatthew G. Knepley         qgeom.detJ     = &fegeom.detJ[q];
13802df84da0SMatthew G. Knepley         PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
1381d3a7d86cSMatthew G. Knepley         if (transform) {
1382d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*Nq];
13839566063dSJacob Faibussowitsch           PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[coordDim*q], PETSC_TRUE, coordDim, &coords[coordDim*q], gcoords, dm->transformCtx));
1384d3a7d86cSMatthew G. Knepley         } else {
1385d3a7d86cSMatthew G. Knepley           gcoords = &coords[coordDim*q];
1386d3a7d86cSMatthew G. Knepley         }
13879566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(funcVal,Nc));
13884bee2e38SMatthew G. Knepley         ierr = (*funcs[field])(coordDim, time, gcoords, n, Nc, funcVal, ctx);
1389e735a8a9SMatthew G. Knepley         if (ierr) {
13909566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
13919566063dSJacob Faibussowitsch           PetscCall(DMRestoreLocalVector(dm, &localX));
13929566063dSJacob Faibussowitsch           PetscCall(PetscFree6(funcVal,coords,fegeom.J,fegeom.invJ,interpolant,fegeom.detJ));
1393e735a8a9SMatthew G. Knepley         }
13949566063dSJacob Faibussowitsch         if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[coordDim*q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
13959566063dSJacob Faibussowitsch         PetscCall(PetscFEInterpolateGradient_Static(fe, 1, &x[fieldOffset], &qgeom, q, interpolant));
13964bee2e38SMatthew G. Knepley         /* Overwrite with the dot product if the normal is given */
13974bee2e38SMatthew G. Knepley         if (n) {
13984bee2e38SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
13994bee2e38SMatthew G. Knepley             PetscScalar sum = 0.0;
14004bee2e38SMatthew G. Knepley             PetscInt    d;
14014bee2e38SMatthew G. Knepley             for (d = 0; d < dim; ++d) sum += interpolant[fc*dim+d]*n[d];
14024bee2e38SMatthew G. Knepley             interpolant[fc] = sum;
14034bee2e38SMatthew G. Knepley           }
14044bee2e38SMatthew G. Knepley         }
14059c3cf19fSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
1406beaa55a6SMatthew G. Knepley           const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
140763a3b9bcSJacob Faibussowitsch           if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    elem %" PetscInt_FMT " fieldDer %" PetscInt_FMT ",%" PetscInt_FMT " diff %g\n", c, field, fc, (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q])));
14084bee2e38SMatthew G. Knepley           elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
140940e14135SMatthew G. Knepley         }
141040e14135SMatthew G. Knepley       }
14119c3cf19fSMatthew G. Knepley       fieldOffset += Nb;
14129c3cf19fSMatthew G. Knepley       qc          += Nc;
141340e14135SMatthew G. Knepley     }
14149566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
141563a3b9bcSJacob Faibussowitsch     if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  elem %" PetscInt_FMT " diff %g\n", c, (double)elemDiff));
141640e14135SMatthew G. Knepley     localDiff += elemDiff;
141740e14135SMatthew G. Knepley   }
14189566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal,coords,fegeom.J,fegeom.invJ,interpolant,fegeom.detJ));
14199566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
14201c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(&localDiff, diff, 1, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
142140e14135SMatthew G. Knepley   *diff = PetscSqrtReal(*diff);
1422cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
1423cb1e1211SMatthew G Knepley }
1424cb1e1211SMatthew G Knepley 
1425c6eecec3SToby Isaac PetscErrorCode DMComputeL2FieldDiff_Plex(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, PetscReal *diff)
142673d901b8SMatthew G. Knepley {
14270f09c10fSMatthew G. Knepley   const PetscInt   debug = ((DM_Plex*)dm->data)->printL2;
1428ca3d3a14SMatthew G. Knepley   DM               tdm;
1429083401c6SMatthew G. Knepley   DMLabel          depthLabel;
143073d901b8SMatthew G. Knepley   PetscSection     section;
1431ca3d3a14SMatthew G. Knepley   Vec              localX, tv;
143273d901b8SMatthew G. Knepley   PetscReal       *localDiff;
1433083401c6SMatthew G. Knepley   PetscInt         dim, depth, dE, Nf, f, Nds, s;
1434ca3d3a14SMatthew G. Knepley   PetscBool        transform;
143573d901b8SMatthew G. Knepley 
143673d901b8SMatthew G. Knepley   PetscFunctionBegin;
14379566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
14389566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &dE));
14399566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
14409566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
14419566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
14429566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
14439566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
14449566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
14459566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
14469566063dSJacob Faibussowitsch   PetscCall(DMLabelGetNumValues(depthLabel, &depth));
1447083401c6SMatthew G. Knepley 
14489566063dSJacob Faibussowitsch   PetscCall(VecSet(localX, 0.0));
14499566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
14509566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
14519566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
14529566063dSJacob Faibussowitsch   PetscCall(DMGetNumDS(dm, &Nds));
14539566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(Nf, &localDiff));
1454083401c6SMatthew G. Knepley   for (s = 0; s < Nds; ++s) {
1455083401c6SMatthew G. Knepley     PetscDS          ds;
1456083401c6SMatthew G. Knepley     DMLabel          label;
1457083401c6SMatthew G. Knepley     IS               fieldIS, pointIS;
1458083401c6SMatthew G. Knepley     const PetscInt  *fields, *points = NULL;
1459083401c6SMatthew G. Knepley     PetscQuadrature  quad;
1460083401c6SMatthew G. Knepley     const PetscReal *quadPoints, *quadWeights;
1461083401c6SMatthew G. Knepley     PetscFEGeom      fegeom;
1462083401c6SMatthew G. Knepley     PetscReal       *coords, *gcoords;
1463083401c6SMatthew G. Knepley     PetscScalar     *funcVal, *interpolant;
14645fedec97SMatthew G. Knepley     PetscBool        isCohesive;
1465083401c6SMatthew G. Knepley     PetscInt         qNc, Nq, totNc, cStart = 0, cEnd, c, dsNf;
146673d901b8SMatthew G. Knepley 
14679566063dSJacob Faibussowitsch     PetscCall(DMGetRegionNumDS(dm, s, &label, &fieldIS, &ds));
14689566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(fieldIS, &fields));
14699566063dSJacob Faibussowitsch     PetscCall(PetscDSIsCohesive(ds, &isCohesive));
14709566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(ds, &dsNf));
14719566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalComponents(ds, &totNc));
14729566063dSJacob Faibussowitsch     PetscCall(PetscDSGetQuadrature(ds, &quad));
14739566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
147463a3b9bcSJacob Faibussowitsch     PetscCheck(!(qNc != 1) || !(qNc != totNc),PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, totNc);
14759566063dSJacob Faibussowitsch     PetscCall(PetscCalloc6(totNc, &funcVal, totNc, &interpolant, dE*(Nq+1), &coords,Nq, &fegeom.detJ, dE*dE*Nq, &fegeom.J, dE*dE*Nq, &fegeom.invJ));
1476083401c6SMatthew G. Knepley     if (!label) {
14779566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1478083401c6SMatthew G. Knepley     } else {
14799566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(label, 1, &pointIS));
14809566063dSJacob Faibussowitsch       PetscCall(ISGetLocalSize(pointIS, &cEnd));
14819566063dSJacob Faibussowitsch       PetscCall(ISGetIndices(pointIS, &points));
1482083401c6SMatthew G. Knepley     }
148373d901b8SMatthew G. Knepley     for (c = cStart; c < cEnd; ++c) {
1484083401c6SMatthew G. Knepley       const PetscInt cell  = points ? points[c] : c;
148573d901b8SMatthew G. Knepley       PetscScalar    *x    = NULL;
14865fedec97SMatthew G. Knepley       const PetscInt *cone;
14875fedec97SMatthew G. Knepley       PetscInt        qc   = 0, fOff = 0, dep;
148873d901b8SMatthew G. Knepley 
14899566063dSJacob Faibussowitsch       PetscCall(DMLabelGetValue(depthLabel, cell, &dep));
1490083401c6SMatthew G. Knepley       if (dep != depth-1) continue;
14915fedec97SMatthew G. Knepley       if (isCohesive) {
14929566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cell, &cone));
14939566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cone[0], quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
149496959cd1SMatthew G. Knepley       } else {
14959566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
149696959cd1SMatthew G. Knepley       }
14979566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, localX, cell, NULL, &x));
14985fedec97SMatthew G. Knepley       for (f = 0; f < dsNf; ++f) {
149915496722SMatthew G. Knepley         PetscObject  obj;
150015496722SMatthew G. Knepley         PetscClassId id;
1501083401c6SMatthew G. Knepley         void * const ctx = ctxs ? ctxs[fields[f]] : NULL;
150215496722SMatthew G. Knepley         PetscInt     Nb, Nc, q, fc;
150315496722SMatthew G. Knepley         PetscReal    elemDiff = 0.0;
15045fedec97SMatthew G. Knepley         PetscBool    cohesive;
150515496722SMatthew G. Knepley 
15069566063dSJacob Faibussowitsch         PetscCall(PetscDSGetCohesive(ds, f, &cohesive));
15075fedec97SMatthew G. Knepley         if (isCohesive && !cohesive) continue;
15089566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
15099566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
15109566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &Nc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
15119566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &Nc));Nb = 1;}
151263a3b9bcSJacob Faibussowitsch         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
151373d901b8SMatthew G. Knepley         if (debug) {
151473d901b8SMatthew G. Knepley           char title[1024];
151563a3b9bcSJacob Faibussowitsch           PetscCall(PetscSNPrintf(title, 1023, "Solution for Field %" PetscInt_FMT, fields[f]));
15169566063dSJacob Faibussowitsch           PetscCall(DMPrintCellVector(cell, title, Nb, &x[fOff]));
151773d901b8SMatthew G. Knepley         }
15187318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
15192a4e142eSMatthew G. Knepley           PetscFEGeom    qgeom;
1520d0609cedSBarry Smith           PetscErrorCode ierr;
15212a4e142eSMatthew G. Knepley 
15222a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
1523083401c6SMatthew G. Knepley           qgeom.J        = &fegeom.J[q*dE*dE];
1524083401c6SMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*dE*dE];
15252a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
152663a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for cell %" PetscInt_FMT ", quadrature point %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
1527d3a7d86cSMatthew G. Knepley           if (transform) {
1528083401c6SMatthew G. Knepley             gcoords = &coords[dE*Nq];
15299566063dSJacob Faibussowitsch             PetscCall(DMPlexBasisTransformApplyReal_Internal(dm, &coords[dE*q], PETSC_TRUE, dE, &coords[dE*q], gcoords, dm->transformCtx));
1530d3a7d86cSMatthew G. Knepley           } else {
1531083401c6SMatthew G. Knepley             gcoords = &coords[dE*q];
1532d3a7d86cSMatthew G. Knepley           }
15332df84da0SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) funcVal[fc] = 0.;
1534083401c6SMatthew G. Knepley           ierr = (*funcs[fields[f]])(dE, time, gcoords, Nc, funcVal, ctx);
1535e735a8a9SMatthew G. Knepley           if (ierr) {
15369566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
15379566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
15389566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
1539e735a8a9SMatthew G. Knepley           }
15409566063dSJacob Faibussowitsch           if (transform) PetscCall(DMPlexBasisTransformApply_Internal(dm, &coords[dE*q], PETSC_FALSE, Nc, funcVal, funcVal, dm->transformCtx));
154196959cd1SMatthew G. Knepley           /* Call once for each face, except for lagrange field */
15429566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID)      PetscCall(PetscFEInterpolate_Static((PetscFE) obj, &x[fOff], &qgeom, q, interpolant));
15439566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV) obj, &x[fOff], q, interpolant));
154463a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, fields[f]);
154515496722SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1546beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
154763a3b9bcSJacob Faibussowitsch             if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "    cell %" PetscInt_FMT " field %" PetscInt_FMT ",%" PetscInt_FMT " point %g %g %g diff %g\n", cell, fields[f], fc, (double)(dE > 0 ? coords[dE*q] : 0.), (double)(dE > 1 ? coords[dE*q+1] : 0.),(double)(dE > 2 ? coords[dE*q+2] : 0.), (double)(PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q])));
15484bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
154973d901b8SMatthew G. Knepley           }
155073d901b8SMatthew G. Knepley         }
1551083401c6SMatthew G. Knepley         fOff += Nb;
15529c3cf19fSMatthew G. Knepley         qc   += Nc;
1553083401c6SMatthew G. Knepley         localDiff[fields[f]] += elemDiff;
155463a3b9bcSJacob Faibussowitsch         if (debug) PetscCall(PetscPrintf(PETSC_COMM_SELF, "  cell %" PetscInt_FMT " field %" PetscInt_FMT " cum diff %g\n", cell, fields[f], (double)localDiff[fields[f]]));
155573d901b8SMatthew G. Knepley       }
15569566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, cell, NULL, &x));
1557083401c6SMatthew G. Knepley     }
1558083401c6SMatthew G. Knepley     if (label) {
15599566063dSJacob Faibussowitsch       PetscCall(ISRestoreIndices(pointIS, &points));
15609566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
1561083401c6SMatthew G. Knepley     }
15629566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(fieldIS, &fields));
15639566063dSJacob Faibussowitsch     PetscCall(PetscFree6(funcVal, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
156473d901b8SMatthew G. Knepley   }
15659566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
15661c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(localDiff, diff, Nf, MPIU_REAL, MPIU_SUM, PetscObjectComm((PetscObject)dm)));
15679566063dSJacob Faibussowitsch   PetscCall(PetscFree(localDiff));
1568083401c6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) diff[f] = PetscSqrtReal(diff[f]);
156973d901b8SMatthew G. Knepley   PetscFunctionReturn(0);
157073d901b8SMatthew G. Knepley }
157173d901b8SMatthew G. Knepley 
1572e729f68cSMatthew G. Knepley /*@C
1573e729f68cSMatthew G. Knepley   DMPlexComputeL2DiffVec - This function computes the cellwise L_2 difference between a function u and an FEM interpolant solution u_h, and stores it in a Vec.
1574e729f68cSMatthew G. Knepley 
1575d083f849SBarry Smith   Collective on dm
1576c0f8e1fdSMatthew G. Knepley 
1577e729f68cSMatthew G. Knepley   Input Parameters:
1578e729f68cSMatthew G. Knepley + dm    - The DM
15790163fd50SMatthew G. Knepley . time  - The time
1580ca3eba1bSToby Isaac . funcs - The functions to evaluate for each field component: NULL means that component does not contribute to error calculation
1581e729f68cSMatthew G. Knepley . ctxs  - Optional array of contexts to pass to each function, or NULL.
1582e729f68cSMatthew G. Knepley - X     - The coefficient vector u_h
1583e729f68cSMatthew G. Knepley 
1584e729f68cSMatthew G. Knepley   Output Parameter:
1585e729f68cSMatthew G. Knepley . D - A Vec which holds the difference ||u - u_h||_2 for each cell
1586e729f68cSMatthew G. Knepley 
1587e729f68cSMatthew G. Knepley   Level: developer
1588e729f68cSMatthew G. Knepley 
1589b698f381SToby Isaac .seealso: DMProjectFunction(), DMComputeL2Diff(), DMPlexComputeL2FieldDiff(), DMComputeL2GradientDiff()
1590e729f68cSMatthew G. Knepley @*/
15910163fd50SMatthew G. Knepley PetscErrorCode DMPlexComputeL2DiffVec(DM dm, PetscReal time, PetscErrorCode (**funcs)(PetscInt, PetscReal, const PetscReal [], PetscInt, PetscScalar *, void *), void **ctxs, Vec X, Vec D)
1592e729f68cSMatthew G. Knepley {
1593e729f68cSMatthew G. Knepley   PetscSection     section;
1594e729f68cSMatthew G. Knepley   PetscQuadrature  quad;
1595e729f68cSMatthew G. Knepley   Vec              localX;
15964bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
1597e729f68cSMatthew G. Knepley   PetscScalar     *funcVal, *interpolant;
15984bee2e38SMatthew G. Knepley   PetscReal       *coords;
1599e729f68cSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1600485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, c, field, fieldOffset;
1601e729f68cSMatthew G. Knepley 
1602e729f68cSMatthew G. Knepley   PetscFunctionBegin;
16039566063dSJacob Faibussowitsch   PetscCall(VecSet(D, 0.0));
16049566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
16059566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
16069566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
16079566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &numFields));
16089566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &localX));
16099566063dSJacob Faibussowitsch   PetscCall(DMProjectFunctionLocal(dm, time, funcs, ctxs, INSERT_BC_VALUES, localX));
16109566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, localX));
16119566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, localX));
1612e729f68cSMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
1613e729f68cSMatthew G. Knepley     PetscObject  obj;
1614e729f68cSMatthew G. Knepley     PetscClassId id;
1615e729f68cSMatthew G. Knepley     PetscInt     Nc;
1616e729f68cSMatthew G. Knepley 
16179566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
16189566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1619e729f68cSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
1620e729f68cSMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
1621e729f68cSMatthew G. Knepley 
16229566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
16239566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
1624e729f68cSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
1625e729f68cSMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
1626e729f68cSMatthew G. Knepley 
16279566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
16289566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
162963a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1630e729f68cSMatthew G. Knepley     numComponents += Nc;
1631e729f68cSMatthew G. Knepley   }
16329566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
163363a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents),PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
16349566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numComponents,&funcVal,numComponents,&interpolant,coordDim*Nq,&coords,Nq,&fegeom.detJ,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ));
16359566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
1636e729f68cSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
1637e729f68cSMatthew G. Knepley     PetscScalar *x = NULL;
16386f288a59SMatthew G. Knepley     PetscScalar  elemDiff = 0.0;
16399c3cf19fSMatthew G. Knepley     PetscInt     qc = 0;
1640e729f68cSMatthew G. Knepley 
16419566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeCellGeometryFEM(dm, c, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
16429566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, NULL, localX, c, NULL, &x));
1643e729f68cSMatthew G. Knepley 
1644e729f68cSMatthew G. Knepley     for (field = 0, fieldOffset = 0; field < numFields; ++field) {
1645e729f68cSMatthew G. Knepley       PetscObject  obj;
1646e729f68cSMatthew G. Knepley       PetscClassId id;
1647e729f68cSMatthew G. Knepley       void * const ctx = ctxs ? ctxs[field] : NULL;
1648e729f68cSMatthew G. Knepley       PetscInt     Nb, Nc, q, fc;
1649e729f68cSMatthew G. Knepley 
16509566063dSJacob Faibussowitsch       PetscCall(DMGetField(dm, field, NULL, &obj));
16519566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
16529566063dSJacob Faibussowitsch       if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &Nc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
16539566063dSJacob Faibussowitsch       else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &Nc));Nb = 1;}
165463a3b9bcSJacob Faibussowitsch       else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
165523f34ed2SToby Isaac       if (funcs[field]) {
16567318780aSToby Isaac         for (q = 0; q < Nq; ++q) {
16572a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
16582a4e142eSMatthew G. Knepley 
16592a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
16602a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q*coordDim*coordDim];
16612a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
16622a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
166363a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], c, q);
16649566063dSJacob Faibussowitsch           PetscCall((*funcs[field])(coordDim, time, &coords[q*coordDim], Nc, funcVal, ctx));
1665c3e24edfSBarry Smith #if defined(needs_fix_with_return_code_argument)
1666e735a8a9SMatthew G. Knepley           if (ierr) {
16679566063dSJacob Faibussowitsch             PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
16689566063dSJacob Faibussowitsch             PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
16699566063dSJacob Faibussowitsch             PetscCall(DMRestoreLocalVector(dm, &localX));
1670e735a8a9SMatthew G. Knepley           }
1671c3e24edfSBarry Smith #endif
16729566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID)      PetscCall(PetscFEInterpolate_Static((PetscFE) obj, &x[fieldOffset], &qgeom, q, interpolant));
16739566063dSJacob Faibussowitsch           else if (id == PETSCFV_CLASSID) PetscCall(PetscFVInterpolate_Static((PetscFV) obj, &x[fieldOffset], q, interpolant));
167463a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
1675e729f68cSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
1676beaa55a6SMatthew G. Knepley             const PetscReal wt = quadWeights[q*qNc+(qNc == 1 ? 0 : qc+fc)];
16774bee2e38SMatthew G. Knepley             elemDiff += PetscSqr(PetscRealPart(interpolant[fc] - funcVal[fc]))*wt*fegeom.detJ[q];
1678e729f68cSMatthew G. Knepley           }
1679e729f68cSMatthew G. Knepley         }
168023f34ed2SToby Isaac       }
1681beaa55a6SMatthew G. Knepley       fieldOffset += Nb;
16829c3cf19fSMatthew G. Knepley       qc          += Nc;
1683e729f68cSMatthew G. Knepley     }
16849566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, NULL, localX, c, NULL, &x));
16859566063dSJacob Faibussowitsch     PetscCall(VecSetValue(D, c - cStart, elemDiff, INSERT_VALUES));
1686e729f68cSMatthew G. Knepley   }
16879566063dSJacob Faibussowitsch   PetscCall(PetscFree6(funcVal,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
16889566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &localX));
16899566063dSJacob Faibussowitsch   PetscCall(VecSqrtAbs(D));
1690e729f68cSMatthew G. Knepley   PetscFunctionReturn(0);
1691e729f68cSMatthew G. Knepley }
1692e729f68cSMatthew G. Knepley 
16935f0b18bfSMatthew G. Knepley /*@
16945f0b18bfSMatthew G. Knepley   DMPlexComputeClementInterpolant - This function computes the L2 projection of the cellwise values of a function u onto P1, and stores it in a Vec.
16955f0b18bfSMatthew G. Knepley 
16965f0b18bfSMatthew G. Knepley   Collective on dm
16975f0b18bfSMatthew G. Knepley 
16985f0b18bfSMatthew G. Knepley   Input Parameters:
16995f0b18bfSMatthew G. Knepley + dm - The DM
17005f0b18bfSMatthew G. Knepley - locX  - The coefficient vector u_h
17015f0b18bfSMatthew G. Knepley 
17025f0b18bfSMatthew G. Knepley   Output Parameter:
17035f0b18bfSMatthew G. Knepley . locC - A Vec which holds the Clement interpolant of the function
17045f0b18bfSMatthew G. Knepley 
17055f0b18bfSMatthew G. Knepley   Notes:
17065f0b18bfSMatthew G. Knepley   u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| where |T_i| is the cell volume
17075f0b18bfSMatthew G. Knepley 
17085f0b18bfSMatthew G. Knepley   Level: developer
17095f0b18bfSMatthew G. Knepley 
17105f0b18bfSMatthew G. Knepley .seealso: DMProjectFunction(), DMComputeL2Diff(), DMPlexComputeL2FieldDiff(), DMComputeL2GradientDiff()
17115f0b18bfSMatthew G. Knepley @*/
17125f0b18bfSMatthew G. Knepley PetscErrorCode DMPlexComputeClementInterpolant(DM dm, Vec locX, Vec locC)
17135f0b18bfSMatthew G. Knepley {
17145f0b18bfSMatthew G. Knepley   PetscInt         debug = ((DM_Plex *) dm->data)->printFEM;
17155f0b18bfSMatthew G. Knepley   DM               dmc;
17165f0b18bfSMatthew G. Knepley   PetscQuadrature  quad;
17175f0b18bfSMatthew G. Knepley   PetscScalar     *interpolant, *valsum;
17185f0b18bfSMatthew G. Knepley   PetscFEGeom      fegeom;
17195f0b18bfSMatthew G. Knepley   PetscReal       *coords;
17205f0b18bfSMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
17215f0b18bfSMatthew G. Knepley   PetscInt         dim, cdim, Nf, f, Nc = 0, Nq, qNc, cStart, cEnd, vStart, vEnd, v;
17225f0b18bfSMatthew G. Knepley 
17235f0b18bfSMatthew G. Knepley   PetscFunctionBegin;
17249566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
17259566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmc));
17269566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
17279566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
17289566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &cdim));
17295f0b18bfSMatthew G. Knepley   fegeom.dimEmbed = cdim;
17309566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
17315f0b18bfSMatthew G. Knepley   PetscCheck(Nf > 0, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
17325f0b18bfSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
17335f0b18bfSMatthew G. Knepley     PetscObject  obj;
17345f0b18bfSMatthew G. Knepley     PetscClassId id;
17355f0b18bfSMatthew G. Knepley     PetscInt     fNc;
17365f0b18bfSMatthew G. Knepley 
17379566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, f, NULL, &obj));
17389566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
17395f0b18bfSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
17405f0b18bfSMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
17415f0b18bfSMatthew G. Knepley 
17429566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
17439566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &fNc));
17445f0b18bfSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
17455f0b18bfSMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
17465f0b18bfSMatthew G. Knepley 
17479566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
17489566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &fNc));
174963a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
17505f0b18bfSMatthew G. Knepley     Nc += fNc;
17515f0b18bfSMatthew G. Knepley   }
17529566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
175363a3b9bcSJacob Faibussowitsch   PetscCheck(qNc == 1,PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " > 1", qNc);
17549566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(Nc*2, &valsum, Nc, &interpolant, cdim*Nq, &coords, Nq, &fegeom.detJ, cdim*cdim*Nq, &fegeom.J, cdim*cdim*Nq, &fegeom.invJ));
17559566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
17569566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
17575f0b18bfSMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
17585f0b18bfSMatthew G. Knepley     PetscScalar volsum = 0.0;
17595f0b18bfSMatthew G. Knepley     PetscInt   *star   = NULL;
17605f0b18bfSMatthew G. Knepley     PetscInt    starSize, st, fc;
17615f0b18bfSMatthew G. Knepley 
17629566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(valsum, Nc));
17639566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
17645f0b18bfSMatthew G. Knepley     for (st = 0; st < starSize*2; st += 2) {
17655f0b18bfSMatthew G. Knepley       const PetscInt cell = star[st];
17665f0b18bfSMatthew G. Knepley       PetscScalar   *val  = &valsum[Nc];
17675f0b18bfSMatthew G. Knepley       PetscScalar   *x    = NULL;
17685f0b18bfSMatthew G. Knepley       PetscReal      vol  = 0.0;
17695f0b18bfSMatthew G. Knepley       PetscInt       foff = 0;
17705f0b18bfSMatthew G. Knepley 
17715f0b18bfSMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
17729566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
17739566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
17745f0b18bfSMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
17755f0b18bfSMatthew G. Knepley         PetscObject  obj;
17765f0b18bfSMatthew G. Knepley         PetscClassId id;
17775f0b18bfSMatthew G. Knepley         PetscInt     Nb, fNc, q;
17785f0b18bfSMatthew G. Knepley 
17799566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(val, Nc));
17809566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, f, NULL, &obj));
17819566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
17829566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &fNc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
17839566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &fNc));Nb = 1;}
178463a3b9bcSJacob Faibussowitsch         else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
17855f0b18bfSMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
17865f0b18bfSMatthew G. Knepley           const PetscReal wt = quadWeights[q]*fegeom.detJ[q];
17875f0b18bfSMatthew G. Knepley           PetscFEGeom     qgeom;
17885f0b18bfSMatthew G. Knepley 
17895f0b18bfSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
17905f0b18bfSMatthew G. Knepley           qgeom.J        = &fegeom.J[q*cdim*cdim];
17915f0b18bfSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*cdim*cdim];
17925f0b18bfSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
179363a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double) fegeom.detJ[q], cell, q);
17949566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID) PetscCall(PetscFEInterpolate_Static((PetscFE) obj, &x[foff], &qgeom, q, interpolant));
179563a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
17965f0b18bfSMatthew G. Knepley           for (fc = 0; fc < fNc; ++fc) val[foff+fc] += interpolant[fc]*wt;
17975f0b18bfSMatthew G. Knepley           vol += wt;
17985f0b18bfSMatthew G. Knepley         }
17995f0b18bfSMatthew G. Knepley         foff += Nb;
18005f0b18bfSMatthew G. Knepley       }
18019566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
18025f0b18bfSMatthew G. Knepley       for (fc = 0; fc < Nc; ++fc) valsum[fc] += val[fc];
18035f0b18bfSMatthew G. Knepley       volsum += vol;
18045f0b18bfSMatthew G. Knepley       if (debug) {
18059566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " value: [", v, cell));
18065f0b18bfSMatthew G. Knepley         for (fc = 0; fc < Nc; ++fc) {
18079566063dSJacob Faibussowitsch           if (fc) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
18089566063dSJacob Faibussowitsch           PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double) PetscRealPart(val[fc])));
18095f0b18bfSMatthew G. Knepley         }
18109566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
18115f0b18bfSMatthew G. Knepley       }
18125f0b18bfSMatthew G. Knepley     }
18135f0b18bfSMatthew G. Knepley     for (fc = 0; fc < Nc; ++fc) valsum[fc] /= volsum;
18149566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
18159566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmc, NULL, locC, v, valsum, INSERT_VALUES));
18165f0b18bfSMatthew G. Knepley   }
18179566063dSJacob Faibussowitsch   PetscCall(PetscFree6(valsum, interpolant, coords, fegeom.detJ, fegeom.J, fegeom.invJ));
18185f0b18bfSMatthew G. Knepley   PetscFunctionReturn(0);
18195f0b18bfSMatthew G. Knepley }
18205f0b18bfSMatthew G. Knepley 
18215f0b18bfSMatthew G. Knepley /*@
18221555c271SMatthew G. Knepley   DMPlexComputeGradientClementInterpolant - This function computes the L2 projection of the cellwise gradient of a function u onto P1, and stores it in a Vec.
18231555c271SMatthew G. Knepley 
1824d083f849SBarry Smith   Collective on dm
1825c0f8e1fdSMatthew G. Knepley 
18261555c271SMatthew G. Knepley   Input Parameters:
18271555c271SMatthew G. Knepley + dm - The DM
18285f0b18bfSMatthew G. Knepley - locX  - The coefficient vector u_h
18291555c271SMatthew G. Knepley 
18301555c271SMatthew G. Knepley   Output Parameter:
18311555c271SMatthew G. Knepley . locC - A Vec which holds the Clement interpolant of the gradient
18321555c271SMatthew G. Knepley 
183395452b02SPatrick Sanan   Notes:
18341555c271SMatthew G. Knepley   \nabla u_h(v_i) = \sum_{T_i \in support(v_i)} |T_i| \nabla u_h(T_i) / \sum_{T_i \in support(v_i)} |T_i| where |T_i| is the cell volume
18351555c271SMatthew G. Knepley 
18361555c271SMatthew G. Knepley   Level: developer
18371555c271SMatthew G. Knepley 
18381555c271SMatthew G. Knepley .seealso: DMProjectFunction(), DMComputeL2Diff(), DMPlexComputeL2FieldDiff(), DMComputeL2GradientDiff()
18391555c271SMatthew G. Knepley @*/
18401555c271SMatthew G. Knepley PetscErrorCode DMPlexComputeGradientClementInterpolant(DM dm, Vec locX, Vec locC)
18411555c271SMatthew G. Knepley {
1842db1066baSMatthew G. Knepley   DM_Plex         *mesh  = (DM_Plex *) dm->data;
1843db1066baSMatthew G. Knepley   PetscInt         debug = mesh->printFEM;
18441555c271SMatthew G. Knepley   DM               dmC;
18451555c271SMatthew G. Knepley   PetscQuadrature  quad;
18461555c271SMatthew G. Knepley   PetscScalar     *interpolant, *gradsum;
18474bee2e38SMatthew G. Knepley   PetscFEGeom      fegeom;
18484bee2e38SMatthew G. Knepley   PetscReal       *coords;
18491555c271SMatthew G. Knepley   const PetscReal *quadPoints, *quadWeights;
1850485ad865SMatthew G. Knepley   PetscInt         dim, coordDim, numFields, numComponents = 0, qNc, Nq, cStart, cEnd, vStart, vEnd, v, field, fieldOffset;
18511555c271SMatthew G. Knepley 
18521555c271SMatthew G. Knepley   PetscFunctionBegin;
18539566063dSJacob Faibussowitsch   PetscCall(PetscCitationsRegister(ClementCitation, &Clementcite));
18549566063dSJacob Faibussowitsch   PetscCall(VecGetDM(locC, &dmC));
18559566063dSJacob Faibussowitsch   PetscCall(VecSet(locC, 0.0));
18569566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
18579566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dm, &coordDim));
18584bee2e38SMatthew G. Knepley   fegeom.dimEmbed = coordDim;
18599566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &numFields));
18605f80ce2aSJacob Faibussowitsch   PetscCheck(numFields,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fields is zero!");
18611555c271SMatthew G. Knepley   for (field = 0; field < numFields; ++field) {
18621555c271SMatthew G. Knepley     PetscObject  obj;
18631555c271SMatthew G. Knepley     PetscClassId id;
18641555c271SMatthew G. Knepley     PetscInt     Nc;
18651555c271SMatthew G. Knepley 
18669566063dSJacob Faibussowitsch     PetscCall(DMGetField(dm, field, NULL, &obj));
18679566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
18681555c271SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
18691555c271SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
18701555c271SMatthew G. Knepley 
18719566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &quad));
18729566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
18731555c271SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
18741555c271SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
18751555c271SMatthew G. Knepley 
18769566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature(fv, &quad));
18779566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
187863a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
18791555c271SMatthew G. Knepley     numComponents += Nc;
18801555c271SMatthew G. Knepley   }
18819566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureGetData(quad, NULL, &qNc, &Nq, &quadPoints, &quadWeights));
188263a3b9bcSJacob Faibussowitsch   PetscCheck(!(qNc != 1) || !(qNc != numComponents),PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_SIZ, "Quadrature components %" PetscInt_FMT " != %" PetscInt_FMT " field components", qNc, numComponents);
18839566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(coordDim*numComponents*2,&gradsum,coordDim*numComponents,&interpolant,coordDim*Nq,&coords,Nq,&fegeom.detJ,coordDim*coordDim*Nq,&fegeom.J,coordDim*coordDim*Nq,&fegeom.invJ));
18849566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthStratum(dm, 0, &vStart, &vEnd));
18859566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, 0, &cStart, &cEnd));
18861555c271SMatthew G. Knepley   for (v = vStart; v < vEnd; ++v) {
18871555c271SMatthew G. Knepley     PetscScalar volsum = 0.0;
18881555c271SMatthew G. Knepley     PetscInt   *star   = NULL;
18891555c271SMatthew G. Knepley     PetscInt    starSize, st, d, fc;
18901555c271SMatthew G. Knepley 
18919566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(gradsum, coordDim*numComponents));
18929566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
18931555c271SMatthew G. Knepley     for (st = 0; st < starSize*2; st += 2) {
18941555c271SMatthew G. Knepley       const PetscInt cell = star[st];
18951555c271SMatthew G. Knepley       PetscScalar   *grad = &gradsum[coordDim*numComponents];
18961555c271SMatthew G. Knepley       PetscScalar   *x    = NULL;
18971555c271SMatthew G. Knepley       PetscReal      vol  = 0.0;
18981555c271SMatthew G. Knepley 
18991555c271SMatthew G. Knepley       if ((cell < cStart) || (cell >= cEnd)) continue;
19009566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dm, cell, quad, coords, fegeom.J, fegeom.invJ, fegeom.detJ));
19019566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, NULL, locX, cell, NULL, &x));
19021555c271SMatthew G. Knepley       for (field = 0, fieldOffset = 0; field < numFields; ++field) {
19031555c271SMatthew G. Knepley         PetscObject  obj;
19041555c271SMatthew G. Knepley         PetscClassId id;
19051555c271SMatthew G. Knepley         PetscInt     Nb, Nc, q, qc = 0;
19061555c271SMatthew G. Knepley 
19079566063dSJacob Faibussowitsch         PetscCall(PetscArrayzero(grad, coordDim*numComponents));
19089566063dSJacob Faibussowitsch         PetscCall(DMGetField(dm, field, NULL, &obj));
19099566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
19109566063dSJacob Faibussowitsch         if (id == PETSCFE_CLASSID)      {PetscCall(PetscFEGetNumComponents((PetscFE) obj, &Nc));PetscCall(PetscFEGetDimension((PetscFE) obj, &Nb));}
19119566063dSJacob Faibussowitsch         else if (id == PETSCFV_CLASSID) {PetscCall(PetscFVGetNumComponents((PetscFV) obj, &Nc));Nb = 1;}
191263a3b9bcSJacob Faibussowitsch         else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
19131555c271SMatthew G. Knepley         for (q = 0; q < Nq; ++q) {
19142a4e142eSMatthew G. Knepley           PetscFEGeom qgeom;
19152a4e142eSMatthew G. Knepley 
19162a4e142eSMatthew G. Knepley           qgeom.dimEmbed = fegeom.dimEmbed;
19172a4e142eSMatthew G. Knepley           qgeom.J        = &fegeom.J[q*coordDim*coordDim];
19182a4e142eSMatthew G. Knepley           qgeom.invJ     = &fegeom.invJ[q*coordDim*coordDim];
19192a4e142eSMatthew G. Knepley           qgeom.detJ     = &fegeom.detJ[q];
192063a3b9bcSJacob Faibussowitsch           PetscCheck(fegeom.detJ[q] > 0.0,PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Invalid determinant %g for element %" PetscInt_FMT ", quadrature points %" PetscInt_FMT, (double)fegeom.detJ[q], cell, q);
19219566063dSJacob Faibussowitsch           if (id == PETSCFE_CLASSID)      PetscCall(PetscFEInterpolateGradient_Static((PetscFE) obj, 1, &x[fieldOffset], &qgeom, q, interpolant));
192263a3b9bcSJacob Faibussowitsch           else SETERRQ(PetscObjectComm((PetscObject)dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, field);
19231555c271SMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
192453d2db2dSJoe Wallwork             const PetscReal wt = quadWeights[q*qNc+qc];
19251555c271SMatthew G. Knepley 
19264bee2e38SMatthew G. Knepley             for (d = 0; d < coordDim; ++d) grad[fc*coordDim+d] += interpolant[fc*dim+d]*wt*fegeom.detJ[q];
19271555c271SMatthew G. Knepley           }
19284bee2e38SMatthew G. Knepley           vol += quadWeights[q*qNc]*fegeom.detJ[q];
19291555c271SMatthew G. Knepley         }
19301555c271SMatthew G. Knepley         fieldOffset += Nb;
19311555c271SMatthew G. Knepley         qc          += Nc;
19321555c271SMatthew G. Knepley       }
19339566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, NULL, locX, cell, NULL, &x));
1934f8527842SMatthew G. Knepley       for (fc = 0; fc < numComponents; ++fc) {
1935f8527842SMatthew G. Knepley         for (d = 0; d < coordDim; ++d) {
1936f8527842SMatthew G. Knepley           gradsum[fc*coordDim+d] += grad[fc*coordDim+d];
1937f8527842SMatthew G. Knepley         }
1938f8527842SMatthew G. Knepley       }
1939f8527842SMatthew G. Knepley       volsum += vol;
1940db1066baSMatthew G. Knepley       if (debug) {
19419566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "Vertex %" PetscInt_FMT " Cell %" PetscInt_FMT " gradient: [", v, cell));
19421555c271SMatthew G. Knepley         for (fc = 0; fc < numComponents; ++fc) {
19431555c271SMatthew G. Knepley           for (d = 0; d < coordDim; ++d) {
19449566063dSJacob Faibussowitsch             if (fc || d > 0) PetscCall(PetscPrintf(PETSC_COMM_SELF, ", "));
19459566063dSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "%g", (double)PetscRealPart(grad[fc*coordDim+d])));
19461555c271SMatthew G. Knepley           }
19471555c271SMatthew G. Knepley         }
19489566063dSJacob Faibussowitsch         PetscCall(PetscPrintf(PETSC_COMM_SELF, "]\n"));
1949db1066baSMatthew G. Knepley       }
19501555c271SMatthew G. Knepley     }
19511555c271SMatthew G. Knepley     for (fc = 0; fc < numComponents; ++fc) {
19521555c271SMatthew G. Knepley       for (d = 0; d < coordDim; ++d) gradsum[fc*coordDim+d] /= volsum;
19531555c271SMatthew G. Knepley     }
19549566063dSJacob Faibussowitsch     PetscCall(DMPlexRestoreTransitiveClosure(dm, v, PETSC_FALSE, &starSize, &star));
19559566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dmC, NULL, locC, v, gradsum, INSERT_VALUES));
19561555c271SMatthew G. Knepley   }
19579566063dSJacob Faibussowitsch   PetscCall(PetscFree6(gradsum,interpolant,coords,fegeom.detJ,fegeom.J,fegeom.invJ));
19581555c271SMatthew G. Knepley   PetscFunctionReturn(0);
19591555c271SMatthew G. Knepley }
19601555c271SMatthew G. Knepley 
1961338f77d5SMatthew G. Knepley static PetscErrorCode DMPlexComputeIntegral_Internal(DM dm, Vec X, PetscInt cStart, PetscInt cEnd, PetscScalar *cintegral, void *user)
196273d901b8SMatthew G. Knepley {
1963338f77d5SMatthew G. Knepley   DM                 dmAux = NULL;
196461aaff12SToby Isaac   PetscDS            prob,    probAux = NULL;
196573d901b8SMatthew G. Knepley   PetscSection       section, sectionAux;
1966338f77d5SMatthew G. Knepley   Vec                locX,    locA;
1967c330f8ffSToby Isaac   PetscInt           dim, numCells = cEnd - cStart, c, f;
1968c330f8ffSToby Isaac   PetscBool          useFVM = PETSC_FALSE;
1969338f77d5SMatthew G. Knepley   /* DS */
1970338f77d5SMatthew G. Knepley   PetscInt           Nf,    totDim,    *uOff, *uOff_x, numConstants;
1971338f77d5SMatthew G. Knepley   PetscInt           NfAux, totDimAux, *aOff;
1972338f77d5SMatthew G. Knepley   PetscScalar       *u, *a;
1973338f77d5SMatthew G. Knepley   const PetscScalar *constants;
1974338f77d5SMatthew G. Knepley   /* Geometry */
1975c330f8ffSToby Isaac   PetscFEGeom       *cgeomFEM;
1976338f77d5SMatthew G. Knepley   DM                 dmGrad;
1977c330f8ffSToby Isaac   PetscQuadrature    affineQuad = NULL;
1978338f77d5SMatthew G. Knepley   Vec                cellGeometryFVM = NULL, faceGeometryFVM = NULL, locGrad = NULL;
1979b5a3613cSMatthew G. Knepley   PetscFVCellGeom   *cgeomFVM;
1980338f77d5SMatthew G. Knepley   const PetscScalar *lgrad;
1981b7260050SToby Isaac   PetscInt           maxDegree;
1982c330f8ffSToby Isaac   DMField            coordField;
1983c330f8ffSToby Isaac   IS                 cellIS;
198473d901b8SMatthew G. Knepley 
198573d901b8SMatthew G. Knepley   PetscFunctionBegin;
19869566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
19879566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
19889566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
19899566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
1990338f77d5SMatthew G. Knepley   /* Determine which discretizations we have */
1991b5a3613cSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
1992b5a3613cSMatthew G. Knepley     PetscObject  obj;
1993b5a3613cSMatthew G. Knepley     PetscClassId id;
1994b5a3613cSMatthew G. Knepley 
19959566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
19969566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
1997338f77d5SMatthew G. Knepley     if (id == PETSCFV_CLASSID) useFVM = PETSC_TRUE;
1998338f77d5SMatthew G. Knepley   }
1999338f77d5SMatthew G. Knepley   /* Get local solution with boundary values */
20009566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &locX));
20019566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
20029566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
20039566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
2004338f77d5SMatthew G. Knepley   /* Read DS information */
20059566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
20069566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
20079566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
20089566063dSJacob Faibussowitsch   PetscCall(ISCreateStride(PETSC_COMM_SELF,numCells,cStart,1,&cellIS));
20099566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
2010338f77d5SMatthew G. Knepley   /* Read Auxiliary DS information */
20119566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
20129a2a23afSMatthew G. Knepley   if (locA) {
20139566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
20149566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
20159566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
20169566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
20179566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
20189566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
2019338f77d5SMatthew G. Knepley   }
2020338f77d5SMatthew G. Knepley   /* Allocate data  arrays */
20219566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(numCells*totDim, &u));
20229566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells*totDimAux, &a));
2023338f77d5SMatthew G. Knepley   /* Read out geometry */
20249566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm,&coordField));
20259566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
2026b7260050SToby Isaac   if (maxDegree <= 1) {
20279566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad));
2028c330f8ffSToby Isaac     if (affineQuad) {
20299566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&cgeomFEM));
2030338f77d5SMatthew G. Knepley     }
2031b5a3613cSMatthew G. Knepley   }
2032b5a3613cSMatthew G. Knepley   if (useFVM) {
2033338f77d5SMatthew G. Knepley     PetscFV   fv = NULL;
2034b5a3613cSMatthew G. Knepley     Vec       grad;
2035b5a3613cSMatthew G. Knepley     PetscInt  fStart, fEnd;
2036b5a3613cSMatthew G. Knepley     PetscBool compGrad;
2037b5a3613cSMatthew G. Knepley 
2038338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
2039338f77d5SMatthew G. Knepley       PetscObject  obj;
2040338f77d5SMatthew G. Knepley       PetscClassId id;
2041338f77d5SMatthew G. Knepley 
20429566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
20439566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
2044338f77d5SMatthew G. Knepley       if (id == PETSCFV_CLASSID) {fv = (PetscFV) obj; break;}
2045338f77d5SMatthew G. Knepley     }
20469566063dSJacob Faibussowitsch     PetscCall(PetscFVGetComputeGradients(fv, &compGrad));
20479566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, PETSC_TRUE));
20489566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGeometryFVM(dm, &cellGeometryFVM, &faceGeometryFVM));
20499566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeGradientFVM(dm, fv, faceGeometryFVM, cellGeometryFVM, &dmGrad));
20509566063dSJacob Faibussowitsch     PetscCall(PetscFVSetComputeGradients(fv, compGrad));
20519566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM));
2052b5a3613cSMatthew G. Knepley     /* Reconstruct and limit cell gradients */
20539566063dSJacob Faibussowitsch     PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
20549566063dSJacob Faibussowitsch     PetscCall(DMGetGlobalVector(dmGrad, &grad));
20559566063dSJacob Faibussowitsch     PetscCall(DMPlexReconstructGradients_Internal(dm, fv, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
2056b5a3613cSMatthew G. Knepley     /* Communicate gradient values */
20579566063dSJacob Faibussowitsch     PetscCall(DMGetLocalVector(dmGrad, &locGrad));
20589566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
20599566063dSJacob Faibussowitsch     PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
20609566063dSJacob Faibussowitsch     PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
2061b5a3613cSMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
20629566063dSJacob Faibussowitsch     PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, 0.0, faceGeometryFVM, cellGeometryFVM, locGrad));
20639566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
2064b5a3613cSMatthew G. Knepley   }
2065338f77d5SMatthew G. Knepley   /* Read out data from inputs */
206673d901b8SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
206773d901b8SMatthew G. Knepley     PetscScalar *x = NULL;
206873d901b8SMatthew G. Knepley     PetscInt     i;
206973d901b8SMatthew G. Knepley 
20709566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, section, locX, c, NULL, &x));
20710f2d7e86SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[c*totDim+i] = x[i];
20729566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, section, locX, c, NULL, &x));
207373d901b8SMatthew G. Knepley     if (dmAux) {
20749566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, locA, c, NULL, &x));
20750f2d7e86SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[c*totDimAux+i] = x[i];
20769566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, locA, c, NULL, &x));
207773d901b8SMatthew G. Knepley     }
207873d901b8SMatthew G. Knepley   }
2079338f77d5SMatthew G. Knepley   /* Do integration for each field */
208073d901b8SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
2081c1f031eeSMatthew G. Knepley     PetscObject  obj;
2082c1f031eeSMatthew G. Knepley     PetscClassId id;
2083c1f031eeSMatthew G. Knepley     PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
208473d901b8SMatthew G. Knepley 
20859566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
20869566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
2087c1f031eeSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
2088c1f031eeSMatthew G. Knepley       PetscFE         fe = (PetscFE) obj;
2089c1f031eeSMatthew G. Knepley       PetscQuadrature q;
2090c330f8ffSToby Isaac       PetscFEGeom     *chunkGeom = NULL;
2091c1f031eeSMatthew G. Knepley       PetscInt        Nq, Nb;
2092c1f031eeSMatthew G. Knepley 
20939566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
20949566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe, &q));
20959566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
20969566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
2097c1f031eeSMatthew G. Knepley       blockSize = Nb*Nq;
209873d901b8SMatthew G. Knepley       batchSize = numBlocks * blockSize;
20999566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
210073d901b8SMatthew G. Knepley       numChunks = numCells / (numBatches*batchSize);
210173d901b8SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
210273d901b8SMatthew G. Knepley       Nr        = numCells % (numBatches*batchSize);
210373d901b8SMatthew G. Knepley       offset    = numCells - Nr;
2104c330f8ffSToby Isaac       if (!affineQuad) {
21059566063dSJacob Faibussowitsch         PetscCall(DMFieldCreateFEGeom(coordField,cellIS,q,PETSC_FALSE,&cgeomFEM));
2106c330f8ffSToby Isaac       }
21079566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom));
21089566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrate(prob, f, Ne, chunkGeom, u, probAux, a, cintegral));
21099566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&chunkGeom));
21109566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrate(prob, f, Nr, chunkGeom, &u[offset*totDim], probAux, &a[offset*totDimAux], &cintegral[offset*Nf]));
21119566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&chunkGeom));
2112c330f8ffSToby Isaac       if (!affineQuad) {
21139566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomDestroy(&cgeomFEM));
2114c330f8ffSToby Isaac       }
2115c1f031eeSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
2116c1f031eeSMatthew G. Knepley       PetscInt       foff;
2117420e96edSMatthew G. Knepley       PetscPointFunc obj_func;
2118b69edc29SMatthew G. Knepley       PetscScalar    lint;
2119c1f031eeSMatthew G. Knepley 
21209566063dSJacob Faibussowitsch       PetscCall(PetscDSGetObjective(prob, f, &obj_func));
21219566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, f, &foff));
2122c1f031eeSMatthew G. Knepley       if (obj_func) {
2123c1f031eeSMatthew G. Knepley         for (c = 0; c < numCells; ++c) {
2124b5a3613cSMatthew G. Knepley           PetscScalar *u_x;
2125b5a3613cSMatthew G. Knepley 
21269566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, c, lgrad, &u_x));
2127338f77d5SMatthew G. Knepley           obj_func(dim, Nf, NfAux, uOff, uOff_x, &u[totDim*c+foff], NULL, u_x, aOff, NULL, &a[totDimAux*c], NULL, NULL, 0.0, cgeomFVM[c].centroid, numConstants, constants, &lint);
2128338f77d5SMatthew G. Knepley           cintegral[c*Nf+f] += PetscRealPart(lint)*cgeomFVM[c].volume;
212973d901b8SMatthew G. Knepley         }
2130c1f031eeSMatthew G. Knepley       }
213163a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
2132c1f031eeSMatthew G. Knepley   }
2133338f77d5SMatthew G. Knepley   /* Cleanup data arrays */
2134b5a3613cSMatthew G. Knepley   if (useFVM) {
21359566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
21369566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM));
21379566063dSJacob Faibussowitsch     PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
21389566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&faceGeometryFVM));
21399566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&cellGeometryFVM));
21409566063dSJacob Faibussowitsch     PetscCall(DMDestroy(&dmGrad));
2141b5a3613cSMatthew G. Knepley   }
21429566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscFree(a));
21439566063dSJacob Faibussowitsch   PetscCall(PetscFree(u));
2144338f77d5SMatthew G. Knepley   /* Cleanup */
2145f99c8401SToby Isaac   if (affineQuad) {
21469566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomDestroy(&cgeomFEM));
2147f99c8401SToby Isaac   }
21489566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&affineQuad));
21499566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cellIS));
21509566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &locX));
2151338f77d5SMatthew G. Knepley   PetscFunctionReturn(0);
2152338f77d5SMatthew G. Knepley }
2153338f77d5SMatthew G. Knepley 
2154338f77d5SMatthew G. Knepley /*@
2155338f77d5SMatthew G. Knepley   DMPlexComputeIntegralFEM - Form the integral over the domain from the global input X using pointwise functions specified by the user
2156338f77d5SMatthew G. Knepley 
2157338f77d5SMatthew G. Knepley   Input Parameters:
2158338f77d5SMatthew G. Knepley + dm - The mesh
2159338f77d5SMatthew G. Knepley . X  - Global input vector
2160338f77d5SMatthew G. Knepley - user - The user context
2161338f77d5SMatthew G. Knepley 
2162338f77d5SMatthew G. Knepley   Output Parameter:
2163338f77d5SMatthew G. Knepley . integral - Integral for each field
2164338f77d5SMatthew G. Knepley 
2165338f77d5SMatthew G. Knepley   Level: developer
2166338f77d5SMatthew G. Knepley 
2167446c23c1SPierre Jolivet .seealso: DMPlexSNESComputeResidualFEM()
2168338f77d5SMatthew G. Knepley @*/
2169b8feb594SMatthew G. Knepley PetscErrorCode DMPlexComputeIntegralFEM(DM dm, Vec X, PetscScalar *integral, void *user)
2170338f77d5SMatthew G. Knepley {
2171338f77d5SMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dm->data;
2172b8feb594SMatthew G. Knepley   PetscScalar   *cintegral, *lintegral;
2173412e9a14SMatthew G. Knepley   PetscInt       Nf, f, cellHeight, cStart, cEnd, cell;
2174338f77d5SMatthew G. Knepley 
2175338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2176338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2177338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2178dadcf809SJacob Faibussowitsch   PetscValidScalarPointer(integral, 3);
21799566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0));
21809566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
21819566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
21829566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2183338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
21849566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &lintegral, (cEnd-cStart)*Nf, &cintegral));
21859566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user));
2186338f77d5SMatthew G. Knepley   /* Sum up values */
2187338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2188338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2189338f77d5SMatthew G. Knepley 
21909566063dSJacob Faibussowitsch     if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c*Nf]));
2191b8feb594SMatthew G. Knepley     for (f = 0; f < Nf; ++f) lintegral[f] += cintegral[c*Nf+f];
2192338f77d5SMatthew G. Knepley   }
21931c2dc1cbSBarry Smith   PetscCall(MPIU_Allreduce(lintegral, integral, Nf, MPIU_SCALAR, MPIU_SUM, PetscObjectComm((PetscObject) dm)));
219473d901b8SMatthew G. Knepley   if (mesh->printFEM) {
21959566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject) dm), "Integral:"));
21969566063dSJacob Faibussowitsch     for (f = 0; f < Nf; ++f) PetscCall(PetscPrintf(PetscObjectComm((PetscObject) dm), " %g", (double) PetscRealPart(integral[f])));
21979566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject) dm), "\n"));
219873d901b8SMatthew G. Knepley   }
21999566063dSJacob Faibussowitsch   PetscCall(PetscFree2(lintegral, cintegral));
22009566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0));
2201338f77d5SMatthew G. Knepley   PetscFunctionReturn(0);
2202338f77d5SMatthew G. Knepley }
2203338f77d5SMatthew G. Knepley 
2204338f77d5SMatthew G. Knepley /*@
2205338f77d5SMatthew G. Knepley   DMPlexComputeCellwiseIntegralFEM - Form the vector of cellwise integrals F from the global input X using pointwise functions specified by the user
2206338f77d5SMatthew G. Knepley 
2207338f77d5SMatthew G. Knepley   Input Parameters:
2208338f77d5SMatthew G. Knepley + dm - The mesh
2209338f77d5SMatthew G. Knepley . X  - Global input vector
2210338f77d5SMatthew G. Knepley - user - The user context
2211338f77d5SMatthew G. Knepley 
2212338f77d5SMatthew G. Knepley   Output Parameter:
2213338f77d5SMatthew G. Knepley . integral - Cellwise integrals for each field
2214338f77d5SMatthew G. Knepley 
2215338f77d5SMatthew G. Knepley   Level: developer
2216338f77d5SMatthew G. Knepley 
2217446c23c1SPierre Jolivet .seealso: DMPlexSNESComputeResidualFEM()
2218338f77d5SMatthew G. Knepley @*/
2219338f77d5SMatthew G. Knepley PetscErrorCode DMPlexComputeCellwiseIntegralFEM(DM dm, Vec X, Vec F, void *user)
2220338f77d5SMatthew G. Knepley {
2221338f77d5SMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dm->data;
2222338f77d5SMatthew G. Knepley   DM             dmF;
2223338f77d5SMatthew G. Knepley   PetscSection   sectionF;
2224338f77d5SMatthew G. Knepley   PetscScalar   *cintegral, *af;
2225412e9a14SMatthew G. Knepley   PetscInt       Nf, f, cellHeight, cStart, cEnd, cell;
2226338f77d5SMatthew G. Knepley 
2227338f77d5SMatthew G. Knepley   PetscFunctionBegin;
2228338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
2229338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
2230338f77d5SMatthew G. Knepley   PetscValidHeaderSpecific(F, VEC_CLASSID, 3);
22319566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0));
22329566063dSJacob Faibussowitsch   PetscCall(DMGetNumFields(dm, &Nf));
22339566063dSJacob Faibussowitsch   PetscCall(DMPlexGetVTKCellHeight(dm, &cellHeight));
22349566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dm, cellHeight, &cStart, &cEnd));
2235338f77d5SMatthew G. Knepley   /* TODO Introduce a loop over large chunks (right now this is a single chunk) */
22369566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1((cEnd-cStart)*Nf, &cintegral));
22379566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeIntegral_Internal(dm, X, cStart, cEnd, cintegral, user));
2238338f77d5SMatthew G. Knepley   /* Put values in F*/
22399566063dSJacob Faibussowitsch   PetscCall(VecGetDM(F, &dmF));
22409566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmF, &sectionF));
22419566063dSJacob Faibussowitsch   PetscCall(VecGetArray(F, &af));
2242338f77d5SMatthew G. Knepley   for (cell = cStart; cell < cEnd; ++cell) {
2243338f77d5SMatthew G. Knepley     const PetscInt c = cell - cStart;
2244338f77d5SMatthew G. Knepley     PetscInt       dof, off;
2245338f77d5SMatthew G. Knepley 
22469566063dSJacob Faibussowitsch     if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, "Cell Integral", Nf, &cintegral[c*Nf]));
22479566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetDof(sectionF, cell, &dof));
22489566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetOffset(sectionF, cell, &off));
224963a3b9bcSJacob Faibussowitsch     PetscCheck(dof == Nf,PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "The number of cell dofs %" PetscInt_FMT " != %" PetscInt_FMT, dof, Nf);
2250338f77d5SMatthew G. Knepley     for (f = 0; f < Nf; ++f) af[off+f] = cintegral[c*Nf+f];
2251338f77d5SMatthew G. Knepley   }
22529566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(F, &af));
22539566063dSJacob Faibussowitsch   PetscCall(PetscFree(cintegral));
22549566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0));
225573d901b8SMatthew G. Knepley   PetscFunctionReturn(0);
225673d901b8SMatthew G. Knepley }
225773d901b8SMatthew G. Knepley 
22589b6f715bSMatthew G. Knepley static PetscErrorCode DMPlexComputeBdIntegral_Internal(DM dm, Vec locX, IS pointIS,
22599b6f715bSMatthew G. Knepley                                                        void (*func)(PetscInt, PetscInt, PetscInt,
226064c72086SMatthew G. Knepley                                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
226164c72086SMatthew G. Knepley                                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
226264c72086SMatthew G. Knepley                                                                     PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]),
226364c72086SMatthew G. Knepley                                                        PetscScalar *fintegral, void *user)
226464c72086SMatthew G. Knepley {
22659b6f715bSMatthew G. Knepley   DM                 plex = NULL, plexA = NULL;
2266a6e0b375SMatthew G. Knepley   DMEnclosureType    encAux;
22679b6f715bSMatthew G. Knepley   PetscDS            prob, probAux = NULL;
22689b6f715bSMatthew G. Knepley   PetscSection       section, sectionAux = NULL;
22699b6f715bSMatthew G. Knepley   Vec                locA = NULL;
22709b6f715bSMatthew G. Knepley   DMField            coordField;
22719b6f715bSMatthew G. Knepley   PetscInt           Nf,        totDim,        *uOff, *uOff_x;
22729b6f715bSMatthew G. Knepley   PetscInt           NfAux = 0, totDimAux = 0, *aOff = NULL;
22739b6f715bSMatthew G. Knepley   PetscScalar       *u, *a = NULL;
227464c72086SMatthew G. Knepley   const PetscScalar *constants;
22759b6f715bSMatthew G. Knepley   PetscInt           numConstants, f;
227664c72086SMatthew G. Knepley 
227764c72086SMatthew G. Knepley   PetscFunctionBegin;
22789566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
22799566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
22809566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
22819566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
22829566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
228364c72086SMatthew G. Knepley   /* Determine which discretizations we have */
22849b6f715bSMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
228564c72086SMatthew G. Knepley     PetscObject  obj;
228664c72086SMatthew G. Knepley     PetscClassId id;
228764c72086SMatthew G. Knepley 
22889566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
22899566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
229063a3b9bcSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID,PetscObjectComm((PetscObject) dm), PETSC_ERR_SUP, "Not supported for FVM (field %" PetscInt_FMT ")", f);
229164c72086SMatthew G. Knepley   }
229264c72086SMatthew G. Knepley   /* Read DS information */
22939566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
22949566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentOffsets(prob, &uOff));
22959566063dSJacob Faibussowitsch   PetscCall(PetscDSGetComponentDerivativeOffsets(prob, &uOff_x));
22969566063dSJacob Faibussowitsch   PetscCall(PetscDSGetConstants(prob, &numConstants, &constants));
229764c72086SMatthew G. Knepley   /* Read Auxiliary DS information */
22989566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
22999b6f715bSMatthew G. Knepley   if (locA) {
23009b6f715bSMatthew G. Knepley     DM dmAux;
23019b6f715bSMatthew G. Knepley 
23029566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
23039566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
23049566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
23059566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
23069566063dSJacob Faibussowitsch     PetscCall(PetscDSGetNumFields(probAux, &NfAux));
23079566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
23089566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
23099566063dSJacob Faibussowitsch     PetscCall(PetscDSGetComponentOffsets(probAux, &aOff));
231064c72086SMatthew G. Knepley   }
23119b6f715bSMatthew G. Knepley   /* Integrate over points */
23129b6f715bSMatthew G. Knepley   {
23139b6f715bSMatthew G. Knepley     PetscFEGeom    *fgeom, *chunkGeom = NULL;
2314b7260050SToby Isaac     PetscInt        maxDegree;
23159b6f715bSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
23169b6f715bSMatthew G. Knepley     const PetscInt *points;
23179b6f715bSMatthew G. Knepley     PetscInt        numFaces, face, Nq, field;
23189b6f715bSMatthew G. Knepley     PetscInt        numChunks, chunkSize, chunk, Nr, offset;
231964c72086SMatthew G. Knepley 
23209566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
23219566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
23229566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(numFaces*totDim, &u, locA ? numFaces*totDimAux : 0, &a));
23239566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField, pointIS, NULL, &maxDegree));
232464c72086SMatthew G. Knepley     for (field = 0; field < Nf; ++field) {
232564c72086SMatthew G. Knepley       PetscFE fe;
232664c72086SMatthew G. Knepley 
23279566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *) &fe));
23289566063dSJacob Faibussowitsch       if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, pointIS, &qGeom));
23299b6f715bSMatthew G. Knepley       if (!qGeom) {
23309566063dSJacob Faibussowitsch         PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
23319566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) qGeom));
23329b6f715bSMatthew G. Knepley       }
23339566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
23349566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
23359b6f715bSMatthew G. Knepley       for (face = 0; face < numFaces; ++face) {
2336f15274beSMatthew Knepley         const PetscInt point = points[face], *support;
23379b6f715bSMatthew G. Knepley         PetscScalar    *x    = NULL;
2338f15274beSMatthew Knepley         PetscInt       i;
23399b6f715bSMatthew G. Knepley 
23409566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, point, &support));
23419566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
23429b6f715bSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
23439566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
23449b6f715bSMatthew G. Knepley         if (locA) {
23459b6f715bSMatthew G. Knepley           PetscInt subp;
23469566063dSJacob Faibussowitsch           PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
23479566063dSJacob Faibussowitsch           PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
23489b6f715bSMatthew G. Knepley           for (i = 0; i < totDimAux; ++i) a[f*totDimAux+i] = x[i];
23499566063dSJacob Faibussowitsch           PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
23509b6f715bSMatthew G. Knepley         }
23519b6f715bSMatthew G. Knepley       }
23529b6f715bSMatthew G. Knepley       /* Get blocking */
23539b6f715bSMatthew G. Knepley       {
23549b6f715bSMatthew G. Knepley         PetscQuadrature q;
23559b6f715bSMatthew G. Knepley         PetscInt        numBatches, batchSize, numBlocks, blockSize;
23569b6f715bSMatthew G. Knepley         PetscInt        Nq, Nb;
23579b6f715bSMatthew G. Knepley 
23589566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
23599566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &q));
23609566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(q, NULL, NULL, &Nq, NULL, NULL));
23619566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
236264c72086SMatthew G. Knepley         blockSize = Nb*Nq;
236364c72086SMatthew G. Knepley         batchSize = numBlocks * blockSize;
23649b6f715bSMatthew G. Knepley         chunkSize = numBatches*batchSize;
23659566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
23669b6f715bSMatthew G. Knepley         numChunks = numFaces / chunkSize;
23679b6f715bSMatthew G. Knepley         Nr        = numFaces % chunkSize;
236864c72086SMatthew G. Knepley         offset    = numFaces - Nr;
236964c72086SMatthew G. Knepley       }
23709b6f715bSMatthew G. Knepley       /* Do integration for each field */
23719b6f715bSMatthew G. Knepley       for (chunk = 0; chunk < numChunks; ++chunk) {
23729566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(fgeom, chunk*chunkSize, (chunk+1)*chunkSize, &chunkGeom));
23739566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateBd(prob, field, func, chunkSize, chunkGeom, u, probAux, a, fintegral));
23749566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
237564c72086SMatthew G. Knepley       }
23769566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom, offset, numFaces, &chunkGeom));
23779566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBd(prob, field, func, Nr, chunkGeom, &u[offset*totDim], probAux, a ? &a[offset*totDimAux] : NULL, &fintegral[offset*Nf]));
23789566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, offset, numFaces, &chunkGeom));
237964c72086SMatthew G. Knepley       /* Cleanup data arrays */
23809566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFEGeom(coordField, pointIS, qGeom, PETSC_TRUE, &fgeom));
23819566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&qGeom));
23829566063dSJacob Faibussowitsch       PetscCall(PetscFree2(u, a));
23839566063dSJacob Faibussowitsch       PetscCall(ISRestoreIndices(pointIS, &points));
238464c72086SMatthew G. Knepley     }
238564c72086SMatthew G. Knepley   }
23869566063dSJacob Faibussowitsch   if (plex)  PetscCall(DMDestroy(&plex));
23879566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
23889b6f715bSMatthew G. Knepley   PetscFunctionReturn(0);
23899b6f715bSMatthew G. Knepley }
23909b6f715bSMatthew G. Knepley 
23919b6f715bSMatthew G. Knepley /*@
23929b6f715bSMatthew G. Knepley   DMPlexComputeBdIntegral - Form the integral over the specified boundary from the global input X using pointwise functions specified by the user
23939b6f715bSMatthew G. Knepley 
23949b6f715bSMatthew G. Knepley   Input Parameters:
23959b6f715bSMatthew G. Knepley + dm      - The mesh
23969b6f715bSMatthew G. Knepley . X       - Global input vector
23979b6f715bSMatthew G. Knepley . label   - The boundary DMLabel
23989b6f715bSMatthew G. Knepley . numVals - The number of label values to use, or PETSC_DETERMINE for all values
23999b6f715bSMatthew G. Knepley . vals    - The label values to use, or PETSC_NULL for all values
240067b8a455SSatish Balay . func    - The function to integrate along the boundary
24019b6f715bSMatthew G. Knepley - user    - The user context
24029b6f715bSMatthew G. Knepley 
24039b6f715bSMatthew G. Knepley   Output Parameter:
24049b6f715bSMatthew G. Knepley . integral - Integral for each field
24059b6f715bSMatthew G. Knepley 
24069b6f715bSMatthew G. Knepley   Level: developer
24079b6f715bSMatthew G. Knepley 
24089b6f715bSMatthew G. Knepley .seealso: DMPlexComputeIntegralFEM(), DMPlexComputeBdResidualFEM()
24099b6f715bSMatthew G. Knepley @*/
24109b6f715bSMatthew G. Knepley PetscErrorCode DMPlexComputeBdIntegral(DM dm, Vec X, DMLabel label, PetscInt numVals, const PetscInt vals[],
24119b6f715bSMatthew G. Knepley                                        void (*func)(PetscInt, PetscInt, PetscInt,
24129b6f715bSMatthew G. Knepley                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
24139b6f715bSMatthew G. Knepley                                                     const PetscInt[], const PetscInt[], const PetscScalar[], const PetscScalar[], const PetscScalar[],
24149b6f715bSMatthew G. Knepley                                                     PetscReal, const PetscReal[], const PetscReal[], PetscInt, const PetscScalar[], PetscScalar[]),
24159b6f715bSMatthew G. Knepley                                        PetscScalar *integral, void *user)
24169b6f715bSMatthew G. Knepley {
24179b6f715bSMatthew G. Knepley   Vec            locX;
24189b6f715bSMatthew G. Knepley   PetscSection   section;
24199b6f715bSMatthew G. Knepley   DMLabel        depthLabel;
24209b6f715bSMatthew G. Knepley   IS             facetIS;
24219b6f715bSMatthew G. Knepley   PetscInt       dim, Nf, f, v;
24229b6f715bSMatthew G. Knepley 
24239b6f715bSMatthew G. Knepley   PetscFunctionBegin;
24249b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
24259b6f715bSMatthew G. Knepley   PetscValidHeaderSpecific(X, VEC_CLASSID, 2);
24269b6f715bSMatthew G. Knepley   PetscValidPointer(label, 3);
2427dadcf809SJacob Faibussowitsch   if (vals) PetscValidIntPointer(vals, 5);
2428dadcf809SJacob Faibussowitsch   PetscValidScalarPointer(integral, 7);
24299566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_IntegralFEM,dm,0,0,0));
24309566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
24319566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
24329566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim-1, &facetIS));
24339566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
24349566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(section, &Nf));
24359b6f715bSMatthew G. Knepley   /* Get local solution with boundary values */
24369566063dSJacob Faibussowitsch   PetscCall(DMGetLocalVector(dm, &locX));
24379566063dSJacob Faibussowitsch   PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_TRUE, locX, 0.0, NULL, NULL, NULL));
24389566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalBegin(dm, X, INSERT_VALUES, locX));
24399566063dSJacob Faibussowitsch   PetscCall(DMGlobalToLocalEnd(dm, X, INSERT_VALUES, locX));
24409b6f715bSMatthew G. Knepley   /* Loop over label values */
24419566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(integral, Nf));
24429b6f715bSMatthew G. Knepley   for (v = 0; v < numVals; ++v) {
24439b6f715bSMatthew G. Knepley     IS           pointIS;
24449b6f715bSMatthew G. Knepley     PetscInt     numFaces, face;
24459b6f715bSMatthew G. Knepley     PetscScalar *fintegral;
24469b6f715bSMatthew G. Knepley 
24479566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, vals[v], &pointIS));
24489b6f715bSMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
24499b6f715bSMatthew G. Knepley     {
24509b6f715bSMatthew G. Knepley       IS isectIS;
24519b6f715bSMatthew G. Knepley 
24529b6f715bSMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
24539566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS, pointIS, &isectIS));
24549566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
24559b6f715bSMatthew G. Knepley       pointIS = isectIS;
24569b6f715bSMatthew G. Knepley     }
24579566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
24589566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(numFaces*Nf, &fintegral));
24599566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdIntegral_Internal(dm, locX, pointIS, func, fintegral, user));
24609b6f715bSMatthew G. Knepley     /* Sum point contributions into integral */
24619b6f715bSMatthew G. Knepley     for (f = 0; f < Nf; ++f) for (face = 0; face < numFaces; ++face) integral[f] += fintegral[face*Nf+f];
24629566063dSJacob Faibussowitsch     PetscCall(PetscFree(fintegral));
24639566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
24649b6f715bSMatthew G. Knepley   }
24659566063dSJacob Faibussowitsch   PetscCall(DMRestoreLocalVector(dm, &locX));
24669566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
24679566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_IntegralFEM,dm,0,0,0));
246864c72086SMatthew G. Knepley   PetscFunctionReturn(0);
246964c72086SMatthew G. Knepley }
247064c72086SMatthew G. Knepley 
2471d69c5d34SMatthew G. Knepley /*@
2472cf51de39SMatthew G. Knepley   DMPlexComputeInterpolatorNested - Form the local portion of the interpolation matrix I from the coarse DM to a uniformly refined DM.
2473d69c5d34SMatthew G. Knepley 
2474d69c5d34SMatthew G. Knepley   Input Parameters:
2475cf51de39SMatthew G. Knepley + dmc  - The coarse mesh
2476cf51de39SMatthew G. Knepley . dmf  - The fine mesh
2477cf51de39SMatthew G. Knepley . isRefined - Flag indicating regular refinement, rather than the same topology
2478d69c5d34SMatthew G. Knepley - user - The user context
2479d69c5d34SMatthew G. Knepley 
2480d69c5d34SMatthew G. Knepley   Output Parameter:
2481934789fcSMatthew G. Knepley . In  - The interpolation matrix
2482d69c5d34SMatthew G. Knepley 
2483d69c5d34SMatthew G. Knepley   Level: developer
2484d69c5d34SMatthew G. Knepley 
248568132eb9SMatthew G. Knepley .seealso: DMPlexComputeInterpolatorGeneral(), DMPlexComputeJacobianFEM()
2486d69c5d34SMatthew G. Knepley @*/
2487cf51de39SMatthew G. Knepley PetscErrorCode DMPlexComputeInterpolatorNested(DM dmc, DM dmf, PetscBool isRefined, Mat In, void *user)
2488d69c5d34SMatthew G. Knepley {
2489d69c5d34SMatthew G. Knepley   DM_Plex          *mesh  = (DM_Plex *) dmc->data;
2490d69c5d34SMatthew G. Knepley   const char       *name  = "Interpolator";
2491d69c5d34SMatthew G. Knepley   PetscFE          *feRef;
249297c42addSMatthew G. Knepley   PetscFV          *fvRef;
2493d69c5d34SMatthew G. Knepley   PetscSection      fsection, fglobalSection;
2494d69c5d34SMatthew G. Knepley   PetscSection      csection, cglobalSection;
2495d69c5d34SMatthew G. Knepley   PetscScalar      *elemMat;
2496485ad865SMatthew G. Knepley   PetscInt          dim, Nf, f, fieldI, fieldJ, offsetI, offsetJ, cStart, cEnd, c;
24972ea9c922SToby Isaac   PetscInt          cTotDim=0, rTotDim = 0;
2498d69c5d34SMatthew G. Knepley 
2499d69c5d34SMatthew G. Knepley   PetscFunctionBegin;
25009566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM,dmc,dmf,0,0));
25019566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
25029566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
25039566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
25049566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
25059566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
25069566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
25079566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
25089566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(Nf, &feRef, Nf, &fvRef));
2509d69c5d34SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
25102ea9c922SToby Isaac     PetscObject  obj, objc;
25112ea9c922SToby Isaac     PetscClassId id, idc;
25122ea9c922SToby Isaac     PetscInt     rNb = 0, Nc = 0, cNb = 0;
2513d69c5d34SMatthew G. Knepley 
25149566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmf, f, NULL, &obj));
25159566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
251697c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
251797c42addSMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
251897c42addSMatthew G. Knepley 
2519cf51de39SMatthew G. Knepley       if (isRefined) {
25209566063dSJacob Faibussowitsch         PetscCall(PetscFERefine(fe, &feRef[f]));
2521cf51de39SMatthew G. Knepley       } else {
25229566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) fe));
2523cf51de39SMatthew G. Knepley         feRef[f] = fe;
2524cf51de39SMatthew G. Knepley       }
25259566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &rNb));
25269566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
252797c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
252897c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV) obj;
252997c42addSMatthew G. Knepley       PetscDualSpace Q;
253097c42addSMatthew G. Knepley 
2531cf51de39SMatthew G. Knepley       if (isRefined) {
25329566063dSJacob Faibussowitsch         PetscCall(PetscFVRefine(fv, &fvRef[f]));
2533cf51de39SMatthew G. Knepley       } else {
25349566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) fv));
2535cf51de39SMatthew G. Knepley         fvRef[f] = fv;
2536cf51de39SMatthew G. Knepley       }
25379566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
25389566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &rNb));
25399566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
25409566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
254197c42addSMatthew G. Knepley     }
25429566063dSJacob Faibussowitsch     PetscCall(DMGetField(dmc, f, NULL, &objc));
25439566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(objc, &idc));
25442ea9c922SToby Isaac     if (idc == PETSCFE_CLASSID) {
25452ea9c922SToby Isaac       PetscFE fe = (PetscFE) objc;
25462ea9c922SToby Isaac 
25479566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &cNb));
25482ea9c922SToby Isaac     } else if (id == PETSCFV_CLASSID) {
25492ea9c922SToby Isaac       PetscFV        fv = (PetscFV) obj;
25502ea9c922SToby Isaac       PetscDualSpace Q;
25512ea9c922SToby Isaac 
25529566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
25539566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &cNb));
2554d69c5d34SMatthew G. Knepley     }
25552ea9c922SToby Isaac     rTotDim += rNb;
25562ea9c922SToby Isaac     cTotDim += cNb;
25572ea9c922SToby Isaac   }
25589566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(rTotDim*cTotDim,&elemMat));
25599566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, rTotDim*cTotDim));
2560d69c5d34SMatthew G. Knepley   for (fieldI = 0, offsetI = 0; fieldI < Nf; ++fieldI) {
2561d69c5d34SMatthew G. Knepley     PetscDualSpace   Qref;
2562d69c5d34SMatthew G. Knepley     PetscQuadrature  f;
2563d69c5d34SMatthew G. Knepley     const PetscReal *qpoints, *qweights;
2564d69c5d34SMatthew G. Knepley     PetscReal       *points;
2565d69c5d34SMatthew G. Knepley     PetscInt         npoints = 0, Nc, Np, fpdim, i, k, p, d;
2566d69c5d34SMatthew G. Knepley 
2567d69c5d34SMatthew G. Knepley     /* Compose points from all dual basis functionals */
256897c42addSMatthew G. Knepley     if (feRef[fieldI]) {
25699566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[fieldI], &Qref));
25709566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[fieldI], &Nc));
257197c42addSMatthew G. Knepley     } else {
25729566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[fieldI], &Qref));
25739566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[fieldI], &Nc));
257497c42addSMatthew G. Knepley     }
25759566063dSJacob Faibussowitsch     PetscCall(PetscDualSpaceGetDimension(Qref, &fpdim));
2576d69c5d34SMatthew G. Knepley     for (i = 0; i < fpdim; ++i) {
25779566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
25789566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, NULL, NULL));
2579d69c5d34SMatthew G. Knepley       npoints += Np;
2580d69c5d34SMatthew G. Knepley     }
25819566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(npoints*dim,&points));
2582d69c5d34SMatthew G. Knepley     for (i = 0, k = 0; i < fpdim; ++i) {
25839566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
25849566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL));
2585d69c5d34SMatthew G. Knepley       for (p = 0; p < Np; ++p, ++k) for (d = 0; d < dim; ++d) points[k*dim+d] = qpoints[p*dim+d];
2586d69c5d34SMatthew G. Knepley     }
2587d69c5d34SMatthew G. Knepley 
2588d69c5d34SMatthew G. Knepley     for (fieldJ = 0, offsetJ = 0; fieldJ < Nf; ++fieldJ) {
258997c42addSMatthew G. Knepley       PetscObject  obj;
259097c42addSMatthew G. Knepley       PetscClassId id;
25919c3cf19fSMatthew G. Knepley       PetscInt     NcJ = 0, cpdim = 0, j, qNc;
2592d69c5d34SMatthew G. Knepley 
25939566063dSJacob Faibussowitsch       PetscCall(DMGetField(dmc, fieldJ, NULL, &obj));
25949566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
259597c42addSMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
259697c42addSMatthew G. Knepley         PetscFE           fe = (PetscFE) obj;
2597ef0bb6c7SMatthew G. Knepley         PetscTabulation T  = NULL;
2598d69c5d34SMatthew G. Knepley 
2599d69c5d34SMatthew G. Knepley         /* Evaluate basis at points */
26009566063dSJacob Faibussowitsch         PetscCall(PetscFEGetNumComponents(fe, &NcJ));
26019566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &cpdim));
2602ffe73a53SMatthew G. Knepley         /* For now, fields only interpolate themselves */
2603ffe73a53SMatthew G. Knepley         if (fieldI == fieldJ) {
260463a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
26059566063dSJacob Faibussowitsch           PetscCall(PetscFECreateTabulation(fe, 1, npoints, points, 0, &T));
2606d69c5d34SMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
26079566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
26089566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
260963a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
2610d69c5d34SMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
261136a6d9c0SMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
2612d172c84bSMatthew G. Knepley                 /*
2613d172c84bSMatthew G. Knepley                    cTotDim:            Total columns in element interpolation matrix, sum of number of dual basis functionals in each field
2614d172c84bSMatthew G. Knepley                    offsetI, offsetJ:   Offsets into the larger element interpolation matrix for different fields
2615d172c84bSMatthew G. Knepley                    fpdim, i, cpdim, j: Dofs for fine and coarse grids, correspond to dual space basis functionals
2616d172c84bSMatthew G. Knepley                    qNC, Nc, Ncj, c:    Number of components in this field
2617d172c84bSMatthew G. Knepley                    Np, p:              Number of quad points in the fine grid functional i
2618d172c84bSMatthew G. Knepley                    k:                  i*Np + p, overall point number for the interpolation
2619d172c84bSMatthew G. Knepley                 */
2620ef0bb6c7SMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i)*cTotDim + offsetJ + j] += T->T[0][k*cpdim*NcJ+j*Nc+c]*qweights[p*qNc+c];
262136a6d9c0SMatthew G. Knepley               }
2622d69c5d34SMatthew G. Knepley             }
2623d69c5d34SMatthew G. Knepley           }
26249566063dSJacob Faibussowitsch           PetscCall(PetscTabulationDestroy(&T));
2625ffe73a53SMatthew G. Knepley         }
262697c42addSMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
262797c42addSMatthew G. Knepley         PetscFV        fv = (PetscFV) obj;
262897c42addSMatthew G. Knepley 
262997c42addSMatthew G. Knepley         /* Evaluate constant function at points */
26309566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &NcJ));
263197c42addSMatthew G. Knepley         cpdim = 1;
263297c42addSMatthew G. Knepley         /* For now, fields only interpolate themselves */
263397c42addSMatthew G. Knepley         if (fieldI == fieldJ) {
263463a3b9bcSJacob Faibussowitsch           PetscCheck(Nc == NcJ,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, Nc, NcJ);
263597c42addSMatthew G. Knepley           for (i = 0, k = 0; i < fpdim; ++i) {
26369566063dSJacob Faibussowitsch             PetscCall(PetscDualSpaceGetFunctional(Qref, i, &f));
26379566063dSJacob Faibussowitsch             PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, NULL, &qweights));
263863a3b9bcSJacob Faibussowitsch             PetscCheck(qNc == NcJ,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, NcJ);
263997c42addSMatthew G. Knepley             for (p = 0; p < Np; ++p, ++k) {
264097c42addSMatthew G. Knepley               for (j = 0; j < cpdim; ++j) {
2641458eb97cSMatthew G. Knepley                 for (c = 0; c < Nc; ++c) elemMat[(offsetI + i)*cTotDim + offsetJ + j] += 1.0*qweights[p*qNc+c];
264297c42addSMatthew G. Knepley               }
264397c42addSMatthew G. Knepley             }
264497c42addSMatthew G. Knepley           }
264597c42addSMatthew G. Knepley         }
264697c42addSMatthew G. Knepley       }
2647d172c84bSMatthew G. Knepley       offsetJ += cpdim;
2648d69c5d34SMatthew G. Knepley     }
2649d172c84bSMatthew G. Knepley     offsetI += fpdim;
26509566063dSJacob Faibussowitsch     PetscCall(PetscFree(points));
2651d69c5d34SMatthew G. Knepley   }
26529566063dSJacob Faibussowitsch   if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(0, name, rTotDim, cTotDim, elemMat));
26537f5b169aSMatthew G. Knepley   /* Preallocate matrix */
26547f5b169aSMatthew G. Knepley   {
2655c094ef40SMatthew G. Knepley     Mat          preallocator;
2656c094ef40SMatthew G. Knepley     PetscScalar *vals;
2657c094ef40SMatthew G. Knepley     PetscInt    *cellCIndices, *cellFIndices;
2658c094ef40SMatthew G. Knepley     PetscInt     locRows, locCols, cell;
26597f5b169aSMatthew G. Knepley 
26609566063dSJacob Faibussowitsch     PetscCall(MatGetLocalSize(In, &locRows, &locCols));
26619566063dSJacob Faibussowitsch     PetscCall(MatCreate(PetscObjectComm((PetscObject) In), &preallocator));
26629566063dSJacob Faibussowitsch     PetscCall(MatSetType(preallocator, MATPREALLOCATOR));
26639566063dSJacob Faibussowitsch     PetscCall(MatSetSizes(preallocator, locRows, locCols, PETSC_DETERMINE, PETSC_DETERMINE));
26649566063dSJacob Faibussowitsch     PetscCall(MatSetUp(preallocator));
26659566063dSJacob Faibussowitsch     PetscCall(PetscCalloc3(rTotDim*cTotDim, &vals,cTotDim,&cellCIndices,rTotDim,&cellFIndices));
26667f5b169aSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
2667cf51de39SMatthew G. Knepley       if (isRefined) {
26689566063dSJacob Faibussowitsch         PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, cell, cellCIndices, cellFIndices));
26699566063dSJacob Faibussowitsch         PetscCall(MatSetValues(preallocator, rTotDim, cellFIndices, cTotDim, cellCIndices, vals, INSERT_VALUES));
2670cf51de39SMatthew G. Knepley       } else {
26719566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, preallocator, cell, vals, INSERT_VALUES));
2672cf51de39SMatthew G. Knepley       }
26737f5b169aSMatthew G. Knepley     }
26749566063dSJacob Faibussowitsch     PetscCall(PetscFree3(vals,cellCIndices,cellFIndices));
26759566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(preallocator, MAT_FINAL_ASSEMBLY));
26769566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(preallocator, MAT_FINAL_ASSEMBLY));
26779566063dSJacob Faibussowitsch     PetscCall(MatPreallocatorPreallocate(preallocator, PETSC_TRUE, In));
26789566063dSJacob Faibussowitsch     PetscCall(MatDestroy(&preallocator));
26797f5b169aSMatthew G. Knepley   }
26807f5b169aSMatthew G. Knepley   /* Fill matrix */
26819566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(In));
2682d69c5d34SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
2683cf51de39SMatthew G. Knepley     if (isRefined) {
26849566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosureRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES));
2685cf51de39SMatthew G. Knepley     } else {
26869566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosureGeneral(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, In, c, elemMat, INSERT_VALUES));
2687cf51de39SMatthew G. Knepley     }
2688d69c5d34SMatthew G. Knepley   }
26899566063dSJacob Faibussowitsch   for (f = 0; f < Nf; ++f) PetscCall(PetscFEDestroy(&feRef[f]));
26909566063dSJacob Faibussowitsch   PetscCall(PetscFree2(feRef,fvRef));
26919566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
26929566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
26939566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
26949318fe57SMatthew G. Knepley   if (mesh->printFEM > 1) {
26959566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)In), "%s:\n", name));
26969566063dSJacob Faibussowitsch     PetscCall(MatChop(In, 1.0e-10));
26979566063dSJacob Faibussowitsch     PetscCall(MatView(In, NULL));
2698d69c5d34SMatthew G. Knepley   }
26999566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM,dmc,dmf,0,0));
2700d69c5d34SMatthew G. Knepley   PetscFunctionReturn(0);
2701d69c5d34SMatthew G. Knepley }
27026c73c22cSMatthew G. Knepley 
2703bd041c0cSMatthew G. Knepley PetscErrorCode DMPlexComputeMassMatrixNested(DM dmc, DM dmf, Mat mass, void *user)
2704bd041c0cSMatthew G. Knepley {
2705bd041c0cSMatthew G. Knepley   SETERRQ(PetscObjectComm((PetscObject) dmc), PETSC_ERR_SUP, "Laziness");
2706bd041c0cSMatthew G. Knepley }
2707bd041c0cSMatthew G. Knepley 
270868132eb9SMatthew G. Knepley /*@
270968132eb9SMatthew G. Knepley   DMPlexComputeInterpolatorGeneral - Form the local portion of the interpolation matrix I from the coarse DM to a non-nested fine DM.
271068132eb9SMatthew G. Knepley 
271168132eb9SMatthew G. Knepley   Input Parameters:
271268132eb9SMatthew G. Knepley + dmf  - The fine mesh
271368132eb9SMatthew G. Knepley . dmc  - The coarse mesh
271468132eb9SMatthew G. Knepley - user - The user context
271568132eb9SMatthew G. Knepley 
271668132eb9SMatthew G. Knepley   Output Parameter:
271768132eb9SMatthew G. Knepley . In  - The interpolation matrix
271868132eb9SMatthew G. Knepley 
271968132eb9SMatthew G. Knepley   Level: developer
272068132eb9SMatthew G. Knepley 
272168132eb9SMatthew G. Knepley .seealso: DMPlexComputeInterpolatorNested(), DMPlexComputeJacobianFEM()
272268132eb9SMatthew G. Knepley @*/
272368132eb9SMatthew G. Knepley PetscErrorCode DMPlexComputeInterpolatorGeneral(DM dmc, DM dmf, Mat In, void *user)
27244ef9d792SMatthew G. Knepley {
272564e98e1dSMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dmf->data;
272664e98e1dSMatthew G. Knepley   const char    *name = "Interpolator";
27274ef9d792SMatthew G. Knepley   PetscDS        prob;
27284ef9d792SMatthew G. Knepley   PetscSection   fsection, csection, globalFSection, globalCSection;
2729e8f14785SLisandro Dalcin   PetscHSetIJ    ht;
27304ef9d792SMatthew G. Knepley   PetscLayout    rLayout;
27314ef9d792SMatthew G. Knepley   PetscInt      *dnz, *onz;
27324ef9d792SMatthew G. Knepley   PetscInt       locRows, rStart, rEnd;
27334ef9d792SMatthew G. Knepley   PetscReal     *x, *v0, *J, *invJ, detJ;
27344ef9d792SMatthew G. Knepley   PetscReal     *v0c, *Jc, *invJc, detJc;
27354ef9d792SMatthew G. Knepley   PetscScalar   *elemMat;
27364ef9d792SMatthew G. Knepley   PetscInt       dim, Nf, field, totDim, cStart, cEnd, cell, ccell;
27374ef9d792SMatthew G. Knepley 
27384ef9d792SMatthew G. Knepley   PetscFunctionBegin;
27399566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InterpolatorFEM,dmc,dmf,0,0));
27409566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
27419566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
27429566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
27439566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
27449566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim,&v0,dim*dim,&J,dim*dim,&invJ));
27459566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim,&v0c,dim*dim,&Jc,dim*dim,&invJc));
27469566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
27479566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
27489566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
27499566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
27509566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd));
27519566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
27529566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
27534ef9d792SMatthew G. Knepley 
27549566063dSJacob Faibussowitsch   PetscCall(MatGetLocalSize(In, &locRows, NULL));
27559566063dSJacob Faibussowitsch   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject) In), &rLayout));
27569566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetLocalSize(rLayout, locRows));
27579566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetBlockSize(rLayout, 1));
27589566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetUp(rLayout));
27599566063dSJacob Faibussowitsch   PetscCall(PetscLayoutGetRange(rLayout, &rStart, &rEnd));
27609566063dSJacob Faibussowitsch   PetscCall(PetscLayoutDestroy(&rLayout));
27619566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(locRows,&dnz,locRows,&onz));
27629566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJCreate(&ht));
27634ef9d792SMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
27644ef9d792SMatthew G. Knepley     PetscObject      obj;
27654ef9d792SMatthew G. Knepley     PetscClassId     id;
2766c0d7054bSMatthew G. Knepley     PetscDualSpace   Q = NULL;
27674ef9d792SMatthew G. Knepley     PetscQuadrature  f;
276817f047d8SMatthew G. Knepley     const PetscReal *qpoints;
276917f047d8SMatthew G. Knepley     PetscInt         Nc, Np, fpdim, i, d;
27704ef9d792SMatthew G. Knepley 
27719566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
27729566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
27734ef9d792SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
27744ef9d792SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
27754ef9d792SMatthew G. Knepley 
27769566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(fe, &Q));
27779566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
27784ef9d792SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
27794ef9d792SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
27804ef9d792SMatthew G. Knepley 
27819566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
27824ef9d792SMatthew G. Knepley       Nc   = 1;
27834ef9d792SMatthew G. Knepley     }
27849566063dSJacob Faibussowitsch     PetscCall(PetscDualSpaceGetDimension(Q, &fpdim));
27854ef9d792SMatthew G. Knepley     /* For each fine grid cell */
27864ef9d792SMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
27874ef9d792SMatthew G. Knepley       PetscInt *findices,   *cindices;
27884ef9d792SMatthew G. Knepley       PetscInt  numFIndices, numCIndices;
27894ef9d792SMatthew G. Knepley 
27909566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
27919566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
279263a3b9bcSJacob Faibussowitsch       PetscCheck(numFIndices == fpdim,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %" PetscInt_FMT " != %" PetscInt_FMT " dual basis vecs", numFIndices, fpdim);
27934ef9d792SMatthew G. Knepley       for (i = 0; i < fpdim; ++i) {
27944ef9d792SMatthew G. Knepley         Vec             pointVec;
27954ef9d792SMatthew G. Knepley         PetscScalar    *pV;
27963a93e3b7SToby Isaac         PetscSF         coarseCellSF = NULL;
27973a93e3b7SToby Isaac         const PetscSFNode *coarseCells;
27989c3cf19fSMatthew G. Knepley         PetscInt        numCoarseCells, q, c;
27994ef9d792SMatthew G. Knepley 
28004ef9d792SMatthew G. Knepley         /* Get points from the dual basis functional quadrature */
28019566063dSJacob Faibussowitsch         PetscCall(PetscDualSpaceGetFunctional(Q, i, &f));
28029566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(f, NULL, NULL, &Np, &qpoints, NULL));
28039566063dSJacob Faibussowitsch         PetscCall(VecCreateSeq(PETSC_COMM_SELF, Np*dim, &pointVec));
28049566063dSJacob Faibussowitsch         PetscCall(VecSetBlockSize(pointVec, dim));
28059566063dSJacob Faibussowitsch         PetscCall(VecGetArray(pointVec, &pV));
28064ef9d792SMatthew G. Knepley         for (q = 0; q < Np; ++q) {
2807c330f8ffSToby Isaac           const PetscReal xi0[3] = {-1., -1., -1.};
2808c330f8ffSToby Isaac 
28094ef9d792SMatthew G. Knepley           /* Transform point to real space */
2810c330f8ffSToby Isaac           CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
28114ef9d792SMatthew G. Knepley           for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
28124ef9d792SMatthew G. Knepley         }
28139566063dSJacob Faibussowitsch         PetscCall(VecRestoreArray(pointVec, &pV));
28144ef9d792SMatthew G. Knepley         /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
28151555c271SMatthew G. Knepley         /* OPT: Pack all quad points from fine cell */
28169566063dSJacob Faibussowitsch         PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
28179566063dSJacob Faibussowitsch         PetscCall(PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view"));
28184ef9d792SMatthew G. Knepley         /* Update preallocation info */
28199566063dSJacob Faibussowitsch         PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
28205f80ce2aSJacob Faibussowitsch         PetscCheck(numCoarseCells == Np,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
28219c3cf19fSMatthew G. Knepley         {
2822e8f14785SLisandro Dalcin           PetscHashIJKey key;
2823e8f14785SLisandro Dalcin           PetscBool      missing;
28244ef9d792SMatthew G. Knepley 
2825e8f14785SLisandro Dalcin           key.i = findices[i];
2826e8f14785SLisandro Dalcin           if (key.i >= 0) {
28274ef9d792SMatthew G. Knepley             /* Get indices for coarse elements */
28284ef9d792SMatthew G. Knepley             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
28299566063dSJacob Faibussowitsch               PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
28304ef9d792SMatthew G. Knepley               for (c = 0; c < numCIndices; ++c) {
2831e8f14785SLisandro Dalcin                 key.j = cindices[c];
2832e8f14785SLisandro Dalcin                 if (key.j < 0) continue;
28339566063dSJacob Faibussowitsch                 PetscCall(PetscHSetIJQueryAdd(ht, key, &missing));
28344ef9d792SMatthew G. Knepley                 if (missing) {
2835e8f14785SLisandro Dalcin                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i-rStart];
2836e8f14785SLisandro Dalcin                   else                                     ++onz[key.i-rStart];
28374ef9d792SMatthew G. Knepley                 }
28384ef9d792SMatthew G. Knepley               }
28399566063dSJacob Faibussowitsch               PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
28404ef9d792SMatthew G. Knepley             }
28414ef9d792SMatthew G. Knepley           }
28428c543595SMatthew G. Knepley         }
28439566063dSJacob Faibussowitsch         PetscCall(PetscSFDestroy(&coarseCellSF));
28449566063dSJacob Faibussowitsch         PetscCall(VecDestroy(&pointVec));
28454ef9d792SMatthew G. Knepley       }
28469566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
28474ef9d792SMatthew G. Knepley     }
28484ef9d792SMatthew G. Knepley   }
28499566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJDestroy(&ht));
28509566063dSJacob Faibussowitsch   PetscCall(MatXAIJSetPreallocation(In, 1, dnz, onz, NULL, NULL));
28519566063dSJacob Faibussowitsch   PetscCall(MatSetOption(In, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE));
28529566063dSJacob Faibussowitsch   PetscCall(PetscFree2(dnz,onz));
28534ef9d792SMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
28544ef9d792SMatthew G. Knepley     PetscObject       obj;
28554ef9d792SMatthew G. Knepley     PetscClassId      id;
2856c0d7054bSMatthew G. Knepley     PetscDualSpace    Q = NULL;
2857ef0bb6c7SMatthew G. Knepley     PetscTabulation T = NULL;
28584ef9d792SMatthew G. Knepley     PetscQuadrature   f;
28594ef9d792SMatthew G. Knepley     const PetscReal  *qpoints, *qweights;
28609c3cf19fSMatthew G. Knepley     PetscInt          Nc, qNc, Np, fpdim, i, d;
28614ef9d792SMatthew G. Knepley 
28629566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
28639566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
28644ef9d792SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
28654ef9d792SMatthew G. Knepley       PetscFE fe = (PetscFE) obj;
28664ef9d792SMatthew G. Knepley 
28679566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(fe, &Q));
28689566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
28699566063dSJacob Faibussowitsch       PetscCall(PetscFECreateTabulation(fe, 1, 1, x, 0, &T));
28704ef9d792SMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
28714ef9d792SMatthew G. Knepley       PetscFV fv = (PetscFV) obj;
28724ef9d792SMatthew G. Knepley 
28739566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fv, &Q));
28744ef9d792SMatthew G. Knepley       Nc   = 1;
287563a3b9bcSJacob Faibussowitsch     } else SETERRQ(PetscObjectComm((PetscObject)dmc),PETSC_ERR_ARG_WRONG,"Unknown discretization type for field %" PetscInt_FMT,field);
28769566063dSJacob Faibussowitsch     PetscCall(PetscDualSpaceGetDimension(Q, &fpdim));
28774ef9d792SMatthew G. Knepley     /* For each fine grid cell */
28784ef9d792SMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
28794ef9d792SMatthew G. Knepley       PetscInt *findices,   *cindices;
28804ef9d792SMatthew G. Knepley       PetscInt  numFIndices, numCIndices;
28814ef9d792SMatthew G. Knepley 
28829566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
28839566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
288463a3b9bcSJacob Faibussowitsch       PetscCheck(numFIndices == fpdim,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of fine indices %" PetscInt_FMT " != %" PetscInt_FMT " dual basis vecs", numFIndices, fpdim);
28854ef9d792SMatthew G. Knepley       for (i = 0; i < fpdim; ++i) {
28864ef9d792SMatthew G. Knepley         Vec             pointVec;
28874ef9d792SMatthew G. Knepley         PetscScalar    *pV;
288812111d7cSToby Isaac         PetscSF         coarseCellSF = NULL;
28893a93e3b7SToby Isaac         const PetscSFNode *coarseCells;
289017f047d8SMatthew G. Knepley         PetscInt        numCoarseCells, cpdim, q, c, j;
28914ef9d792SMatthew G. Knepley 
28924ef9d792SMatthew G. Knepley         /* Get points from the dual basis functional quadrature */
28939566063dSJacob Faibussowitsch         PetscCall(PetscDualSpaceGetFunctional(Q, i, &f));
28949566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(f, NULL, &qNc, &Np, &qpoints, &qweights));
289563a3b9bcSJacob Faibussowitsch         PetscCheck(qNc == Nc,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in quadrature %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, qNc, Nc);
28969566063dSJacob Faibussowitsch         PetscCall(VecCreateSeq(PETSC_COMM_SELF, Np*dim, &pointVec));
28979566063dSJacob Faibussowitsch         PetscCall(VecSetBlockSize(pointVec, dim));
28989566063dSJacob Faibussowitsch         PetscCall(VecGetArray(pointVec, &pV));
28994ef9d792SMatthew G. Knepley         for (q = 0; q < Np; ++q) {
2900c330f8ffSToby Isaac           const PetscReal xi0[3] = {-1., -1., -1.};
2901c330f8ffSToby Isaac 
29024ef9d792SMatthew G. Knepley           /* Transform point to real space */
2903c330f8ffSToby Isaac           CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
29044ef9d792SMatthew G. Knepley           for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
29054ef9d792SMatthew G. Knepley         }
29069566063dSJacob Faibussowitsch         PetscCall(VecRestoreArray(pointVec, &pV));
29074ef9d792SMatthew G. Knepley         /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
29081555c271SMatthew G. Knepley         /* OPT: Read this out from preallocation information */
29099566063dSJacob Faibussowitsch         PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
29104ef9d792SMatthew G. Knepley         /* Update preallocation info */
29119566063dSJacob Faibussowitsch         PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
29125f80ce2aSJacob Faibussowitsch         PetscCheck(numCoarseCells == Np,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
29139566063dSJacob Faibussowitsch         PetscCall(VecGetArray(pointVec, &pV));
29144ef9d792SMatthew G. Knepley         for (ccell = 0; ccell < numCoarseCells; ++ccell) {
2915826eb36dSMatthew G. Knepley           PetscReal pVReal[3];
2916c330f8ffSToby Isaac           const PetscReal xi0[3] = {-1., -1., -1.};
2917826eb36dSMatthew G. Knepley 
29189566063dSJacob Faibussowitsch           PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
29194ef9d792SMatthew G. Knepley           /* Transform points from real space to coarse reference space */
29209566063dSJacob Faibussowitsch           PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
2921e2d86523SMatthew G. Knepley           for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell*dim+d]);
2922c330f8ffSToby Isaac           CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
29234ef9d792SMatthew G. Knepley 
29244ef9d792SMatthew G. Knepley           if (id == PETSCFE_CLASSID) {
29254ef9d792SMatthew G. Knepley             PetscFE fe = (PetscFE) obj;
29264ef9d792SMatthew G. Knepley 
29274ef9d792SMatthew G. Knepley             /* Evaluate coarse basis on contained point */
29289566063dSJacob Faibussowitsch             PetscCall(PetscFEGetDimension(fe, &cpdim));
29299566063dSJacob Faibussowitsch             PetscCall(PetscFEComputeTabulation(fe, 1, x, 0, T));
29309566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
29314ef9d792SMatthew G. Knepley             /* Get elemMat entries by multiplying by weight */
29324ef9d792SMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
2933ef0bb6c7SMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j*Nc + c]*qweights[ccell*qNc + c];
29344ef9d792SMatthew G. Knepley             }
29354ef9d792SMatthew G. Knepley           } else {
29364ef9d792SMatthew G. Knepley             cpdim = 1;
29374ef9d792SMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
29389c3cf19fSMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0*qweights[ccell*qNc + c];
29394ef9d792SMatthew G. Knepley             }
29404ef9d792SMatthew G. Knepley           }
29414ef9d792SMatthew G. Knepley           /* Update interpolator */
29429566063dSJacob Faibussowitsch           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
294363a3b9bcSJacob Faibussowitsch           PetscCheck(numCIndices == cpdim,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
29449566063dSJacob Faibussowitsch           PetscCall(MatSetValues(In, 1, &findices[i], numCIndices, cindices, elemMat, INSERT_VALUES));
29459566063dSJacob Faibussowitsch           PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
29464ef9d792SMatthew G. Knepley         }
29479566063dSJacob Faibussowitsch         PetscCall(VecRestoreArray(pointVec, &pV));
29489566063dSJacob Faibussowitsch         PetscCall(PetscSFDestroy(&coarseCellSF));
29499566063dSJacob Faibussowitsch         PetscCall(VecDestroy(&pointVec));
29504ef9d792SMatthew G. Knepley       }
29519566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
29524ef9d792SMatthew G. Knepley     }
29539566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
29544ef9d792SMatthew G. Knepley   }
29559566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0,J,invJ));
29569566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c,Jc,invJc));
29579566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
29589566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(In, MAT_FINAL_ASSEMBLY));
29599566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(In, MAT_FINAL_ASSEMBLY));
29609566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InterpolatorFEM,dmc,dmf,0,0));
29614ef9d792SMatthew G. Knepley   PetscFunctionReturn(0);
29624ef9d792SMatthew G. Knepley }
29634ef9d792SMatthew G. Knepley 
296446fa42a0SMatthew G. Knepley /*@
2965bd041c0cSMatthew G. Knepley   DMPlexComputeMassMatrixGeneral - Form the local portion of the mass matrix M from the coarse DM to a non-nested fine DM.
2966bd041c0cSMatthew G. Knepley 
2967bd041c0cSMatthew G. Knepley   Input Parameters:
2968bd041c0cSMatthew G. Knepley + dmf  - The fine mesh
2969bd041c0cSMatthew G. Knepley . dmc  - The coarse mesh
2970bd041c0cSMatthew G. Knepley - user - The user context
2971bd041c0cSMatthew G. Knepley 
2972bd041c0cSMatthew G. Knepley   Output Parameter:
2973bd041c0cSMatthew G. Knepley . mass  - The mass matrix
2974bd041c0cSMatthew G. Knepley 
2975bd041c0cSMatthew G. Knepley   Level: developer
2976bd041c0cSMatthew G. Knepley 
2977bd041c0cSMatthew G. Knepley .seealso: DMPlexComputeMassMatrixNested(), DMPlexComputeInterpolatorNested(), DMPlexComputeInterpolatorGeneral(), DMPlexComputeJacobianFEM()
2978bd041c0cSMatthew G. Knepley @*/
2979bd041c0cSMatthew G. Knepley PetscErrorCode DMPlexComputeMassMatrixGeneral(DM dmc, DM dmf, Mat mass, void *user)
2980bd041c0cSMatthew G. Knepley {
2981bd041c0cSMatthew G. Knepley   DM_Plex       *mesh = (DM_Plex *) dmf->data;
2982bd041c0cSMatthew G. Knepley   const char    *name = "Mass Matrix";
2983bd041c0cSMatthew G. Knepley   PetscDS        prob;
2984bd041c0cSMatthew G. Knepley   PetscSection   fsection, csection, globalFSection, globalCSection;
2985e8f14785SLisandro Dalcin   PetscHSetIJ    ht;
2986bd041c0cSMatthew G. Knepley   PetscLayout    rLayout;
2987bd041c0cSMatthew G. Knepley   PetscInt      *dnz, *onz;
2988bd041c0cSMatthew G. Knepley   PetscInt       locRows, rStart, rEnd;
2989bd041c0cSMatthew G. Knepley   PetscReal     *x, *v0, *J, *invJ, detJ;
2990bd041c0cSMatthew G. Knepley   PetscReal     *v0c, *Jc, *invJc, detJc;
2991bd041c0cSMatthew G. Knepley   PetscScalar   *elemMat;
2992bd041c0cSMatthew G. Knepley   PetscInt       dim, Nf, field, totDim, cStart, cEnd, cell, ccell;
2993bd041c0cSMatthew G. Knepley 
2994bd041c0cSMatthew G. Knepley   PetscFunctionBegin;
29959566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateDim(dmc, &dim));
29969566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
29979566063dSJacob Faibussowitsch   PetscCall(PetscDSGetWorkspace(prob, &x, NULL, NULL, NULL, NULL));
29989566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
29999566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim,&v0,dim*dim,&J,dim*dim,&invJ));
30009566063dSJacob Faibussowitsch   PetscCall(PetscMalloc3(dim,&v0c,dim*dim,&Jc,dim*dim,&invJc));
30019566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
30029566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &globalFSection));
30039566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
30049566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &globalCSection));
30059566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dmf, 0, &cStart, &cEnd));
30069566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
30079566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(totDim, &elemMat));
3008bd041c0cSMatthew G. Knepley 
30099566063dSJacob Faibussowitsch   PetscCall(MatGetLocalSize(mass, &locRows, NULL));
30109566063dSJacob Faibussowitsch   PetscCall(PetscLayoutCreate(PetscObjectComm((PetscObject) mass), &rLayout));
30119566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetLocalSize(rLayout, locRows));
30129566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetBlockSize(rLayout, 1));
30139566063dSJacob Faibussowitsch   PetscCall(PetscLayoutSetUp(rLayout));
30149566063dSJacob Faibussowitsch   PetscCall(PetscLayoutGetRange(rLayout, &rStart, &rEnd));
30159566063dSJacob Faibussowitsch   PetscCall(PetscLayoutDestroy(&rLayout));
30169566063dSJacob Faibussowitsch   PetscCall(PetscCalloc2(locRows,&dnz,locRows,&onz));
30179566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJCreate(&ht));
3018bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
3019bd041c0cSMatthew G. Knepley     PetscObject      obj;
3020bd041c0cSMatthew G. Knepley     PetscClassId     id;
3021bd041c0cSMatthew G. Knepley     PetscQuadrature  quad;
3022bd041c0cSMatthew G. Knepley     const PetscReal *qpoints;
3023bd041c0cSMatthew G. Knepley     PetscInt         Nq, Nc, i, d;
3024bd041c0cSMatthew G. Knepley 
30259566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
30269566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
30279566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscFEGetQuadrature((PetscFE) obj, &quad));
30289566063dSJacob Faibussowitsch     else                       PetscCall(PetscFVGetQuadrature((PetscFV) obj, &quad));
30299566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, NULL));
3030bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
3031bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3032bd041c0cSMatthew G. Knepley       Vec                pointVec;
3033bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
3034bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
3035bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
3036bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, q, c;
3037bd041c0cSMatthew G. Knepley       PetscInt          *findices,   *cindices;
3038bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
3039bd041c0cSMatthew G. Knepley 
30409566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
30419566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3042bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
30439566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq*dim, &pointVec));
30449566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
30459566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3046bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
3047c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3048c330f8ffSToby Isaac 
3049bd041c0cSMatthew G. Knepley         /* Transform point to real space */
3050c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
3051bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
3052bd041c0cSMatthew G. Knepley       }
30539566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
3054bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
30559566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
30569566063dSJacob Faibussowitsch       PetscCall(PetscSFViewFromOptions(coarseCellSF, NULL, "-interp_sf_view"));
3057bd041c0cSMatthew G. Knepley       /* Update preallocation info */
30589566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
30595f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
3060bd041c0cSMatthew G. Knepley       {
3061e8f14785SLisandro Dalcin         PetscHashIJKey key;
3062e8f14785SLisandro Dalcin         PetscBool      missing;
3063bd041c0cSMatthew G. Knepley 
3064bd041c0cSMatthew G. Knepley         for (i = 0; i < numFIndices; ++i) {
3065e8f14785SLisandro Dalcin           key.i = findices[i];
3066e8f14785SLisandro Dalcin           if (key.i >= 0) {
3067bd041c0cSMatthew G. Knepley             /* Get indices for coarse elements */
3068bd041c0cSMatthew G. Knepley             for (ccell = 0; ccell < numCoarseCells; ++ccell) {
30699566063dSJacob Faibussowitsch               PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3070bd041c0cSMatthew G. Knepley               for (c = 0; c < numCIndices; ++c) {
3071e8f14785SLisandro Dalcin                 key.j = cindices[c];
3072e8f14785SLisandro Dalcin                 if (key.j < 0) continue;
30739566063dSJacob Faibussowitsch                 PetscCall(PetscHSetIJQueryAdd(ht, key, &missing));
3074bd041c0cSMatthew G. Knepley                 if (missing) {
3075e8f14785SLisandro Dalcin                   if ((key.j >= rStart) && (key.j < rEnd)) ++dnz[key.i-rStart];
3076e8f14785SLisandro Dalcin                   else                                     ++onz[key.i-rStart];
3077bd041c0cSMatthew G. Knepley                 }
3078bd041c0cSMatthew G. Knepley               }
30799566063dSJacob Faibussowitsch               PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3080bd041c0cSMatthew G. Knepley             }
3081bd041c0cSMatthew G. Knepley           }
3082bd041c0cSMatthew G. Knepley         }
3083bd041c0cSMatthew G. Knepley       }
30849566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
30859566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
30869566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
3087bd041c0cSMatthew G. Knepley     }
3088bd041c0cSMatthew G. Knepley   }
30899566063dSJacob Faibussowitsch   PetscCall(PetscHSetIJDestroy(&ht));
30909566063dSJacob Faibussowitsch   PetscCall(MatXAIJSetPreallocation(mass, 1, dnz, onz, NULL, NULL));
30919566063dSJacob Faibussowitsch   PetscCall(MatSetOption(mass, MAT_NEW_NONZERO_ALLOCATION_ERR,PETSC_TRUE));
30929566063dSJacob Faibussowitsch   PetscCall(PetscFree2(dnz,onz));
3093bd041c0cSMatthew G. Knepley   for (field = 0; field < Nf; ++field) {
3094bd041c0cSMatthew G. Knepley     PetscObject       obj;
3095bd041c0cSMatthew G. Knepley     PetscClassId      id;
3096ef0bb6c7SMatthew G. Knepley     PetscTabulation T, Tfine;
3097bd041c0cSMatthew G. Knepley     PetscQuadrature   quad;
3098bd041c0cSMatthew G. Knepley     const PetscReal  *qpoints, *qweights;
3099bd041c0cSMatthew G. Knepley     PetscInt          Nq, Nc, i, d;
3100bd041c0cSMatthew G. Knepley 
31019566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
31029566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
3103ef0bb6c7SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
31049566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature((PetscFE) obj, &quad));
31059566063dSJacob Faibussowitsch       PetscCall(PetscFEGetCellTabulation((PetscFE) obj, 1, &Tfine));
31069566063dSJacob Faibussowitsch       PetscCall(PetscFECreateTabulation((PetscFE) obj, 1, 1, x, 0, &T));
3107ef0bb6c7SMatthew G. Knepley     } else {
31089566063dSJacob Faibussowitsch       PetscCall(PetscFVGetQuadrature((PetscFV) obj, &quad));
3109ef0bb6c7SMatthew G. Knepley     }
31109566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(quad, NULL, &Nc, &Nq, &qpoints, &qweights));
3111bd041c0cSMatthew G. Knepley     /* For each fine grid cell */
3112bd041c0cSMatthew G. Knepley     for (cell = cStart; cell < cEnd; ++cell) {
3113bd041c0cSMatthew G. Knepley       Vec                pointVec;
3114bd041c0cSMatthew G. Knepley       PetscScalar       *pV;
3115bd041c0cSMatthew G. Knepley       PetscSF            coarseCellSF = NULL;
3116bd041c0cSMatthew G. Knepley       const PetscSFNode *coarseCells;
3117bd041c0cSMatthew G. Knepley       PetscInt           numCoarseCells, cpdim, q, c, j;
3118bd041c0cSMatthew G. Knepley       PetscInt          *findices,   *cindices;
3119bd041c0cSMatthew G. Knepley       PetscInt           numFIndices, numCIndices;
3120bd041c0cSMatthew G. Knepley 
31219566063dSJacob Faibussowitsch       PetscCall(DMPlexGetClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
31229566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeCellGeometryFEM(dmf, cell, NULL, v0, J, invJ, &detJ));
3123bd041c0cSMatthew G. Knepley       /* Get points from the quadrature */
31249566063dSJacob Faibussowitsch       PetscCall(VecCreateSeq(PETSC_COMM_SELF, Nq*dim, &pointVec));
31259566063dSJacob Faibussowitsch       PetscCall(VecSetBlockSize(pointVec, dim));
31269566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3127bd041c0cSMatthew G. Knepley       for (q = 0; q < Nq; ++q) {
3128c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3129c330f8ffSToby Isaac 
3130bd041c0cSMatthew G. Knepley         /* Transform point to real space */
3131c330f8ffSToby Isaac         CoordinatesRefToReal(dim, dim, xi0, v0, J, &qpoints[q*dim], x);
3132bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pV[q*dim+d] = x[d];
3133bd041c0cSMatthew G. Knepley       }
31349566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
3135bd041c0cSMatthew G. Knepley       /* Get set of coarse cells that overlap points (would like to group points by coarse cell) */
31369566063dSJacob Faibussowitsch       PetscCall(DMLocatePoints(dmc, pointVec, DM_POINTLOCATION_NEAREST, &coarseCellSF));
3137bd041c0cSMatthew G. Knepley       /* Update matrix */
31389566063dSJacob Faibussowitsch       PetscCall(PetscSFGetGraph(coarseCellSF, NULL, &numCoarseCells, NULL, &coarseCells));
31395f80ce2aSJacob Faibussowitsch       PetscCheck(numCoarseCells == Nq,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Not all closure points located");
31409566063dSJacob Faibussowitsch       PetscCall(VecGetArray(pointVec, &pV));
3141bd041c0cSMatthew G. Knepley       for (ccell = 0; ccell < numCoarseCells; ++ccell) {
3142bd041c0cSMatthew G. Knepley         PetscReal pVReal[3];
3143c330f8ffSToby Isaac         const PetscReal xi0[3] = {-1., -1., -1.};
3144c330f8ffSToby Isaac 
31459566063dSJacob Faibussowitsch         PetscCall(DMPlexGetClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3146bd041c0cSMatthew G. Knepley         /* Transform points from real space to coarse reference space */
31479566063dSJacob Faibussowitsch         PetscCall(DMPlexComputeCellGeometryFEM(dmc, coarseCells[ccell].index, NULL, v0c, Jc, invJc, &detJc));
3148bd041c0cSMatthew G. Knepley         for (d = 0; d < dim; ++d) pVReal[d] = PetscRealPart(pV[ccell*dim+d]);
3149c330f8ffSToby Isaac         CoordinatesRealToRef(dim, dim, xi0, v0c, invJc, pVReal, x);
3150bd041c0cSMatthew G. Knepley 
3151bd041c0cSMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
3152bd041c0cSMatthew G. Knepley           PetscFE fe = (PetscFE) obj;
3153bd041c0cSMatthew G. Knepley 
3154bd041c0cSMatthew G. Knepley           /* Evaluate coarse basis on contained point */
31559566063dSJacob Faibussowitsch           PetscCall(PetscFEGetDimension(fe, &cpdim));
31569566063dSJacob Faibussowitsch           PetscCall(PetscFEComputeTabulation(fe, 1, x, 0, T));
3157bd041c0cSMatthew G. Knepley           /* Get elemMat entries by multiplying by weight */
3158bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
31599566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3160bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3161ef0bb6c7SMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += T->T[0][j*Nc + c]*Tfine->T[0][(ccell*numFIndices + i)*Nc + c]*qweights[ccell*Nc + c]*detJ;
3162bd041c0cSMatthew G. Knepley             }
3163bd041c0cSMatthew G. Knepley             /* Update interpolator */
31649566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
316563a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
31669566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3167bd041c0cSMatthew G. Knepley           }
3168bd041c0cSMatthew G. Knepley         } else {
3169bd041c0cSMatthew G. Knepley           cpdim = 1;
3170bd041c0cSMatthew G. Knepley           for (i = 0; i < numFIndices; ++i) {
31719566063dSJacob Faibussowitsch             PetscCall(PetscArrayzero(elemMat, cpdim));
3172bd041c0cSMatthew G. Knepley             for (j = 0; j < cpdim; ++j) {
3173bd041c0cSMatthew G. Knepley               for (c = 0; c < Nc; ++c) elemMat[j] += 1.0*1.0*qweights[ccell*Nc + c]*detJ;
3174bd041c0cSMatthew G. Knepley             }
3175bd041c0cSMatthew G. Knepley             /* Update interpolator */
31769566063dSJacob Faibussowitsch             if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, 1, numCIndices, elemMat));
317763a3b9bcSJacob Faibussowitsch             PetscCall(PetscPrintf(PETSC_COMM_SELF, "Nq: %" PetscInt_FMT " %" PetscInt_FMT " Nf: %" PetscInt_FMT " %" PetscInt_FMT " Nc: %" PetscInt_FMT " %" PetscInt_FMT "\n", ccell, Nq, i, numFIndices, j, numCIndices));
317863a3b9bcSJacob Faibussowitsch             PetscCheck(numCIndices == cpdim,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Number of element matrix columns %" PetscInt_FMT " != %" PetscInt_FMT, numCIndices, cpdim);
31799566063dSJacob Faibussowitsch             PetscCall(MatSetValues(mass, 1, &findices[i], numCIndices, cindices, elemMat, ADD_VALUES));
3180bd041c0cSMatthew G. Knepley           }
3181bd041c0cSMatthew G. Knepley         }
31829566063dSJacob Faibussowitsch         PetscCall(DMPlexRestoreClosureIndices(dmc, csection, globalCSection, coarseCells[ccell].index, PETSC_FALSE, &numCIndices, &cindices, NULL, NULL));
3183bd041c0cSMatthew G. Knepley       }
31849566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(pointVec, &pV));
31859566063dSJacob Faibussowitsch       PetscCall(PetscSFDestroy(&coarseCellSF));
31869566063dSJacob Faibussowitsch       PetscCall(VecDestroy(&pointVec));
31879566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreClosureIndices(dmf, fsection, globalFSection, cell, PETSC_FALSE, &numFIndices, &findices, NULL, NULL));
3188bd041c0cSMatthew G. Knepley     }
31899566063dSJacob Faibussowitsch     if (id == PETSCFE_CLASSID) PetscCall(PetscTabulationDestroy(&T));
3190bd041c0cSMatthew G. Knepley   }
31919566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0,J,invJ));
31929566063dSJacob Faibussowitsch   PetscCall(PetscFree3(v0c,Jc,invJc));
31939566063dSJacob Faibussowitsch   PetscCall(PetscFree(elemMat));
31949566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(mass, MAT_FINAL_ASSEMBLY));
31959566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(mass, MAT_FINAL_ASSEMBLY));
3196bd041c0cSMatthew G. Knepley   PetscFunctionReturn(0);
3197bd041c0cSMatthew G. Knepley }
3198bd041c0cSMatthew G. Knepley 
3199bd041c0cSMatthew G. Knepley /*@
320046fa42a0SMatthew G. Knepley   DMPlexComputeInjectorFEM - Compute a mapping from coarse unknowns to fine unknowns
320146fa42a0SMatthew G. Knepley 
320246fa42a0SMatthew G. Knepley   Input Parameters:
320346fa42a0SMatthew G. Knepley + dmc  - The coarse mesh
320446fa42a0SMatthew G. Knepley - dmf  - The fine mesh
320546fa42a0SMatthew G. Knepley - user - The user context
320646fa42a0SMatthew G. Knepley 
320746fa42a0SMatthew G. Knepley   Output Parameter:
320846fa42a0SMatthew G. Knepley . sc   - The mapping
320946fa42a0SMatthew G. Knepley 
321046fa42a0SMatthew G. Knepley   Level: developer
321146fa42a0SMatthew G. Knepley 
321246fa42a0SMatthew G. Knepley .seealso: DMPlexComputeInterpolatorNested(), DMPlexComputeJacobianFEM()
321346fa42a0SMatthew G. Knepley @*/
32147c927364SMatthew G. Knepley PetscErrorCode DMPlexComputeInjectorFEM(DM dmc, DM dmf, VecScatter *sc, void *user)
32157c927364SMatthew G. Knepley {
3216e9d4ef1bSMatthew G. Knepley   PetscDS        prob;
32177c927364SMatthew G. Knepley   PetscFE       *feRef;
321897c42addSMatthew G. Knepley   PetscFV       *fvRef;
32197c927364SMatthew G. Knepley   Vec            fv, cv;
32207c927364SMatthew G. Knepley   IS             fis, cis;
32217c927364SMatthew G. Knepley   PetscSection   fsection, fglobalSection, csection, cglobalSection;
32227c927364SMatthew G. Knepley   PetscInt      *cmap, *cellCIndices, *cellFIndices, *cindices, *findices;
3223485ad865SMatthew G. Knepley   PetscInt       cTotDim, fTotDim = 0, Nf, f, field, cStart, cEnd, c, dim, d, startC, endC, offsetC, offsetF, m;
32246f3d3cbcSMatthew G. Knepley   PetscBool     *needAvg;
32257c927364SMatthew G. Knepley 
32267c927364SMatthew G. Knepley   PetscFunctionBegin;
32279566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_InjectorFEM,dmc,dmf,0,0));
32289566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dmf, &dim));
32299566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmf, &fsection));
32309566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmf, &fglobalSection));
32319566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dmc, &csection));
32329566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dmc, &cglobalSection));
32339566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetNumFields(fsection, &Nf));
32349566063dSJacob Faibussowitsch   PetscCall(DMPlexGetSimplexOrBoxCells(dmc, 0, &cStart, &cEnd));
32359566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dmc, &prob));
32369566063dSJacob Faibussowitsch   PetscCall(PetscCalloc3(Nf,&feRef,Nf,&fvRef,Nf,&needAvg));
32377c927364SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
323897c42addSMatthew G. Knepley     PetscObject  obj;
323997c42addSMatthew G. Knepley     PetscClassId id;
3240aa7890ccSMatthew G. Knepley     PetscInt     fNb = 0, Nc = 0;
32417c927364SMatthew G. Knepley 
32429566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
32439566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
324497c42addSMatthew G. Knepley     if (id == PETSCFE_CLASSID) {
324597c42addSMatthew G. Knepley       PetscFE    fe = (PetscFE) obj;
32466f3d3cbcSMatthew G. Knepley       PetscSpace sp;
32479b2fc754SMatthew G. Knepley       PetscInt   maxDegree;
324897c42addSMatthew G. Knepley 
32499566063dSJacob Faibussowitsch       PetscCall(PetscFERefine(fe, &feRef[f]));
32509566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feRef[f], &fNb));
32519566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(fe, &Nc));
32529566063dSJacob Faibussowitsch       PetscCall(PetscFEGetBasisSpace(fe, &sp));
32539566063dSJacob Faibussowitsch       PetscCall(PetscSpaceGetDegree(sp, NULL, &maxDegree));
32549b2fc754SMatthew G. Knepley       if (!maxDegree) needAvg[f] = PETSC_TRUE;
325597c42addSMatthew G. Knepley     } else if (id == PETSCFV_CLASSID) {
325697c42addSMatthew G. Knepley       PetscFV        fv = (PetscFV) obj;
325797c42addSMatthew G. Knepley       PetscDualSpace Q;
325897c42addSMatthew G. Knepley 
32599566063dSJacob Faibussowitsch       PetscCall(PetscFVRefine(fv, &fvRef[f]));
32609566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[f], &Q));
32619566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(Q, &fNb));
32629566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &Nc));
32636f3d3cbcSMatthew G. Knepley       needAvg[f] = PETSC_TRUE;
326497c42addSMatthew G. Knepley     }
3265d172c84bSMatthew G. Knepley     fTotDim += fNb;
32667c927364SMatthew G. Knepley   }
32679566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &cTotDim));
32689566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(cTotDim,&cmap));
32697c927364SMatthew G. Knepley   for (field = 0, offsetC = 0, offsetF = 0; field < Nf; ++field) {
32707c927364SMatthew G. Knepley     PetscFE        feC;
327197c42addSMatthew G. Knepley     PetscFV        fvC;
32727c927364SMatthew G. Knepley     PetscDualSpace QF, QC;
3273d172c84bSMatthew G. Knepley     PetscInt       order = -1, NcF, NcC, fpdim, cpdim;
32747c927364SMatthew G. Knepley 
327597c42addSMatthew G. Knepley     if (feRef[field]) {
32769566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *) &feC));
32779566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feC, &NcC));
32789566063dSJacob Faibussowitsch       PetscCall(PetscFEGetNumComponents(feRef[field], &NcF));
32799566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feRef[field], &QF));
32809566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetOrder(QF, &order));
32819566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
32829566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDualSpace(feC, &QC));
32839566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
328497c42addSMatthew G. Knepley     } else {
32859566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, field, (PetscObject *) &fvC));
32869566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvC, &NcC));
32879566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fvRef[field], &NcF));
32889566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvRef[field], &QF));
32899566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QF, &fpdim));
32909566063dSJacob Faibussowitsch       PetscCall(PetscFVGetDualSpace(fvC, &QC));
32919566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetDimension(QC, &cpdim));
329297c42addSMatthew G. Knepley     }
329363a3b9bcSJacob Faibussowitsch     PetscCheck(NcF == NcC,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of components in fine space field %" PetscInt_FMT " does not match coarse field %" PetscInt_FMT, NcF, NcC);
32947c927364SMatthew G. Knepley     for (c = 0; c < cpdim; ++c) {
32957c927364SMatthew G. Knepley       PetscQuadrature  cfunc;
3296d172c84bSMatthew G. Knepley       const PetscReal *cqpoints, *cqweights;
3297d172c84bSMatthew G. Knepley       PetscInt         NqcC, NpC;
329897c42addSMatthew G. Knepley       PetscBool        found = PETSC_FALSE;
32997c927364SMatthew G. Knepley 
33009566063dSJacob Faibussowitsch       PetscCall(PetscDualSpaceGetFunctional(QC, c, &cfunc));
33019566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(cfunc, NULL, &NqcC, &NpC, &cqpoints, &cqweights));
330263a3b9bcSJacob Faibussowitsch       PetscCheck(NqcC == NcC,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcC, NcC);
3303*1dca8a05SBarry Smith       PetscCheck(NpC == 1 || !feRef[field],PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Do not know how to do injection for moments");
33047c927364SMatthew G. Knepley       for (f = 0; f < fpdim; ++f) {
33057c927364SMatthew G. Knepley         PetscQuadrature  ffunc;
3306d172c84bSMatthew G. Knepley         const PetscReal *fqpoints, *fqweights;
33077c927364SMatthew G. Knepley         PetscReal        sum = 0.0;
3308d172c84bSMatthew G. Knepley         PetscInt         NqcF, NpF;
33097c927364SMatthew G. Knepley 
33109566063dSJacob Faibussowitsch         PetscCall(PetscDualSpaceGetFunctional(QF, f, &ffunc));
33119566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(ffunc, NULL, &NqcF, &NpF, &fqpoints, &fqweights));
331263a3b9bcSJacob Faibussowitsch         PetscCheck(NqcF == NcF,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Number of quadrature components %" PetscInt_FMT " must match number of field components %" PetscInt_FMT, NqcF, NcF);
33137c927364SMatthew G. Knepley         if (NpC != NpF) continue;
33147c927364SMatthew G. Knepley         for (d = 0; d < dim; ++d) sum += PetscAbsReal(cqpoints[d] - fqpoints[d]);
33157c927364SMatthew G. Knepley         if (sum > 1.0e-9) continue;
3316d172c84bSMatthew G. Knepley         for (d = 0; d < NcC; ++d) sum += PetscAbsReal(cqweights[d]*fqweights[d]);
3317d172c84bSMatthew G. Knepley         if (sum < 1.0e-9) continue;
3318d172c84bSMatthew G. Knepley         cmap[offsetC+c] = offsetF+f;
331997c42addSMatthew G. Knepley         found = PETSC_TRUE;
33207c927364SMatthew G. Knepley         break;
33217c927364SMatthew G. Knepley       }
332297c42addSMatthew G. Knepley       if (!found) {
332397c42addSMatthew G. Knepley         /* TODO We really want the average here, but some asshole put VecScatter in the interface */
3324d172c84bSMatthew G. Knepley         if (fvRef[field] || (feRef[field] && order == 0)) {
3325d172c84bSMatthew G. Knepley           cmap[offsetC+c] = offsetF+0;
332697c42addSMatthew G. Knepley         } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Could not locate matching functional for injection");
332797c42addSMatthew G. Knepley       }
33287c927364SMatthew G. Knepley     }
3329d172c84bSMatthew G. Knepley     offsetC += cpdim;
3330d172c84bSMatthew G. Knepley     offsetF += fpdim;
33317c927364SMatthew G. Knepley   }
33329566063dSJacob Faibussowitsch   for (f = 0; f < Nf; ++f) {PetscCall(PetscFEDestroy(&feRef[f]));PetscCall(PetscFVDestroy(&fvRef[f]));}
33339566063dSJacob Faibussowitsch   PetscCall(PetscFree3(feRef,fvRef,needAvg));
33347c927364SMatthew G. Knepley 
33359566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmf, &fv));
33369566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalVector(dmc, &cv));
33379566063dSJacob Faibussowitsch   PetscCall(VecGetOwnershipRange(cv, &startC, &endC));
33389566063dSJacob Faibussowitsch   PetscCall(PetscSectionGetConstrainedStorageSize(cglobalSection, &m));
33399566063dSJacob Faibussowitsch   PetscCall(PetscMalloc2(cTotDim,&cellCIndices,fTotDim,&cellFIndices));
33409566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m,&cindices));
33419566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(m,&findices));
33427c927364SMatthew G. Knepley   for (d = 0; d < m; ++d) cindices[d] = findices[d] = -1;
33437c927364SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
33449566063dSJacob Faibussowitsch     PetscCall(DMPlexMatGetClosureIndicesRefined(dmf, fsection, fglobalSection, dmc, csection, cglobalSection, c, cellCIndices, cellFIndices));
33457c927364SMatthew G. Knepley     for (d = 0; d < cTotDim; ++d) {
33460bd915a7SMatthew G. Knepley       if ((cellCIndices[d] < startC) || (cellCIndices[d] >= endC)) continue;
334763a3b9bcSJacob Faibussowitsch       PetscCheck(!(findices[cellCIndices[d]-startC] >= 0) || !(findices[cellCIndices[d]-startC] != cellFIndices[cmap[d]]),PETSC_COMM_SELF, PETSC_ERR_PLIB, "Coarse dof %" PetscInt_FMT " maps to both %" PetscInt_FMT " and %" PetscInt_FMT, cindices[cellCIndices[d]-startC], findices[cellCIndices[d]-startC], cellFIndices[cmap[d]]);
33487c927364SMatthew G. Knepley       cindices[cellCIndices[d]-startC] = cellCIndices[d];
33497c927364SMatthew G. Knepley       findices[cellCIndices[d]-startC] = cellFIndices[cmap[d]];
33507c927364SMatthew G. Knepley     }
33517c927364SMatthew G. Knepley   }
33529566063dSJacob Faibussowitsch   PetscCall(PetscFree(cmap));
33539566063dSJacob Faibussowitsch   PetscCall(PetscFree2(cellCIndices,cellFIndices));
33547c927364SMatthew G. Knepley 
33559566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, cindices, PETSC_OWN_POINTER, &cis));
33569566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, m, findices, PETSC_OWN_POINTER, &fis));
33579566063dSJacob Faibussowitsch   PetscCall(VecScatterCreate(cv, cis, fv, fis, sc));
33589566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cis));
33599566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&fis));
33609566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmf, &fv));
33619566063dSJacob Faibussowitsch   PetscCall(DMRestoreGlobalVector(dmc, &cv));
33629566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_InjectorFEM,dmc,dmf,0,0));
3363cb1e1211SMatthew G Knepley   PetscFunctionReturn(0);
3364cb1e1211SMatthew G Knepley }
3365a1cf66bbSMatthew G. Knepley 
33662f856554SMatthew G. Knepley /*@C
33672f856554SMatthew G. Knepley   DMPlexGetCellFields - Retrieve the field values values for a chunk of cells
33682f856554SMatthew G. Knepley 
33692f856554SMatthew G. Knepley   Input Parameters:
33702f856554SMatthew G. Knepley + dm     - The DM
33712f856554SMatthew G. Knepley . cellIS - The cells to include
33722f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
33732f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
33742f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
33752f856554SMatthew G. Knepley 
33762f856554SMatthew G. Knepley   Output Parameters:
33772f856554SMatthew G. Knepley + u   - The field coefficients
33782f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
33792f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
33802f856554SMatthew G. Knepley 
33812f856554SMatthew G. Knepley   Level: developer
33822f856554SMatthew G. Knepley 
33832f856554SMatthew G. Knepley .seealso: DMPlexGetFaceFields()
33842f856554SMatthew G. Knepley @*/
33852f856554SMatthew G. Knepley PetscErrorCode DMPlexGetCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
33862f856554SMatthew G. Knepley {
33872f856554SMatthew G. Knepley   DM              plex, plexA = NULL;
3388a6e0b375SMatthew G. Knepley   DMEnclosureType encAux;
33892f856554SMatthew G. Knepley   PetscSection    section, sectionAux;
33902f856554SMatthew G. Knepley   PetscDS         prob;
33912f856554SMatthew G. Knepley   const PetscInt *cells;
33922f856554SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, totDim, totDimAux, c;
33932f856554SMatthew G. Knepley 
33942f856554SMatthew G. Knepley   PetscFunctionBegin;
33952f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
3396064a246eSJacob Faibussowitsch   PetscValidHeaderSpecific(locX, VEC_CLASSID, 3);
3397064a246eSJacob Faibussowitsch   if (locX_t) {PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 4);}
3398064a246eSJacob Faibussowitsch   if (locA)   {PetscValidHeaderSpecific(locA, VEC_CLASSID, 5);}
3399064a246eSJacob Faibussowitsch   PetscValidPointer(u, 6);
3400064a246eSJacob Faibussowitsch   PetscValidPointer(u_t, 7);
3401064a246eSJacob Faibussowitsch   PetscValidPointer(a, 8);
34029566063dSJacob Faibussowitsch   PetscCall(DMPlexConvertPlex(dm, &plex, PETSC_FALSE));
34039566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
34049566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
34059566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob));
34069566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
34072f856554SMatthew G. Knepley   if (locA) {
34082f856554SMatthew G. Knepley     DM      dmAux;
34092f856554SMatthew G. Knepley     PetscDS probAux;
34102f856554SMatthew G. Knepley 
34119566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
34129566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
34139566063dSJacob Faibussowitsch     PetscCall(DMPlexConvertPlex(dmAux, &plexA, PETSC_FALSE));
34149566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
34159566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
34169566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
34172f856554SMatthew G. Knepley   }
34182f856554SMatthew G. Knepley   numCells = cEnd - cStart;
34199566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, u));
34209566063dSJacob Faibussowitsch   if (locX_t) PetscCall(DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, u_t)); else {*u_t = NULL;}
34219566063dSJacob Faibussowitsch   if (locA)   PetscCall(DMGetWorkArray(dm, numCells*totDimAux, MPIU_SCALAR, a)); else {*a = NULL;}
34222f856554SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
34232f856554SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
34242f856554SMatthew G. Knepley     const PetscInt cind = c - cStart;
34252f856554SMatthew G. Knepley     PetscScalar   *x = NULL, *x_t = NULL, *ul = *u, *ul_t = *u_t, *al = *a;
34262f856554SMatthew G. Knepley     PetscInt       i;
34272f856554SMatthew G. Knepley 
34289566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, locX, cell, NULL, &x));
34292f856554SMatthew G. Knepley     for (i = 0; i < totDim; ++i) ul[cind*totDim+i] = x[i];
34309566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, locX, cell, NULL, &x));
34312f856554SMatthew G. Knepley     if (locX_t) {
34329566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX_t, cell, NULL, &x_t));
34332f856554SMatthew G. Knepley       for (i = 0; i < totDim; ++i) ul_t[cind*totDim+i] = x_t[i];
34349566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, cell, NULL, &x_t));
34352f856554SMatthew G. Knepley     }
34362f856554SMatthew G. Knepley     if (locA) {
34372f856554SMatthew G. Knepley       PetscInt subcell;
34389566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, cell, &subcell));
34399566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subcell, NULL, &x));
34402f856554SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) al[cind*totDimAux+i] = x[i];
34419566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subcell, NULL, &x));
34422f856554SMatthew G. Knepley     }
34432f856554SMatthew G. Knepley   }
34449566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
34459566063dSJacob Faibussowitsch   if (locA) PetscCall(DMDestroy(&plexA));
34469566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
34472f856554SMatthew G. Knepley   PetscFunctionReturn(0);
34482f856554SMatthew G. Knepley }
34492f856554SMatthew G. Knepley 
34502f856554SMatthew G. Knepley /*@C
34512f856554SMatthew G. Knepley   DMPlexRestoreCellFields - Restore the field values values for a chunk of cells
34522f856554SMatthew G. Knepley 
34532f856554SMatthew G. Knepley   Input Parameters:
34542f856554SMatthew G. Knepley + dm     - The DM
34552f856554SMatthew G. Knepley . cellIS - The cells to include
34562f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
34572f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
34582f856554SMatthew G. Knepley - locA   - A local vector with auxiliary fields, or NULL
34592f856554SMatthew G. Knepley 
34602f856554SMatthew G. Knepley   Output Parameters:
34612f856554SMatthew G. Knepley + u   - The field coefficients
34622f856554SMatthew G. Knepley . u_t - The fields derivative coefficients
34632f856554SMatthew G. Knepley - a   - The auxiliary field coefficients
34642f856554SMatthew G. Knepley 
34652f856554SMatthew G. Knepley   Level: developer
34662f856554SMatthew G. Knepley 
34672f856554SMatthew G. Knepley .seealso: DMPlexGetFaceFields()
34682f856554SMatthew G. Knepley @*/
34692f856554SMatthew G. Knepley PetscErrorCode DMPlexRestoreCellFields(DM dm, IS cellIS, Vec locX, Vec locX_t, Vec locA, PetscScalar **u, PetscScalar **u_t, PetscScalar **a)
34702f856554SMatthew G. Knepley {
34712f856554SMatthew G. Knepley   PetscFunctionBegin;
34729566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u));
34739566063dSJacob Faibussowitsch   if (locX_t) PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, u_t));
34749566063dSJacob Faibussowitsch   if (locA)   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, a));
34752f856554SMatthew G. Knepley   PetscFunctionReturn(0);
34762f856554SMatthew G. Knepley }
34772f856554SMatthew G. Knepley 
3478faff4f7fSMatthew G. Knepley /*
3479faff4f7fSMatthew G. Knepley   Get the auxiliary field vectors for the negative side (s = 0) and positive side (s = 1) of the interfaace
3480faff4f7fSMatthew G. Knepley */
3481148442b3SMatthew G. Knepley static PetscErrorCode DMPlexGetHybridAuxFields(DM dm, DM dmAux[], PetscDS dsAux[], IS cellIS, Vec locA[], PetscScalar *a[])
34826528b96dSMatthew G. Knepley {
348304c51a94SMatthew G. Knepley   DM              plexA[2];
3484148442b3SMatthew G. Knepley   DMEnclosureType encAux[2];
348504c51a94SMatthew G. Knepley   PetscSection    sectionAux[2];
34866528b96dSMatthew G. Knepley   const PetscInt *cells;
3487148442b3SMatthew G. Knepley   PetscInt        cStart, cEnd, numCells, c, s, totDimAux[2];
34886528b96dSMatthew G. Knepley 
34896528b96dSMatthew G. Knepley   PetscFunctionBegin;
3490148442b3SMatthew G. Knepley   PetscValidPointer(locA, 5);
349104c51a94SMatthew G. Knepley   if (!locA[0] || !locA[1]) PetscFunctionReturn(0);
3492148442b3SMatthew G. Knepley   PetscValidPointer(dmAux, 2);
3493148442b3SMatthew G. Knepley   PetscValidPointer(dsAux, 3);
3494148442b3SMatthew G. Knepley   PetscValidPointer(a, 6);
34959566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
34966528b96dSMatthew G. Knepley   numCells = cEnd - cStart;
3497148442b3SMatthew G. Knepley   for (s = 0; s < 2; ++s) {
3498148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(dmAux[s], DM_CLASSID, 2);
3499148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(dsAux[s], PETSCDS_CLASSID, 3);
3500148442b3SMatthew G. Knepley     PetscValidHeaderSpecific(locA[s], VEC_CLASSID, 5);
35019566063dSJacob Faibussowitsch     PetscCall(DMPlexConvertPlex(dmAux[s], &plexA[s], PETSC_FALSE));
35029566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux[s], dm, &encAux[s]));
35039566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux[s], &sectionAux[s]));
35049566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[s], &totDimAux[s]));
35059566063dSJacob Faibussowitsch     PetscCall(DMGetWorkArray(dmAux[s], numCells*totDimAux[s], MPIU_SCALAR, &a[s]));
350604c51a94SMatthew G. Knepley   }
3507148442b3SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
35086528b96dSMatthew G. Knepley     const PetscInt  cell = cells ? cells[c] : c;
35096528b96dSMatthew G. Knepley     const PetscInt  cind = c - cStart;
35106528b96dSMatthew G. Knepley     const PetscInt *cone, *ornt;
35116528b96dSMatthew G. Knepley 
35129566063dSJacob Faibussowitsch     PetscCall(DMPlexGetCone(dm, cell, &cone));
35139566063dSJacob Faibussowitsch     PetscCall(DMPlexGetConeOrientation(dm, cell, &ornt));
3514*1dca8a05SBarry Smith     PetscCheck(!ornt[0],PETSC_COMM_SELF, PETSC_ERR_SUP, "Face %" PetscInt_FMT " in hybrid cell %" PetscInt_FMT " has orientation %" PetscInt_FMT " != 0", cone[0], cell, ornt[0]);
3515148442b3SMatthew G. Knepley     for (s = 0; s < 2; ++s) {
35165fedec97SMatthew G. Knepley       const PetscInt *support;
3517148442b3SMatthew G. Knepley       PetscScalar    *x = NULL, *al = a[s];
3518148442b3SMatthew G. Knepley       const PetscInt  tdA = totDimAux[s];
35195fedec97SMatthew G. Knepley       PetscInt        ssize, scell;
3520148442b3SMatthew G. Knepley       PetscInt        subface, Na, i;
35216528b96dSMatthew G. Knepley 
35229566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, cone[s], &support));
35239566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupportSize(dm, cone[s], &ssize));
352463a3b9bcSJacob Faibussowitsch       PetscCheck(ssize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[s], cell, ssize);
35255fedec97SMatthew G. Knepley       if      (support[0] == cell) scell = support[1];
35265fedec97SMatthew G. Knepley       else if (support[1] == cell) scell = support[0];
352763a3b9bcSJacob Faibussowitsch       else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[s], cell);
35285fedec97SMatthew G. Knepley 
35299566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(plexA[s], dm, encAux[s], scell, &subface));
35309566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexA[s], sectionAux[s], locA[s], subface, &Na, &x));
35316528b96dSMatthew G. Knepley       for (i = 0; i < Na; ++i) al[cind*tdA+i] = x[i];
35329566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexA[s], sectionAux[s], locA[s], subface, &Na, &x));
35336528b96dSMatthew G. Knepley     }
35346528b96dSMatthew G. Knepley   }
35359566063dSJacob Faibussowitsch   for (s = 0; s < 2; ++s) PetscCall(DMDestroy(&plexA[s]));
35369566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
35376528b96dSMatthew G. Knepley   PetscFunctionReturn(0);
35386528b96dSMatthew G. Knepley }
35396528b96dSMatthew G. Knepley 
354004c51a94SMatthew G. Knepley static PetscErrorCode DMPlexRestoreHybridAuxFields(DM dmAux[], PetscDS dsAux[], IS cellIS, Vec locA[], PetscScalar *a[])
35416528b96dSMatthew G. Knepley {
35426528b96dSMatthew G. Knepley   PetscFunctionBegin;
354304c51a94SMatthew G. Knepley   if (!locA[0] || !locA[1]) PetscFunctionReturn(0);
35449566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dmAux[0], 0, MPIU_SCALAR, &a[0]));
35459566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dmAux[1], 0, MPIU_SCALAR, &a[1]));
35466528b96dSMatthew G. Knepley   PetscFunctionReturn(0);
35476528b96dSMatthew G. Knepley }
35486528b96dSMatthew G. Knepley 
35492f856554SMatthew G. Knepley /*@C
35502f856554SMatthew G. Knepley   DMPlexGetFaceFields - Retrieve the field values values for a chunk of faces
35512f856554SMatthew G. Knepley 
35522f856554SMatthew G. Knepley   Input Parameters:
35532f856554SMatthew G. Knepley + dm     - The DM
35542f856554SMatthew G. Knepley . fStart - The first face to include
35552f856554SMatthew G. Knepley . fEnd   - The first face to exclude
35562f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
35572f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
35582f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
35592f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
35602f856554SMatthew G. Knepley - locaGrad - A local vector with field gradients, or NULL
35612f856554SMatthew G. Knepley 
35622f856554SMatthew G. Knepley   Output Parameters:
35632f856554SMatthew G. Knepley + Nface - The number of faces with field values
35642f856554SMatthew G. Knepley . uL - The field values at the left side of the face
35652f856554SMatthew G. Knepley - uR - The field values at the right side of the face
35662f856554SMatthew G. Knepley 
35672f856554SMatthew G. Knepley   Level: developer
35682f856554SMatthew G. Knepley 
35692f856554SMatthew G. Knepley .seealso: DMPlexGetCellFields()
35702f856554SMatthew G. Knepley @*/
35712f856554SMatthew G. Knepley PetscErrorCode DMPlexGetFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
35722f856554SMatthew G. Knepley {
35732f856554SMatthew G. Knepley   DM                 dmFace, dmCell, dmGrad = NULL;
35742f856554SMatthew G. Knepley   PetscSection       section;
35752f856554SMatthew G. Knepley   PetscDS            prob;
35762f856554SMatthew G. Knepley   DMLabel            ghostLabel;
35772f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom, *x, *lgrad;
35782f856554SMatthew G. Knepley   PetscBool         *isFE;
35792f856554SMatthew G. Knepley   PetscInt           dim, Nf, f, Nc, numFaces = fEnd - fStart, iface, face;
35802f856554SMatthew G. Knepley 
35812f856554SMatthew G. Knepley   PetscFunctionBegin;
35822f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
35832f856554SMatthew G. Knepley   PetscValidHeaderSpecific(locX, VEC_CLASSID, 4);
35842f856554SMatthew G. Knepley   if (locX_t) {PetscValidHeaderSpecific(locX_t, VEC_CLASSID, 5);}
35852f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 6);
35862f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 7);
35872f856554SMatthew G. Knepley   if (locGrad) {PetscValidHeaderSpecific(locGrad, VEC_CLASSID, 8);}
3588064a246eSJacob Faibussowitsch   PetscValidPointer(uL, 10);
3589064a246eSJacob Faibussowitsch   PetscValidPointer(uR, 11);
35909566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
35919566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
35929566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
35939566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
35949566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalComponents(prob, &Nc));
35959566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(Nf, &isFE));
35962f856554SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
35972f856554SMatthew G. Knepley     PetscObject  obj;
35982f856554SMatthew G. Knepley     PetscClassId id;
35992f856554SMatthew G. Knepley 
36009566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
36019566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
36022f856554SMatthew G. Knepley     if (id == PETSCFE_CLASSID)      {isFE[f] = PETSC_TRUE;}
36032f856554SMatthew G. Knepley     else if (id == PETSCFV_CLASSID) {isFE[f] = PETSC_FALSE;}
36042f856554SMatthew G. Knepley     else                            {isFE[f] = PETSC_FALSE;}
36052f856554SMatthew G. Knepley   }
36069566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
36079566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(locX, &x));
36089566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
36099566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
36109566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
36119566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
36122f856554SMatthew G. Knepley   if (locGrad) {
36139566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locGrad, &dmGrad));
36149566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(locGrad, &lgrad));
36152f856554SMatthew G. Knepley   }
36169566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces*Nc, MPIU_SCALAR, uL));
36179566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces*Nc, MPIU_SCALAR, uR));
36182f856554SMatthew G. Knepley   /* Right now just eat the extra work for FE (could make a cell loop) */
36192f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
36202f856554SMatthew G. Knepley     const PetscInt        *cells;
36212f856554SMatthew G. Knepley     PetscFVFaceGeom       *fg;
36222f856554SMatthew G. Knepley     PetscFVCellGeom       *cgL, *cgR;
36232f856554SMatthew G. Knepley     PetscScalar           *xL, *xR, *gL, *gR;
36242f856554SMatthew G. Knepley     PetscScalar           *uLl = *uL, *uRl = *uR;
36252f856554SMatthew G. Knepley     PetscInt               ghost, nsupp, nchild;
36262f856554SMatthew G. Knepley 
36279566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
36289566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
36299566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
36302f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
36319566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
36329566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
36339566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
36349566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
36352f856554SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
36362f856554SMatthew G. Knepley       PetscInt off;
36372f856554SMatthew G. Knepley 
36389566063dSJacob Faibussowitsch       PetscCall(PetscDSGetComponentOffset(prob, f, &off));
36392f856554SMatthew G. Knepley       if (isFE[f]) {
36402f856554SMatthew G. Knepley         const PetscInt *cone;
36412f856554SMatthew G. Knepley         PetscInt        comp, coneSizeL, coneSizeR, faceLocL, faceLocR, ldof, rdof, d;
36422f856554SMatthew G. Knepley 
36432f856554SMatthew G. Knepley         xL = xR = NULL;
36449566063dSJacob Faibussowitsch         PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
36459566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **) &xL));
36469566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **) &xR));
36479566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[0], &cone));
36489566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[0], &coneSizeL));
36492f856554SMatthew G. Knepley         for (faceLocL = 0; faceLocL < coneSizeL; ++faceLocL) if (cone[faceLocL] == face) break;
36509566063dSJacob Faibussowitsch         PetscCall(DMPlexGetCone(dm, cells[1], &cone));
36519566063dSJacob Faibussowitsch         PetscCall(DMPlexGetConeSize(dm, cells[1], &coneSizeR));
36522f856554SMatthew G. Knepley         for (faceLocR = 0; faceLocR < coneSizeR; ++faceLocR) if (cone[faceLocR] == face) break;
3653*1dca8a05SBarry Smith         PetscCheck(faceLocL != coneSizeL || faceLocR != coneSizeR,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Could not find face %" PetscInt_FMT " in cone of cell %" PetscInt_FMT " or cell %" PetscInt_FMT, face, cells[0], cells[1]);
36542f856554SMatthew G. Knepley         /* Check that FEM field has values in the right cell (sometimes its an FV ghost cell) */
36552f856554SMatthew G. Knepley         /* TODO: this is a hack that might not be right for nonconforming */
36562f856554SMatthew G. Knepley         if (faceLocL < coneSizeL) {
36579566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocL, xL, &uLl[iface*Nc+off]));
36589566063dSJacob Faibussowitsch           if (rdof == ldof && faceLocR < coneSizeR) PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface*Nc+off]));
36592f856554SMatthew G. Knepley           else              {for (d = 0; d < comp; ++d) uRl[iface*Nc+off+d] = uLl[iface*Nc+off+d];}
36602f856554SMatthew G. Knepley         }
36612f856554SMatthew G. Knepley         else {
36629566063dSJacob Faibussowitsch           PetscCall(PetscFEEvaluateFaceFields_Internal(prob, f, faceLocR, xR, &uRl[iface*Nc+off]));
36639566063dSJacob Faibussowitsch           PetscCall(PetscSectionGetFieldComponents(section, f, &comp));
36642f856554SMatthew G. Knepley           for (d = 0; d < comp; ++d) uLl[iface*Nc+off+d] = uRl[iface*Nc+off+d];
36652f856554SMatthew G. Knepley         }
36669566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[0], &ldof, (PetscScalar **) &xL));
36679566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, locX, cells[1], &rdof, (PetscScalar **) &xR));
36682f856554SMatthew G. Knepley       } else {
36692f856554SMatthew G. Knepley         PetscFV  fv;
36702f856554SMatthew G. Knepley         PetscInt numComp, c;
36712f856554SMatthew G. Knepley 
36729566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, (PetscObject *) &fv));
36739566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &numComp));
36749566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[0], f, x, &xL));
36759566063dSJacob Faibussowitsch         PetscCall(DMPlexPointLocalFieldRead(dm, cells[1], f, x, &xR));
36762f856554SMatthew G. Knepley         if (dmGrad) {
36772f856554SMatthew G. Knepley           PetscReal dxL[3], dxR[3];
36782f856554SMatthew G. Knepley 
36799566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[0], lgrad, &gL));
36809566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalRead(dmGrad, cells[1], lgrad, &gR));
36812f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgL->centroid, fg->centroid, dxL);
36822f856554SMatthew G. Knepley           DMPlex_WaxpyD_Internal(dim, -1, cgR->centroid, fg->centroid, dxR);
36832f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
36842f856554SMatthew G. Knepley             uLl[iface*Nc+off+c] = xL[c] + DMPlex_DotD_Internal(dim, &gL[c*dim], dxL);
36852f856554SMatthew G. Knepley             uRl[iface*Nc+off+c] = xR[c] + DMPlex_DotD_Internal(dim, &gR[c*dim], dxR);
36862f856554SMatthew G. Knepley           }
36872f856554SMatthew G. Knepley         } else {
36882f856554SMatthew G. Knepley           for (c = 0; c < numComp; ++c) {
36892f856554SMatthew G. Knepley             uLl[iface*Nc+off+c] = xL[c];
36902f856554SMatthew G. Knepley             uRl[iface*Nc+off+c] = xR[c];
36912f856554SMatthew G. Knepley           }
36922f856554SMatthew G. Knepley         }
36932f856554SMatthew G. Knepley       }
36942f856554SMatthew G. Knepley     }
36952f856554SMatthew G. Knepley     ++iface;
36962f856554SMatthew G. Knepley   }
36972f856554SMatthew G. Knepley   *Nface = iface;
36989566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(locX, &x));
36999566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
37009566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
37012f856554SMatthew G. Knepley   if (locGrad) {
37029566063dSJacob Faibussowitsch     PetscCall(VecRestoreArrayRead(locGrad, &lgrad));
37032f856554SMatthew G. Knepley   }
37049566063dSJacob Faibussowitsch   PetscCall(PetscFree(isFE));
37052f856554SMatthew G. Knepley   PetscFunctionReturn(0);
37062f856554SMatthew G. Knepley }
37072f856554SMatthew G. Knepley 
37082f856554SMatthew G. Knepley /*@C
37092f856554SMatthew G. Knepley   DMPlexRestoreFaceFields - Restore the field values values for a chunk of faces
37102f856554SMatthew G. Knepley 
37112f856554SMatthew G. Knepley   Input Parameters:
37122f856554SMatthew G. Knepley + dm     - The DM
37132f856554SMatthew G. Knepley . fStart - The first face to include
37142f856554SMatthew G. Knepley . fEnd   - The first face to exclude
37152f856554SMatthew G. Knepley . locX   - A local vector with the solution fields
37162f856554SMatthew G. Knepley . locX_t - A local vector with solution field time derivatives, or NULL
37172f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
37182f856554SMatthew G. Knepley . cellGeometry - A local vector with cell geometry
37192f856554SMatthew G. Knepley - locaGrad - A local vector with field gradients, or NULL
37202f856554SMatthew G. Knepley 
37212f856554SMatthew G. Knepley   Output Parameters:
37222f856554SMatthew G. Knepley + Nface - The number of faces with field values
37232f856554SMatthew G. Knepley . uL - The field values at the left side of the face
37242f856554SMatthew G. Knepley - uR - The field values at the right side of the face
37252f856554SMatthew G. Knepley 
37262f856554SMatthew G. Knepley   Level: developer
37272f856554SMatthew G. Knepley 
37282f856554SMatthew G. Knepley .seealso: DMPlexGetFaceFields()
37292f856554SMatthew G. Knepley @*/
37302f856554SMatthew G. Knepley PetscErrorCode DMPlexRestoreFaceFields(DM dm, PetscInt fStart, PetscInt fEnd, Vec locX, Vec locX_t, Vec faceGeometry, Vec cellGeometry, Vec locGrad, PetscInt *Nface, PetscScalar **uL, PetscScalar **uR)
37312f856554SMatthew G. Knepley {
37322f856554SMatthew G. Knepley   PetscFunctionBegin;
37339566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uL));
37349566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_SCALAR, uR));
37352f856554SMatthew G. Knepley   PetscFunctionReturn(0);
37362f856554SMatthew G. Knepley }
37372f856554SMatthew G. Knepley 
37382f856554SMatthew G. Knepley /*@C
37392f856554SMatthew G. Knepley   DMPlexGetFaceGeometry - Retrieve the geometric values for a chunk of faces
37402f856554SMatthew G. Knepley 
37412f856554SMatthew G. Knepley   Input Parameters:
37422f856554SMatthew G. Knepley + dm     - The DM
37432f856554SMatthew G. Knepley . fStart - The first face to include
37442f856554SMatthew G. Knepley . fEnd   - The first face to exclude
37452f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
37462f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
37472f856554SMatthew G. Knepley 
37482f856554SMatthew G. Knepley   Output Parameters:
37492f856554SMatthew G. Knepley + Nface - The number of faces with field values
37502f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
37512f856554SMatthew G. Knepley - vol   - The cell volume
37522f856554SMatthew G. Knepley 
37532f856554SMatthew G. Knepley   Level: developer
37542f856554SMatthew G. Knepley 
37552f856554SMatthew G. Knepley .seealso: DMPlexGetCellFields()
37562f856554SMatthew G. Knepley @*/
37572f856554SMatthew G. Knepley PetscErrorCode DMPlexGetFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
37582f856554SMatthew G. Knepley {
37592f856554SMatthew G. Knepley   DM                 dmFace, dmCell;
37602f856554SMatthew G. Knepley   DMLabel            ghostLabel;
37612f856554SMatthew G. Knepley   const PetscScalar *facegeom, *cellgeom;
37622f856554SMatthew G. Knepley   PetscInt           dim, numFaces = fEnd - fStart, iface, face;
37632f856554SMatthew G. Knepley 
37642f856554SMatthew G. Knepley   PetscFunctionBegin;
37652f856554SMatthew G. Knepley   PetscValidHeaderSpecific(dm, DM_CLASSID, 1);
37662f856554SMatthew G. Knepley   PetscValidHeaderSpecific(faceGeometry, VEC_CLASSID, 4);
37672f856554SMatthew G. Knepley   PetscValidHeaderSpecific(cellGeometry, VEC_CLASSID, 5);
3768064a246eSJacob Faibussowitsch   PetscValidPointer(fgeom, 7);
3769064a246eSJacob Faibussowitsch   PetscValidPointer(vol, 8);
37709566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
37719566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
37729566063dSJacob Faibussowitsch   PetscCall(VecGetDM(faceGeometry, &dmFace));
37739566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(faceGeometry, &facegeom));
37749566063dSJacob Faibussowitsch   PetscCall(VecGetDM(cellGeometry, &dmCell));
37759566063dSJacob Faibussowitsch   PetscCall(VecGetArrayRead(cellGeometry, &cellgeom));
37769566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(numFaces, fgeom));
37779566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, numFaces*2, MPIU_SCALAR, vol));
37782f856554SMatthew G. Knepley   for (face = fStart, iface = 0; face < fEnd; ++face) {
37792f856554SMatthew G. Knepley     const PetscInt        *cells;
37802f856554SMatthew G. Knepley     PetscFVFaceGeom       *fg;
37812f856554SMatthew G. Knepley     PetscFVCellGeom       *cgL, *cgR;
37822f856554SMatthew G. Knepley     PetscFVFaceGeom       *fgeoml = *fgeom;
37832f856554SMatthew G. Knepley     PetscReal             *voll   = *vol;
37842f856554SMatthew G. Knepley     PetscInt               ghost, d, nchild, nsupp;
37852f856554SMatthew G. Knepley 
37869566063dSJacob Faibussowitsch     PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
37879566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
37889566063dSJacob Faibussowitsch     PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
37892f856554SMatthew G. Knepley     if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
37909566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmFace, face, facegeom, &fg));
37919566063dSJacob Faibussowitsch     PetscCall(DMPlexGetSupport(dm, face, &cells));
37929566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[0], cellgeom, &cgL));
37939566063dSJacob Faibussowitsch     PetscCall(DMPlexPointLocalRead(dmCell, cells[1], cellgeom, &cgR));
37942f856554SMatthew G. Knepley     for (d = 0; d < dim; ++d) {
37952f856554SMatthew G. Knepley       fgeoml[iface].centroid[d] = fg->centroid[d];
37962f856554SMatthew G. Knepley       fgeoml[iface].normal[d]   = fg->normal[d];
37972f856554SMatthew G. Knepley     }
37982f856554SMatthew G. Knepley     voll[iface*2+0] = cgL->volume;
37992f856554SMatthew G. Knepley     voll[iface*2+1] = cgR->volume;
38002f856554SMatthew G. Knepley     ++iface;
38012f856554SMatthew G. Knepley   }
38022f856554SMatthew G. Knepley   *Nface = iface;
38039566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(faceGeometry, &facegeom));
38049566063dSJacob Faibussowitsch   PetscCall(VecRestoreArrayRead(cellGeometry, &cellgeom));
38052f856554SMatthew G. Knepley   PetscFunctionReturn(0);
38062f856554SMatthew G. Knepley }
38072f856554SMatthew G. Knepley 
38082f856554SMatthew G. Knepley /*@C
38092f856554SMatthew G. Knepley   DMPlexRestoreFaceGeometry - Restore the field values values for a chunk of faces
38102f856554SMatthew G. Knepley 
38112f856554SMatthew G. Knepley   Input Parameters:
38122f856554SMatthew G. Knepley + dm     - The DM
38132f856554SMatthew G. Knepley . fStart - The first face to include
38142f856554SMatthew G. Knepley . fEnd   - The first face to exclude
38152f856554SMatthew G. Knepley . faceGeometry - A local vector with face geometry
38162f856554SMatthew G. Knepley - cellGeometry - A local vector with cell geometry
38172f856554SMatthew G. Knepley 
38182f856554SMatthew G. Knepley   Output Parameters:
38192f856554SMatthew G. Knepley + Nface - The number of faces with field values
38202f856554SMatthew G. Knepley . fgeom - The extract the face centroid and normal
38212f856554SMatthew G. Knepley - vol   - The cell volume
38222f856554SMatthew G. Knepley 
38232f856554SMatthew G. Knepley   Level: developer
38242f856554SMatthew G. Knepley 
38252f856554SMatthew G. Knepley .seealso: DMPlexGetFaceFields()
38262f856554SMatthew G. Knepley @*/
38272f856554SMatthew G. Knepley PetscErrorCode DMPlexRestoreFaceGeometry(DM dm, PetscInt fStart, PetscInt fEnd, Vec faceGeometry, Vec cellGeometry, PetscInt *Nface, PetscFVFaceGeom **fgeom, PetscReal **vol)
38282f856554SMatthew G. Knepley {
38292f856554SMatthew G. Knepley   PetscFunctionBegin;
38309566063dSJacob Faibussowitsch   PetscCall(PetscFree(*fgeom));
38319566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, 0, MPIU_REAL, vol));
38322f856554SMatthew G. Knepley   PetscFunctionReturn(0);
38332f856554SMatthew G. Knepley }
38342f856554SMatthew G. Knepley 
3835a1cf66bbSMatthew G. Knepley PetscErrorCode DMSNESGetFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
3836a1cf66bbSMatthew G. Knepley {
3837a1cf66bbSMatthew G. Knepley   char            composeStr[33] = {0};
3838a1cf66bbSMatthew G. Knepley   PetscObjectId   id;
3839a1cf66bbSMatthew G. Knepley   PetscContainer  container;
3840a1cf66bbSMatthew G. Knepley 
3841a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
38429566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetId((PetscObject)quad,&id));
384363a3b9bcSJacob Faibussowitsch   PetscCall(PetscSNPrintf(composeStr, 32, "DMSNESGetFEGeom_%" PetscInt64_FMT "\n", id));
38449566063dSJacob Faibussowitsch   PetscCall(PetscObjectQuery((PetscObject) pointIS, composeStr, (PetscObject *) &container));
3845a1cf66bbSMatthew G. Knepley   if (container) {
38469566063dSJacob Faibussowitsch     PetscCall(PetscContainerGetPointer(container, (void **) geom));
3847a1cf66bbSMatthew G. Knepley   } else {
38489566063dSJacob Faibussowitsch     PetscCall(DMFieldCreateFEGeom(coordField, pointIS, quad, faceData, geom));
38499566063dSJacob Faibussowitsch     PetscCall(PetscContainerCreate(PETSC_COMM_SELF,&container));
38509566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetPointer(container, (void *) *geom));
38519566063dSJacob Faibussowitsch     PetscCall(PetscContainerSetUserDestroy(container, PetscContainerUserDestroy_PetscFEGeom));
38529566063dSJacob Faibussowitsch     PetscCall(PetscObjectCompose((PetscObject) pointIS, composeStr, (PetscObject) container));
38539566063dSJacob Faibussowitsch     PetscCall(PetscContainerDestroy(&container));
3854a1cf66bbSMatthew G. Knepley   }
3855a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
3856a1cf66bbSMatthew G. Knepley }
3857a1cf66bbSMatthew G. Knepley 
3858a1cf66bbSMatthew G. Knepley PetscErrorCode DMSNESRestoreFEGeom(DMField coordField, IS pointIS, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
3859a1cf66bbSMatthew G. Knepley {
3860a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
3861a1cf66bbSMatthew G. Knepley   *geom = NULL;
3862a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
3863a1cf66bbSMatthew G. Knepley }
3864a1cf66bbSMatthew G. Knepley 
386592d50984SMatthew G. Knepley PetscErrorCode DMPlexComputeResidual_Patch_Internal(DM dm, PetscSection section, IS cellIS, PetscReal t, Vec locX, Vec locX_t, Vec locF, void *user)
386692d50984SMatthew G. Knepley {
386792d50984SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *) dm->data;
386892d50984SMatthew G. Knepley   const char      *name       = "Residual";
386992d50984SMatthew G. Knepley   DM               dmAux      = NULL;
387092d50984SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
387192d50984SMatthew G. Knepley   PetscDS          prob       = NULL;
387292d50984SMatthew G. Knepley   PetscDS          probAux    = NULL;
387392d50984SMatthew G. Knepley   PetscBool        useFEM     = PETSC_FALSE;
387492d50984SMatthew G. Knepley   PetscBool        isImplicit = (locX_t || t == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
387592d50984SMatthew G. Knepley   DMField          coordField = NULL;
3876c0006e53SPatrick Farrell   Vec              locA;
3877c0006e53SPatrick Farrell   PetscScalar     *u = NULL, *u_t, *a, *uL = NULL, *uR = NULL;
387892d50984SMatthew G. Knepley   IS               chunkIS;
387992d50984SMatthew G. Knepley   const PetscInt  *cells;
388092d50984SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
3881364207b6SKarl Rupp   PetscInt         Nf, f, totDim, totDimAux, numChunks, cellChunkSize, chunk, fStart, fEnd;
388292d50984SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
388306ad1575SMatthew G. Knepley   PetscFormKey key;
388492d50984SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
388592d50984SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
388692d50984SMatthew G. Knepley 
388792d50984SMatthew G. Knepley   PetscFunctionBegin;
38889566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0));
388992d50984SMatthew G. Knepley   /* FEM+FVM */
389092d50984SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
38919566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
38929566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
38939566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
38949566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
38959566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &locA));
389692d50984SMatthew G. Knepley   if (locA) {
38979566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
38989566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
38999566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
390092d50984SMatthew G. Knepley   }
390192d50984SMatthew G. Knepley   /* 2: Get geometric data */
390292d50984SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
390392d50984SMatthew G. Knepley     PetscObject  obj;
390492d50984SMatthew G. Knepley     PetscClassId id;
390592d50984SMatthew G. Knepley     PetscBool    fimp;
390692d50984SMatthew G. Knepley 
39079566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(prob, f, &fimp));
390892d50984SMatthew G. Knepley     if (isImplicit != fimp) continue;
39099566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, f, &obj));
39109566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
391192d50984SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {useFEM = PETSC_TRUE;}
39125f80ce2aSJacob Faibussowitsch     PetscCheck(id != PETSCFV_CLASSID,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Use of FVM with PCPATCH not yet implemented");
391392d50984SMatthew G. Knepley   }
391492d50984SMatthew G. Knepley   if (useFEM) {
39159566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
39169566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
391792d50984SMatthew G. Knepley     if (maxDegree <= 1) {
39189566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad));
391992d50984SMatthew G. Knepley       if (affineQuad) {
39209566063dSJacob Faibussowitsch         PetscCall(DMSNESGetFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
392192d50984SMatthew G. Knepley       }
392292d50984SMatthew G. Knepley     } else {
39239566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf,&quads,Nf,&geoms));
392492d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
392592d50984SMatthew G. Knepley         PetscObject  obj;
392692d50984SMatthew G. Knepley         PetscClassId id;
392792d50984SMatthew G. Knepley         PetscBool    fimp;
392892d50984SMatthew G. Knepley 
39299566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(prob, f, &fimp));
393092d50984SMatthew G. Knepley         if (isImplicit != fimp) continue;
39319566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
39329566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
393392d50984SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
393492d50984SMatthew G. Knepley           PetscFE fe = (PetscFE) obj;
393592d50984SMatthew G. Knepley 
39369566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
39379566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
39389566063dSJacob Faibussowitsch           PetscCall(DMSNESGetFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
393992d50984SMatthew G. Knepley         }
394092d50984SMatthew G. Knepley       }
394192d50984SMatthew G. Knepley     }
394292d50984SMatthew G. Knepley   }
394392d50984SMatthew G. Knepley   /* Loop over chunks */
39449566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
39459566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
39469566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
394792d50984SMatthew G. Knepley   numCells      = cEnd - cStart;
394892d50984SMatthew G. Knepley   numChunks     = 1;
394992d50984SMatthew G. Knepley   cellChunkSize = numCells/numChunks;
395092d50984SMatthew G. Knepley   numChunks     = PetscMin(1,numCells);
39516528b96dSMatthew G. Knepley   key.label     = NULL;
39526528b96dSMatthew G. Knepley   key.value     = 0;
395306ad1575SMatthew G. Knepley   key.part      = 0;
395492d50984SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
3955c0006e53SPatrick Farrell     PetscScalar     *elemVec, *fluxL = NULL, *fluxR = NULL;
3956c0006e53SPatrick Farrell     PetscReal       *vol = NULL;
3957c0006e53SPatrick Farrell     PetscFVFaceGeom *fgeom = NULL;
395892d50984SMatthew G. Knepley     PetscInt         cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
3959c0006e53SPatrick Farrell     PetscInt         numFaces = 0;
396092d50984SMatthew G. Knepley 
396192d50984SMatthew G. Knepley     /* Extract field coefficients */
396292d50984SMatthew G. Knepley     if (useFEM) {
39639566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
39649566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
39659566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
39669566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells*totDim));
396792d50984SMatthew G. Knepley     }
396892d50984SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
396992d50984SMatthew G. Knepley     /* Loop over fields */
397092d50984SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
397192d50984SMatthew G. Knepley       PetscObject  obj;
397292d50984SMatthew G. Knepley       PetscClassId id;
397392d50984SMatthew G. Knepley       PetscBool    fimp;
397492d50984SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
397592d50984SMatthew G. Knepley 
39766528b96dSMatthew G. Knepley       key.field = f;
39779566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(prob, f, &fimp));
397892d50984SMatthew G. Knepley       if (isImplicit != fimp) continue;
39799566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, f, &obj));
39809566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
398192d50984SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
398292d50984SMatthew G. Knepley         PetscFE         fe = (PetscFE) obj;
398392d50984SMatthew G. Knepley         PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
398492d50984SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
398592d50984SMatthew G. Knepley         PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
398692d50984SMatthew G. Knepley         PetscInt        Nq, Nb;
398792d50984SMatthew G. Knepley 
39889566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
39899566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
39909566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
399192d50984SMatthew G. Knepley         blockSize = Nb;
399292d50984SMatthew G. Knepley         batchSize = numBlocks * blockSize;
39939566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
399492d50984SMatthew G. Knepley         numChunks = numCells / (numBatches*batchSize);
399592d50984SMatthew G. Knepley         Ne        = numChunks*numBatches*batchSize;
399692d50984SMatthew G. Knepley         Nr        = numCells % (numBatches*batchSize);
399792d50984SMatthew G. Knepley         offset    = numCells - Nr;
399892d50984SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
399992d50984SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
40009566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom,0,offset,&chunkGeom));
40019566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(prob, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
40029566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom,offset,numCells,&chunkGeom));
40039566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(prob, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, &elemVec[offset*totDim]));
40049566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom,offset,numCells,&chunkGeom));
400592d50984SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
400692d50984SMatthew G. Knepley         PetscFV fv = (PetscFV) obj;
400792d50984SMatthew G. Knepley 
400892d50984SMatthew G. Knepley         Ne = numFaces;
400992d50984SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
401092d50984SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
40119566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, prob, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
401263a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
401392d50984SMatthew G. Knepley     }
401492d50984SMatthew G. Knepley     /* Loop over domain */
401592d50984SMatthew G. Knepley     if (useFEM) {
401692d50984SMatthew G. Knepley       /* Add elemVec to locX */
401792d50984SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
401892d50984SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
401992d50984SMatthew G. Knepley         const PetscInt cind = c - cStart;
402092d50984SMatthew G. Knepley 
40219566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]));
402292d50984SMatthew G. Knepley         if (ghostLabel) {
402392d50984SMatthew G. Knepley           PetscInt ghostVal;
402492d50984SMatthew G. Knepley 
40259566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel,cell,&ghostVal));
402692d50984SMatthew G. Knepley           if (ghostVal > 0) continue;
402792d50984SMatthew G. Knepley         }
40289566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES));
402992d50984SMatthew G. Knepley       }
403092d50984SMatthew G. Knepley     }
403192d50984SMatthew G. Knepley     /* Handle time derivative */
403292d50984SMatthew G. Knepley     if (locX_t) {
403392d50984SMatthew G. Knepley       PetscScalar *x_t, *fa;
403492d50984SMatthew G. Knepley 
40359566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
40369566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
403792d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
403892d50984SMatthew G. Knepley         PetscFV      fv;
403992d50984SMatthew G. Knepley         PetscObject  obj;
404092d50984SMatthew G. Knepley         PetscClassId id;
404192d50984SMatthew G. Knepley         PetscInt     pdim, d;
404292d50984SMatthew G. Knepley 
40439566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, f, &obj));
40449566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
404592d50984SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
404692d50984SMatthew G. Knepley         fv   = (PetscFV) obj;
40479566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
404892d50984SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
404992d50984SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
405092d50984SMatthew G. Knepley           PetscScalar   *u_t, *r;
405192d50984SMatthew G. Knepley 
405292d50984SMatthew G. Knepley           if (ghostLabel) {
405392d50984SMatthew G. Knepley             PetscInt ghostVal;
405492d50984SMatthew G. Knepley 
40559566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
405692d50984SMatthew G. Knepley             if (ghostVal > 0) continue;
405792d50984SMatthew G. Knepley           }
40589566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
40599566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
406092d50984SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
406192d50984SMatthew G. Knepley         }
406292d50984SMatthew G. Knepley       }
40639566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
40649566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
406592d50984SMatthew G. Knepley     }
406692d50984SMatthew G. Knepley     if (useFEM) {
40679566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
40689566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
406992d50984SMatthew G. Knepley     }
407092d50984SMatthew G. Knepley   }
40719566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
40729566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
407392d50984SMatthew G. Knepley   /* TODO Could include boundary residual here (see DMPlexComputeResidual_Internal) */
407492d50984SMatthew G. Knepley   if (useFEM) {
407592d50984SMatthew G. Knepley     if (maxDegree <= 1) {
40769566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
40779566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
407892d50984SMatthew G. Knepley     } else {
407992d50984SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
40809566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
40819566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
408292d50984SMatthew G. Knepley       }
40839566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads,geoms));
408492d50984SMatthew G. Knepley     }
408592d50984SMatthew G. Knepley   }
40869566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0));
408792d50984SMatthew G. Knepley   PetscFunctionReturn(0);
408892d50984SMatthew G. Knepley }
408992d50984SMatthew G. Knepley 
4090a1cf66bbSMatthew G. Knepley /*
4091a1cf66bbSMatthew G. Knepley   We always assemble JacP, and if the matrix is different from Jac and two different sets of point functions are provided, we also assemble Jac
4092a1cf66bbSMatthew G. Knepley 
4093a1cf66bbSMatthew G. Knepley   X   - The local solution vector
4094a5b23f4aSJose E. Roman   X_t - The local solution time derivative vector, or NULL
4095a1cf66bbSMatthew G. Knepley */
4096a1cf66bbSMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Patch_Internal(DM dm, PetscSection section, PetscSection globalSection, IS cellIS,
4097a1cf66bbSMatthew G. Knepley                                                     PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP, void *ctx)
4098a1cf66bbSMatthew G. Knepley {
4099a1cf66bbSMatthew G. Knepley   DM_Plex         *mesh  = (DM_Plex *) dm->data;
4100a1cf66bbSMatthew G. Knepley   const char      *name = "Jacobian", *nameP = "JacobianPre";
4101a1cf66bbSMatthew G. Knepley   DM               dmAux = NULL;
4102a1cf66bbSMatthew G. Knepley   PetscDS          prob,   probAux = NULL;
4103a1cf66bbSMatthew G. Knepley   PetscSection     sectionAux = NULL;
4104a1cf66bbSMatthew G. Knepley   Vec              A;
4105a1cf66bbSMatthew G. Knepley   DMField          coordField;
4106a1cf66bbSMatthew G. Knepley   PetscFEGeom     *cgeomFEM;
4107a1cf66bbSMatthew G. Knepley   PetscQuadrature  qGeom = NULL;
4108a1cf66bbSMatthew G. Knepley   Mat              J = Jac, JP = JacP;
4109a1cf66bbSMatthew G. Knepley   PetscScalar     *work, *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL, *elemMatP = NULL, *elemMatD = NULL;
4110e432b41dSStefano Zampini   PetscBool        hasJac, hasPrec, hasDyn, assembleJac, *isFE, hasFV = PETSC_FALSE;
4111a1cf66bbSMatthew G. Knepley   const PetscInt  *cells;
411206ad1575SMatthew G. Knepley   PetscFormKey key;
41139b2fc754SMatthew G. Knepley   PetscInt         Nf, fieldI, fieldJ, maxDegree, numCells, cStart, cEnd, numChunks, chunkSize, chunk, totDim, totDimAux = 0, sz, wsz, off = 0, offCell = 0;
4114a1cf66bbSMatthew G. Knepley 
4115a1cf66bbSMatthew G. Knepley   PetscFunctionBegin;
41169566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
41179566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
41189566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0));
41199566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
41209566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, NULL, 0, 0, &A));
41219a2a23afSMatthew G. Knepley   if (A) {
41229566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
41239566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(dmAux, &sectionAux));
41249566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
4125a1cf66bbSMatthew G. Knepley   }
4126a1cf66bbSMatthew G. Knepley   /* Get flags */
41279566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
41289566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, Nf, MPIU_BOOL, &isFE));
4129a1cf66bbSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
4130a1cf66bbSMatthew G. Knepley     PetscObject  disc;
4131a1cf66bbSMatthew G. Knepley     PetscClassId id;
41329566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &disc));
41339566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(disc, &id));
4134a1cf66bbSMatthew G. Knepley     if (id == PETSCFE_CLASSID)      {isFE[fieldI] = PETSC_TRUE;}
4135a1cf66bbSMatthew G. Knepley     else if (id == PETSCFV_CLASSID) {hasFV = PETSC_TRUE; isFE[fieldI] = PETSC_FALSE;}
4136a1cf66bbSMatthew G. Knepley   }
41379566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
41389566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
41399566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
4140a1cf66bbSMatthew G. Knepley   assembleJac = hasJac && hasPrec && (Jac != JacP) ? PETSC_TRUE : PETSC_FALSE;
4141a1cf66bbSMatthew G. Knepley   hasDyn      = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
41429566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE)); /* No allocated space for FV stuff, so ignore the zero entries */
41439566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
41449566063dSJacob Faibussowitsch   if (probAux) PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
4145a1cf66bbSMatthew G. Knepley   /* Compute batch sizes */
4146a1cf66bbSMatthew G. Knepley   if (isFE[0]) {
4147a1cf66bbSMatthew G. Knepley     PetscFE         fe;
4148a1cf66bbSMatthew G. Knepley     PetscQuadrature q;
4149a1cf66bbSMatthew G. Knepley     PetscInt        numQuadPoints, numBatches, batchSize, numBlocks, blockSize, Nb;
4150a1cf66bbSMatthew G. Knepley 
41519566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *) &fe));
41529566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &q));
41539566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(q, NULL, NULL, &numQuadPoints, NULL, NULL));
41549566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
41559566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
4156a1cf66bbSMatthew G. Knepley     blockSize = Nb*numQuadPoints;
4157a1cf66bbSMatthew G. Knepley     batchSize = numBlocks  * blockSize;
4158a1cf66bbSMatthew G. Knepley     chunkSize = numBatches * batchSize;
4159a1cf66bbSMatthew G. Knepley     numChunks = numCells / chunkSize + numCells % chunkSize;
41609566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
4161a1cf66bbSMatthew G. Knepley   } else {
4162a1cf66bbSMatthew G. Knepley     chunkSize = numCells;
4163a1cf66bbSMatthew G. Knepley     numChunks = 1;
4164a1cf66bbSMatthew G. Knepley   }
4165a1cf66bbSMatthew G. Knepley   /* Get work space */
4166a1cf66bbSMatthew G. Knepley   wsz  = (((X?1:0) + (X_t?1:0) + (dmAux?1:0))*totDim + ((hasJac?1:0) + (hasPrec?1:0) + (hasDyn?1:0))*totDim*totDim)*chunkSize;
41679566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, wsz, MPIU_SCALAR, &work));
41689566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(work, wsz));
4169a1cf66bbSMatthew G. Knepley   off      = 0;
4170a1cf66bbSMatthew G. Knepley   u        = X       ? (sz = chunkSize*totDim,        off += sz, work+off-sz) : NULL;
4171a1cf66bbSMatthew G. Knepley   u_t      = X_t     ? (sz = chunkSize*totDim,        off += sz, work+off-sz) : NULL;
4172a1cf66bbSMatthew G. Knepley   a        = dmAux   ? (sz = chunkSize*totDimAux,     off += sz, work+off-sz) : NULL;
4173a1cf66bbSMatthew G. Knepley   elemMat  = hasJac  ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
4174a1cf66bbSMatthew G. Knepley   elemMatP = hasPrec ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
4175a1cf66bbSMatthew G. Knepley   elemMatD = hasDyn  ? (sz = chunkSize*totDim*totDim, off += sz, work+off-sz) : NULL;
417663a3b9bcSJacob Faibussowitsch   PetscCheck(off == wsz,PETSC_COMM_SELF, PETSC_ERR_PLIB, "Error is workspace size %" PetscInt_FMT " should be %" PetscInt_FMT, off, wsz);
4177a1cf66bbSMatthew G. Knepley   /* Setup geometry */
41789566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
41799566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
41809566063dSJacob Faibussowitsch   if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, &qGeom));
4181a1cf66bbSMatthew G. Knepley   if (!qGeom) {
4182a1cf66bbSMatthew G. Knepley     PetscFE fe;
4183a1cf66bbSMatthew G. Knepley 
41849566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, 0, (PetscObject *) &fe));
41859566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &qGeom));
41869566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject) qGeom));
4187a1cf66bbSMatthew G. Knepley   }
41889566063dSJacob Faibussowitsch   PetscCall(DMSNESGetFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
4189a1cf66bbSMatthew G. Knepley   /* Compute volume integrals */
41909566063dSJacob Faibussowitsch   if (assembleJac) PetscCall(MatZeroEntries(J));
41919566063dSJacob Faibussowitsch   PetscCall(MatZeroEntries(JP));
41926528b96dSMatthew G. Knepley   key.label = NULL;
41936528b96dSMatthew G. Knepley   key.value = 0;
419406ad1575SMatthew G. Knepley   key.part  = 0;
4195a1cf66bbSMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk, offCell += chunkSize) {
4196a1cf66bbSMatthew G. Knepley     const PetscInt   Ncell = PetscMin(chunkSize, numCells - offCell);
4197a1cf66bbSMatthew G. Knepley     PetscInt         c;
4198a1cf66bbSMatthew G. Knepley 
4199a1cf66bbSMatthew G. Knepley     /* Extract values */
4200a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4201a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c+offCell] : c+offCell;
4202a1cf66bbSMatthew G. Knepley       PetscScalar   *x = NULL,  *x_t = NULL;
4203a1cf66bbSMatthew G. Knepley       PetscInt       i;
4204a1cf66bbSMatthew G. Knepley 
4205a1cf66bbSMatthew G. Knepley       if (X) {
42069566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x));
4207a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u[c*totDim+i] = x[i];
42089566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x));
4209a1cf66bbSMatthew G. Knepley       }
4210a1cf66bbSMatthew G. Knepley       if (X_t) {
42119566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t));
4212a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[c*totDim+i] = x_t[i];
42139566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t));
4214a1cf66bbSMatthew G. Knepley       }
4215a1cf66bbSMatthew G. Knepley       if (dmAux) {
42169566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(dmAux, sectionAux, A, cell, NULL, &x));
4217a1cf66bbSMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[c*totDimAux+i] = x[i];
42189566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(dmAux, sectionAux, A, cell, NULL, &x));
4219a1cf66bbSMatthew G. Knepley       }
4220a1cf66bbSMatthew G. Knepley     }
4221a1cf66bbSMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
4222a1cf66bbSMatthew G. Knepley       PetscFE fe;
42239566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
4224a1cf66bbSMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
42256528b96dSMatthew G. Knepley         key.field = fieldI*Nf + fieldJ;
42269566063dSJacob Faibussowitsch         if (hasJac)  PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN,     key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMat));
42279566063dSJacob Faibussowitsch         if (hasPrec) PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatP));
42289566063dSJacob Faibussowitsch         if (hasDyn)  PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ncell, cgeomFEM, u, u_t, probAux, a, t, X_tShift, elemMatD));
4229a1cf66bbSMatthew G. Knepley       }
4230a1cf66bbSMatthew G. Knepley       /* For finite volume, add the identity */
4231a1cf66bbSMatthew G. Knepley       if (!isFE[fieldI]) {
4232a1cf66bbSMatthew G. Knepley         PetscFV  fv;
4233a1cf66bbSMatthew G. Knepley         PetscInt eOffset = 0, Nc, fc, foff;
4234a1cf66bbSMatthew G. Knepley 
42359566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(prob, fieldI, &foff));
42369566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fv));
42379566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &Nc));
4238a1cf66bbSMatthew G. Knepley         for (c = 0; c < chunkSize; ++c, eOffset += totDim*totDim) {
4239a1cf66bbSMatthew G. Knepley           for (fc = 0; fc < Nc; ++fc) {
4240a1cf66bbSMatthew G. Knepley             const PetscInt i = foff + fc;
4241a1cf66bbSMatthew G. Knepley             if (hasJac)  {elemMat [eOffset+i*totDim+i] = 1.0;}
4242a1cf66bbSMatthew G. Knepley             if (hasPrec) {elemMatP[eOffset+i*totDim+i] = 1.0;}
4243a1cf66bbSMatthew G. Knepley           }
4244a1cf66bbSMatthew G. Knepley         }
4245a1cf66bbSMatthew G. Knepley       }
4246a1cf66bbSMatthew G. Knepley     }
4247a1cf66bbSMatthew G. Knepley     /*   Add contribution from X_t */
4248a1cf66bbSMatthew G. Knepley     if (hasDyn) {for (c = 0; c < chunkSize*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];}
4249a1cf66bbSMatthew G. Knepley     /* Insert values into matrix */
4250a1cf66bbSMatthew G. Knepley     for (c = 0; c < Ncell; ++c) {
4251a1cf66bbSMatthew G. Knepley       const PetscInt cell = cells ? cells[c+offCell] : c+offCell;
4252a1cf66bbSMatthew G. Knepley       if (mesh->printFEM > 1) {
42539566063dSJacob Faibussowitsch         if (hasJac)  PetscCall(DMPrintCellMatrix(cell, name,  totDim, totDim, &elemMat[(c-cStart)*totDim*totDim]));
42549566063dSJacob Faibussowitsch         if (hasPrec) PetscCall(DMPrintCellMatrix(cell, nameP, totDim, totDim, &elemMatP[(c-cStart)*totDim*totDim]));
4255a1cf66bbSMatthew G. Knepley       }
42569566063dSJacob Faibussowitsch       if (assembleJac) PetscCall(DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[(c-cStart)*totDim*totDim], ADD_VALUES));
42579566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JP, cell, &elemMat[(c-cStart)*totDim*totDim], ADD_VALUES));
4258a1cf66bbSMatthew G. Knepley     }
4259a1cf66bbSMatthew G. Knepley   }
4260a1cf66bbSMatthew G. Knepley   /* Cleanup */
42619566063dSJacob Faibussowitsch   PetscCall(DMSNESRestoreFEGeom(coordField, cellIS, qGeom, PETSC_FALSE, &cgeomFEM));
42629566063dSJacob Faibussowitsch   PetscCall(PetscQuadratureDestroy(&qGeom));
42639566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
42649566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, Nf, MPIU_BOOL, &isFE));
42659566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, ((1 + (X_t?1:0) + (dmAux?1:0))*totDim + ((hasJac?1:0) + (hasPrec?1:0) + (hasDyn?1:0))*totDim*totDim)*chunkSize, MPIU_SCALAR, &work));
4266a1cf66bbSMatthew G. Knepley   /* Compute boundary integrals */
42679566063dSJacob Faibussowitsch   /* PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, ctx)); */
4268a1cf66bbSMatthew G. Knepley   /* Assemble matrix */
42699566063dSJacob Faibussowitsch   if (assembleJac) {PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));}
42709566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
42719566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0));
4272a1cf66bbSMatthew G. Knepley   PetscFunctionReturn(0);
4273a1cf66bbSMatthew G. Knepley }
42743e9753d6SMatthew G. Knepley 
42753e9753d6SMatthew G. Knepley /******** FEM Assembly Function ********/
42763e9753d6SMatthew G. Knepley 
42773e9753d6SMatthew G. Knepley static PetscErrorCode DMConvertPlex_Internal(DM dm, DM *plex, PetscBool copy)
42783e9753d6SMatthew G. Knepley {
42793e9753d6SMatthew G. Knepley   PetscBool      isPlex;
42803e9753d6SMatthew G. Knepley 
42813e9753d6SMatthew G. Knepley   PetscFunctionBegin;
42829566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject) dm, DMPLEX, &isPlex));
42833e9753d6SMatthew G. Knepley   if (isPlex) {
42843e9753d6SMatthew G. Knepley     *plex = dm;
42859566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject) dm));
42863e9753d6SMatthew G. Knepley   } else {
42879566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject) dm, "dm_plex", (PetscObject *) plex));
42883e9753d6SMatthew G. Knepley     if (!*plex) {
42899566063dSJacob Faibussowitsch       PetscCall(DMConvert(dm,DMPLEX,plex));
42909566063dSJacob Faibussowitsch       PetscCall(PetscObjectCompose((PetscObject) dm, "dm_plex", (PetscObject) *plex));
42913e9753d6SMatthew G. Knepley       if (copy) {
42929566063dSJacob Faibussowitsch         PetscCall(DMCopyAuxiliaryVec(dm, *plex));
42933e9753d6SMatthew G. Knepley       }
42943e9753d6SMatthew G. Knepley     } else {
42959566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject) *plex));
42963e9753d6SMatthew G. Knepley     }
42973e9753d6SMatthew G. Knepley   }
42983e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
42993e9753d6SMatthew G. Knepley }
43003e9753d6SMatthew G. Knepley 
43013e9753d6SMatthew G. Knepley /*@
43023e9753d6SMatthew G. Knepley   DMPlexGetGeometryFVM - Return precomputed geometric data
43033e9753d6SMatthew G. Knepley 
43043e9753d6SMatthew G. Knepley   Collective on DM
43053e9753d6SMatthew G. Knepley 
43063e9753d6SMatthew G. Knepley   Input Parameter:
43073e9753d6SMatthew G. Knepley . dm - The DM
43083e9753d6SMatthew G. Knepley 
43093e9753d6SMatthew G. Knepley   Output Parameters:
43103e9753d6SMatthew G. Knepley + facegeom - The values precomputed from face geometry
43113e9753d6SMatthew G. Knepley . cellgeom - The values precomputed from cell geometry
43123e9753d6SMatthew G. Knepley - minRadius - The minimum radius over the mesh of an inscribed sphere in a cell
43133e9753d6SMatthew G. Knepley 
43143e9753d6SMatthew G. Knepley   Level: developer
43153e9753d6SMatthew G. Knepley 
4316446c23c1SPierre Jolivet .seealso: DMTSSetRHSFunctionLocal()
43173e9753d6SMatthew G. Knepley @*/
43183e9753d6SMatthew G. Knepley PetscErrorCode DMPlexGetGeometryFVM(DM dm, Vec *facegeom, Vec *cellgeom, PetscReal *minRadius)
43193e9753d6SMatthew G. Knepley {
43203e9753d6SMatthew G. Knepley   DM             plex;
43213e9753d6SMatthew G. Knepley 
43223e9753d6SMatthew G. Knepley   PetscFunctionBegin;
43233e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
43249566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm,&plex,PETSC_TRUE));
43259566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, NULL, cellgeom, facegeom, NULL));
43269566063dSJacob Faibussowitsch   if (minRadius) PetscCall(DMPlexGetMinRadius(plex, minRadius));
43279566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
43283e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
43293e9753d6SMatthew G. Knepley }
43303e9753d6SMatthew G. Knepley 
43313e9753d6SMatthew G. Knepley /*@
43323e9753d6SMatthew G. Knepley   DMPlexGetGradientDM - Return gradient data layout
43333e9753d6SMatthew G. Knepley 
43343e9753d6SMatthew G. Knepley   Collective on DM
43353e9753d6SMatthew G. Knepley 
43363e9753d6SMatthew G. Knepley   Input Parameters:
43373e9753d6SMatthew G. Knepley + dm - The DM
43383e9753d6SMatthew G. Knepley - fv - The PetscFV
43393e9753d6SMatthew G. Knepley 
43403e9753d6SMatthew G. Knepley   Output Parameter:
43413e9753d6SMatthew G. Knepley . dmGrad - The layout for gradient values
43423e9753d6SMatthew G. Knepley 
43433e9753d6SMatthew G. Knepley   Level: developer
43443e9753d6SMatthew G. Knepley 
4345446c23c1SPierre Jolivet .seealso: DMPlexGetGeometryFVM()
43463e9753d6SMatthew G. Knepley @*/
43473e9753d6SMatthew G. Knepley PetscErrorCode DMPlexGetGradientDM(DM dm, PetscFV fv, DM *dmGrad)
43483e9753d6SMatthew G. Knepley {
43493e9753d6SMatthew G. Knepley   DM             plex;
43503e9753d6SMatthew G. Knepley   PetscBool      computeGradients;
43513e9753d6SMatthew G. Knepley 
43523e9753d6SMatthew G. Knepley   PetscFunctionBegin;
43533e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(dm,DM_CLASSID,1);
43543e9753d6SMatthew G. Knepley   PetscValidHeaderSpecific(fv,PETSCFV_CLASSID,2);
43553e9753d6SMatthew G. Knepley   PetscValidPointer(dmGrad,3);
43569566063dSJacob Faibussowitsch   PetscCall(PetscFVGetComputeGradients(fv, &computeGradients));
43573e9753d6SMatthew G. Knepley   if (!computeGradients) {*dmGrad = NULL; PetscFunctionReturn(0);}
43589566063dSJacob Faibussowitsch   PetscCall(DMConvertPlex_Internal(dm,&plex,PETSC_TRUE));
43599566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDataFVM(plex, fv, NULL, NULL, dmGrad));
43609566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
43613e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
43623e9753d6SMatthew G. Knepley }
43633e9753d6SMatthew G. Knepley 
43640c290148SMatthew G. Knepley static PetscErrorCode DMPlexComputeBdResidual_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF, DMField coordField, IS facetIS)
43653e9753d6SMatthew G. Knepley {
43663e9753d6SMatthew G. Knepley   DM_Plex         *mesh = (DM_Plex *) dm->data;
43673e9753d6SMatthew G. Knepley   DM               plex = NULL, plexA = NULL;
43683e9753d6SMatthew G. Knepley   DMEnclosureType  encAux;
43693e9753d6SMatthew G. Knepley   PetscDS          prob, probAux = NULL;
43703e9753d6SMatthew G. Knepley   PetscSection     section, sectionAux = NULL;
43713e9753d6SMatthew G. Knepley   Vec              locA = NULL;
43723e9753d6SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t = NULL, *a = NULL, *elemVec = NULL;
43733e9753d6SMatthew G. Knepley   PetscInt         totDim, totDimAux = 0;
43743e9753d6SMatthew G. Knepley 
43753e9753d6SMatthew G. Knepley   PetscFunctionBegin;
43769566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
43779566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
43789566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
43799566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
43809566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
43813e9753d6SMatthew G. Knepley   if (locA) {
43823e9753d6SMatthew G. Knepley     DM dmAux;
43833e9753d6SMatthew G. Knepley 
43849566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
43859566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
43869566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
43879566063dSJacob Faibussowitsch     PetscCall(DMGetDS(plexA, &probAux));
43889566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
43899566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
43903e9753d6SMatthew G. Knepley   }
43910c290148SMatthew G. Knepley   {
43923e9753d6SMatthew G. Knepley     PetscFEGeom     *fgeom;
43933e9753d6SMatthew G. Knepley     PetscInt         maxDegree;
43943e9753d6SMatthew G. Knepley     PetscQuadrature  qGeom = NULL;
43953e9753d6SMatthew G. Knepley     IS               pointIS;
43963e9753d6SMatthew G. Knepley     const PetscInt  *points;
43973e9753d6SMatthew G. Knepley     PetscInt         numFaces, face, Nq;
43983e9753d6SMatthew G. Knepley 
43999566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(key.label, key.value, &pointIS));
44000c290148SMatthew G. Knepley     if (!pointIS) goto end; /* No points with that id on this process */
44013e9753d6SMatthew G. Knepley     {
44023e9753d6SMatthew G. Knepley       IS isectIS;
44033e9753d6SMatthew G. Knepley 
44043e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a priori if one is a superset of the other */
44059566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS,pointIS,&isectIS));
44069566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
44073e9753d6SMatthew G. Knepley       pointIS = isectIS;
44083e9753d6SMatthew G. Knepley     }
44099566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS,&numFaces));
44109566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS,&points));
44119566063dSJacob Faibussowitsch     PetscCall(PetscMalloc4(numFaces*totDim, &u, locX_t ? numFaces*totDim : 0, &u_t, numFaces*totDim, &elemVec, locA ? numFaces*totDimAux : 0, &a));
44129566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,pointIS,NULL,&maxDegree));
44133e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
44149566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,pointIS,&qGeom));
44153e9753d6SMatthew G. Knepley     }
44163e9753d6SMatthew G. Knepley     if (!qGeom) {
44173e9753d6SMatthew G. Knepley       PetscFE fe;
44183e9753d6SMatthew G. Knepley 
44199566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *) &fe));
44209566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
44219566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
44223e9753d6SMatthew G. Knepley     }
44239566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
44249566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom));
44253e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
4426f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
44273e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
4428f15274beSMatthew Knepley       PetscInt       i;
44293e9753d6SMatthew G. Knepley 
44309566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
44319566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
44323e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
44339566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
44343e9753d6SMatthew G. Knepley       if (locX_t) {
44359566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
44363e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face*totDim+i] = x[i];
44379566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
44383e9753d6SMatthew G. Knepley       }
44393e9753d6SMatthew G. Knepley       if (locA) {
44403e9753d6SMatthew G. Knepley         PetscInt subp;
44413e9753d6SMatthew G. Knepley 
44429566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
44439566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
44443e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face*totDimAux+i] = x[i];
44459566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
44463e9753d6SMatthew G. Knepley       }
44473e9753d6SMatthew G. Knepley     }
44489566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(elemVec, numFaces*totDim));
44493e9753d6SMatthew G. Knepley     {
44503e9753d6SMatthew G. Knepley       PetscFE         fe;
44513e9753d6SMatthew G. Knepley       PetscInt        Nb;
44523e9753d6SMatthew G. Knepley       PetscFEGeom     *chunkGeom = NULL;
44533e9753d6SMatthew G. Knepley       /* Conforming batches */
44543e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
44553e9753d6SMatthew G. Knepley       /* Remainder */
44563e9753d6SMatthew G. Knepley       PetscInt        Nr, offset;
44573e9753d6SMatthew G. Knepley 
44589566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, key.field, (PetscObject *) &fe));
44599566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
44609566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
44613e9753d6SMatthew G. Knepley       /* TODO: documentation is unclear about what is going on with these numbers: how should Nb / Nq factor in ? */
44623e9753d6SMatthew G. Knepley       blockSize = Nb;
44633e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
44649566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
44653e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches*batchSize);
44663e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
44673e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches*batchSize);
44683e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
44699566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom,0,offset,&chunkGeom));
44709566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, elemVec));
44719566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom, 0, offset, &chunkGeom));
44729566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom,offset,numFaces,&chunkGeom));
44739566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateBdResidual(prob, wf, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, a ? &a[offset*totDimAux] : NULL, t, &elemVec[offset*totDim]));
44749566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom,offset,numFaces,&chunkGeom));
44753e9753d6SMatthew G. Knepley     }
44763e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
44773e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
44783e9753d6SMatthew G. Knepley 
44799566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(point, "BdResidual", totDim, &elemVec[face*totDim]));
44809566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
44819566063dSJacob Faibussowitsch       PetscCall(DMPlexVecSetClosure(plex, NULL, locF, support[0], &elemVec[face*totDim], ADD_ALL_VALUES));
44823e9753d6SMatthew G. Knepley     }
44839566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom));
44849566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
44859566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
44869566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
44879566063dSJacob Faibussowitsch     PetscCall(PetscFree4(u, u_t, elemVec, a));
44883e9753d6SMatthew G. Knepley   }
44890c290148SMatthew G. Knepley   end:
44909566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
44919566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexA));
44923e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
44933e9753d6SMatthew G. Knepley }
44943e9753d6SMatthew G. Knepley 
44950c290148SMatthew G. Knepley PetscErrorCode DMPlexComputeBdResidualSingle(DM dm, PetscReal t, PetscWeakForm wf, PetscFormKey key, Vec locX, Vec locX_t, Vec locF)
44963e9753d6SMatthew G. Knepley {
44973e9753d6SMatthew G. Knepley   DMField        coordField;
44983e9753d6SMatthew G. Knepley   DMLabel        depthLabel;
44993e9753d6SMatthew G. Knepley   IS             facetIS;
45003e9753d6SMatthew G. Knepley   PetscInt       dim;
45013e9753d6SMatthew G. Knepley 
45023e9753d6SMatthew G. Knepley   PetscFunctionBegin;
45039566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
45049566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
45059566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim-1, &facetIS));
45069566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
45079566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS));
45089566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
45093e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
45103e9753d6SMatthew G. Knepley }
45113e9753d6SMatthew G. Knepley 
45123e9753d6SMatthew G. Knepley PetscErrorCode DMPlexComputeBdResidual_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
45133e9753d6SMatthew G. Knepley {
45143e9753d6SMatthew G. Knepley   PetscDS        prob;
45153e9753d6SMatthew G. Knepley   PetscInt       numBd, bd;
45163e9753d6SMatthew G. Knepley   DMField        coordField = NULL;
45173e9753d6SMatthew G. Knepley   IS             facetIS    = NULL;
45183e9753d6SMatthew G. Knepley   DMLabel        depthLabel;
45193e9753d6SMatthew G. Knepley   PetscInt       dim;
45203e9753d6SMatthew G. Knepley 
45213e9753d6SMatthew G. Knepley   PetscFunctionBegin;
45229566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
45239566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
45249566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
45259566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel,dim - 1,&facetIS));
45269566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
45273e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
452845480ffeSMatthew G. Knepley     PetscWeakForm           wf;
45293e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
45303e9753d6SMatthew G. Knepley     DMLabel                 label;
45313e9753d6SMatthew G. Knepley     const PetscInt         *values;
45320c290148SMatthew G. Knepley     PetscInt                field, numValues, v;
45333e9753d6SMatthew G. Knepley     PetscObject             obj;
45343e9753d6SMatthew G. Knepley     PetscClassId            id;
45350c290148SMatthew G. Knepley     PetscFormKey            key;
45363e9753d6SMatthew G. Knepley 
45379566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &field, NULL, NULL, NULL, NULL, NULL));
45389566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, field, &obj));
45399566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
45403e9753d6SMatthew G. Knepley     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
45413e9753d6SMatthew G. Knepley     if (!facetIS) {
45423e9753d6SMatthew G. Knepley       DMLabel  depthLabel;
45433e9753d6SMatthew G. Knepley       PetscInt dim;
45443e9753d6SMatthew G. Knepley 
45459566063dSJacob Faibussowitsch       PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
45469566063dSJacob Faibussowitsch       PetscCall(DMGetDimension(dm, &dim));
45479566063dSJacob Faibussowitsch       PetscCall(DMLabelGetStratumIS(depthLabel, dim - 1, &facetIS));
45483e9753d6SMatthew G. Knepley     }
45499566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
45500c290148SMatthew G. Knepley     for (v = 0; v < numValues; ++v) {
45510c290148SMatthew G. Knepley       key.label = label;
45520c290148SMatthew G. Knepley       key.value = values[v];
45530c290148SMatthew G. Knepley       key.field = field;
45540c290148SMatthew G. Knepley       key.part  = 0;
45559566063dSJacob Faibussowitsch       PetscCall(DMPlexComputeBdResidual_Single_Internal(dm, t, wf, key, locX, locX_t, locF, coordField, facetIS));
45560c290148SMatthew G. Knepley     }
45573e9753d6SMatthew G. Knepley   }
45589566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
45593e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
45603e9753d6SMatthew G. Knepley }
45613e9753d6SMatthew G. Knepley 
456206ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeResidual_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
45633e9753d6SMatthew G. Knepley {
45643e9753d6SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *) dm->data;
45653e9753d6SMatthew G. Knepley   const char      *name       = "Residual";
45663e9753d6SMatthew G. Knepley   DM               dmAux      = NULL;
45673e9753d6SMatthew G. Knepley   DM               dmGrad     = NULL;
45683e9753d6SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
45696528b96dSMatthew G. Knepley   PetscDS          ds         = NULL;
45706528b96dSMatthew G. Knepley   PetscDS          dsAux      = NULL;
45713e9753d6SMatthew G. Knepley   PetscSection     section    = NULL;
45723e9753d6SMatthew G. Knepley   PetscBool        useFEM     = PETSC_FALSE;
45733e9753d6SMatthew G. Knepley   PetscBool        useFVM     = PETSC_FALSE;
45743e9753d6SMatthew G. Knepley   PetscBool        isImplicit = (locX_t || time == PETSC_MIN_REAL) ? PETSC_TRUE : PETSC_FALSE;
45753e9753d6SMatthew G. Knepley   PetscFV          fvm        = NULL;
45763e9753d6SMatthew G. Knepley   PetscFVCellGeom *cgeomFVM   = NULL;
45773e9753d6SMatthew G. Knepley   PetscFVFaceGeom *fgeomFVM   = NULL;
45783e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
45793e9753d6SMatthew G. Knepley   Vec              locA, cellGeometryFVM = NULL, faceGeometryFVM = NULL, grad, locGrad = NULL;
45803e9753d6SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a, *uL, *uR;
45813e9753d6SMatthew G. Knepley   IS               chunkIS;
45823e9753d6SMatthew G. Knepley   const PetscInt  *cells;
45833e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
45843e9753d6SMatthew G. Knepley   PetscInt         Nf, f, totDim, totDimAux, numChunks, cellChunkSize, faceChunkSize, chunk, fStart, fEnd;
45853e9753d6SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
45863e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
45873e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
45883e9753d6SMatthew G. Knepley 
45893e9753d6SMatthew G. Knepley   PetscFunctionBegin;
45909566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0));
45913e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
45923e9753d6SMatthew G. Knepley   /* TODO The FVM geometry is over-manipulated. Make the precalc functions return exactly what we need */
45933e9753d6SMatthew G. Knepley   /* FEM+FVM */
45949566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
45959566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
45963e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
45979566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
45989566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
45999566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &ds));
46009566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
46019566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
46029566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &locA));
46033e9753d6SMatthew G. Knepley   if (locA) {
46043e9753d6SMatthew G. Knepley     PetscInt subcell;
46059566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
46069566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosurePoint(dmAux, dm, DM_ENC_UNKNOWN, cStart, &subcell));
46079566063dSJacob Faibussowitsch     PetscCall(DMGetCellDS(dmAux, subcell, &dsAux));
46089566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux, &totDimAux));
46093e9753d6SMatthew G. Knepley   }
46103e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
46113e9753d6SMatthew G. Knepley   for (f = 0; f < Nf; ++f) {
46123e9753d6SMatthew G. Knepley     PetscObject  obj;
46133e9753d6SMatthew G. Knepley     PetscClassId id;
46143e9753d6SMatthew G. Knepley     PetscBool    fimp;
46153e9753d6SMatthew G. Knepley 
46169566063dSJacob Faibussowitsch     PetscCall(PetscDSGetImplicit(ds, f, &fimp));
46173e9753d6SMatthew G. Knepley     if (isImplicit != fimp) continue;
46189566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(ds, f, &obj));
46199566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
46203e9753d6SMatthew G. Knepley     if (id == PETSCFE_CLASSID) {useFEM = PETSC_TRUE;}
46213e9753d6SMatthew G. Knepley     if (id == PETSCFV_CLASSID) {useFVM = PETSC_TRUE; fvm = (PetscFV) obj;}
46223e9753d6SMatthew G. Knepley   }
46233e9753d6SMatthew G. Knepley   if (useFEM) {
46249566063dSJacob Faibussowitsch     PetscCall(DMGetCoordinateField(dm, &coordField));
46259566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
46263e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
46279566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&affineQuad));
46283e9753d6SMatthew G. Knepley       if (affineQuad) {
46299566063dSJacob Faibussowitsch         PetscCall(DMSNESGetFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
46303e9753d6SMatthew G. Knepley       }
46313e9753d6SMatthew G. Knepley     } else {
46329566063dSJacob Faibussowitsch       PetscCall(PetscCalloc2(Nf,&quads,Nf,&geoms));
46333e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
46343e9753d6SMatthew G. Knepley         PetscObject  obj;
46353e9753d6SMatthew G. Knepley         PetscClassId id;
46363e9753d6SMatthew G. Knepley         PetscBool    fimp;
46373e9753d6SMatthew G. Knepley 
46389566063dSJacob Faibussowitsch         PetscCall(PetscDSGetImplicit(ds, f, &fimp));
46393e9753d6SMatthew G. Knepley         if (isImplicit != fimp) continue;
46409566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
46419566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
46423e9753d6SMatthew G. Knepley         if (id == PETSCFE_CLASSID) {
46433e9753d6SMatthew G. Knepley           PetscFE fe = (PetscFE) obj;
46443e9753d6SMatthew G. Knepley 
46459566063dSJacob Faibussowitsch           PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
46469566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject)quads[f]));
46479566063dSJacob Faibussowitsch           PetscCall(DMSNESGetFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
46483e9753d6SMatthew G. Knepley         }
46493e9753d6SMatthew G. Knepley       }
46503e9753d6SMatthew G. Knepley     }
46513e9753d6SMatthew G. Knepley   }
46523e9753d6SMatthew G. Knepley   if (useFVM) {
46539566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGeometryFVM(dm, &faceGeometryFVM, &cellGeometryFVM, NULL));
46549566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(faceGeometryFVM, (const PetscScalar **) &fgeomFVM));
46559566063dSJacob Faibussowitsch     PetscCall(VecGetArrayRead(cellGeometryFVM, (const PetscScalar **) &cgeomFVM));
46563e9753d6SMatthew G. Knepley     /* Reconstruct and limit cell gradients */
46579566063dSJacob Faibussowitsch     PetscCall(DMPlexGetGradientDM(dm, fvm, &dmGrad));
46583e9753d6SMatthew G. Knepley     if (dmGrad) {
46599566063dSJacob Faibussowitsch       PetscCall(DMPlexGetHeightStratum(dm, 1, &fStart, &fEnd));
46609566063dSJacob Faibussowitsch       PetscCall(DMGetGlobalVector(dmGrad, &grad));
46619566063dSJacob Faibussowitsch       PetscCall(DMPlexReconstructGradients_Internal(dm, fvm, fStart, fEnd, faceGeometryFVM, cellGeometryFVM, locX, grad));
46623e9753d6SMatthew G. Knepley       /* Communicate gradient values */
46639566063dSJacob Faibussowitsch       PetscCall(DMGetLocalVector(dmGrad, &locGrad));
46649566063dSJacob Faibussowitsch       PetscCall(DMGlobalToLocalBegin(dmGrad, grad, INSERT_VALUES, locGrad));
46659566063dSJacob Faibussowitsch       PetscCall(DMGlobalToLocalEnd(dmGrad, grad, INSERT_VALUES, locGrad));
46669566063dSJacob Faibussowitsch       PetscCall(DMRestoreGlobalVector(dmGrad, &grad));
46673e9753d6SMatthew G. Knepley     }
46683e9753d6SMatthew G. Knepley     /* Handle non-essential (e.g. outflow) boundary values */
46699566063dSJacob Faibussowitsch     PetscCall(DMPlexInsertBoundaryValues(dm, PETSC_FALSE, locX, time, faceGeometryFVM, cellGeometryFVM, locGrad));
46703e9753d6SMatthew G. Knepley   }
46713e9753d6SMatthew G. Knepley   /* Loop over chunks */
46729566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISCreate(PETSC_COMM_SELF, &chunkIS));
46733e9753d6SMatthew G. Knepley   numCells      = cEnd - cStart;
46743e9753d6SMatthew G. Knepley   numChunks     = 1;
46753e9753d6SMatthew G. Knepley   cellChunkSize = numCells/numChunks;
46763e9753d6SMatthew G. Knepley   faceChunkSize = (fEnd - fStart)/numChunks;
46773e9753d6SMatthew G. Knepley   numChunks     = PetscMin(1,numCells);
46783e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
46793e9753d6SMatthew G. Knepley     PetscScalar     *elemVec, *fluxL, *fluxR;
46803e9753d6SMatthew G. Knepley     PetscReal       *vol;
46813e9753d6SMatthew G. Knepley     PetscFVFaceGeom *fgeom;
46823e9753d6SMatthew G. Knepley     PetscInt         cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
46833e9753d6SMatthew G. Knepley     PetscInt         fS = fStart+chunk*faceChunkSize, fE = PetscMin(fS+faceChunkSize, fEnd), numFaces = 0, face;
46843e9753d6SMatthew G. Knepley 
46853e9753d6SMatthew G. Knepley     /* Extract field coefficients */
46863e9753d6SMatthew G. Knepley     if (useFEM) {
46879566063dSJacob Faibussowitsch       PetscCall(ISGetPointSubrange(chunkIS, cS, cE, cells));
46889566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
46899566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
46909566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(elemVec, numCells*totDim));
46913e9753d6SMatthew G. Knepley     }
46923e9753d6SMatthew G. Knepley     if (useFVM) {
46939566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
46949566063dSJacob Faibussowitsch       PetscCall(DMPlexGetFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
46959566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxL));
46969566063dSJacob Faibussowitsch       PetscCall(DMGetWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxR));
46979566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxL, numFaces*totDim));
46989566063dSJacob Faibussowitsch       PetscCall(PetscArrayzero(fluxR, numFaces*totDim));
46993e9753d6SMatthew G. Knepley     }
47003e9753d6SMatthew G. Knepley     /* TODO We will interlace both our field coefficients (u, u_t, uL, uR, etc.) and our output (elemVec, fL, fR). I think this works */
47013e9753d6SMatthew G. Knepley     /* Loop over fields */
47023e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
47033e9753d6SMatthew G. Knepley       PetscObject  obj;
47043e9753d6SMatthew G. Knepley       PetscClassId id;
47053e9753d6SMatthew G. Knepley       PetscBool    fimp;
47063e9753d6SMatthew G. Knepley       PetscInt     numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset;
47073e9753d6SMatthew G. Knepley 
47086528b96dSMatthew G. Knepley       key.field = f;
47099566063dSJacob Faibussowitsch       PetscCall(PetscDSGetImplicit(ds, f, &fimp));
47103e9753d6SMatthew G. Knepley       if (isImplicit != fimp) continue;
47119566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, &obj));
47129566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId(obj, &id));
47133e9753d6SMatthew G. Knepley       if (id == PETSCFE_CLASSID) {
47143e9753d6SMatthew G. Knepley         PetscFE         fe = (PetscFE) obj;
47153e9753d6SMatthew G. Knepley         PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
47163e9753d6SMatthew G. Knepley         PetscFEGeom    *chunkGeom = NULL;
47173e9753d6SMatthew G. Knepley         PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
47183e9753d6SMatthew G. Knepley         PetscInt        Nq, Nb;
47193e9753d6SMatthew G. Knepley 
47209566063dSJacob Faibussowitsch         PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
47219566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
47229566063dSJacob Faibussowitsch         PetscCall(PetscFEGetDimension(fe, &Nb));
47233e9753d6SMatthew G. Knepley         blockSize = Nb;
47243e9753d6SMatthew G. Knepley         batchSize = numBlocks * blockSize;
47259566063dSJacob Faibussowitsch         PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
47263e9753d6SMatthew G. Knepley         numChunks = numCells / (numBatches*batchSize);
47273e9753d6SMatthew G. Knepley         Ne        = numChunks*numBatches*batchSize;
47283e9753d6SMatthew G. Knepley         Nr        = numCells % (numBatches*batchSize);
47293e9753d6SMatthew G. Knepley         offset    = numCells - Nr;
47303e9753d6SMatthew G. Knepley         /* Integrate FE residual to get elemVec (need fields at quadrature points) */
47313e9753d6SMatthew G. Knepley         /*   For FV, I think we use a P0 basis and the cell coefficients (for subdivided cells, we can tweak the basis tabulation to be the indicator function) */
47329566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom,0,offset,&chunkGeom));
47339566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(ds, key, Ne, chunkGeom, u, u_t, dsAux, a, t, elemVec));
47349566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomGetChunk(geom,offset,numCells,&chunkGeom));
47359566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateResidual(ds, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux, &a[offset*totDimAux], t, &elemVec[offset*totDim]));
47369566063dSJacob Faibussowitsch         PetscCall(PetscFEGeomRestoreChunk(geom,offset,numCells,&chunkGeom));
47373e9753d6SMatthew G. Knepley       } else if (id == PETSCFV_CLASSID) {
47383e9753d6SMatthew G. Knepley         PetscFV fv = (PetscFV) obj;
47393e9753d6SMatthew G. Knepley 
47403e9753d6SMatthew G. Knepley         Ne = numFaces;
47413e9753d6SMatthew G. Knepley         /* Riemann solve over faces (need fields at face centroids) */
47423e9753d6SMatthew G. Knepley         /*   We need to evaluate FE fields at those coordinates */
47439566063dSJacob Faibussowitsch         PetscCall(PetscFVIntegrateRHSFunction(fv, ds, f, Ne, fgeom, vol, uL, uR, fluxL, fluxR));
474463a3b9bcSJacob Faibussowitsch       } else SETERRQ(PetscObjectComm((PetscObject) dm), PETSC_ERR_ARG_WRONG, "Unknown discretization type for field %" PetscInt_FMT, f);
47453e9753d6SMatthew G. Knepley     }
47463e9753d6SMatthew G. Knepley     /* Loop over domain */
47473e9753d6SMatthew G. Knepley     if (useFEM) {
47483e9753d6SMatthew G. Knepley       /* Add elemVec to locX */
47493e9753d6SMatthew G. Knepley       for (c = cS; c < cE; ++c) {
47503e9753d6SMatthew G. Knepley         const PetscInt cell = cells ? cells[c] : c;
47513e9753d6SMatthew G. Knepley         const PetscInt cind = c - cStart;
47523e9753d6SMatthew G. Knepley 
47539566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]));
47543e9753d6SMatthew G. Knepley         if (ghostLabel) {
47553e9753d6SMatthew G. Knepley           PetscInt ghostVal;
47563e9753d6SMatthew G. Knepley 
47579566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel,cell,&ghostVal));
47583e9753d6SMatthew G. Knepley           if (ghostVal > 0) continue;
47593e9753d6SMatthew G. Knepley         }
47609566063dSJacob Faibussowitsch         PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES));
47613e9753d6SMatthew G. Knepley       }
47623e9753d6SMatthew G. Knepley     }
47633e9753d6SMatthew G. Knepley     if (useFVM) {
47643e9753d6SMatthew G. Knepley       PetscScalar *fa;
47653e9753d6SMatthew G. Knepley       PetscInt     iface;
47663e9753d6SMatthew G. Knepley 
47679566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
47683e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
47693e9753d6SMatthew G. Knepley         PetscFV      fv;
47703e9753d6SMatthew G. Knepley         PetscObject  obj;
47713e9753d6SMatthew G. Knepley         PetscClassId id;
47723e9753d6SMatthew G. Knepley         PetscInt     foff, pdim;
47733e9753d6SMatthew G. Knepley 
47749566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
47759566063dSJacob Faibussowitsch         PetscCall(PetscDSGetFieldOffset(ds, f, &foff));
47769566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
47773e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
47783e9753d6SMatthew G. Knepley         fv   = (PetscFV) obj;
47799566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
47803e9753d6SMatthew G. Knepley         /* Accumulate fluxes to cells */
47813e9753d6SMatthew G. Knepley         for (face = fS, iface = 0; face < fE; ++face) {
47823e9753d6SMatthew G. Knepley           const PetscInt *scells;
47833e9753d6SMatthew G. Knepley           PetscScalar    *fL = NULL, *fR = NULL;
47843e9753d6SMatthew G. Knepley           PetscInt        ghost, d, nsupp, nchild;
47853e9753d6SMatthew G. Knepley 
47869566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel, face, &ghost));
47879566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupportSize(dm, face, &nsupp));
47889566063dSJacob Faibussowitsch           PetscCall(DMPlexGetTreeChildren(dm, face, &nchild, NULL));
47893e9753d6SMatthew G. Knepley           if (ghost >= 0 || nsupp > 2 || nchild > 0) continue;
47909566063dSJacob Faibussowitsch           PetscCall(DMPlexGetSupport(dm, face, &scells));
47919566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel,scells[0],&ghost));
47929566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[0], f, fa, &fL));
47939566063dSJacob Faibussowitsch           PetscCall(DMLabelGetValue(ghostLabel,scells[1],&ghost));
47949566063dSJacob Faibussowitsch           if (ghost <= 0) PetscCall(DMPlexPointLocalFieldRef(dm, scells[1], f, fa, &fR));
47953e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) {
47963e9753d6SMatthew G. Knepley             if (fL) fL[d] -= fluxL[iface*totDim+foff+d];
47973e9753d6SMatthew G. Knepley             if (fR) fR[d] += fluxR[iface*totDim+foff+d];
47983e9753d6SMatthew G. Knepley           }
47993e9753d6SMatthew G. Knepley           ++iface;
48003e9753d6SMatthew G. Knepley         }
48013e9753d6SMatthew G. Knepley       }
48029566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
48033e9753d6SMatthew G. Knepley     }
48043e9753d6SMatthew G. Knepley     /* Handle time derivative */
48053e9753d6SMatthew G. Knepley     if (locX_t) {
48063e9753d6SMatthew G. Knepley       PetscScalar *x_t, *fa;
48073e9753d6SMatthew G. Knepley 
48089566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locF, &fa));
48099566063dSJacob Faibussowitsch       PetscCall(VecGetArray(locX_t, &x_t));
48103e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
48113e9753d6SMatthew G. Knepley         PetscFV      fv;
48123e9753d6SMatthew G. Knepley         PetscObject  obj;
48133e9753d6SMatthew G. Knepley         PetscClassId id;
48143e9753d6SMatthew G. Knepley         PetscInt     pdim, d;
48153e9753d6SMatthew G. Knepley 
48169566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, f, &obj));
48179566063dSJacob Faibussowitsch         PetscCall(PetscObjectGetClassId(obj, &id));
48183e9753d6SMatthew G. Knepley         if (id != PETSCFV_CLASSID) continue;
48193e9753d6SMatthew G. Knepley         fv   = (PetscFV) obj;
48209566063dSJacob Faibussowitsch         PetscCall(PetscFVGetNumComponents(fv, &pdim));
48213e9753d6SMatthew G. Knepley         for (c = cS; c < cE; ++c) {
48223e9753d6SMatthew G. Knepley           const PetscInt cell = cells ? cells[c] : c;
48233e9753d6SMatthew G. Knepley           PetscScalar   *u_t, *r;
48243e9753d6SMatthew G. Knepley 
48253e9753d6SMatthew G. Knepley           if (ghostLabel) {
48263e9753d6SMatthew G. Knepley             PetscInt ghostVal;
48273e9753d6SMatthew G. Knepley 
48289566063dSJacob Faibussowitsch             PetscCall(DMLabelGetValue(ghostLabel, cell, &ghostVal));
48293e9753d6SMatthew G. Knepley             if (ghostVal > 0) continue;
48303e9753d6SMatthew G. Knepley           }
48319566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRead(dm, cell, f, x_t, &u_t));
48329566063dSJacob Faibussowitsch           PetscCall(DMPlexPointLocalFieldRef(dm, cell, f, fa, &r));
48333e9753d6SMatthew G. Knepley           for (d = 0; d < pdim; ++d) r[d] += u_t[d];
48343e9753d6SMatthew G. Knepley         }
48353e9753d6SMatthew G. Knepley       }
48369566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locX_t, &x_t));
48379566063dSJacob Faibussowitsch       PetscCall(VecRestoreArray(locF, &fa));
48383e9753d6SMatthew G. Knepley     }
48393e9753d6SMatthew G. Knepley     if (useFEM) {
48409566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreCellFields(dm, chunkIS, locX, locX_t, locA, &u, &u_t, &a));
48419566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
48423e9753d6SMatthew G. Knepley     }
48433e9753d6SMatthew G. Knepley     if (useFVM) {
48449566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceFields(dm, fS, fE, locX, locX_t, faceGeometryFVM, cellGeometryFVM, locGrad, &numFaces, &uL, &uR));
48459566063dSJacob Faibussowitsch       PetscCall(DMPlexRestoreFaceGeometry(dm, fS, fE, faceGeometryFVM, cellGeometryFVM, &numFaces, &fgeom, &vol));
48469566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxL));
48479566063dSJacob Faibussowitsch       PetscCall(DMRestoreWorkArray(dm, numFaces*totDim, MPIU_SCALAR, &fluxR));
48489566063dSJacob Faibussowitsch       if (dmGrad) PetscCall(DMRestoreLocalVector(dmGrad, &locGrad));
48493e9753d6SMatthew G. Knepley     }
48503e9753d6SMatthew G. Knepley   }
48519566063dSJacob Faibussowitsch   if (useFEM) PetscCall(ISDestroy(&chunkIS));
48529566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
48533e9753d6SMatthew G. Knepley 
48543e9753d6SMatthew G. Knepley   if (useFEM) {
48559566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdResidual_Internal(dm, locX, locX_t, t, locF, user));
48563e9753d6SMatthew G. Knepley 
48573e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
48589566063dSJacob Faibussowitsch       PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
48599566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureDestroy(&affineQuad));
48603e9753d6SMatthew G. Knepley     } else {
48613e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
48629566063dSJacob Faibussowitsch         PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
48639566063dSJacob Faibussowitsch         PetscCall(PetscQuadratureDestroy(&quads[f]));
48643e9753d6SMatthew G. Knepley       }
48659566063dSJacob Faibussowitsch       PetscCall(PetscFree2(quads,geoms));
48663e9753d6SMatthew G. Knepley     }
48673e9753d6SMatthew G. Knepley   }
48683e9753d6SMatthew G. Knepley 
48693e9753d6SMatthew G. Knepley   /* FEM */
48703e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
48713e9753d6SMatthew G. Knepley   /* 2: Get geometric data */
48723e9753d6SMatthew G. Knepley   /* 3: Handle boundary values */
48733e9753d6SMatthew G. Knepley   /* 4: Loop over domain */
48743e9753d6SMatthew G. Knepley   /*   Extract coefficients */
48753e9753d6SMatthew G. Knepley   /* Loop over fields */
48763e9753d6SMatthew G. Knepley   /*   Set tiling for FE*/
48773e9753d6SMatthew G. Knepley   /*   Integrate FE residual to get elemVec */
48783e9753d6SMatthew G. Knepley   /*     Loop over subdomain */
48793e9753d6SMatthew G. Knepley   /*       Loop over quad points */
48803e9753d6SMatthew G. Knepley   /*         Transform coords to real space */
48813e9753d6SMatthew G. Knepley   /*         Evaluate field and aux fields at point */
48823e9753d6SMatthew G. Knepley   /*         Evaluate residual at point */
48833e9753d6SMatthew G. Knepley   /*         Transform residual to real space */
48843e9753d6SMatthew G. Knepley   /*       Add residual to elemVec */
48853e9753d6SMatthew G. Knepley   /* Loop over domain */
48863e9753d6SMatthew G. Knepley   /*   Add elemVec to locX */
48873e9753d6SMatthew G. Knepley 
48883e9753d6SMatthew G. Knepley   /* FVM */
48893e9753d6SMatthew G. Knepley   /* Get geometric data */
48903e9753d6SMatthew G. Knepley   /* If using gradients */
48913e9753d6SMatthew G. Knepley   /*   Compute gradient data */
48923e9753d6SMatthew G. Knepley   /*   Loop over domain faces */
48933e9753d6SMatthew G. Knepley   /*     Count computational faces */
48943e9753d6SMatthew G. Knepley   /*     Reconstruct cell gradient */
48953e9753d6SMatthew G. Knepley   /*   Loop over domain cells */
48963e9753d6SMatthew G. Knepley   /*     Limit cell gradients */
48973e9753d6SMatthew G. Knepley   /* Handle boundary values */
48983e9753d6SMatthew G. Knepley   /* Loop over domain faces */
48993e9753d6SMatthew G. Knepley   /*   Read out field, centroid, normal, volume for each side of face */
49003e9753d6SMatthew G. Knepley   /* Riemann solve over faces */
49013e9753d6SMatthew G. Knepley   /* Loop over domain faces */
49023e9753d6SMatthew G. Knepley   /*   Accumulate fluxes to cells */
49033e9753d6SMatthew G. Knepley   /* TODO Change printFEM to printDisc here */
49043e9753d6SMatthew G. Knepley   if (mesh->printFEM) {
49053e9753d6SMatthew G. Knepley     Vec         locFbc;
49063e9753d6SMatthew G. Knepley     PetscInt    pStart, pEnd, p, maxDof;
49073e9753d6SMatthew G. Knepley     PetscScalar *zeroes;
49083e9753d6SMatthew G. Knepley 
49099566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(locF,&locFbc));
49109566063dSJacob Faibussowitsch     PetscCall(VecCopy(locF,locFbc));
49119566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetChart(section,&pStart,&pEnd));
49129566063dSJacob Faibussowitsch     PetscCall(PetscSectionGetMaxDof(section,&maxDof));
49139566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(maxDof,&zeroes));
49143e9753d6SMatthew G. Knepley     for (p = pStart; p < pEnd; p++) {
49159566063dSJacob Faibussowitsch       PetscCall(VecSetValuesSection(locFbc,section,p,zeroes,INSERT_BC_VALUES));
49163e9753d6SMatthew G. Knepley     }
49179566063dSJacob Faibussowitsch     PetscCall(PetscFree(zeroes));
49189566063dSJacob Faibussowitsch     PetscCall(DMPrintLocalVec(dm, name, mesh->printTol, locFbc));
49199566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&locFbc));
49203e9753d6SMatthew G. Knepley   }
49219566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0));
49223e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
49233e9753d6SMatthew G. Knepley }
49243e9753d6SMatthew G. Knepley 
49256528b96dSMatthew G. Knepley /*
49266528b96dSMatthew G. Knepley   1) Allow multiple kernels for BdResidual for hybrid DS
49276528b96dSMatthew G. Knepley 
49286528b96dSMatthew G. Knepley   DONE 2) Get out dsAux for either side at the same time as cohesive cell dsAux
49296528b96dSMatthew G. Knepley 
49306528b96dSMatthew G. Knepley   DONE 3) Change DMGetCellFields() to get different aux data a[] for each side
49316528b96dSMatthew G. Knepley      - I think I just need to replace a[] with the closure from each face
49326528b96dSMatthew G. Knepley 
49336528b96dSMatthew G. Knepley   4) Run both kernels for each non-hybrid field with correct dsAux, and then hybrid field as before
49346528b96dSMatthew G. Knepley */
493506ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeResidual_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal time, Vec locX, Vec locX_t, PetscReal t, Vec locF, void *user)
49363e9753d6SMatthew G. Knepley {
49373e9753d6SMatthew G. Knepley   DM_Plex         *mesh       = (DM_Plex *) dm->data;
49383e9753d6SMatthew G. Knepley   const char      *name       = "Hybrid Residual";
493904c51a94SMatthew G. Knepley   DM               dmAux[3]   = {NULL, NULL, NULL};
49403e9753d6SMatthew G. Knepley   DMLabel          ghostLabel = NULL;
49416528b96dSMatthew G. Knepley   PetscDS          ds         = NULL;
49426528b96dSMatthew G. Knepley   PetscDS          dsAux[3]   = {NULL, NULL, NULL};
494304c51a94SMatthew G. Knepley   Vec              locA[3]    = {NULL, NULL, NULL};
49443e9753d6SMatthew G. Knepley   PetscSection     section    = NULL;
49453e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
49466528b96dSMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a[3];
49473e9753d6SMatthew G. Knepley   PetscScalar     *elemVec;
49483e9753d6SMatthew G. Knepley   IS               chunkIS;
49493e9753d6SMatthew G. Knepley   const PetscInt  *cells;
49503e9753d6SMatthew G. Knepley   PetscInt        *faces;
49513e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
49526528b96dSMatthew G. Knepley   PetscInt         Nf, f, totDim, totDimAux[3], numChunks, cellChunkSize, chunk;
49533e9753d6SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
49543e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
49553e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
49563e9753d6SMatthew G. Knepley 
49573e9753d6SMatthew G. Knepley   PetscFunctionBegin;
49585fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
49595fedec97SMatthew G. Knepley     const char *name;
49609566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject) key[0].label, &name));
496163a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
49625fedec97SMatthew G. Knepley   }
49639566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_ResidualFEM,dm,0,0,0));
49643e9753d6SMatthew G. Knepley   /* TODO The places where we have to use isFE are probably the member functions for the PetscDisc class */
49653e9753d6SMatthew G. Knepley   /* FEM */
49669566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
49679566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
49683e9753d6SMatthew G. Knepley   /* 1: Get sizes from dm and dmAux */
49699566063dSJacob Faibussowitsch   PetscCall(DMGetSection(dm, &section));
49709566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
49719566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cStart, &ds));
49729566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
49739566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
49749566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
497504c51a94SMatthew G. Knepley   if (locA[2]) {
49769566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
49779566063dSJacob Faibussowitsch     PetscCall(DMGetCellDS(dmAux[2], cStart, &dsAux[2]));
49789566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
49796528b96dSMatthew G. Knepley     {
49806528b96dSMatthew G. Knepley       const PetscInt *cone;
49816528b96dSMatthew G. Knepley       PetscInt        c;
49826528b96dSMatthew G. Knepley 
49839566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(dm, cStart, &cone));
49846528b96dSMatthew G. Knepley       for (c = 0; c < 2; ++c) {
49856528b96dSMatthew G. Knepley         const PetscInt *support;
49866528b96dSMatthew G. Knepley         PetscInt ssize, s;
49876528b96dSMatthew G. Knepley 
49889566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
49899566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
499063a3b9bcSJacob Faibussowitsch         PetscCheck(ssize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cStart, ssize);
49916528b96dSMatthew G. Knepley         if      (support[0] == cStart) s = 1;
49926528b96dSMatthew G. Knepley         else if (support[1] == cStart) s = 0;
499363a3b9bcSJacob Faibussowitsch         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cStart);
49949566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
49959566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
499604c51a94SMatthew G. Knepley         else         {dmAux[c] = dmAux[2];}
49979566063dSJacob Faibussowitsch         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c]));
49989566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
49996528b96dSMatthew G. Knepley       }
50006528b96dSMatthew G. Knepley     }
50013e9753d6SMatthew G. Knepley   }
50023e9753d6SMatthew G. Knepley   /* 2: Setup geometric data */
50039566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
50049566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
50053e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
50069566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
50073e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
50083e9753d6SMatthew G. Knepley       PetscFE fe;
50093e9753d6SMatthew G. Knepley 
50109566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *) &fe));
50113e9753d6SMatthew G. Knepley       if (fe) {
50129566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
50139566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) quads[f]));
50143e9753d6SMatthew G. Knepley       }
50153e9753d6SMatthew G. Knepley     }
50163e9753d6SMatthew G. Knepley   }
50173e9753d6SMatthew G. Knepley   /* Loop over chunks */
50183e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
50193e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal) numCells)/cellChunkSize);
50209566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(2*cellChunkSize, &faces));
50219566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS));
50223e9753d6SMatthew G. Knepley   /* Extract field coefficients */
50233e9753d6SMatthew G. Knepley   /* NOTE This needs the end cap faces to have identical orientations */
50249566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
50259566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHybridAuxFields(dm, dmAux, dsAux, cellIS, locA, a));
50269566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, cellChunkSize*totDim, MPIU_SCALAR, &elemVec));
50273e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
50283e9753d6SMatthew G. Knepley     PetscInt cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
50293e9753d6SMatthew G. Knepley 
50309566063dSJacob Faibussowitsch     PetscCall(PetscMemzero(elemVec, cellChunkSize*totDim * sizeof(PetscScalar)));
50313e9753d6SMatthew G. Knepley     /* Get faces */
50323e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
50333e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
50343e9753d6SMatthew G. Knepley       const PetscInt *cone;
50359566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(dm, cell, &cone));
50363e9753d6SMatthew G. Knepley       faces[(c-cS)*2+0] = cone[0];
50373e9753d6SMatthew G. Knepley       faces[(c-cS)*2+1] = cone[1];
50383e9753d6SMatthew G. Knepley     }
50399566063dSJacob Faibussowitsch     PetscCall(ISGeneralSetIndices(chunkIS, cellChunkSize, faces, PETSC_USE_POINTER));
50403e9753d6SMatthew G. Knepley     /* Get geometric data */
50413e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
50429566063dSJacob Faibussowitsch       if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad));
50439566063dSJacob Faibussowitsch       if (affineQuad)  PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom));
50443e9753d6SMatthew G. Knepley     } else {
50453e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
50469566063dSJacob Faibussowitsch         if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]));
50473e9753d6SMatthew G. Knepley       }
50483e9753d6SMatthew G. Knepley     }
50493e9753d6SMatthew G. Knepley     /* Loop over fields */
50503e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
50513e9753d6SMatthew G. Knepley       PetscFE         fe;
50523e9753d6SMatthew G. Knepley       PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[f];
5053148442b3SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
50543e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[f];
50553e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
50565fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
50573e9753d6SMatthew G. Knepley 
50589566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *) &fe));
50593e9753d6SMatthew G. Knepley       if (!fe) continue;
50609566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
50619566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
50629566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
50633e9753d6SMatthew G. Knepley       blockSize = Nb;
50643e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
50659566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
50663e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches*batchSize);
50673e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
50683e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches*batchSize);
50693e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
50709566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom,0,offset,&chunkGeom));
50719566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom,offset,numCells,&remGeom));
50729566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, f, &isCohesiveField));
50735fedec97SMatthew G. Knepley       chunkGeom->isCohesive = remGeom->isCohesive = PETSC_TRUE;
50746528b96dSMatthew G. Knepley       key[0].field = f;
50756528b96dSMatthew G. Knepley       key[1].field = f;
50765fedec97SMatthew G. Knepley       key[2].field = f;
50779566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, elemVec));
50789566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[0], 0, Nr, remGeom,  &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[0], &a[0][offset*totDimAux[0]], t, &elemVec[offset*totDim]));
50799566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, elemVec));
50809566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[1], 1, Nr, remGeom,  &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[1], &a[1][offset*totDimAux[1]], t, &elemVec[offset*totDim]));
50819566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, elemVec));
50829566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateHybridResidual(ds, key[2], 2, Nr, remGeom,  &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[2], &a[2][offset*totDimAux[2]], t, &elemVec[offset*totDim]));
50839566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom,offset,numCells,&remGeom));
50849566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom,0,offset,&chunkGeom));
50853e9753d6SMatthew G. Knepley     }
50863e9753d6SMatthew G. Knepley     /* Add elemVec to locX */
50873e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
50883e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
50893e9753d6SMatthew G. Knepley       const PetscInt cind = c - cStart;
50903e9753d6SMatthew G. Knepley 
50919566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellVector(cell, name, totDim, &elemVec[cind*totDim]));
50923e9753d6SMatthew G. Knepley       if (ghostLabel) {
50933e9753d6SMatthew G. Knepley         PetscInt ghostVal;
50943e9753d6SMatthew G. Knepley 
50959566063dSJacob Faibussowitsch         PetscCall(DMLabelGetValue(ghostLabel,cell,&ghostVal));
50963e9753d6SMatthew G. Knepley         if (ghostVal > 0) continue;
50973e9753d6SMatthew G. Knepley       }
50989566063dSJacob Faibussowitsch       PetscCall(DMPlexVecSetClosure(dm, section, locF, cell, &elemVec[cind*totDim], ADD_ALL_VALUES));
50993e9753d6SMatthew G. Knepley     }
51003e9753d6SMatthew G. Knepley   }
51019566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
51029566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreHybridAuxFields(dmAux, dsAux, cellIS, locA, a));
51039566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, numCells*totDim, MPIU_SCALAR, &elemVec));
51049566063dSJacob Faibussowitsch   PetscCall(PetscFree(faces));
51059566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&chunkIS));
51069566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
51073e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
51089566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
51099566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&affineQuad));
51103e9753d6SMatthew G. Knepley   } else {
51113e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
51129566063dSJacob Faibussowitsch       if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE,&geoms[f]));
51139566063dSJacob Faibussowitsch       if (quads) PetscCall(PetscQuadratureDestroy(&quads[f]));
51143e9753d6SMatthew G. Knepley     }
51159566063dSJacob Faibussowitsch     PetscCall(PetscFree2(quads,geoms));
51163e9753d6SMatthew G. Knepley   }
51179566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_ResidualFEM,dm,0,0,0));
51183e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
51193e9753d6SMatthew G. Knepley }
51203e9753d6SMatthew G. Knepley 
512145480ffeSMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobian_Single_Internal(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt fieldI, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP, DMField coordField, IS facetIS)
51223e9753d6SMatthew G. Knepley {
51233e9753d6SMatthew G. Knepley   DM_Plex        *mesh = (DM_Plex *) dm->data;
51243e9753d6SMatthew G. Knepley   DM              plex = NULL, plexA = NULL, tdm;
51253e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
51263e9753d6SMatthew G. Knepley   PetscDS         prob, probAux = NULL;
51273e9753d6SMatthew G. Knepley   PetscSection    section, sectionAux = NULL;
5128e432b41dSStefano Zampini   PetscSection    globalSection;
51293e9753d6SMatthew G. Knepley   Vec             locA = NULL, tv;
51303e9753d6SMatthew G. Knepley   PetscScalar    *u = NULL, *u_t = NULL, *a = NULL, *elemMat = NULL;
51313e9753d6SMatthew G. Knepley   PetscInt        v;
51323e9753d6SMatthew G. Knepley   PetscInt        Nf, totDim, totDimAux = 0;
5133e432b41dSStefano Zampini   PetscBool       transform;
51343e9753d6SMatthew G. Knepley 
51353e9753d6SMatthew G. Knepley   PetscFunctionBegin;
51369566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
51379566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
51389566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
51399566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
51409566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
51419566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
51429566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
51439566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
51449566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, label, values[0], 0, &locA));
51453e9753d6SMatthew G. Knepley   if (locA) {
51463e9753d6SMatthew G. Knepley     DM dmAux;
51473e9753d6SMatthew G. Knepley 
51489566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA, &dmAux));
51499566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
51509566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexA));
51519566063dSJacob Faibussowitsch     PetscCall(DMGetDS(plexA, &probAux));
51529566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
51539566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexA, &sectionAux));
51543e9753d6SMatthew G. Knepley   }
51553e9753d6SMatthew G. Knepley 
51569566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
51573e9753d6SMatthew G. Knepley   for (v = 0; v < numValues; ++v) {
51583e9753d6SMatthew G. Knepley     PetscFEGeom     *fgeom;
51593e9753d6SMatthew G. Knepley     PetscInt         maxDegree;
51603e9753d6SMatthew G. Knepley     PetscQuadrature  qGeom = NULL;
51613e9753d6SMatthew G. Knepley     IS               pointIS;
51623e9753d6SMatthew G. Knepley     const PetscInt  *points;
516306ad1575SMatthew G. Knepley     PetscFormKey key;
51643e9753d6SMatthew G. Knepley     PetscInt         numFaces, face, Nq;
51653e9753d6SMatthew G. Knepley 
516645480ffeSMatthew G. Knepley     key.label = label;
516745480ffeSMatthew G. Knepley     key.value = values[v];
516806ad1575SMatthew G. Knepley     key.part  = 0;
51699566063dSJacob Faibussowitsch     PetscCall(DMLabelGetStratumIS(label, values[v], &pointIS));
51703e9753d6SMatthew G. Knepley     if (!pointIS) continue; /* No points with that id on this process */
51713e9753d6SMatthew G. Knepley     {
51723e9753d6SMatthew G. Knepley       IS isectIS;
51733e9753d6SMatthew G. Knepley 
51743e9753d6SMatthew G. Knepley       /* TODO: Special cases of ISIntersect where it is quick to check a prior if one is a superset of the other */
51759566063dSJacob Faibussowitsch       PetscCall(ISIntersect_Caching_Internal(facetIS,pointIS,&isectIS));
51769566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&pointIS));
51773e9753d6SMatthew G. Knepley       pointIS = isectIS;
51783e9753d6SMatthew G. Knepley     }
51799566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(pointIS, &numFaces));
51809566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(pointIS, &points));
51819566063dSJacob Faibussowitsch     PetscCall(PetscMalloc4(numFaces*totDim, &u, locX_t ? numFaces*totDim : 0, &u_t, numFaces*totDim*totDim, &elemMat, locA ? numFaces*totDimAux : 0, &a));
51829566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,pointIS,NULL,&maxDegree));
51833e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
51849566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,pointIS,&qGeom));
51853e9753d6SMatthew G. Knepley     }
51863e9753d6SMatthew G. Knepley     if (!qGeom) {
51873e9753d6SMatthew G. Knepley       PetscFE fe;
51883e9753d6SMatthew G. Knepley 
51899566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
51909566063dSJacob Faibussowitsch       PetscCall(PetscFEGetFaceQuadrature(fe, &qGeom));
51919566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
51923e9753d6SMatthew G. Knepley     }
51939566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
51949566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom));
51953e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
5196f15274beSMatthew Knepley       const PetscInt point = points[face], *support;
51973e9753d6SMatthew G. Knepley       PetscScalar   *x     = NULL;
5198f15274beSMatthew Knepley       PetscInt       i;
51993e9753d6SMatthew G. Knepley 
52009566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(dm, point, &support));
52019566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, locX, support[0], NULL, &x));
52023e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u[face*totDim+i] = x[i];
52039566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, locX, support[0], NULL, &x));
52043e9753d6SMatthew G. Knepley       if (locX_t) {
52059566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plex, section, locX_t, support[0], NULL, &x));
52063e9753d6SMatthew G. Knepley         for (i = 0; i < totDim; ++i) u_t[face*totDim+i] = x[i];
52079566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plex, section, locX_t, support[0], NULL, &x));
52083e9753d6SMatthew G. Knepley       }
52093e9753d6SMatthew G. Knepley       if (locA) {
52103e9753d6SMatthew G. Knepley         PetscInt subp;
52119566063dSJacob Faibussowitsch         PetscCall(DMGetEnclosurePoint(plexA, dm, encAux, support[0], &subp));
52129566063dSJacob Faibussowitsch         PetscCall(DMPlexVecGetClosure(plexA, sectionAux, locA, subp, NULL, &x));
52133e9753d6SMatthew G. Knepley         for (i = 0; i < totDimAux; ++i) a[face*totDimAux+i] = x[i];
52149566063dSJacob Faibussowitsch         PetscCall(DMPlexVecRestoreClosure(plexA, sectionAux, locA, subp, NULL, &x));
52153e9753d6SMatthew G. Knepley       }
52163e9753d6SMatthew G. Knepley     }
52179566063dSJacob Faibussowitsch     PetscCall(PetscArrayzero(elemMat, numFaces*totDim*totDim));
52183e9753d6SMatthew G. Knepley     {
52193e9753d6SMatthew G. Knepley       PetscFE         fe;
52203e9753d6SMatthew G. Knepley       PetscInt        Nb;
52213e9753d6SMatthew G. Knepley       /* Conforming batches */
52223e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
52233e9753d6SMatthew G. Knepley       /* Remainder */
52243e9753d6SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL;
52253e9753d6SMatthew G. Knepley       PetscInt        fieldJ, Nr, offset;
52263e9753d6SMatthew G. Knepley 
52279566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
52289566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(fe, &Nb));
52299566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
52303e9753d6SMatthew G. Knepley       blockSize = Nb;
52313e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
52329566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
52333e9753d6SMatthew G. Knepley       numChunks = numFaces / (numBatches*batchSize);
52343e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
52353e9753d6SMatthew G. Knepley       Nr        = numFaces % (numBatches*batchSize);
52363e9753d6SMatthew G. Knepley       offset    = numFaces - Nr;
52379566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom,0,offset,&chunkGeom));
52383e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
523945480ffeSMatthew G. Knepley         key.field = fieldI*Nf+fieldJ;
52409566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateBdJacobian(prob, wf, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
52413e9753d6SMatthew G. Knepley       }
52429566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(fgeom,offset,numFaces,&chunkGeom));
52433e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
524445480ffeSMatthew G. Knepley         key.field = fieldI*Nf+fieldJ;
52459566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateBdJacobian(prob, wf, key, Nr, chunkGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, a ? &a[offset*totDimAux] : NULL, t, X_tShift, &elemMat[offset*totDim*totDim]));
52463e9753d6SMatthew G. Knepley       }
52479566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(fgeom,offset,numFaces,&chunkGeom));
52483e9753d6SMatthew G. Knepley     }
52493e9753d6SMatthew G. Knepley     for (face = 0; face < numFaces; ++face) {
52503e9753d6SMatthew G. Knepley       const PetscInt point = points[face], *support;
52513e9753d6SMatthew G. Knepley 
52523e9753d6SMatthew G. Knepley       /* Transform to global basis before insertion in Jacobian */
52539566063dSJacob Faibussowitsch       PetscCall(DMPlexGetSupport(plex, point, &support));
52549566063dSJacob Faibussowitsch       if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, support[0], PETSC_TRUE, totDim, &elemMat[face*totDim*totDim]));
52559566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(point, "BdJacobian", totDim, totDim, &elemMat[face*totDim*totDim]));
52569566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, support[0], &elemMat[face*totDim*totDim], ADD_VALUES));
52573e9753d6SMatthew G. Knepley     }
52589566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,pointIS,qGeom,PETSC_TRUE,&fgeom));
52599566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
52609566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(pointIS, &points));
52619566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&pointIS));
52629566063dSJacob Faibussowitsch     PetscCall(PetscFree4(u, u_t, elemMat, a));
52633e9753d6SMatthew G. Knepley   }
52649566063dSJacob Faibussowitsch   if (plex)  PetscCall(DMDestroy(&plex));
52659566063dSJacob Faibussowitsch   if (plexA) PetscCall(DMDestroy(&plexA));
52663e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
52673e9753d6SMatthew G. Knepley }
52683e9753d6SMatthew G. Knepley 
526945480ffeSMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobianSingle(DM dm, PetscReal t, PetscWeakForm wf, DMLabel label, PetscInt numValues, const PetscInt values[], PetscInt field, Vec locX, Vec locX_t, PetscReal X_tShift, Mat Jac, Mat JacP)
52703e9753d6SMatthew G. Knepley {
52713e9753d6SMatthew G. Knepley   DMField        coordField;
52723e9753d6SMatthew G. Knepley   DMLabel        depthLabel;
52733e9753d6SMatthew G. Knepley   IS             facetIS;
52743e9753d6SMatthew G. Knepley   PetscInt       dim;
52753e9753d6SMatthew G. Knepley 
52763e9753d6SMatthew G. Knepley   PetscFunctionBegin;
52779566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
52789566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
52799566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim-1, &facetIS));
52809566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
52819566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, field, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS));
52829566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
52833e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
52843e9753d6SMatthew G. Knepley }
52853e9753d6SMatthew G. Knepley 
52863e9753d6SMatthew G. Knepley PetscErrorCode DMPlexComputeBdJacobian_Internal(DM dm, Vec locX, Vec locX_t, PetscReal t, PetscReal X_tShift, Mat Jac, Mat JacP, void *user)
52873e9753d6SMatthew G. Knepley {
52883e9753d6SMatthew G. Knepley   PetscDS          prob;
52893e9753d6SMatthew G. Knepley   PetscInt         dim, numBd, bd;
52903e9753d6SMatthew G. Knepley   DMLabel          depthLabel;
52913e9753d6SMatthew G. Knepley   DMField          coordField = NULL;
52923e9753d6SMatthew G. Knepley   IS               facetIS;
52933e9753d6SMatthew G. Knepley 
52943e9753d6SMatthew G. Knepley   PetscFunctionBegin;
52959566063dSJacob Faibussowitsch   PetscCall(DMGetDS(dm, &prob));
52969566063dSJacob Faibussowitsch   PetscCall(DMPlexGetDepthLabel(dm, &depthLabel));
52979566063dSJacob Faibussowitsch   PetscCall(DMGetDimension(dm, &dim));
52989566063dSJacob Faibussowitsch   PetscCall(DMLabelGetStratumIS(depthLabel, dim-1, &facetIS));
52999566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumBoundary(prob, &numBd));
53009566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
53013e9753d6SMatthew G. Knepley   for (bd = 0; bd < numBd; ++bd) {
530245480ffeSMatthew G. Knepley     PetscWeakForm           wf;
53033e9753d6SMatthew G. Knepley     DMBoundaryConditionType type;
53043e9753d6SMatthew G. Knepley     DMLabel                 label;
53053e9753d6SMatthew G. Knepley     const PetscInt         *values;
53063e9753d6SMatthew G. Knepley     PetscInt                fieldI, numValues;
53073e9753d6SMatthew G. Knepley     PetscObject             obj;
53083e9753d6SMatthew G. Knepley     PetscClassId            id;
53093e9753d6SMatthew G. Knepley 
53109566063dSJacob Faibussowitsch     PetscCall(PetscDSGetBoundary(prob, bd, &wf, &type, NULL, &label, &numValues, &values, &fieldI, NULL, NULL, NULL, NULL, NULL));
53119566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, &obj));
53129566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId(obj, &id));
53133e9753d6SMatthew G. Knepley     if ((id != PETSCFE_CLASSID) || (type & DM_BC_ESSENTIAL)) continue;
53149566063dSJacob Faibussowitsch     PetscCall(DMPlexComputeBdJacobian_Single_Internal(dm, t, wf, label, numValues, values, fieldI, locX, locX_t, X_tShift, Jac, JacP, coordField, facetIS));
53153e9753d6SMatthew G. Knepley   }
53169566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&facetIS));
53173e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
53183e9753d6SMatthew G. Knepley }
53193e9753d6SMatthew G. Knepley 
532006ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Mat Jac, Mat JacP,void *user)
53213e9753d6SMatthew G. Knepley {
53223e9753d6SMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *) dm->data;
53233e9753d6SMatthew G. Knepley   const char     *name  = "Jacobian";
53249a2a23afSMatthew G. Knepley   DM              dmAux = NULL, plex, tdm;
53253e9753d6SMatthew G. Knepley   DMEnclosureType encAux;
53263e9753d6SMatthew G. Knepley   Vec             A, tv;
53273e9753d6SMatthew G. Knepley   DMField         coordField;
53283e9753d6SMatthew G. Knepley   PetscDS         prob, probAux = NULL;
5329e432b41dSStefano Zampini   PetscSection    section, globalSection, sectionAux;
53303e9753d6SMatthew G. Knepley   PetscScalar    *elemMat, *elemMatP, *elemMatD, *u, *u_t, *a = NULL;
53313e9753d6SMatthew G. Knepley   const PetscInt *cells;
53323e9753d6SMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
53333e9753d6SMatthew G. Knepley   PetscInt        totDim, totDimAux, cStart, cEnd, numCells, c;
5334e432b41dSStefano Zampini   PetscBool       hasJac, hasPrec, hasDyn, hasFV = PETSC_FALSE, transform;
53353e9753d6SMatthew G. Knepley 
53363e9753d6SMatthew G. Knepley   PetscFunctionBegin;
53379566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0));
53389566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
53399566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
53409566063dSJacob Faibussowitsch   PetscCall(DMHasBasisTransform(dm, &transform));
53419566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformDM_Internal(dm, &tdm));
53429566063dSJacob Faibussowitsch   PetscCall(DMGetBasisTransformVec_Internal(dm, &tv));
53439566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
53449566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
53459566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob));
53469566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
53479566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
53489566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobian(prob, &hasJac));
53499566063dSJacob Faibussowitsch   PetscCall(PetscDSHasJacobianPreconditioner(prob, &hasPrec));
53503e9753d6SMatthew G. Knepley   /* user passed in the same matrix, avoid double contributions and
53513e9753d6SMatthew G. Knepley      only assemble the Jacobian */
53523e9753d6SMatthew G. Knepley   if (hasJac && Jac == JacP) hasPrec = PETSC_FALSE;
53539566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
53543e9753d6SMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
53559566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
53569a2a23afSMatthew G. Knepley   if (A) {
53579566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
53589566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
53599566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plex));
53609566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plex, &sectionAux));
53619566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
53629566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
53633e9753d6SMatthew G. Knepley   }
53649566063dSJacob Faibussowitsch   PetscCall(PetscMalloc5(numCells*totDim,&u,X_t ? numCells*totDim : 0,&u_t,hasJac ? numCells*totDim*totDim : 0,&elemMat,hasPrec ? numCells*totDim*totDim : 0, &elemMatP,hasDyn ? numCells*totDim*totDim : 0, &elemMatD));
53659566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells*totDimAux, &a));
53669566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
53673e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
53683e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
53693e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
53703e9753d6SMatthew G. Knepley     PetscScalar   *x = NULL,  *x_t = NULL;
53713e9753d6SMatthew G. Knepley     PetscInt       i;
53723e9753d6SMatthew G. Knepley 
53739566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(dm, section, X, cell, NULL, &x));
53743e9753d6SMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind*totDim+i] = x[i];
53759566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(dm, section, X, cell, NULL, &x));
53763e9753d6SMatthew G. Knepley     if (X_t) {
53779566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(dm, section, X_t, cell, NULL, &x_t));
53783e9753d6SMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind*totDim+i] = x_t[i];
53799566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(dm, section, X_t, cell, NULL, &x_t));
53803e9753d6SMatthew G. Knepley     }
53813e9753d6SMatthew G. Knepley     if (dmAux) {
53823e9753d6SMatthew G. Knepley       PetscInt subcell;
53839566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
53849566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, sectionAux, A, subcell, NULL, &x));
53853e9753d6SMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind*totDimAux+i] = x[i];
53869566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, sectionAux, A, subcell, NULL, &x));
53873e9753d6SMatthew G. Knepley     }
53883e9753d6SMatthew G. Knepley   }
53899566063dSJacob Faibussowitsch   if (hasJac)  PetscCall(PetscArrayzero(elemMat,  numCells*totDim*totDim));
53909566063dSJacob Faibussowitsch   if (hasPrec) PetscCall(PetscArrayzero(elemMatP, numCells*totDim*totDim));
53919566063dSJacob Faibussowitsch   if (hasDyn)  PetscCall(PetscArrayzero(elemMatD, numCells*totDim*totDim));
53923e9753d6SMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
53933e9753d6SMatthew G. Knepley     PetscClassId    id;
53943e9753d6SMatthew G. Knepley     PetscFE         fe;
53953e9753d6SMatthew G. Knepley     PetscQuadrature qGeom = NULL;
53963e9753d6SMatthew G. Knepley     PetscInt        Nb;
53973e9753d6SMatthew G. Knepley     /* Conforming batches */
53983e9753d6SMatthew G. Knepley     PetscInt        numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
53993e9753d6SMatthew G. Knepley     /* Remainder */
54003e9753d6SMatthew G. Knepley     PetscInt        Nr, offset, Nq;
54013e9753d6SMatthew G. Knepley     PetscInt        maxDegree;
54023e9753d6SMatthew G. Knepley     PetscFEGeom     *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
54033e9753d6SMatthew G. Knepley 
54049566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
54059566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetClassId((PetscObject) fe, &id));
54063e9753d6SMatthew G. Knepley     if (id == PETSCFV_CLASSID) {hasFV = PETSC_TRUE; continue;}
54079566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
54089566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
54099566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
54103e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
54119566063dSJacob Faibussowitsch       PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&qGeom));
54123e9753d6SMatthew G. Knepley     }
54133e9753d6SMatthew G. Knepley     if (!qGeom) {
54149566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe,&qGeom));
54159566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
54163e9753d6SMatthew G. Knepley     }
54179566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
54189566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM));
54193e9753d6SMatthew G. Knepley     blockSize = Nb;
54203e9753d6SMatthew G. Knepley     batchSize = numBlocks * blockSize;
54219566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
54223e9753d6SMatthew G. Knepley     numChunks = numCells / (numBatches*batchSize);
54233e9753d6SMatthew G. Knepley     Ne        = numChunks*numBatches*batchSize;
54243e9753d6SMatthew G. Knepley     Nr        = numCells % (numBatches*batchSize);
54253e9753d6SMatthew G. Knepley     offset    = numCells - Nr;
54269566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom));
54279566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&remGeom));
54283e9753d6SMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
54296528b96dSMatthew G. Knepley       key.field = fieldI*Nf+fieldJ;
54303e9753d6SMatthew G. Knepley       if (hasJac) {
54319566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
54329566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMat[offset*totDim*totDim]));
54333e9753d6SMatthew G. Knepley       }
54343e9753d6SMatthew G. Knepley       if (hasPrec) {
54359566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatP));
54369566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_PRE, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatP[offset*totDim*totDim]));
54373e9753d6SMatthew G. Knepley       }
54383e9753d6SMatthew G. Knepley       if (hasDyn) {
54399566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
54409566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatD[offset*totDim*totDim]));
54413e9753d6SMatthew G. Knepley       }
54423e9753d6SMatthew G. Knepley     }
54439566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&remGeom));
54449566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,0,offset,&chunkGeom));
54459566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM));
54469566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
54473e9753d6SMatthew G. Knepley   }
54483e9753d6SMatthew G. Knepley   /*   Add contribution from X_t */
54493e9753d6SMatthew G. Knepley   if (hasDyn) {for (c = 0; c < numCells*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];}
54503e9753d6SMatthew G. Knepley   if (hasFV) {
54513e9753d6SMatthew G. Knepley     PetscClassId id;
54523e9753d6SMatthew G. Knepley     PetscFV      fv;
54533e9753d6SMatthew G. Knepley     PetscInt     offsetI, NcI, NbI = 1, fc, f;
54543e9753d6SMatthew G. Knepley 
54553e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
54569566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fv));
54579566063dSJacob Faibussowitsch       PetscCall(PetscDSGetFieldOffset(prob, fieldI, &offsetI));
54589566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetClassId((PetscObject) fv, &id));
54593e9753d6SMatthew G. Knepley       if (id != PETSCFV_CLASSID) continue;
54603e9753d6SMatthew G. Knepley       /* Put in the identity */
54619566063dSJacob Faibussowitsch       PetscCall(PetscFVGetNumComponents(fv, &NcI));
54623e9753d6SMatthew G. Knepley       for (c = cStart; c < cEnd; ++c) {
54633e9753d6SMatthew G. Knepley         const PetscInt cind    = c - cStart;
54643e9753d6SMatthew G. Knepley         const PetscInt eOffset = cind*totDim*totDim;
54653e9753d6SMatthew G. Knepley         for (fc = 0; fc < NcI; ++fc) {
54663e9753d6SMatthew G. Knepley           for (f = 0; f < NbI; ++f) {
54673e9753d6SMatthew G. Knepley             const PetscInt i = offsetI + f*NcI+fc;
54683e9753d6SMatthew G. Knepley             if (hasPrec) {
54693e9753d6SMatthew G. Knepley               if (hasJac) {elemMat[eOffset+i*totDim+i] = 1.0;}
54703e9753d6SMatthew G. Knepley               elemMatP[eOffset+i*totDim+i] = 1.0;
54713e9753d6SMatthew G. Knepley             } else {elemMat[eOffset+i*totDim+i] = 1.0;}
54723e9753d6SMatthew G. Knepley           }
54733e9753d6SMatthew G. Knepley         }
54743e9753d6SMatthew G. Knepley       }
54753e9753d6SMatthew G. Knepley     }
54763e9753d6SMatthew G. Knepley     /* No allocated space for FV stuff, so ignore the zero entries */
54779566063dSJacob Faibussowitsch     PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_TRUE));
54783e9753d6SMatthew G. Knepley   }
54793e9753d6SMatthew G. Knepley   /* Insert values into matrix */
54803e9753d6SMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
54813e9753d6SMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
54823e9753d6SMatthew G. Knepley     const PetscInt cind = c - cStart;
54833e9753d6SMatthew G. Knepley 
54843e9753d6SMatthew G. Knepley     /* Transform to global basis before insertion in Jacobian */
54859566063dSJacob Faibussowitsch     if (transform) PetscCall(DMPlexBasisTransformPointTensor_Internal(dm, tdm, tv, cell, PETSC_TRUE, totDim, &elemMat[cind*totDim*totDim]));
54863e9753d6SMatthew G. Knepley     if (hasPrec) {
54873e9753d6SMatthew G. Knepley       if (hasJac) {
54889566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
54899566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(dm, section, globalSection, Jac, cell, &elemMat[cind*totDim*totDim], ADD_VALUES));
54903e9753d6SMatthew G. Knepley       }
54919566063dSJacob Faibussowitsch       if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind*totDim*totDim]));
54929566063dSJacob Faibussowitsch       PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES));
54933e9753d6SMatthew G. Knepley     } else {
54943e9753d6SMatthew G. Knepley       if (hasJac) {
54959566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
54969566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(dm, section, globalSection, JacP, cell, &elemMat[cind*totDim*totDim], ADD_VALUES));
54973e9753d6SMatthew G. Knepley       }
54983e9753d6SMatthew G. Knepley     }
54993e9753d6SMatthew G. Knepley   }
55009566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
55019566063dSJacob Faibussowitsch   if (hasFV) PetscCall(MatSetOption(JacP, MAT_IGNORE_ZERO_ENTRIES, PETSC_FALSE));
55029566063dSJacob Faibussowitsch   PetscCall(PetscFree5(u,u_t,elemMat,elemMatP,elemMatD));
55033e9753d6SMatthew G. Knepley   if (dmAux) {
55049566063dSJacob Faibussowitsch     PetscCall(PetscFree(a));
55059566063dSJacob Faibussowitsch     PetscCall(DMDestroy(&plex));
55063e9753d6SMatthew G. Knepley   }
55073e9753d6SMatthew G. Knepley   /* Compute boundary integrals */
55089566063dSJacob Faibussowitsch   PetscCall(DMPlexComputeBdJacobian_Internal(dm, X, X_t, t, X_tShift, Jac, JacP, user));
55093e9753d6SMatthew G. Knepley   /* Assemble matrix */
55103e9753d6SMatthew G. Knepley   if (hasJac && hasPrec) {
55119566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
55129566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
55133e9753d6SMatthew G. Knepley   }
55149566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
55159566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
55169566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0));
55173e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
55183e9753d6SMatthew G. Knepley }
55193e9753d6SMatthew G. Knepley 
552006ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Hybrid_Internal(DM dm, PetscFormKey key[], IS cellIS, PetscReal t, PetscReal X_tShift, Vec locX, Vec locX_t, Mat Jac, Mat JacP, void *user)
55213e9753d6SMatthew G. Knepley {
55223e9753d6SMatthew G. Knepley   DM_Plex         *mesh          = (DM_Plex *) dm->data;
55233e9753d6SMatthew G. Knepley   const char      *name          = "Hybrid Jacobian";
5524148442b3SMatthew G. Knepley   DM               dmAux[3]      = {NULL, NULL, NULL};
5525148442b3SMatthew G. Knepley   DMLabel          ghostLabel    = NULL;
55263e9753d6SMatthew G. Knepley   DM               plex          = NULL;
55273e9753d6SMatthew G. Knepley   DM               plexA         = NULL;
5528148442b3SMatthew G. Knepley   PetscDS          ds            = NULL;
5529148442b3SMatthew G. Knepley   PetscDS          dsAux[3]      = {NULL, NULL, NULL};
5530148442b3SMatthew G. Knepley   Vec              locA[3]       = {NULL, NULL, NULL};
55313e9753d6SMatthew G. Knepley   PetscSection     section       = NULL;
5532148442b3SMatthew G. Knepley   PetscSection     sectionAux[3] = {NULL, NULL, NULL};
55333e9753d6SMatthew G. Knepley   DMField          coordField    = NULL;
5534148442b3SMatthew G. Knepley   PetscScalar     *u = NULL, *u_t, *a[3];
55353e9753d6SMatthew G. Knepley   PetscScalar     *elemMat, *elemMatP;
5536e432b41dSStefano Zampini   PetscSection     globalSection;
55373e9753d6SMatthew G. Knepley   IS               chunkIS;
55383e9753d6SMatthew G. Knepley   const PetscInt  *cells;
55393e9753d6SMatthew G. Knepley   PetscInt        *faces;
55403e9753d6SMatthew G. Knepley   PetscInt         cStart, cEnd, numCells;
5541148442b3SMatthew G. Knepley   PetscInt         Nf, fieldI, fieldJ, totDim, totDimAux[3], numChunks, cellChunkSize, chunk;
55423e9753d6SMatthew G. Knepley   PetscInt         maxDegree = PETSC_MAX_INT;
55433e9753d6SMatthew G. Knepley   PetscQuadrature  affineQuad = NULL, *quads = NULL;
55443e9753d6SMatthew G. Knepley   PetscFEGeom     *affineGeom = NULL, **geoms = NULL;
5545e432b41dSStefano Zampini   PetscBool        hasBdJac, hasBdPrec;
55463e9753d6SMatthew G. Knepley 
55473e9753d6SMatthew G. Knepley   PetscFunctionBegin;
55485fedec97SMatthew G. Knepley   if ((key[0].label == key[1].label) && (key[0].value == key[1].value) && (key[0].part == key[1].part)) {
55495fedec97SMatthew G. Knepley     const char *name;
55509566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetName((PetscObject) key[0].label, &name));
555163a3b9bcSJacob Faibussowitsch     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Form keys for each side of a cohesive surface must be different (%s, %" PetscInt_FMT ", %" PetscInt_FMT ")", name, key[0].value, key[0].part);
55525fedec97SMatthew G. Knepley   }
55539566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0));
55549566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
55559566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
55569566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
55579566063dSJacob Faibussowitsch   PetscCall(DMGetSection(dm, &section));
55589566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
55599566063dSJacob Faibussowitsch   PetscCall(DMGetLabel(dm, "ghost", &ghostLabel));
55609566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cStart, &ds));
55619566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(ds, &Nf));
55629566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(ds, &totDim));
55639566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobian(ds, &hasBdJac));
55649566063dSJacob Faibussowitsch   PetscCall(PetscDSHasBdJacobianPreconditioner(ds, &hasBdPrec));
55659566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key[2].label, key[2].value, key[2].part, &locA[2]));
5566148442b3SMatthew G. Knepley   if (locA[2]) {
55679566063dSJacob Faibussowitsch     PetscCall(VecGetDM(locA[2], &dmAux[2]));
55689566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux[2], DMPLEX, &plexA));
55699566063dSJacob Faibussowitsch     PetscCall(DMGetSection(dmAux[2], &sectionAux[2]));
55709566063dSJacob Faibussowitsch     PetscCall(DMGetCellDS(dmAux[2], cStart, &dsAux[2]));
55719566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(dsAux[2], &totDimAux[2]));
5572148442b3SMatthew G. Knepley     {
5573148442b3SMatthew G. Knepley       const PetscInt *cone;
5574148442b3SMatthew G. Knepley       PetscInt        c;
5575148442b3SMatthew G. Knepley 
55769566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(dm, cStart, &cone));
5577148442b3SMatthew G. Knepley       for (c = 0; c < 2; ++c) {
5578148442b3SMatthew G. Knepley         const PetscInt *support;
5579148442b3SMatthew G. Knepley         PetscInt ssize, s;
5580148442b3SMatthew G. Knepley 
55819566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupport(dm, cone[c], &support));
55829566063dSJacob Faibussowitsch         PetscCall(DMPlexGetSupportSize(dm, cone[c], &ssize));
558363a3b9bcSJacob Faibussowitsch         PetscCheck(ssize == 2,PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " from cell %" PetscInt_FMT " has support size %" PetscInt_FMT " != 2", cone[c], cStart, ssize);
5584148442b3SMatthew G. Knepley         if      (support[0] == cStart) s = 1;
5585148442b3SMatthew G. Knepley         else if (support[1] == cStart) s = 0;
558663a3b9bcSJacob Faibussowitsch         else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Face %" PetscInt_FMT " does not have cell %" PetscInt_FMT " in its support", cone[c], cStart);
55879566063dSJacob Faibussowitsch         PetscCall(DMGetAuxiliaryVec(dm, key[c].label, key[c].value, key[c].part, &locA[c]));
55889566063dSJacob Faibussowitsch         if (locA[c]) PetscCall(VecGetDM(locA[c], &dmAux[c]));
5589148442b3SMatthew G. Knepley         else         {dmAux[c] = dmAux[2];}
55909566063dSJacob Faibussowitsch         PetscCall(DMGetCellDS(dmAux[c], support[s], &dsAux[c]));
55919566063dSJacob Faibussowitsch         PetscCall(PetscDSGetTotalDimension(dsAux[c], &totDimAux[c]));
5592148442b3SMatthew G. Knepley       }
5593148442b3SMatthew G. Knepley     }
55943e9753d6SMatthew G. Knepley   }
55959566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
55969566063dSJacob Faibussowitsch   PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
55973e9753d6SMatthew G. Knepley   if (maxDegree > 1) {
55983e9753d6SMatthew G. Knepley     PetscInt f;
55999566063dSJacob Faibussowitsch     PetscCall(PetscCalloc2(Nf, &quads, Nf, &geoms));
56003e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
56013e9753d6SMatthew G. Knepley       PetscFE fe;
56023e9753d6SMatthew G. Knepley 
56039566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *) &fe));
56043e9753d6SMatthew G. Knepley       if (fe) {
56059566063dSJacob Faibussowitsch         PetscCall(PetscFEGetQuadrature(fe, &quads[f]));
56069566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) quads[f]));
56073e9753d6SMatthew G. Knepley       }
56083e9753d6SMatthew G. Knepley     }
56093e9753d6SMatthew G. Knepley   }
56103e9753d6SMatthew G. Knepley   cellChunkSize = numCells;
56113e9753d6SMatthew G. Knepley   numChunks     = !numCells ? 0 : PetscCeilReal(((PetscReal) numCells)/cellChunkSize);
56129566063dSJacob Faibussowitsch   PetscCall(PetscCalloc1(2*cellChunkSize, &faces));
56139566063dSJacob Faibussowitsch   PetscCall(ISCreateGeneral(PETSC_COMM_SELF, cellChunkSize, faces, PETSC_USE_POINTER, &chunkIS));
56149566063dSJacob Faibussowitsch   PetscCall(DMPlexGetCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
56159566063dSJacob Faibussowitsch   PetscCall(DMPlexGetHybridAuxFields(dm, dmAux, dsAux, cellIS, locA, a));
56169566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, hasBdJac  ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMat));
56179566063dSJacob Faibussowitsch   PetscCall(DMGetWorkArray(dm, hasBdPrec ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMatP));
56183e9753d6SMatthew G. Knepley   for (chunk = 0; chunk < numChunks; ++chunk) {
56193e9753d6SMatthew G. Knepley     PetscInt cS = cStart+chunk*cellChunkSize, cE = PetscMin(cS+cellChunkSize, cEnd), numCells = cE - cS, c;
56203e9753d6SMatthew G. Knepley 
56219566063dSJacob Faibussowitsch     if (hasBdJac)  PetscCall(PetscMemzero(elemMat,  numCells*totDim*totDim * sizeof(PetscScalar)));
56229566063dSJacob Faibussowitsch     if (hasBdPrec) PetscCall(PetscMemzero(elemMatP, numCells*totDim*totDim * sizeof(PetscScalar)));
56233e9753d6SMatthew G. Knepley     /* Get faces */
56243e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
56253e9753d6SMatthew G. Knepley       const PetscInt  cell = cells ? cells[c] : c;
56263e9753d6SMatthew G. Knepley       const PetscInt *cone;
56279566063dSJacob Faibussowitsch       PetscCall(DMPlexGetCone(plex, cell, &cone));
56283e9753d6SMatthew G. Knepley       faces[(c-cS)*2+0] = cone[0];
56293e9753d6SMatthew G. Knepley       faces[(c-cS)*2+1] = cone[1];
56303e9753d6SMatthew G. Knepley     }
56319566063dSJacob Faibussowitsch     PetscCall(ISGeneralSetIndices(chunkIS, cellChunkSize, faces, PETSC_USE_POINTER));
56323e9753d6SMatthew G. Knepley     if (maxDegree <= 1) {
56339566063dSJacob Faibussowitsch       if (!affineQuad) PetscCall(DMFieldCreateDefaultQuadrature(coordField, chunkIS, &affineQuad));
56349566063dSJacob Faibussowitsch       if (affineQuad)  PetscCall(DMSNESGetFEGeom(coordField, chunkIS, affineQuad, PETSC_TRUE, &affineGeom));
56353e9753d6SMatthew G. Knepley     } else {
56363e9753d6SMatthew G. Knepley       PetscInt f;
56373e9753d6SMatthew G. Knepley       for (f = 0; f < Nf; ++f) {
56389566063dSJacob Faibussowitsch         if (quads[f]) PetscCall(DMSNESGetFEGeom(coordField, chunkIS, quads[f], PETSC_TRUE, &geoms[f]));
56393e9753d6SMatthew G. Knepley       }
56403e9753d6SMatthew G. Knepley     }
56413e9753d6SMatthew G. Knepley 
56423e9753d6SMatthew G. Knepley     for (fieldI = 0; fieldI < Nf; ++fieldI) {
56433e9753d6SMatthew G. Knepley       PetscFE         feI;
56443e9753d6SMatthew G. Knepley       PetscFEGeom    *geom = affineGeom ? affineGeom : geoms[fieldI];
56453e9753d6SMatthew G. Knepley       PetscFEGeom    *chunkGeom = NULL, *remGeom = NULL;
56463e9753d6SMatthew G. Knepley       PetscQuadrature quad = affineQuad ? affineQuad : quads[fieldI];
56473e9753d6SMatthew G. Knepley       PetscInt        numChunks, numBatches, batchSize, numBlocks, blockSize, Ne, Nr, offset, Nq, Nb;
56485fedec97SMatthew G. Knepley       PetscBool       isCohesiveField;
56493e9753d6SMatthew G. Knepley 
56509566063dSJacob Faibussowitsch       PetscCall(PetscDSGetDiscretization(ds, fieldI, (PetscObject *) &feI));
56513e9753d6SMatthew G. Knepley       if (!feI) continue;
56529566063dSJacob Faibussowitsch       PetscCall(PetscFEGetTileSizes(feI, NULL, &numBlocks, NULL, &numBatches));
56539566063dSJacob Faibussowitsch       PetscCall(PetscQuadratureGetData(quad, NULL, NULL, &Nq, NULL, NULL));
56549566063dSJacob Faibussowitsch       PetscCall(PetscFEGetDimension(feI, &Nb));
56553e9753d6SMatthew G. Knepley       blockSize = Nb;
56563e9753d6SMatthew G. Knepley       batchSize = numBlocks * blockSize;
56579566063dSJacob Faibussowitsch       PetscCall(PetscFESetTileSizes(feI, blockSize, numBlocks, batchSize, numBatches));
56583e9753d6SMatthew G. Knepley       numChunks = numCells / (numBatches*batchSize);
56593e9753d6SMatthew G. Knepley       Ne        = numChunks*numBatches*batchSize;
56603e9753d6SMatthew G. Knepley       Nr        = numCells % (numBatches*batchSize);
56613e9753d6SMatthew G. Knepley       offset    = numCells - Nr;
56629566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom,0,offset,&chunkGeom));
56639566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomGetChunk(geom,offset,numCells,&remGeom));
56649566063dSJacob Faibussowitsch       PetscCall(PetscDSGetCohesive(ds, fieldI, &isCohesiveField));
56653e9753d6SMatthew G. Knepley       for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
56663e9753d6SMatthew G. Knepley         PetscFE feJ;
56673e9753d6SMatthew G. Knepley 
56689566063dSJacob Faibussowitsch         PetscCall(PetscDSGetDiscretization(ds, fieldJ, (PetscObject *) &feJ));
56693e9753d6SMatthew G. Knepley         if (!feJ) continue;
5670148442b3SMatthew G. Knepley         key[0].field = fieldI*Nf+fieldJ;
5671148442b3SMatthew G. Knepley         key[1].field = fieldI*Nf+fieldJ;
56725fedec97SMatthew G. Knepley         key[2].field = fieldI*Nf+fieldJ;
5673148442b3SMatthew G. Knepley         if (hasBdJac) {
56749566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMat));
56759566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[0], 0, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[0], &a[0][offset*totDimAux[0]], t, X_tShift, &elemMat[offset*totDim*totDim]));
56769566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMat));
56779566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[1], 1, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[1], &a[1][offset*totDimAux[1]], t, X_tShift, &elemMat[offset*totDim*totDim]));
5678148442b3SMatthew G. Knepley         }
5679148442b3SMatthew G. Knepley         if (hasBdPrec) {
56809566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[0], 0, Ne, chunkGeom, u, u_t, dsAux[0], a[0], t, X_tShift, elemMatP));
56819566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[0], 0, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[0], &a[0][offset*totDimAux[0]], t, X_tShift, &elemMatP[offset*totDim*totDim]));
56829566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[1], 1, Ne, chunkGeom, u, u_t, dsAux[1], a[1], t, X_tShift, elemMatP));
56839566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[1], 1, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[1], &a[1][offset*totDimAux[1]], t, X_tShift, &elemMatP[offset*totDim*totDim]));
5684148442b3SMatthew G. Knepley         }
56855fedec97SMatthew G. Knepley         if (hasBdJac) {
56869566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMat));
56879566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN, key[2], 2, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[2], &a[2][offset*totDimAux[2]], t, X_tShift, &elemMat[offset*totDim*totDim]));
5688148442b3SMatthew G. Knepley         }
56895fedec97SMatthew G. Knepley         if (hasBdPrec) {
56909566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[2], 2, Ne, chunkGeom, u, u_t, dsAux[2], a[2], t, X_tShift, elemMatP));
56919566063dSJacob Faibussowitsch           PetscCall(PetscFEIntegrateHybridJacobian(ds, PETSCFE_JACOBIAN_PRE, key[2], 2, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, dsAux[2], &a[2][offset*totDimAux[2]], t, X_tShift, &elemMatP[offset*totDim*totDim]));
56923e9753d6SMatthew G. Knepley         }
56933e9753d6SMatthew G. Knepley       }
56949566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom,offset,numCells,&remGeom));
56959566063dSJacob Faibussowitsch       PetscCall(PetscFEGeomRestoreChunk(geom,0,offset,&chunkGeom));
56963e9753d6SMatthew G. Knepley     }
56973e9753d6SMatthew G. Knepley     /* Insert values into matrix */
56983e9753d6SMatthew G. Knepley     for (c = cS; c < cE; ++c) {
56993e9753d6SMatthew G. Knepley       const PetscInt cell = cells ? cells[c] : c;
57003e9753d6SMatthew G. Knepley       const PetscInt cind = c - cS;
57013e9753d6SMatthew G. Knepley 
57023e9753d6SMatthew G. Knepley       if (hasBdPrec) {
57033e9753d6SMatthew G. Knepley         if (hasBdJac) {
57049566063dSJacob Faibussowitsch           if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
57059566063dSJacob Faibussowitsch           PetscCall(DMPlexMatSetClosure(plex, section, globalSection, Jac, cell, &elemMat[cind*totDim*totDim], ADD_VALUES));
57063e9753d6SMatthew G. Knepley         }
57079566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMatP[cind*totDim*totDim]));
57089566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMatP[cind*totDim*totDim], ADD_VALUES));
57093e9753d6SMatthew G. Knepley       } else if (hasBdJac) {
57109566063dSJacob Faibussowitsch         if (mesh->printFEM > 1) PetscCall(DMPrintCellMatrix(cell, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
57119566063dSJacob Faibussowitsch         PetscCall(DMPlexMatSetClosure(plex, section, globalSection, JacP, cell, &elemMat[cind*totDim*totDim], ADD_VALUES));
57123e9753d6SMatthew G. Knepley       }
57133e9753d6SMatthew G. Knepley     }
57143e9753d6SMatthew G. Knepley   }
57159566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreCellFields(dm, cellIS, locX, locX_t, locA[2], &u, &u_t, &a[2]));
57169566063dSJacob Faibussowitsch   PetscCall(DMPlexRestoreHybridAuxFields(dmAux, dsAux, cellIS, locA, a));
57179566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, hasBdJac  ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMat));
57189566063dSJacob Faibussowitsch   PetscCall(DMRestoreWorkArray(dm, hasBdPrec ? cellChunkSize*totDim*totDim : 0, MPIU_SCALAR, &elemMatP));
57199566063dSJacob Faibussowitsch   PetscCall(PetscFree(faces));
57209566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&chunkIS));
57219566063dSJacob Faibussowitsch   PetscCall(ISRestorePointRange(cellIS, &cStart, &cEnd, &cells));
57223e9753d6SMatthew G. Knepley   if (maxDegree <= 1) {
57239566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,affineQuad,PETSC_FALSE,&affineGeom));
57249566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&affineQuad));
57253e9753d6SMatthew G. Knepley   } else {
57263e9753d6SMatthew G. Knepley     PetscInt f;
57273e9753d6SMatthew G. Knepley     for (f = 0; f < Nf; ++f) {
57289566063dSJacob Faibussowitsch       if (geoms) PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,quads[f],PETSC_FALSE, &geoms[f]));
57299566063dSJacob Faibussowitsch       if (quads) PetscCall(PetscQuadratureDestroy(&quads[f]));
57303e9753d6SMatthew G. Knepley     }
57319566063dSJacob Faibussowitsch     PetscCall(PetscFree2(quads,geoms));
57323e9753d6SMatthew G. Knepley   }
57339566063dSJacob Faibussowitsch   if (dmAux[2]) PetscCall(DMDestroy(&plexA));
57349566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
57353e9753d6SMatthew G. Knepley   /* Assemble matrix */
57363e9753d6SMatthew G. Knepley   if (hasBdJac && hasBdPrec) {
57379566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(Jac, MAT_FINAL_ASSEMBLY));
57389566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(Jac, MAT_FINAL_ASSEMBLY));
57393e9753d6SMatthew G. Knepley   }
57409566063dSJacob Faibussowitsch   PetscCall(MatAssemblyBegin(JacP, MAT_FINAL_ASSEMBLY));
57419566063dSJacob Faibussowitsch   PetscCall(MatAssemblyEnd(JacP, MAT_FINAL_ASSEMBLY));
57429566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0));
57433e9753d6SMatthew G. Knepley   PetscFunctionReturn(0);
57443e9753d6SMatthew G. Knepley }
57458e3a2eefSMatthew G. Knepley 
57468e3a2eefSMatthew G. Knepley /*
57478e3a2eefSMatthew G. Knepley   DMPlexComputeJacobian_Action_Internal - Form the local portion of the Jacobian action Z = J(X) Y at the local solution X using pointwise functions specified by the user.
57488e3a2eefSMatthew G. Knepley 
57498e3a2eefSMatthew G. Knepley   Input Parameters:
57508e3a2eefSMatthew G. Knepley + dm     - The mesh
57518e3a2eefSMatthew G. Knepley . key    - The PetscWeakFormKey indcating where integration should happen
575206ad1575SMatthew G. Knepley . cellIS - The cells to integrate over
57538e3a2eefSMatthew G. Knepley . t      - The time
57548e3a2eefSMatthew G. Knepley . X_tShift - The multiplier for the Jacobian with repsect to X_t
57558e3a2eefSMatthew G. Knepley . X      - Local solution vector
57568e3a2eefSMatthew G. Knepley . X_t    - Time-derivative of the local solution vector
57578e3a2eefSMatthew G. Knepley . Y      - Local input vector
575806ad1575SMatthew G. Knepley - user   - the user context
57598e3a2eefSMatthew G. Knepley 
57608e3a2eefSMatthew G. Knepley   Output Parameter:
57618e3a2eefSMatthew G. Knepley . Z - Local output vector
57628e3a2eefSMatthew G. Knepley 
57638e3a2eefSMatthew G. Knepley   Note:
57648e3a2eefSMatthew G. Knepley   We form the residual one batch of elements at a time. This allows us to offload work onto an accelerator,
57658e3a2eefSMatthew G. Knepley   like a GPU, or vectorize on a multicore machine.
57668e3a2eefSMatthew G. Knepley */
576706ad1575SMatthew G. Knepley PetscErrorCode DMPlexComputeJacobian_Action_Internal(DM dm, PetscFormKey key, IS cellIS, PetscReal t, PetscReal X_tShift, Vec X, Vec X_t, Vec Y, Vec Z, void *user)
57688e3a2eefSMatthew G. Knepley {
57698e3a2eefSMatthew G. Knepley   DM_Plex        *mesh  = (DM_Plex *) dm->data;
57708e3a2eefSMatthew G. Knepley   const char     *name  = "Jacobian";
57718e3a2eefSMatthew G. Knepley   DM              dmAux = NULL, plex, plexAux = NULL;
57728e3a2eefSMatthew G. Knepley   DMEnclosureType encAux;
57738e3a2eefSMatthew G. Knepley   Vec             A;
57748e3a2eefSMatthew G. Knepley   DMField         coordField;
57758e3a2eefSMatthew G. Knepley   PetscDS         prob, probAux = NULL;
57768e3a2eefSMatthew G. Knepley   PetscQuadrature quad;
57778e3a2eefSMatthew G. Knepley   PetscSection    section, globalSection, sectionAux;
57788e3a2eefSMatthew G. Knepley   PetscScalar    *elemMat, *elemMatD, *u, *u_t, *a = NULL, *y, *z;
57798e3a2eefSMatthew G. Knepley   const PetscInt *cells;
57808e3a2eefSMatthew G. Knepley   PetscInt        Nf, fieldI, fieldJ;
57818e3a2eefSMatthew G. Knepley   PetscInt        totDim, totDimAux = 0, cStart, cEnd, numCells, c;
57828e3a2eefSMatthew G. Knepley   PetscBool       hasDyn;
57838e3a2eefSMatthew G. Knepley 
57848e3a2eefSMatthew G. Knepley   PetscFunctionBegin;
57859566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(DMPLEX_JacobianFEM,dm,0,0,0));
57869566063dSJacob Faibussowitsch   PetscCall(DMConvert(dm, DMPLEX, &plex));
57878e3a2eefSMatthew G. Knepley   if (!cellIS) {
57888e3a2eefSMatthew G. Knepley     PetscInt depth;
57898e3a2eefSMatthew G. Knepley 
57909566063dSJacob Faibussowitsch     PetscCall(DMPlexGetDepth(plex, &depth));
57919566063dSJacob Faibussowitsch     PetscCall(DMGetStratumIS(plex, "dim", depth, &cellIS));
57929566063dSJacob Faibussowitsch     if (!cellIS) PetscCall(DMGetStratumIS(plex, "depth", depth, &cellIS));
57938e3a2eefSMatthew G. Knepley   } else {
57949566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject) cellIS));
57958e3a2eefSMatthew G. Knepley   }
57969566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(cellIS, &numCells));
57979566063dSJacob Faibussowitsch   PetscCall(ISGetPointRange(cellIS, &cStart, &cEnd, &cells));
57989566063dSJacob Faibussowitsch   PetscCall(DMGetLocalSection(dm, &section));
57999566063dSJacob Faibussowitsch   PetscCall(DMGetGlobalSection(dm, &globalSection));
58009566063dSJacob Faibussowitsch   PetscCall(DMGetCellDS(dm, cells ? cells[cStart] : cStart, &prob));
58019566063dSJacob Faibussowitsch   PetscCall(PetscDSGetNumFields(prob, &Nf));
58029566063dSJacob Faibussowitsch   PetscCall(PetscDSGetTotalDimension(prob, &totDim));
58039566063dSJacob Faibussowitsch   PetscCall(PetscDSHasDynamicJacobian(prob, &hasDyn));
58048e3a2eefSMatthew G. Knepley   hasDyn = hasDyn && (X_tShift != 0.0) ? PETSC_TRUE : PETSC_FALSE;
58059566063dSJacob Faibussowitsch   PetscCall(DMGetAuxiliaryVec(dm, key.label, key.value, key.part, &A));
58068e3a2eefSMatthew G. Knepley   if (A) {
58079566063dSJacob Faibussowitsch     PetscCall(VecGetDM(A, &dmAux));
58089566063dSJacob Faibussowitsch     PetscCall(DMGetEnclosureRelation(dmAux, dm, &encAux));
58099566063dSJacob Faibussowitsch     PetscCall(DMConvert(dmAux, DMPLEX, &plexAux));
58109566063dSJacob Faibussowitsch     PetscCall(DMGetLocalSection(plexAux, &sectionAux));
58119566063dSJacob Faibussowitsch     PetscCall(DMGetDS(dmAux, &probAux));
58129566063dSJacob Faibussowitsch     PetscCall(PetscDSGetTotalDimension(probAux, &totDimAux));
58138e3a2eefSMatthew G. Knepley   }
58149566063dSJacob Faibussowitsch   PetscCall(VecSet(Z, 0.0));
58159566063dSJacob Faibussowitsch   PetscCall(PetscMalloc6(numCells*totDim,&u,X_t ? numCells*totDim : 0,&u_t,numCells*totDim*totDim,&elemMat,hasDyn ? numCells*totDim*totDim : 0, &elemMatD,numCells*totDim,&y,totDim,&z));
58169566063dSJacob Faibussowitsch   if (dmAux) PetscCall(PetscMalloc1(numCells*totDimAux, &a));
58179566063dSJacob Faibussowitsch   PetscCall(DMGetCoordinateField(dm, &coordField));
58188e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
58198e3a2eefSMatthew G. Knepley     const PetscInt cell = cells ? cells[c] : c;
58208e3a2eefSMatthew G. Knepley     const PetscInt cind = c - cStart;
58218e3a2eefSMatthew G. Knepley     PetscScalar   *x = NULL,  *x_t = NULL;
58228e3a2eefSMatthew G. Knepley     PetscInt       i;
58238e3a2eefSMatthew G. Knepley 
58249566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, X, cell, NULL, &x));
58258e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) u[cind*totDim+i] = x[i];
58269566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, X, cell, NULL, &x));
58278e3a2eefSMatthew G. Knepley     if (X_t) {
58289566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plex, section, X_t, cell, NULL, &x_t));
58298e3a2eefSMatthew G. Knepley       for (i = 0; i < totDim; ++i) u_t[cind*totDim+i] = x_t[i];
58309566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plex, section, X_t, cell, NULL, &x_t));
58318e3a2eefSMatthew G. Knepley     }
58328e3a2eefSMatthew G. Knepley     if (dmAux) {
58338e3a2eefSMatthew G. Knepley       PetscInt subcell;
58349566063dSJacob Faibussowitsch       PetscCall(DMGetEnclosurePoint(dmAux, dm, encAux, cell, &subcell));
58359566063dSJacob Faibussowitsch       PetscCall(DMPlexVecGetClosure(plexAux, sectionAux, A, subcell, NULL, &x));
58368e3a2eefSMatthew G. Knepley       for (i = 0; i < totDimAux; ++i) a[cind*totDimAux+i] = x[i];
58379566063dSJacob Faibussowitsch       PetscCall(DMPlexVecRestoreClosure(plexAux, sectionAux, A, subcell, NULL, &x));
58388e3a2eefSMatthew G. Knepley     }
58399566063dSJacob Faibussowitsch     PetscCall(DMPlexVecGetClosure(plex, section, Y, cell, NULL, &x));
58408e3a2eefSMatthew G. Knepley     for (i = 0; i < totDim; ++i) y[cind*totDim+i] = x[i];
58419566063dSJacob Faibussowitsch     PetscCall(DMPlexVecRestoreClosure(plex, section, Y, cell, NULL, &x));
58428e3a2eefSMatthew G. Knepley   }
58439566063dSJacob Faibussowitsch   PetscCall(PetscArrayzero(elemMat, numCells*totDim*totDim));
58449566063dSJacob Faibussowitsch   if (hasDyn)  PetscCall(PetscArrayzero(elemMatD, numCells*totDim*totDim));
58458e3a2eefSMatthew G. Knepley   for (fieldI = 0; fieldI < Nf; ++fieldI) {
58468e3a2eefSMatthew G. Knepley     PetscFE  fe;
58478e3a2eefSMatthew G. Knepley     PetscInt Nb;
58488e3a2eefSMatthew G. Knepley     /* Conforming batches */
58498e3a2eefSMatthew G. Knepley     PetscInt numChunks, numBatches, numBlocks, Ne, blockSize, batchSize;
58508e3a2eefSMatthew G. Knepley     /* Remainder */
58518e3a2eefSMatthew G. Knepley     PetscInt Nr, offset, Nq;
58528e3a2eefSMatthew G. Knepley     PetscQuadrature qGeom = NULL;
58538e3a2eefSMatthew G. Knepley     PetscInt    maxDegree;
58548e3a2eefSMatthew G. Knepley     PetscFEGeom *cgeomFEM, *chunkGeom = NULL, *remGeom = NULL;
58558e3a2eefSMatthew G. Knepley 
58569566063dSJacob Faibussowitsch     PetscCall(PetscDSGetDiscretization(prob, fieldI, (PetscObject *) &fe));
58579566063dSJacob Faibussowitsch     PetscCall(PetscFEGetQuadrature(fe, &quad));
58589566063dSJacob Faibussowitsch     PetscCall(PetscFEGetDimension(fe, &Nb));
58599566063dSJacob Faibussowitsch     PetscCall(PetscFEGetTileSizes(fe, NULL, &numBlocks, NULL, &numBatches));
58609566063dSJacob Faibussowitsch     PetscCall(DMFieldGetDegree(coordField,cellIS,NULL,&maxDegree));
58619566063dSJacob Faibussowitsch     if (maxDegree <= 1) PetscCall(DMFieldCreateDefaultQuadrature(coordField,cellIS,&qGeom));
58628e3a2eefSMatthew G. Knepley     if (!qGeom) {
58639566063dSJacob Faibussowitsch       PetscCall(PetscFEGetQuadrature(fe,&qGeom));
58649566063dSJacob Faibussowitsch       PetscCall(PetscObjectReference((PetscObject)qGeom));
58658e3a2eefSMatthew G. Knepley     }
58669566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureGetData(qGeom, NULL, NULL, &Nq, NULL, NULL));
58679566063dSJacob Faibussowitsch     PetscCall(DMSNESGetFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM));
58688e3a2eefSMatthew G. Knepley     blockSize = Nb;
58698e3a2eefSMatthew G. Knepley     batchSize = numBlocks * blockSize;
58709566063dSJacob Faibussowitsch     PetscCall(PetscFESetTileSizes(fe, blockSize, numBlocks, batchSize, numBatches));
58718e3a2eefSMatthew G. Knepley     numChunks = numCells / (numBatches*batchSize);
58728e3a2eefSMatthew G. Knepley     Ne        = numChunks*numBatches*batchSize;
58738e3a2eefSMatthew G. Knepley     Nr        = numCells % (numBatches*batchSize);
58748e3a2eefSMatthew G. Knepley     offset    = numCells - Nr;
58759566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM,0,offset,&chunkGeom));
58769566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomGetChunk(cgeomFEM,offset,numCells,&remGeom));
58778e3a2eefSMatthew G. Knepley     for (fieldJ = 0; fieldJ < Nf; ++fieldJ) {
58788e3a2eefSMatthew G. Knepley       key.field = fieldI*Nf + fieldJ;
58799566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMat));
58809566063dSJacob Faibussowitsch       PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMat[offset*totDim*totDim]));
58818e3a2eefSMatthew G. Knepley       if (hasDyn) {
58829566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Ne, chunkGeom, u, u_t, probAux, a, t, X_tShift, elemMatD));
58839566063dSJacob Faibussowitsch         PetscCall(PetscFEIntegrateJacobian(prob, PETSCFE_JACOBIAN_DYN, key, Nr, remGeom, &u[offset*totDim], u_t ? &u_t[offset*totDim] : NULL, probAux, &a[offset*totDimAux], t, X_tShift, &elemMatD[offset*totDim*totDim]));
58848e3a2eefSMatthew G. Knepley       }
58858e3a2eefSMatthew G. Knepley     }
58869566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,offset,numCells,&remGeom));
58879566063dSJacob Faibussowitsch     PetscCall(PetscFEGeomRestoreChunk(cgeomFEM,0,offset,&chunkGeom));
58889566063dSJacob Faibussowitsch     PetscCall(DMSNESRestoreFEGeom(coordField,cellIS,qGeom,PETSC_FALSE,&cgeomFEM));
58899566063dSJacob Faibussowitsch     PetscCall(PetscQuadratureDestroy(&qGeom));
58908e3a2eefSMatthew G. Knepley   }
58918e3a2eefSMatthew G. Knepley   if (hasDyn) {
58928e3a2eefSMatthew G. Knepley     for (c = 0; c < numCells*totDim*totDim; ++c) elemMat[c] += X_tShift*elemMatD[c];
58938e3a2eefSMatthew G. Knepley   }
58948e3a2eefSMatthew G. Knepley   for (c = cStart; c < cEnd; ++c) {
58958e3a2eefSMatthew G. Knepley     const PetscInt     cell = cells ? cells[c] : c;
58968e3a2eefSMatthew G. Knepley     const PetscInt     cind = c - cStart;
58978e3a2eefSMatthew G. Knepley     const PetscBLASInt M = totDim, one = 1;
58988e3a2eefSMatthew G. Knepley     const PetscScalar  a = 1.0, b = 0.0;
58998e3a2eefSMatthew G. Knepley 
59008e3a2eefSMatthew G. Knepley     PetscStackCallBLAS("BLASgemv", BLASgemv_("N", &M, &M, &a, &elemMat[cind*totDim*totDim], &M, &y[cind*totDim], &one, &b, z, &one));
59018e3a2eefSMatthew G. Knepley     if (mesh->printFEM > 1) {
59029566063dSJacob Faibussowitsch       PetscCall(DMPrintCellMatrix(c, name, totDim, totDim, &elemMat[cind*totDim*totDim]));
59039566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Y",  totDim, &y[cind*totDim]));
59049566063dSJacob Faibussowitsch       PetscCall(DMPrintCellVector(c, "Z",  totDim, z));
59058e3a2eefSMatthew G. Knepley     }
59069566063dSJacob Faibussowitsch     PetscCall(DMPlexVecSetClosure(dm, section, Z, cell, z, ADD_VALUES));
59078e3a2eefSMatthew G. Knepley   }
59089566063dSJacob Faibussowitsch   PetscCall(PetscFree6(u,u_t,elemMat,elemMatD,y,z));
59098e3a2eefSMatthew G. Knepley   if (mesh->printFEM) {
59109566063dSJacob Faibussowitsch     PetscCall(PetscPrintf(PetscObjectComm((PetscObject)Z), "Z:\n"));
59119566063dSJacob Faibussowitsch     PetscCall(VecView(Z, NULL));
59128e3a2eefSMatthew G. Knepley   }
59139566063dSJacob Faibussowitsch   PetscCall(PetscFree(a));
59149566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&cellIS));
59159566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plexAux));
59169566063dSJacob Faibussowitsch   PetscCall(DMDestroy(&plex));
59179566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(DMPLEX_JacobianFEM,dm,0,0,0));
59188e3a2eefSMatthew G. Knepley   PetscFunctionReturn(0);
59198e3a2eefSMatthew G. Knepley }
5920