xref: /petsc/src/ksp/pc/impls/redundant/redundant.c (revision d71ae5a4db6382e7f06317b8d368875286fe9008)
1dba47a55SKris Buschelman 
24b9ad928SBarry Smith /*
33f457be1SHong Zhang   This file defines a "solve the problem redundantly on each subgroup of processor" preconditioner.
44b9ad928SBarry Smith */
5af0996ceSBarry Smith #include <petsc/private/pcimpl.h>
6c6db04a5SJed Brown #include <petscksp.h> /*I "petscksp.h" I*/
74b9ad928SBarry Smith 
84b9ad928SBarry Smith typedef struct {
93e065800SHong Zhang   KSP                ksp;
104b9ad928SBarry Smith   PC                 pc;                    /* actual preconditioner used on each processor */
11ce94432eSBarry Smith   Vec                xsub, ysub;            /* vectors of a subcommunicator to hold parallel vectors of PetscObjectComm((PetscObject)pc) */
123f457be1SHong Zhang   Vec                xdup, ydup;            /* parallel vector that congregates xsub or ysub facilitating vector scattering */
13b3804887SHong Zhang   Mat                pmats;                 /* matrix and optional preconditioner matrix belong to a subcommunicator */
143f457be1SHong Zhang   VecScatter         scatterin, scatterout; /* scatter used to move all values to each processor group (subcommunicator) */
15ace3abfcSBarry Smith   PetscBool          useparallelmat;
16c540e29cSHong Zhang   PetscSubcomm       psubcomm;
171fbd8f88SHong Zhang   PetscInt           nsubcomm; /* num of data structure PetscSubcomm */
18753b7fb9SBarry Smith   PetscBool          shifttypeset;
19753b7fb9SBarry Smith   MatFactorShiftType shifttype;
204b9ad928SBarry Smith } PC_Redundant;
214b9ad928SBarry Smith 
22*d71ae5a4SJacob Faibussowitsch PetscErrorCode PCFactorSetShiftType_Redundant(PC pc, MatFactorShiftType shifttype)
23*d71ae5a4SJacob Faibussowitsch {
24753b7fb9SBarry Smith   PC_Redundant *red = (PC_Redundant *)pc->data;
25753b7fb9SBarry Smith 
26753b7fb9SBarry Smith   PetscFunctionBegin;
27753b7fb9SBarry Smith   if (red->ksp) {
28753b7fb9SBarry Smith     PC pc;
299566063dSJacob Faibussowitsch     PetscCall(KSPGetPC(red->ksp, &pc));
309566063dSJacob Faibussowitsch     PetscCall(PCFactorSetShiftType(pc, shifttype));
31753b7fb9SBarry Smith   } else {
32753b7fb9SBarry Smith     red->shifttypeset = PETSC_TRUE;
33753b7fb9SBarry Smith     red->shifttype    = shifttype;
34753b7fb9SBarry Smith   }
35753b7fb9SBarry Smith   PetscFunctionReturn(0);
36753b7fb9SBarry Smith }
37753b7fb9SBarry Smith 
38*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCView_Redundant(PC pc, PetscViewer viewer)
39*d71ae5a4SJacob Faibussowitsch {
404b9ad928SBarry Smith   PC_Redundant *red = (PC_Redundant *)pc->data;
41ace3abfcSBarry Smith   PetscBool     iascii, isstring;
4203ccd0b4SBarry Smith   PetscViewer   subviewer;
434b9ad928SBarry Smith 
444b9ad928SBarry Smith   PetscFunctionBegin;
459566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &iascii));
469566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERSTRING, &isstring));
4732077d6dSBarry Smith   if (iascii) {
4803ccd0b4SBarry Smith     if (!red->psubcomm) {
499566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer, "  Not yet setup\n"));
5003ccd0b4SBarry Smith     } else {
5163a3b9bcSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer, "  First (color=0) of %" PetscInt_FMT " PCs follows\n", red->nsubcomm));
529566063dSJacob Faibussowitsch       PetscCall(PetscViewerGetSubViewer(viewer, ((PetscObject)red->pc)->comm, &subviewer));
53f5dd71faSBarry Smith       if (!red->psubcomm->color) { /* only view first redundant pc */
549566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPushTab(subviewer));
559566063dSJacob Faibussowitsch         PetscCall(KSPView(red->ksp, subviewer));
569566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPopTab(subviewer));
574b9ad928SBarry Smith       }
589566063dSJacob Faibussowitsch       PetscCall(PetscViewerRestoreSubViewer(viewer, ((PetscObject)red->pc)->comm, &subviewer));
594b9ad928SBarry Smith     }
6003ccd0b4SBarry Smith   } else if (isstring) {
619566063dSJacob Faibussowitsch     PetscCall(PetscViewerStringSPrintf(viewer, " Redundant solver preconditioner"));
624b9ad928SBarry Smith   }
634b9ad928SBarry Smith   PetscFunctionReturn(0);
644b9ad928SBarry Smith }
654b9ad928SBarry Smith 
6619b3b6edSHong Zhang #include <../src/mat/impls/aij/mpi/mpiaij.h>
67*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCSetUp_Redundant(PC pc)
68*d71ae5a4SJacob Faibussowitsch {
694b9ad928SBarry Smith   PC_Redundant *red = (PC_Redundant *)pc->data;
701b81debcSHong Zhang   PetscInt      mstart, mend, mlocal, M;
7113f74950SBarry Smith   PetscMPIInt   size;
72ce94432eSBarry Smith   MPI_Comm      comm, subcomm;
73ddc54837SHong Zhang   Vec           x;
743f457be1SHong Zhang 
754b9ad928SBarry Smith   PetscFunctionBegin;
769566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetComm((PetscObject)pc, &comm));
77ddc54837SHong Zhang 
78ddc54837SHong Zhang   /* if pmatrix set by user is sequential then we do not need to gather the parallel matrix */
799566063dSJacob Faibussowitsch   PetscCallMPI(MPI_Comm_size(comm, &size));
80ddc54837SHong Zhang   if (size == 1) red->useparallelmat = PETSC_FALSE;
811fbd8f88SHong Zhang 
824b9ad928SBarry Smith   if (!pc->setupcalled) {
831b81debcSHong Zhang     PetscInt mloc_sub;
8475024027SHong Zhang     if (!red->psubcomm) { /* create red->psubcomm, new ksp and pc over subcomm */
8575024027SHong Zhang       KSP ksp;
869566063dSJacob Faibussowitsch       PetscCall(PCRedundantGetKSP(pc, &ksp));
871b81debcSHong Zhang     }
8875024027SHong Zhang     subcomm = PetscSubcommChild(red->psubcomm);
891fbd8f88SHong Zhang 
901b81debcSHong Zhang     if (red->useparallelmat) {
911b81debcSHong Zhang       /* grab the parallel matrix and put it into processors of a subcomminicator */
929566063dSJacob Faibussowitsch       PetscCall(MatCreateRedundantMatrix(pc->pmat, red->psubcomm->n, subcomm, MAT_INITIAL_MATRIX, &red->pmats));
93b85f2e9bSHong Zhang 
949566063dSJacob Faibussowitsch       PetscCallMPI(MPI_Comm_size(subcomm, &size));
95b85f2e9bSHong Zhang       if (size > 1) {
9608cecb0aSPierre Jolivet         PetscBool foundpack, issbaij;
979566063dSJacob Faibussowitsch         PetscCall(PetscObjectTypeCompare((PetscObject)red->pmats, MATMPISBAIJ, &issbaij));
9808cecb0aSPierre Jolivet         if (!issbaij) {
999566063dSJacob Faibussowitsch           PetscCall(MatGetFactorAvailable(red->pmats, NULL, MAT_FACTOR_LU, &foundpack));
10008cecb0aSPierre Jolivet         } else {
1019566063dSJacob Faibussowitsch           PetscCall(MatGetFactorAvailable(red->pmats, NULL, MAT_FACTOR_CHOLESKY, &foundpack));
10208cecb0aSPierre Jolivet         }
103b85f2e9bSHong Zhang         if (!foundpack) { /* reset default ksp and pc */
1049566063dSJacob Faibussowitsch           PetscCall(KSPSetType(red->ksp, KSPGMRES));
1059566063dSJacob Faibussowitsch           PetscCall(PCSetType(red->pc, PCBJACOBI));
106c1619fb6SBarry Smith         } else {
1079566063dSJacob Faibussowitsch           PetscCall(PCFactorSetMatSolverType(red->pc, NULL));
108b85f2e9bSHong Zhang         }
109b85f2e9bSHong Zhang       }
110b85f2e9bSHong Zhang 
1119566063dSJacob Faibussowitsch       PetscCall(KSPSetOperators(red->ksp, red->pmats, red->pmats));
1124b9ad928SBarry Smith 
1131b81debcSHong Zhang       /* get working vectors xsub and ysub */
1149566063dSJacob Faibussowitsch       PetscCall(MatCreateVecs(red->pmats, &red->xsub, &red->ysub));
1152fa5cd67SKarl Rupp 
1168b96b0d2SHong Zhang       /* create working vectors xdup and ydup.
1178b96b0d2SHong Zhang        xdup concatenates all xsub's contigously to form a mpi vector over dupcomm  (see PetscSubcommCreate_interlaced())
1188b96b0d2SHong Zhang        ydup concatenates all ysub and has empty local arrays because ysub's arrays will be place into it.
119ce94432eSBarry Smith        Note: we use communicator dupcomm, not PetscObjectComm((PetscObject)pc)! */
1209566063dSJacob Faibussowitsch       PetscCall(MatGetLocalSize(red->pmats, &mloc_sub, NULL));
1219566063dSJacob Faibussowitsch       PetscCall(VecCreateMPI(PetscSubcommContiguousParent(red->psubcomm), mloc_sub, PETSC_DECIDE, &red->xdup));
1229566063dSJacob Faibussowitsch       PetscCall(VecCreateMPIWithArray(PetscSubcommContiguousParent(red->psubcomm), 1, mloc_sub, PETSC_DECIDE, NULL, &red->ydup));
1233f457be1SHong Zhang 
124f68be91cSHong Zhang       /* create vecscatters */
125f68be91cSHong Zhang       if (!red->scatterin) { /* efficiency of scatterin is independent from psubcomm_type! */
1263f457be1SHong Zhang         IS        is1, is2;
1273f457be1SHong Zhang         PetscInt *idx1, *idx2, i, j, k;
12845fc02eaSBarry Smith 
1299566063dSJacob Faibussowitsch         PetscCall(MatCreateVecs(pc->pmat, &x, NULL));
1309566063dSJacob Faibussowitsch         PetscCall(VecGetSize(x, &M));
1319566063dSJacob Faibussowitsch         PetscCall(VecGetOwnershipRange(x, &mstart, &mend));
1321b81debcSHong Zhang         mlocal = mend - mstart;
1339566063dSJacob Faibussowitsch         PetscCall(PetscMalloc2(red->psubcomm->n * mlocal, &idx1, red->psubcomm->n * mlocal, &idx2));
1343f457be1SHong Zhang         j = 0;
1351fbd8f88SHong Zhang         for (k = 0; k < red->psubcomm->n; k++) {
1363f457be1SHong Zhang           for (i = mstart; i < mend; i++) {
1373f457be1SHong Zhang             idx1[j]   = i;
138ddc54837SHong Zhang             idx2[j++] = i + M * k;
1393f457be1SHong Zhang           }
1403f457be1SHong Zhang         }
1419566063dSJacob Faibussowitsch         PetscCall(ISCreateGeneral(comm, red->psubcomm->n * mlocal, idx1, PETSC_COPY_VALUES, &is1));
1429566063dSJacob Faibussowitsch         PetscCall(ISCreateGeneral(comm, red->psubcomm->n * mlocal, idx2, PETSC_COPY_VALUES, &is2));
1439566063dSJacob Faibussowitsch         PetscCall(VecScatterCreate(x, is1, red->xdup, is2, &red->scatterin));
1449566063dSJacob Faibussowitsch         PetscCall(ISDestroy(&is1));
1459566063dSJacob Faibussowitsch         PetscCall(ISDestroy(&is2));
1463f457be1SHong Zhang 
1476909748bSHong Zhang         /* Impl below is good for PETSC_SUBCOMM_INTERLACED (no inter-process communication) and PETSC_SUBCOMM_CONTIGUOUS (communication within subcomm) */
1489566063dSJacob Faibussowitsch         PetscCall(ISCreateStride(comm, mlocal, mstart + red->psubcomm->color * M, 1, &is1));
1499566063dSJacob Faibussowitsch         PetscCall(ISCreateStride(comm, mlocal, mstart, 1, &is2));
1509566063dSJacob Faibussowitsch         PetscCall(VecScatterCreate(red->xdup, is1, x, is2, &red->scatterout));
1519566063dSJacob Faibussowitsch         PetscCall(ISDestroy(&is1));
1529566063dSJacob Faibussowitsch         PetscCall(ISDestroy(&is2));
1539566063dSJacob Faibussowitsch         PetscCall(PetscFree2(idx1, idx2));
1549566063dSJacob Faibussowitsch         PetscCall(VecDestroy(&x));
1551b81debcSHong Zhang       }
156ab661555SHong Zhang     } else { /* !red->useparallelmat */
1579566063dSJacob Faibussowitsch       PetscCall(KSPSetOperators(red->ksp, pc->mat, pc->pmat));
1581b81debcSHong Zhang     }
159ab661555SHong Zhang   } else { /* pc->setupcalled */
1604b9ad928SBarry Smith     if (red->useparallelmat) {
161ab661555SHong Zhang       MatReuse reuse;
1621b81debcSHong Zhang       /* grab the parallel matrix and put it into processors of a subcomminicator */
1631b81debcSHong Zhang       /*--------------------------------------------------------------------------*/
164ab661555SHong Zhang       if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
1654b9ad928SBarry Smith         /* destroy old matrices */
1669566063dSJacob Faibussowitsch         PetscCall(MatDestroy(&red->pmats));
167ab661555SHong Zhang         reuse = MAT_INITIAL_MATRIX;
1684b9ad928SBarry Smith       } else {
169ab661555SHong Zhang         reuse = MAT_REUSE_MATRIX;
170ab661555SHong Zhang       }
1719566063dSJacob Faibussowitsch       PetscCall(MatCreateRedundantMatrix(pc->pmat, red->psubcomm->n, PetscSubcommChild(red->psubcomm), reuse, &red->pmats));
1729566063dSJacob Faibussowitsch       PetscCall(KSPSetOperators(red->ksp, red->pmats, red->pmats));
173ab661555SHong Zhang     } else { /* !red->useparallelmat */
1749566063dSJacob Faibussowitsch       PetscCall(KSPSetOperators(red->ksp, pc->mat, pc->pmat));
1754b9ad928SBarry Smith     }
176ab661555SHong Zhang   }
1771b81debcSHong Zhang 
1781baa6e33SBarry Smith   if (pc->setfromoptionscalled) PetscCall(KSPSetFromOptions(red->ksp));
1799566063dSJacob Faibussowitsch   PetscCall(KSPSetUp(red->ksp));
1804b9ad928SBarry Smith   PetscFunctionReturn(0);
1814b9ad928SBarry Smith }
1824b9ad928SBarry Smith 
183*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCApply_Redundant(PC pc, Vec x, Vec y)
184*d71ae5a4SJacob Faibussowitsch {
1854b9ad928SBarry Smith   PC_Redundant *red = (PC_Redundant *)pc->data;
1863f457be1SHong Zhang   PetscScalar  *array;
1874b9ad928SBarry Smith 
1884b9ad928SBarry Smith   PetscFunctionBegin;
189ddc54837SHong Zhang   if (!red->useparallelmat) {
1909566063dSJacob Faibussowitsch     PetscCall(KSPSolve(red->ksp, x, y));
1919566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(red->ksp, pc, y));
192ddc54837SHong Zhang     PetscFunctionReturn(0);
193ddc54837SHong Zhang   }
194ddc54837SHong Zhang 
1953f457be1SHong Zhang   /* scatter x to xdup */
1969566063dSJacob Faibussowitsch   PetscCall(VecScatterBegin(red->scatterin, x, red->xdup, INSERT_VALUES, SCATTER_FORWARD));
1979566063dSJacob Faibussowitsch   PetscCall(VecScatterEnd(red->scatterin, x, red->xdup, INSERT_VALUES, SCATTER_FORWARD));
1983f457be1SHong Zhang 
1993f457be1SHong Zhang   /* place xdup's local array into xsub */
2009566063dSJacob Faibussowitsch   PetscCall(VecGetArray(red->xdup, &array));
2019566063dSJacob Faibussowitsch   PetscCall(VecPlaceArray(red->xsub, (const PetscScalar *)array));
2024b9ad928SBarry Smith 
2034b9ad928SBarry Smith   /* apply preconditioner on each processor */
2049566063dSJacob Faibussowitsch   PetscCall(KSPSolve(red->ksp, red->xsub, red->ysub));
2059566063dSJacob Faibussowitsch   PetscCall(KSPCheckSolve(red->ksp, pc, red->ysub));
2069566063dSJacob Faibussowitsch   PetscCall(VecResetArray(red->xsub));
2079566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(red->xdup, &array));
2084b9ad928SBarry Smith 
2093f457be1SHong Zhang   /* place ysub's local array into ydup */
2109566063dSJacob Faibussowitsch   PetscCall(VecGetArray(red->ysub, &array));
2119566063dSJacob Faibussowitsch   PetscCall(VecPlaceArray(red->ydup, (const PetscScalar *)array));
2123f457be1SHong Zhang 
2133f457be1SHong Zhang   /* scatter ydup to y */
2149566063dSJacob Faibussowitsch   PetscCall(VecScatterBegin(red->scatterout, red->ydup, y, INSERT_VALUES, SCATTER_FORWARD));
2159566063dSJacob Faibussowitsch   PetscCall(VecScatterEnd(red->scatterout, red->ydup, y, INSERT_VALUES, SCATTER_FORWARD));
2169566063dSJacob Faibussowitsch   PetscCall(VecResetArray(red->ydup));
2179566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(red->ysub, &array));
2184b9ad928SBarry Smith   PetscFunctionReturn(0);
2194b9ad928SBarry Smith }
2204b9ad928SBarry Smith 
221*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCApplyTranspose_Redundant(PC pc, Vec x, Vec y)
222*d71ae5a4SJacob Faibussowitsch {
223d88bfacbSStefano Zampini   PC_Redundant *red = (PC_Redundant *)pc->data;
224d88bfacbSStefano Zampini   PetscScalar  *array;
225d88bfacbSStefano Zampini 
226d88bfacbSStefano Zampini   PetscFunctionBegin;
227d88bfacbSStefano Zampini   if (!red->useparallelmat) {
2289566063dSJacob Faibussowitsch     PetscCall(KSPSolveTranspose(red->ksp, x, y));
2299566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(red->ksp, pc, y));
230d88bfacbSStefano Zampini     PetscFunctionReturn(0);
231d88bfacbSStefano Zampini   }
232d88bfacbSStefano Zampini 
233d88bfacbSStefano Zampini   /* scatter x to xdup */
2349566063dSJacob Faibussowitsch   PetscCall(VecScatterBegin(red->scatterin, x, red->xdup, INSERT_VALUES, SCATTER_FORWARD));
2359566063dSJacob Faibussowitsch   PetscCall(VecScatterEnd(red->scatterin, x, red->xdup, INSERT_VALUES, SCATTER_FORWARD));
236d88bfacbSStefano Zampini 
237d88bfacbSStefano Zampini   /* place xdup's local array into xsub */
2389566063dSJacob Faibussowitsch   PetscCall(VecGetArray(red->xdup, &array));
2399566063dSJacob Faibussowitsch   PetscCall(VecPlaceArray(red->xsub, (const PetscScalar *)array));
240d88bfacbSStefano Zampini 
241d88bfacbSStefano Zampini   /* apply preconditioner on each processor */
2429566063dSJacob Faibussowitsch   PetscCall(KSPSolveTranspose(red->ksp, red->xsub, red->ysub));
2439566063dSJacob Faibussowitsch   PetscCall(KSPCheckSolve(red->ksp, pc, red->ysub));
2449566063dSJacob Faibussowitsch   PetscCall(VecResetArray(red->xsub));
2459566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(red->xdup, &array));
246d88bfacbSStefano Zampini 
247d88bfacbSStefano Zampini   /* place ysub's local array into ydup */
2489566063dSJacob Faibussowitsch   PetscCall(VecGetArray(red->ysub, &array));
2499566063dSJacob Faibussowitsch   PetscCall(VecPlaceArray(red->ydup, (const PetscScalar *)array));
250d88bfacbSStefano Zampini 
251d88bfacbSStefano Zampini   /* scatter ydup to y */
2529566063dSJacob Faibussowitsch   PetscCall(VecScatterBegin(red->scatterout, red->ydup, y, INSERT_VALUES, SCATTER_FORWARD));
2539566063dSJacob Faibussowitsch   PetscCall(VecScatterEnd(red->scatterout, red->ydup, y, INSERT_VALUES, SCATTER_FORWARD));
2549566063dSJacob Faibussowitsch   PetscCall(VecResetArray(red->ydup));
2559566063dSJacob Faibussowitsch   PetscCall(VecRestoreArray(red->ysub, &array));
256d88bfacbSStefano Zampini   PetscFunctionReturn(0);
257d88bfacbSStefano Zampini }
258d88bfacbSStefano Zampini 
259*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCReset_Redundant(PC pc)
260*d71ae5a4SJacob Faibussowitsch {
2614b9ad928SBarry Smith   PC_Redundant *red = (PC_Redundant *)pc->data;
2624b9ad928SBarry Smith 
2634b9ad928SBarry Smith   PetscFunctionBegin;
2641b81debcSHong Zhang   if (red->useparallelmat) {
2659566063dSJacob Faibussowitsch     PetscCall(VecScatterDestroy(&red->scatterin));
2669566063dSJacob Faibussowitsch     PetscCall(VecScatterDestroy(&red->scatterout));
2679566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&red->ysub));
2689566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&red->xsub));
2699566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&red->xdup));
2709566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&red->ydup));
2711b81debcSHong Zhang   }
2729566063dSJacob Faibussowitsch   PetscCall(MatDestroy(&red->pmats));
2739566063dSJacob Faibussowitsch   PetscCall(KSPReset(red->ksp));
2741ea5a559SBarry Smith   PetscFunctionReturn(0);
2751ea5a559SBarry Smith }
2761ea5a559SBarry Smith 
277*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCDestroy_Redundant(PC pc)
278*d71ae5a4SJacob Faibussowitsch {
2791ea5a559SBarry Smith   PC_Redundant *red = (PC_Redundant *)pc->data;
2801ea5a559SBarry Smith 
2811ea5a559SBarry Smith   PetscFunctionBegin;
2829566063dSJacob Faibussowitsch   PetscCall(PCReset_Redundant(pc));
2839566063dSJacob Faibussowitsch   PetscCall(KSPDestroy(&red->ksp));
2849566063dSJacob Faibussowitsch   PetscCall(PetscSubcommDestroy(&red->psubcomm));
2852e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCRedundantSetScatter_C", NULL));
2862e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCRedundantSetNumber_C", NULL));
2872e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCRedundantGetKSP_C", NULL));
2882e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCRedundantGetOperators_C", NULL));
2892e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCFactorSetShiftType_C", NULL));
2909566063dSJacob Faibussowitsch   PetscCall(PetscFree(pc->data));
2914b9ad928SBarry Smith   PetscFunctionReturn(0);
2924b9ad928SBarry Smith }
2934b9ad928SBarry Smith 
294*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCSetFromOptions_Redundant(PC pc, PetscOptionItems *PetscOptionsObject)
295*d71ae5a4SJacob Faibussowitsch {
296a98ce0f4SHong Zhang   PC_Redundant *red = (PC_Redundant *)pc->data;
297a98ce0f4SHong Zhang 
2984b9ad928SBarry Smith   PetscFunctionBegin;
299d0609cedSBarry Smith   PetscOptionsHeadBegin(PetscOptionsObject, "Redundant options");
3009566063dSJacob Faibussowitsch   PetscCall(PetscOptionsInt("-pc_redundant_number", "Number of redundant pc", "PCRedundantSetNumber", red->nsubcomm, &red->nsubcomm, NULL));
301d0609cedSBarry Smith   PetscOptionsHeadEnd();
3024b9ad928SBarry Smith   PetscFunctionReturn(0);
3034b9ad928SBarry Smith }
3044b9ad928SBarry Smith 
305*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCRedundantSetNumber_Redundant(PC pc, PetscInt nreds)
306*d71ae5a4SJacob Faibussowitsch {
30709a6bc64SHong Zhang   PC_Redundant *red = (PC_Redundant *)pc->data;
30809a6bc64SHong Zhang 
30909a6bc64SHong Zhang   PetscFunctionBegin;
31009a6bc64SHong Zhang   red->nsubcomm = nreds;
31109a6bc64SHong Zhang   PetscFunctionReturn(0);
31209a6bc64SHong Zhang }
31309a6bc64SHong Zhang 
31409a6bc64SHong Zhang /*@
31509a6bc64SHong Zhang    PCRedundantSetNumber - Sets the number of redundant preconditioner contexts.
31609a6bc64SHong Zhang 
317f1580f4eSBarry Smith    Logically Collective on pc
31809a6bc64SHong Zhang 
31909a6bc64SHong Zhang    Input Parameters:
32009a6bc64SHong Zhang +  pc - the preconditioner context
3219b21d695SBarry Smith -  nredundant - number of redundant preconditioner contexts; for example if you are using 64 MPI processes and
3229b21d695SBarry Smith                               use an nredundant of 4 there will be 4 parallel solves each on 16 = 64/4 processes.
32309a6bc64SHong Zhang 
32409a6bc64SHong Zhang    Level: advanced
32509a6bc64SHong Zhang 
326f1580f4eSBarry Smith .seealso: `PCREDUNDANT`
32709a6bc64SHong Zhang @*/
328*d71ae5a4SJacob Faibussowitsch PetscErrorCode PCRedundantSetNumber(PC pc, PetscInt nredundant)
329*d71ae5a4SJacob Faibussowitsch {
33009a6bc64SHong Zhang   PetscFunctionBegin;
3310700a824SBarry Smith   PetscValidHeaderSpecific(pc, PC_CLASSID, 1);
33263a3b9bcSJacob Faibussowitsch   PetscCheck(nredundant > 0, PetscObjectComm((PetscObject)pc), PETSC_ERR_ARG_WRONG, "num of redundant pc %" PetscInt_FMT " must be positive", nredundant);
333cac4c232SBarry Smith   PetscTryMethod(pc, "PCRedundantSetNumber_C", (PC, PetscInt), (pc, nredundant));
33409a6bc64SHong Zhang   PetscFunctionReturn(0);
33509a6bc64SHong Zhang }
33609a6bc64SHong Zhang 
337*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCRedundantSetScatter_Redundant(PC pc, VecScatter in, VecScatter out)
338*d71ae5a4SJacob Faibussowitsch {
3394b9ad928SBarry Smith   PC_Redundant *red = (PC_Redundant *)pc->data;
3404b9ad928SBarry Smith 
3414b9ad928SBarry Smith   PetscFunctionBegin;
3429566063dSJacob Faibussowitsch   PetscCall(PetscObjectReference((PetscObject)in));
3439566063dSJacob Faibussowitsch   PetscCall(VecScatterDestroy(&red->scatterin));
3442fa5cd67SKarl Rupp 
345c3122656SLisandro Dalcin   red->scatterin = in;
3462fa5cd67SKarl Rupp 
3479566063dSJacob Faibussowitsch   PetscCall(PetscObjectReference((PetscObject)out));
3489566063dSJacob Faibussowitsch   PetscCall(VecScatterDestroy(&red->scatterout));
349c3122656SLisandro Dalcin   red->scatterout = out;
3504b9ad928SBarry Smith   PetscFunctionReturn(0);
3514b9ad928SBarry Smith }
3524b9ad928SBarry Smith 
3534b9ad928SBarry Smith /*@
3544b9ad928SBarry Smith    PCRedundantSetScatter - Sets the scatter used to copy values into the
3554b9ad928SBarry Smith      redundant local solve and the scatter to move them back into the global
3564b9ad928SBarry Smith      vector.
3574b9ad928SBarry Smith 
358f1580f4eSBarry Smith    Logically Collective on pc
3594b9ad928SBarry Smith 
3604b9ad928SBarry Smith    Input Parameters:
3614b9ad928SBarry Smith +  pc - the preconditioner context
3624b9ad928SBarry Smith .  in - the scatter to move the values in
3634b9ad928SBarry Smith -  out - the scatter to move them out
3644b9ad928SBarry Smith 
3654b9ad928SBarry Smith    Level: advanced
3664b9ad928SBarry Smith 
367f1580f4eSBarry Smith .seealso: `PCREDUNDANT`
3684b9ad928SBarry Smith @*/
369*d71ae5a4SJacob Faibussowitsch PetscErrorCode PCRedundantSetScatter(PC pc, VecScatter in, VecScatter out)
370*d71ae5a4SJacob Faibussowitsch {
3714b9ad928SBarry Smith   PetscFunctionBegin;
3720700a824SBarry Smith   PetscValidHeaderSpecific(pc, PC_CLASSID, 1);
37397929ea7SJunchao Zhang   PetscValidHeaderSpecific(in, PETSCSF_CLASSID, 2);
37497929ea7SJunchao Zhang   PetscValidHeaderSpecific(out, PETSCSF_CLASSID, 3);
375cac4c232SBarry Smith   PetscTryMethod(pc, "PCRedundantSetScatter_C", (PC, VecScatter, VecScatter), (pc, in, out));
3764b9ad928SBarry Smith   PetscFunctionReturn(0);
3774b9ad928SBarry Smith }
3784b9ad928SBarry Smith 
379*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCRedundantGetKSP_Redundant(PC pc, KSP *innerksp)
380*d71ae5a4SJacob Faibussowitsch {
3814b9ad928SBarry Smith   PC_Redundant *red = (PC_Redundant *)pc->data;
38275024027SHong Zhang   MPI_Comm      comm, subcomm;
38375024027SHong Zhang   const char   *prefix;
38408cecb0aSPierre Jolivet   PetscBool     issbaij;
3854b9ad928SBarry Smith 
3864b9ad928SBarry Smith   PetscFunctionBegin;
38775024027SHong Zhang   if (!red->psubcomm) {
3889566063dSJacob Faibussowitsch     PetscCall(PCGetOptionsPrefix(pc, &prefix));
389e5acf8a4SHong Zhang 
3909566063dSJacob Faibussowitsch     PetscCall(PetscObjectGetComm((PetscObject)pc, &comm));
3919566063dSJacob Faibussowitsch     PetscCall(PetscSubcommCreate(comm, &red->psubcomm));
3929566063dSJacob Faibussowitsch     PetscCall(PetscSubcommSetNumber(red->psubcomm, red->nsubcomm));
3939566063dSJacob Faibussowitsch     PetscCall(PetscSubcommSetType(red->psubcomm, PETSC_SUBCOMM_CONTIGUOUS));
394e5acf8a4SHong Zhang 
3959566063dSJacob Faibussowitsch     PetscCall(PetscSubcommSetOptionsPrefix(red->psubcomm, prefix));
3969566063dSJacob Faibussowitsch     PetscCall(PetscSubcommSetFromOptions(red->psubcomm));
39775024027SHong Zhang 
39875024027SHong Zhang     /* create a new PC that processors in each subcomm have copy of */
39975024027SHong Zhang     subcomm = PetscSubcommChild(red->psubcomm);
40075024027SHong Zhang 
4019566063dSJacob Faibussowitsch     PetscCall(KSPCreate(subcomm, &red->ksp));
4029566063dSJacob Faibussowitsch     PetscCall(KSPSetErrorIfNotConverged(red->ksp, pc->erroriffailure));
4039566063dSJacob Faibussowitsch     PetscCall(PetscObjectIncrementTabLevel((PetscObject)red->ksp, (PetscObject)pc, 1));
4049566063dSJacob Faibussowitsch     PetscCall(KSPSetType(red->ksp, KSPPREONLY));
4059566063dSJacob Faibussowitsch     PetscCall(KSPGetPC(red->ksp, &red->pc));
4069566063dSJacob Faibussowitsch     PetscCall(PetscObjectTypeCompare((PetscObject)pc->pmat, MATSEQSBAIJ, &issbaij));
40748a46eb9SPierre Jolivet     if (!issbaij) PetscCall(PetscObjectTypeCompare((PetscObject)pc->pmat, MATMPISBAIJ, &issbaij));
40808cecb0aSPierre Jolivet     if (!issbaij) {
4099566063dSJacob Faibussowitsch       PetscCall(PCSetType(red->pc, PCLU));
41008cecb0aSPierre Jolivet     } else {
4119566063dSJacob Faibussowitsch       PetscCall(PCSetType(red->pc, PCCHOLESKY));
41208cecb0aSPierre Jolivet     }
413753b7fb9SBarry Smith     if (red->shifttypeset) {
4149566063dSJacob Faibussowitsch       PetscCall(PCFactorSetShiftType(red->pc, red->shifttype));
415753b7fb9SBarry Smith       red->shifttypeset = PETSC_FALSE;
416753b7fb9SBarry Smith     }
4179566063dSJacob Faibussowitsch     PetscCall(KSPSetOptionsPrefix(red->ksp, prefix));
4189566063dSJacob Faibussowitsch     PetscCall(KSPAppendOptionsPrefix(red->ksp, "redundant_"));
41975024027SHong Zhang   }
42083ab6a24SBarry Smith   *innerksp = red->ksp;
4214b9ad928SBarry Smith   PetscFunctionReturn(0);
4224b9ad928SBarry Smith }
4234b9ad928SBarry Smith 
4244b9ad928SBarry Smith /*@
425f1580f4eSBarry Smith    PCRedundantGetKSP - Gets the less parallel `KSP` created by the redundant `PC`.
4264b9ad928SBarry Smith 
4274b9ad928SBarry Smith    Not Collective
4284b9ad928SBarry Smith 
4294b9ad928SBarry Smith    Input Parameter:
4304b9ad928SBarry Smith .  pc - the preconditioner context
4314b9ad928SBarry Smith 
4324b9ad928SBarry Smith    Output Parameter:
433f1580f4eSBarry Smith .  innerksp - the `KSP` on the smaller set of processes
4344b9ad928SBarry Smith 
4354b9ad928SBarry Smith    Level: advanced
4364b9ad928SBarry Smith 
437f1580f4eSBarry Smith .seealso: `PCREDUNDANT`
4384b9ad928SBarry Smith @*/
439*d71ae5a4SJacob Faibussowitsch PetscErrorCode PCRedundantGetKSP(PC pc, KSP *innerksp)
440*d71ae5a4SJacob Faibussowitsch {
4414b9ad928SBarry Smith   PetscFunctionBegin;
4420700a824SBarry Smith   PetscValidHeaderSpecific(pc, PC_CLASSID, 1);
44383ab6a24SBarry Smith   PetscValidPointer(innerksp, 2);
444cac4c232SBarry Smith   PetscUseMethod(pc, "PCRedundantGetKSP_C", (PC, KSP *), (pc, innerksp));
4454b9ad928SBarry Smith   PetscFunctionReturn(0);
4464b9ad928SBarry Smith }
4474b9ad928SBarry Smith 
448*d71ae5a4SJacob Faibussowitsch static PetscErrorCode PCRedundantGetOperators_Redundant(PC pc, Mat *mat, Mat *pmat)
449*d71ae5a4SJacob Faibussowitsch {
4504b9ad928SBarry Smith   PC_Redundant *red = (PC_Redundant *)pc->data;
4514b9ad928SBarry Smith 
4524b9ad928SBarry Smith   PetscFunctionBegin;
453b3804887SHong Zhang   if (mat) *mat = red->pmats;
454b3804887SHong Zhang   if (pmat) *pmat = red->pmats;
4554b9ad928SBarry Smith   PetscFunctionReturn(0);
4564b9ad928SBarry Smith }
4574b9ad928SBarry Smith 
4584b9ad928SBarry Smith /*@
4594b9ad928SBarry Smith    PCRedundantGetOperators - gets the sequential matrix and preconditioner matrix
4604b9ad928SBarry Smith 
4614b9ad928SBarry Smith    Not Collective
4624b9ad928SBarry Smith 
4634b9ad928SBarry Smith    Input Parameter:
4644b9ad928SBarry Smith .  pc - the preconditioner context
4654b9ad928SBarry Smith 
4664b9ad928SBarry Smith    Output Parameters:
4674b9ad928SBarry Smith +  mat - the matrix
4684b9ad928SBarry Smith -  pmat - the (possibly different) preconditioner matrix
4694b9ad928SBarry Smith 
4704b9ad928SBarry Smith    Level: advanced
4714b9ad928SBarry Smith 
472f1580f4eSBarry Smith .seealso: `PCREDUNDANT`
4734b9ad928SBarry Smith @*/
474*d71ae5a4SJacob Faibussowitsch PetscErrorCode PCRedundantGetOperators(PC pc, Mat *mat, Mat *pmat)
475*d71ae5a4SJacob Faibussowitsch {
4764b9ad928SBarry Smith   PetscFunctionBegin;
4770700a824SBarry Smith   PetscValidHeaderSpecific(pc, PC_CLASSID, 1);
4784482741eSBarry Smith   if (mat) PetscValidPointer(mat, 2);
4794482741eSBarry Smith   if (pmat) PetscValidPointer(pmat, 3);
480cac4c232SBarry Smith   PetscUseMethod(pc, "PCRedundantGetOperators_C", (PC, Mat *, Mat *), (pc, mat, pmat));
4814b9ad928SBarry Smith   PetscFunctionReturn(0);
4824b9ad928SBarry Smith }
4834b9ad928SBarry Smith 
48437a17b4dSBarry Smith /*MC
485f1580f4eSBarry Smith      PCREDUNDANT - Runs a `KSP` solver with preconditioner for the entire problem on subgroups of processors
48637a17b4dSBarry Smith 
487f1580f4eSBarry Smith   Options Database Key:
4889b21d695SBarry Smith .  -pc_redundant_number <n> - number of redundant solves, for example if you are using 64 MPI processes and
4899b21d695SBarry Smith                               use an n of 4 there will be 4 parallel solves each on 16 = 64/4 processes.
49009391456SBarry Smith 
49137a17b4dSBarry Smith    Level: intermediate
49237a17b4dSBarry Smith 
49395452b02SPatrick Sanan    Notes:
494f1580f4eSBarry Smith    Options for the redundant preconditioners can be set using the options database prefix -redundant_
49583ab6a24SBarry Smith 
496f1580f4eSBarry Smith    The default `KSP` is preonly and the default `PC` is `PCLU` or `PCCHOLESKY` if Pmat is of type `MATSBAIJ`.
497753b7fb9SBarry Smith 
498f1580f4eSBarry Smith    `PCFactorSetShiftType()` applied to this `PC` will convey they shift type into the inner `PC` if it is factorization based.
499f1580f4eSBarry Smith 
500f1580f4eSBarry Smith    Developer Note:
501f1580f4eSBarry Smith    `PCSetInitialGuessNonzero()` is not used by this class but likely should be.
5029cfaa89bSBarry Smith 
503db781477SPatrick Sanan .seealso: `PCCreate()`, `PCSetType()`, `PCType`, `PCRedundantSetScatter()`,
504f1580f4eSBarry Smith           `PCRedundantGetKSP()`, `PCRedundantGetOperators()`, `PCRedundantSetNumber()`, `PCREDISTRIBUTE`
50537a17b4dSBarry Smith M*/
50637a17b4dSBarry Smith 
507*d71ae5a4SJacob Faibussowitsch PETSC_EXTERN PetscErrorCode PCCreate_Redundant(PC pc)
508*d71ae5a4SJacob Faibussowitsch {
5094b9ad928SBarry Smith   PC_Redundant *red;
51069db28dcSHong Zhang   PetscMPIInt   size;
5113f457be1SHong Zhang 
5124b9ad928SBarry Smith   PetscFunctionBegin;
5134dfa11a4SJacob Faibussowitsch   PetscCall(PetscNew(&red));
5149566063dSJacob Faibussowitsch   PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)pc), &size));
5152fa5cd67SKarl Rupp 
51669db28dcSHong Zhang   red->nsubcomm       = size;
5174b9ad928SBarry Smith   red->useparallelmat = PETSC_TRUE;
5181fbd8f88SHong Zhang   pc->data            = (void *)red;
5194b9ad928SBarry Smith 
5204b9ad928SBarry Smith   pc->ops->apply          = PCApply_Redundant;
521d88bfacbSStefano Zampini   pc->ops->applytranspose = PCApplyTranspose_Redundant;
5224b9ad928SBarry Smith   pc->ops->setup          = PCSetUp_Redundant;
5234b9ad928SBarry Smith   pc->ops->destroy        = PCDestroy_Redundant;
5241ea5a559SBarry Smith   pc->ops->reset          = PCReset_Redundant;
5254b9ad928SBarry Smith   pc->ops->setfromoptions = PCSetFromOptions_Redundant;
5264b9ad928SBarry Smith   pc->ops->view           = PCView_Redundant;
5272fa5cd67SKarl Rupp 
5289566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCRedundantSetScatter_C", PCRedundantSetScatter_Redundant));
5299566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCRedundantSetNumber_C", PCRedundantSetNumber_Redundant));
5309566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCRedundantGetKSP_C", PCRedundantGetKSP_Redundant));
5319566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCRedundantGetOperators_C", PCRedundantGetOperators_Redundant));
5329566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCFactorSetShiftType_C", PCFactorSetShiftType_Redundant));
5334b9ad928SBarry Smith   PetscFunctionReturn(0);
5344b9ad928SBarry Smith }
535