16618991cSMark Adams /* 26618991cSMark Adams GAMG geometric-algebric multigrid PC - Mark Adams 2011 36618991cSMark Adams */ 46618991cSMark Adams #include <petsc/private/matimpl.h> 56618991cSMark Adams #include <../src/ksp/pc/impls/gamg/gamg.h> /*I "petscpc.h" I*/ 685cd6069SMark Adams #include <petsc/private/kspimpl.h> 76618991cSMark Adams 804c3f3b8SBarry Smith // PetscClangLinter pragma disable: -fdoc-sowing-chars 96618991cSMark Adams /* 10bae903cbSmarkadams4 PCGAMGGetDataWithGhosts - Get array of local + ghost data with local data 11bae903cbSmarkadams4 hacks into Mat MPIAIJ so this must have size > 1 126618991cSMark Adams 1304c3f3b8SBarry Smith Input Parameters: 14feefa0e1SJacob Faibussowitsch + Gmat - MPIAIJ matrix for scatters 156618991cSMark Adams . data_sz - number of data terms per node (# cols in output) 165e116b59SBarry Smith - data_in - column-oriented local data of size nloc*data_sz 17bae903cbSmarkadams4 1804c3f3b8SBarry Smith Output Parameters: 19feefa0e1SJacob Faibussowitsch + a_stride - number of rows of output (locals+ghosts) 2004c3f3b8SBarry Smith - a_data_out - output data with ghosts of size stride*data_sz 21bae903cbSmarkadams4 226618991cSMark Adams */ 23d71ae5a4SJacob Faibussowitsch PetscErrorCode PCGAMGGetDataWithGhosts(Mat Gmat, PetscInt data_sz, PetscReal data_in[], PetscInt *a_stride, PetscReal **a_data_out) 24d71ae5a4SJacob Faibussowitsch { 256618991cSMark Adams Vec tmp_crds; 269e9caa57Smarkadams4 Mat_MPIAIJ *mpimat; 276618991cSMark Adams PetscInt nnodes, num_ghosts, dir, kk, jj, my0, Iend, nloc; 286618991cSMark Adams PetscScalar *data_arr; 296618991cSMark Adams PetscReal *datas; 306618991cSMark Adams PetscBool isMPIAIJ; 316618991cSMark Adams 326618991cSMark Adams PetscFunctionBegin; 339e9caa57Smarkadams4 PetscValidHeaderSpecific(Gmat, MAT_CLASSID, 1); 349e9caa57Smarkadams4 mpimat = (Mat_MPIAIJ *)Gmat->data; 359566063dSJacob Faibussowitsch PetscCall(PetscObjectBaseTypeCompare((PetscObject)Gmat, MATMPIAIJ, &isMPIAIJ)); 369566063dSJacob Faibussowitsch PetscCall(MatGetOwnershipRange(Gmat, &my0, &Iend)); 376618991cSMark Adams nloc = Iend - my0; 389566063dSJacob Faibussowitsch PetscCall(VecGetLocalSize(mpimat->lvec, &num_ghosts)); 396618991cSMark Adams nnodes = num_ghosts + nloc; 406618991cSMark Adams *a_stride = nnodes; 419566063dSJacob Faibussowitsch PetscCall(MatCreateVecs(Gmat, &tmp_crds, NULL)); 426618991cSMark Adams 439566063dSJacob Faibussowitsch PetscCall(PetscMalloc1(data_sz * nnodes, &datas)); 446618991cSMark Adams for (dir = 0; dir < data_sz; dir++) { 456618991cSMark Adams /* set local, and global */ 466618991cSMark Adams for (kk = 0; kk < nloc; kk++) { 476618991cSMark Adams PetscInt gid = my0 + kk; 48*835f2295SStefano Zampini PetscScalar crd = data_in[dir * nloc + kk]; /* col oriented */ 49bae903cbSmarkadams4 datas[dir * nnodes + kk] = PetscRealPart(crd); // get local part now 506618991cSMark Adams 519566063dSJacob Faibussowitsch PetscCall(VecSetValues(tmp_crds, 1, &gid, &crd, INSERT_VALUES)); 526618991cSMark Adams } 539566063dSJacob Faibussowitsch PetscCall(VecAssemblyBegin(tmp_crds)); 549566063dSJacob Faibussowitsch PetscCall(VecAssemblyEnd(tmp_crds)); 55bae903cbSmarkadams4 /* scatter / gather ghost data and add to end of output data */ 569566063dSJacob Faibussowitsch PetscCall(VecScatterBegin(mpimat->Mvctx, tmp_crds, mpimat->lvec, INSERT_VALUES, SCATTER_FORWARD)); 579566063dSJacob Faibussowitsch PetscCall(VecScatterEnd(mpimat->Mvctx, tmp_crds, mpimat->lvec, INSERT_VALUES, SCATTER_FORWARD)); 589566063dSJacob Faibussowitsch PetscCall(VecGetArray(mpimat->lvec, &data_arr)); 596618991cSMark Adams for (kk = nloc, jj = 0; jj < num_ghosts; kk++, jj++) datas[dir * nnodes + kk] = PetscRealPart(data_arr[jj]); 609566063dSJacob Faibussowitsch PetscCall(VecRestoreArray(mpimat->lvec, &data_arr)); 616618991cSMark Adams } 629566063dSJacob Faibussowitsch PetscCall(VecDestroy(&tmp_crds)); 636618991cSMark Adams *a_data_out = datas; 643ba16761SJacob Faibussowitsch PetscFunctionReturn(PETSC_SUCCESS); 656618991cSMark Adams } 66