xref: /petsc/src/ksp/pc/impls/gamg/classical.c (revision f9a65ec8fcde71d2ca904ed86418f1620c01ae3c) 
1 #include <../src/ksp/pc/impls/gamg/gamg.h>        /*I "petscpc.h" I*/
2 #include <petsc-private/kspimpl.h>
3 
4 typedef struct {
5   PetscReal dummy; /* empty struct; save for later */
6 } PC_GAMG_Classical;
7 
8 
9 #undef __FUNCT__
10 #define __FUNCT__ "PCGAMGClassicalCreateGhostVector_Private"
11 PetscErrorCode PCGAMGClassicalCreateGhostVector_Private(Mat G,Vec *gvec,PetscInt **global)
12 {
13   Mat_MPIAIJ     *aij = (Mat_MPIAIJ*)G->data;
14   PetscErrorCode ierr;
15   PetscBool      isMPIAIJ;
16 
17   PetscFunctionBegin;
18   ierr = PetscObjectTypeCompare((PetscObject)G, MATMPIAIJ, &isMPIAIJ); CHKERRQ(ierr);
19   if (isMPIAIJ) {
20     if (gvec)ierr = VecDuplicate(aij->lvec,gvec);CHKERRQ(ierr);
21     if (global)*global = aij->garray;
22   } else {
23     /* no off-processor nodes */
24     if (gvec)*gvec = NULL;
25     if (global)*global = NULL;
26   }
27   PetscFunctionReturn(0);
28 }
29 
30 #undef __FUNCT__
31 #define __FUNCT__ "PCGAMGClassicalGraphSplitting_Private"
32 /*
33  Split the relevant graph into diagonal and off-diagonal parts in local numbering; for now this
34  a roundabout private interface to the mats' internal diag and offdiag mats.
35  */
36 PetscErrorCode PCGAMGClassicalGraphSplitting_Private(Mat G,Mat *Gd, Mat *Go)
37 {
38   Mat_MPIAIJ     *aij = (Mat_MPIAIJ*)G->data;
39   PetscErrorCode ierr;
40   PetscBool      isMPIAIJ;
41   PetscFunctionBegin;
42   ierr = PetscObjectTypeCompare((PetscObject)G, MATMPIAIJ, &isMPIAIJ ); CHKERRQ(ierr);
43   if (isMPIAIJ) {
44     *Gd = aij->A;
45     *Go = aij->B;
46   } else {
47     *Gd = G;
48     *Go = NULL;
49   }
50   PetscFunctionReturn(0);
51 }
52 
53 #undef __FUNCT__
54 #define __FUNCT__ "PCGAMGGraph_Classical"
55 PetscErrorCode PCGAMGGraph_Classical(PC pc,const Mat A,Mat *G)
56 {
57   PetscInt          s,f,n,idx,lidx,gidx;
58   PetscInt          r,c,ncols;
59   const PetscInt    *rcol;
60   const PetscScalar *rval;
61   PetscInt          *gcol;
62   PetscScalar       *gval;
63   PetscReal         rmax;
64   PetscInt          cmax = 0;
65   PC_MG             *mg;
66   PC_GAMG           *gamg;
67   PetscErrorCode    ierr;
68   PetscInt          *gsparse,*lsparse;
69   PetscScalar       *Amax;
70   MatType           mtype;
71 
72   PetscFunctionBegin;
73   mg   = (PC_MG *)pc->data;
74   gamg = (PC_GAMG *)mg->innerctx;
75 
76   ierr = MatGetOwnershipRange(A,&s,&f);CHKERRQ(ierr);
77   n=f-s;
78   ierr = PetscMalloc(sizeof(PetscInt)*n,&lsparse);CHKERRQ(ierr);
79   ierr = PetscMalloc(sizeof(PetscInt)*n,&gsparse);CHKERRQ(ierr);
80   ierr = PetscMalloc(sizeof(PetscScalar)*n,&Amax);CHKERRQ(ierr);
81 
82   for (r = 0;r < n;r++) {
83     lsparse[r] = 0;
84     gsparse[r] = 0;
85   }
86 
87   for (r = s;r < f;r++) {
88     /* determine the maximum off-diagonal in each row */
89     rmax = 0.;
90     ierr = MatGetRow(A,r,&ncols,&rcol,&rval);CHKERRQ(ierr);
91     for (c = 0; c < ncols; c++) {
92       if (PetscRealPart(-rval[c]) > rmax && rcol[c] != r) {
93         rmax = PetscRealPart(-rval[c]);
94       }
95     }
96     Amax[r-s] = rmax;
97     if (ncols > cmax) cmax = ncols;
98     lidx = 0;
99     gidx = 0;
100     /* create the local and global sparsity patterns */
101     for (c = 0; c < ncols; c++) {
102       if (PetscRealPart(-rval[c]) > gamg->threshold*PetscRealPart(Amax[r-s])) {
103         if (rcol[c] < f && rcol[c] >= s) {
104           lidx++;
105         } else {
106           gidx++;
107         }
108       }
109     }
110     ierr = MatRestoreRow(A,r,&ncols,&rcol,&rval);CHKERRQ(ierr);
111     lsparse[r-s] = lidx;
112     gsparse[r-s] = gidx;
113   }
114   ierr = PetscMalloc(sizeof(PetscScalar)*cmax,&gval);CHKERRQ(ierr);
115   ierr = PetscMalloc(sizeof(PetscInt)*cmax,&gcol);CHKERRQ(ierr);
116 
117   ierr = MatCreate(PetscObjectComm((PetscObject)A),G); CHKERRQ(ierr);
118   ierr = MatGetType(A,&mtype);CHKERRQ(ierr);
119   ierr = MatSetType(*G,mtype);CHKERRQ(ierr);
120   ierr = MatSetSizes(*G,n,n,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
121   ierr = MatMPIAIJSetPreallocation(*G,0,lsparse,0,gsparse);CHKERRQ(ierr);
122   ierr = MatSeqAIJSetPreallocation(*G,0,lsparse);CHKERRQ(ierr);
123   for (r = s;r < f;r++) {
124     ierr = MatGetRow(A,r,&ncols,&rcol,&rval);CHKERRQ(ierr);
125     idx = 0;
126     for (c = 0; c < ncols; c++) {
127       /* classical strength of connection */
128       if (PetscRealPart(-rval[c]) > gamg->threshold*PetscRealPart(Amax[r-s])) {
129         gcol[idx] = rcol[c];
130         gval[idx] = rval[c];
131         idx++;
132       }
133     }
134     ierr = MatSetValues(*G,1,&r,idx,gcol,gval,INSERT_VALUES);CHKERRQ(ierr);
135     ierr = MatRestoreRow(A,r,&ncols,&rcol,&rval);CHKERRQ(ierr);
136   }
137   ierr = MatAssemblyBegin(*G, MAT_FINAL_ASSEMBLY); CHKERRQ(ierr);
138   ierr = MatAssemblyEnd(*G, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
139 
140   ierr = PetscFree(gval);CHKERRQ(ierr);
141   ierr = PetscFree(gcol);CHKERRQ(ierr);
142   ierr = PetscFree(lsparse);CHKERRQ(ierr);
143   ierr = PetscFree(gsparse);CHKERRQ(ierr);
144   ierr = PetscFree(Amax);CHKERRQ(ierr);
145   PetscFunctionReturn(0);
146 }
147 
148 
149 #undef __FUNCT__
150 #define __FUNCT__ "PCGAMGCoarsen_Classical"
151 PetscErrorCode PCGAMGCoarsen_Classical(PC pc,Mat *G,PetscCoarsenData **agg_lists)
152 {
153   PetscErrorCode   ierr;
154   MatCoarsen       crs;
155   MPI_Comm         fcomm = ((PetscObject)pc)->comm;
156 
157   PetscFunctionBegin;
158 
159 
160   /* construct the graph if necessary */
161   if (!G) {
162     SETERRQ(fcomm,PETSC_ERR_ARG_WRONGSTATE,"Must set Graph in PC in PCGAMG before coarsening");
163   }
164 
165   ierr = MatCoarsenCreate(fcomm,&crs);CHKERRQ(ierr);
166   ierr = MatCoarsenSetFromOptions(crs);CHKERRQ(ierr);
167   ierr = MatCoarsenSetAdjacency(crs,*G);CHKERRQ(ierr);
168   ierr = MatCoarsenSetStrictAggs(crs,PETSC_TRUE);CHKERRQ(ierr);
169   ierr = MatCoarsenApply(crs);CHKERRQ(ierr);
170   ierr = MatCoarsenGetData(crs,agg_lists);CHKERRQ(ierr);
171   ierr = MatCoarsenDestroy(&crs);CHKERRQ(ierr);
172 
173   PetscFunctionReturn(0);
174 }
175 
176 #undef __FUNCT__
177 #define __FUNCT__ "PCGAMGClassicalGhost_Private"
178 /*
179  Find all ghost nodes that are coarse and output the fine/coarse splitting for those as well
180 
181  Input:
182  G - graph;
183  gvec - Global Vector
184  avec - Local part of the scattered vec
185  bvec - Global part of the scattered vec
186 
187  Output:
188  findx - indirection t
189 
190  */
191 PetscErrorCode PCGAMGClassicalGhost_Private(Mat G,Vec v,Vec gv)
192 {
193   PetscErrorCode ierr;
194   Mat_MPIAIJ     *aij = (Mat_MPIAIJ*)G->data;
195   PetscBool      isMPIAIJ;
196 
197   PetscFunctionBegin;
198   ierr = PetscObjectTypeCompare((PetscObject)G, MATMPIAIJ, &isMPIAIJ ); CHKERRQ(ierr);
199   if (isMPIAIJ) {
200     ierr = VecScatterBegin(aij->Mvctx,v,gv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
201     ierr = VecScatterEnd(aij->Mvctx,v,gv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
202   }
203   PetscFunctionReturn(0);
204 }
205 
206 #undef __FUNCT__
207 #define __FUNCT__ "PCGAMGProlongator_Classical"
208 PetscErrorCode PCGAMGProlongator_Classical(PC pc, const Mat A, const Mat G, PetscCoarsenData *agg_lists,Mat *P)
209 {
210   PetscErrorCode    ierr;
211   MPI_Comm          comm;
212   PetscReal         *Amax_pos,*Amax_neg;
213   Mat               lA,gA;                     /* on and off diagonal matrices */
214   PetscInt          fn;                        /* fine local blocked sizes */
215   PetscInt          cn;                        /* coarse local blocked sizes */
216   PetscInt          gn;                        /* size of the off-diagonal fine vector */
217   PetscInt          fs,fe;                     /* fine (row) ownership range*/
218   PetscInt          cs,ce;                     /* coarse (column) ownership range */
219   PetscInt          i,j;                       /* indices! */
220   PetscBool         iscoarse;                  /* flag for determining if a node is coarse */
221   PetscInt          *lcid,*gcid;               /* on and off-processor coarse unknown IDs */
222   PetscInt          *lsparse,*gsparse;         /* on and off-processor sparsity patterns for prolongator */
223   PetscScalar       pij;
224   const PetscScalar *rval;
225   const PetscInt    *rcol;
226   PetscScalar       g_pos,g_neg,a_pos,a_neg,diag,invdiag,alpha,beta;
227   Vec               F;   /* vec of coarse size */
228   Vec               C;   /* vec of fine size */
229   Vec               gF;  /* vec of off-diagonal fine size */
230   MatType           mtype;
231   PetscInt          c_indx;
232   PetscScalar       c_scalar;
233   PetscInt          ncols,col;
234   PetscInt          row_f,row_c;
235   PetscInt          cmax=0,idx;
236   PetscScalar       *pvals;
237   PetscInt          *pcols;
238   PC_MG             *mg          = (PC_MG*)pc->data;
239   PC_GAMG           *gamg        = (PC_GAMG*)mg->innerctx;
240 
241   PetscFunctionBegin;
242   comm = ((PetscObject)pc)->comm;
243   ierr = MatGetOwnershipRange(A,&fs,&fe); CHKERRQ(ierr);
244   fn = (fe - fs);
245 
246   ierr = MatGetVecs(A,&F,NULL);CHKERRQ(ierr);
247 
248   /* get the number of local unknowns and the indices of the local unknowns */
249 
250   ierr = PetscMalloc(sizeof(PetscInt)*fn,&lsparse);CHKERRQ(ierr);
251   ierr = PetscMalloc(sizeof(PetscInt)*fn,&gsparse);CHKERRQ(ierr);
252   ierr = PetscMalloc(sizeof(PetscInt)*fn,&lcid);CHKERRQ(ierr);
253   ierr = PetscMalloc(sizeof(PetscReal)*fn,&Amax_pos);CHKERRQ(ierr);
254   ierr = PetscMalloc(sizeof(PetscReal)*fn,&Amax_neg);CHKERRQ(ierr);
255 
256   /* count the number of coarse unknowns */
257   cn = 0;
258   for (i=0;i<fn;i++) {
259     /* filter out singletons */
260     ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse); CHKERRQ(ierr);
261     lcid[i] = -1;
262     if (!iscoarse) {
263       cn++;
264     }
265   }
266 
267    /* create the coarse vector */
268   ierr = VecCreateMPI(comm,cn,PETSC_DECIDE,&C);CHKERRQ(ierr);
269   ierr = VecGetOwnershipRange(C,&cs,&ce);CHKERRQ(ierr);
270 
271   /* construct a global vector indicating the global indices of the coarse unknowns */
272   cn = 0;
273   for (i=0;i<fn;i++) {
274     ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse); CHKERRQ(ierr);
275     if (!iscoarse) {
276       lcid[i] = cs+cn;
277       cn++;
278     } else {
279       lcid[i] = -1;
280     }
281     *((PetscInt *)&c_scalar) = lcid[i];
282     c_indx = fs+i;
283     ierr = VecSetValues(F,1,&c_indx,&c_scalar,INSERT_VALUES);CHKERRQ(ierr);
284   }
285 
286   ierr = VecAssemblyBegin(F);CHKERRQ(ierr);
287   ierr = VecAssemblyEnd(F);CHKERRQ(ierr);
288 
289   /* determine the biggest off-diagonal entries in each row */
290   for (i=fs;i<fe;i++) {
291     Amax_pos[i-fs] = 0.;
292     Amax_neg[i-fs] = 0.;
293     ierr = MatGetRow(A,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
294     for(j=0;j<ncols;j++){
295       if ((PetscRealPart(-rval[j]) > Amax_neg[i-fs]) && i != rcol[j]) Amax_neg[i-fs] = PetscAbsScalar(rval[j]);
296       if ((PetscRealPart(rval[j])  > Amax_pos[i-fs]) && i != rcol[j]) Amax_pos[i-fs] = PetscAbsScalar(rval[j]);
297     }
298     if (ncols > cmax) cmax = ncols;
299     ierr = MatRestoreRow(A,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
300   }
301   ierr = PetscMalloc(sizeof(PetscInt)*cmax,&pcols);CHKERRQ(ierr);
302   ierr = PetscMalloc(sizeof(PetscScalar)*cmax,&pvals);CHKERRQ(ierr);
303 
304   /* split the operator into two */
305   ierr = PCGAMGClassicalGraphSplitting_Private(A,&lA,&gA);CHKERRQ(ierr);
306 
307   /* scatter to the ghost vector */
308   ierr = PCGAMGClassicalCreateGhostVector_Private(A,&gF,NULL);CHKERRQ(ierr);
309   ierr = PCGAMGClassicalGhost_Private(A,F,gF);CHKERRQ(ierr);
310 
311   if (gA) {
312     ierr = VecGetSize(gF,&gn);CHKERRQ(ierr);
313     ierr = PetscMalloc(sizeof(PetscInt)*gn,&gcid);CHKERRQ(ierr);
314     for (i=0;i<gn;i++) {
315       ierr = VecGetValues(gF,1,&i,&c_scalar);CHKERRQ(ierr);
316       gcid[i] = *((PetscInt *)&c_scalar);
317     }
318   }
319 
320   ierr = VecDestroy(&F);CHKERRQ(ierr);
321   ierr = VecDestroy(&gF);CHKERRQ(ierr);
322   ierr = VecDestroy(&C);CHKERRQ(ierr);
323 
324   /* count the on and off processor sparsity patterns for the prolongator */
325   for (i=0;i<fn;i++) {
326     /* on */
327     lsparse[i] = 0;
328     gsparse[i] = 0;
329     if (lcid[i] >= 0) {
330       lsparse[i] = 1;
331       gsparse[i] = 0;
332     } else {
333       ierr = MatGetRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
334       for (j = 0;j < ncols;j++) {
335         col = rcol[j];
336         if (lcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
337           lsparse[i] += 1;
338         }
339       }
340       ierr = MatRestoreRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
341       /* off */
342       if (gA) {
343         ierr = MatGetRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
344         for (j = 0; j < ncols; j++) {
345           col = rcol[j];
346           if (gcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
347             gsparse[i] += 1;
348           }
349         }
350         ierr = MatRestoreRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
351       }
352     }
353   }
354 
355   /* preallocate and create the prolongator */
356   ierr = MatCreate(comm,P); CHKERRQ(ierr);
357   ierr = MatGetType(G,&mtype);CHKERRQ(ierr);
358   ierr = MatSetType(*P,mtype);CHKERRQ(ierr);
359 
360   ierr = MatSetSizes(*P,fn,cn,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
361   ierr = MatMPIAIJSetPreallocation(*P,0,lsparse,0,gsparse);CHKERRQ(ierr);
362   ierr = MatSeqAIJSetPreallocation(*P,0,lsparse);CHKERRQ(ierr);
363 
364   /* loop over local fine nodes -- get the diagonal, the sum of positive and negative strong and weak weights, and set up the row */
365   for (i = 0;i < fn;i++) {
366     /* determine on or off */
367     row_f = i + fs;
368     row_c = lcid[i];
369     if (row_c >= 0) {
370       pij = 1.;
371       ierr = MatSetValues(*P,1,&row_f,1,&row_c,&pij,INSERT_VALUES);CHKERRQ(ierr);
372     } else {
373       g_pos = 0.;
374       g_neg = 0.;
375       a_pos = 0.;
376       a_neg = 0.;
377       diag = 0.;
378 
379       /* local connections */
380       ierr = MatGetRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
381       for (j = 0; j < ncols; j++) {
382         col = rcol[j];
383         if (lcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
384           if (PetscRealPart(rval[j]) > 0.) {
385             g_pos += rval[j];
386           } else {
387             g_neg += rval[j];
388           }
389         }
390         if (col != i) {
391           if (PetscRealPart(rval[j]) > 0.) {
392             a_pos += rval[j];
393           } else {
394             a_neg += rval[j];
395           }
396         } else {
397           diag = rval[j];
398         }
399       }
400       ierr = MatRestoreRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
401 
402       /* ghosted connections */
403       if (gA) {
404         ierr = MatGetRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
405         for (j = 0; j < ncols; j++) {
406           col = rcol[j];
407           if (gcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
408             if (PetscRealPart(rval[j]) > 0.) {
409               g_pos += rval[j];
410             } else {
411               g_neg += rval[j];
412             }
413           }
414           if (PetscRealPart(rval[j]) > 0.) {
415             a_pos += rval[j];
416           } else {
417             a_neg += rval[j];
418           }
419         }
420         ierr = MatRestoreRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
421       }
422 
423       if (g_neg == 0.) {
424         alpha = 0.;
425       } else {
426         alpha = -a_neg/g_neg;
427       }
428 
429       if (g_pos == 0.) {
430         diag += a_pos;
431         beta = 0.;
432       } else {
433         beta = -a_pos/g_pos;
434       }
435       if (diag == 0.) {
436         invdiag = 0.;
437       } else invdiag = 1. / diag;
438       /* on */
439       ierr = MatGetRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
440       idx = 0;
441       for (j = 0;j < ncols;j++) {
442         col = rcol[j];
443         if (lcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
444           row_f = i + fs;
445           row_c = lcid[col];
446           /* set the values for on-processor ones */
447           if (PetscRealPart(rval[j]) < 0.) {
448             pij = rval[j]*alpha*invdiag;
449           } else {
450             pij = rval[j]*beta*invdiag;
451           }
452           if (PetscAbsScalar(pij) != 0.) {
453             pvals[idx] = pij;
454             pcols[idx] = row_c;
455             idx++;
456           }
457         }
458       }
459       ierr = MatRestoreRow(lA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
460       /* off */
461       if (gA) {
462         ierr = MatGetRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
463         for (j = 0; j < ncols; j++) {
464           col = rcol[j];
465           if (gcid[col] >= 0 && (PetscRealPart(rval[j]) > gamg->threshold*Amax_pos[i] || PetscRealPart(-rval[j]) > gamg->threshold*Amax_neg[i])) {
466             row_f = i + fs;
467             row_c = gcid[col];
468             /* set the values for on-processor ones */
469             if (PetscRealPart(rval[j]) < 0.) {
470               pij = rval[j]*alpha*invdiag;
471             } else {
472               pij = rval[j]*beta*invdiag;
473             }
474             if (PetscAbsScalar(pij) != 0.) {
475               pvals[idx] = pij;
476               pcols[idx] = row_c;
477               idx++;
478             }
479           }
480         }
481         ierr = MatRestoreRow(gA,i,&ncols,&rcol,&rval);CHKERRQ(ierr);
482       }
483       ierr = MatSetValues(*P,1,&row_f,idx,pcols,pvals,INSERT_VALUES);CHKERRQ(ierr);
484     }
485   }
486 
487   ierr = MatAssemblyBegin(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
488   ierr = MatAssemblyEnd(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
489 
490   ierr = PetscFree(lsparse);CHKERRQ(ierr);
491   ierr = PetscFree(gsparse);CHKERRQ(ierr);
492   ierr = PetscFree(pcols);CHKERRQ(ierr);
493   ierr = PetscFree(pvals);CHKERRQ(ierr);
494   ierr = PetscFree(Amax_pos);CHKERRQ(ierr);
495   ierr = PetscFree(Amax_neg);CHKERRQ(ierr);
496   ierr = PetscFree(lcid);CHKERRQ(ierr);
497   if (gA) {ierr = PetscFree(gcid);CHKERRQ(ierr);}
498 
499   PetscFunctionReturn(0);
500 }
501 
502 #undef __FUNCT__
503 #define __FUNCT__ "PCGAMGProlongator_Standard_Classical"
504 PetscErrorCode PCGAMGProlongator_Standard_Classical(PC pc, const Mat A, const Mat G, PetscCoarsenData *agg_lists,Mat *P)
505 {
506   PetscErrorCode    ierr;
507   Mat               *lA;
508   Vec               lv,v,cv;
509   PetscScalar       *lcid;
510   IS                lis;
511   PetscInt          fs,fe,cs,ce,nl,i,j,k,li,lni,ci;
512   VecScatter        lscat;
513   PetscInt          fn,cn,cid,c_indx;
514   PetscBool         iscoarse;
515   PetscScalar       c_scalar;
516   const PetscScalar *vcol;
517   const PetscInt    *icol;
518   const PetscInt    *gidx;
519   PetscInt          ncols;
520   PetscInt          *lsparse,*gsparse;
521   MatType           mtype;
522   PetscInt          maxcols;
523   PetscReal         g_pos,g_neg,a_pos,a_neg,diag,invdiag,alpha,beta;
524   /* PetscReal         jdiag,invjdiag; */
525   PetscReal         *amax_pos,*amax_neg;
526   PetscScalar       *pvcol,vi;
527   PetscInt          *picol;
528   PetscInt          pncols;
529   PetscScalar       *pcontrib,pentry;
530   PC_MG             *mg          = (PC_MG*)pc->data;
531   PC_GAMG           *gamg        = (PC_GAMG*)mg->innerctx;
532 
533   PetscFunctionBegin;
534 
535   ierr = MatGetOwnershipRange(A,&fs,&fe);CHKERRQ(ierr);
536   fn = fe-fs;
537   ierr = MatGetVecs(A,NULL,&v);CHKERRQ(ierr);
538   ierr = ISCreateStride(PETSC_COMM_SELF,fe-fs,fs,1,&lis);CHKERRQ(ierr);
539   /* increase the overlap by two to get neighbors of neighbors */
540   ierr = MatIncreaseOverlap(A,1,&lis,2);CHKERRQ(ierr);
541   ierr = ISSort(lis);CHKERRQ(ierr);
542   /* get the local part of A */
543   ierr = MatGetSubMatrices(A,1,&lis,&lis,MAT_INITIAL_MATRIX,&lA);CHKERRQ(ierr);
544   /* build the scatter out of it */
545   ierr = ISGetLocalSize(lis,&nl);CHKERRQ(ierr);
546   ierr = VecCreateSeq(PETSC_COMM_SELF,nl,&lv);CHKERRQ(ierr);
547   ierr = VecScatterCreate(v,lis,lv,NULL,&lscat);CHKERRQ(ierr);
548 
549   ierr = PetscMalloc(sizeof(PetscInt)*fn,&lsparse);CHKERRQ(ierr);
550   ierr = PetscMalloc(sizeof(PetscInt)*fn,&gsparse);CHKERRQ(ierr);
551   ierr = PetscMalloc(sizeof(PetscScalar)*nl,&amax_pos);CHKERRQ(ierr);
552   ierr = PetscMalloc(sizeof(PetscScalar)*nl,&amax_neg);CHKERRQ(ierr);
553   ierr = PetscMalloc(sizeof(PetscScalar)*nl,&pcontrib);CHKERRQ(ierr);
554 
555   /* create coarse vector */
556   cn = 0;
557   for (i=0;i<fn;i++) {
558     ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse);CHKERRQ(ierr);
559     if (!iscoarse) {
560       cn++;
561     }
562   }
563   ierr = VecCreateMPI(PetscObjectComm((PetscObject)A),cn,PETSC_DECIDE,&cv);CHKERRQ(ierr);
564   ierr = VecGetOwnershipRange(cv,&cs,&ce);CHKERRQ(ierr);
565   cn = 0;
566   for (i=0;i<fn;i++) {
567     ierr = PetscCDEmptyAt(agg_lists,i,&iscoarse); CHKERRQ(ierr);
568     if (!iscoarse) {
569       cid = cs+cn;
570       cn++;
571     } else {
572       cid = -1;
573     }
574     c_scalar = (PetscScalar)cid;
575     c_indx = fs+i;
576     ierr = VecSetValues(v,1,&c_indx,&c_scalar,INSERT_VALUES);CHKERRQ(ierr);
577   }
578   ierr = VecScatterBegin(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
579   ierr = VecScatterEnd(lscat,v,lv,INSERT_VALUES,SCATTER_FORWARD);CHKERRQ(ierr);
580   /* count to preallocate the prolongator */
581   ierr = ISGetIndices(lis,&gidx);CHKERRQ(ierr);
582   ierr = VecGetArray(lv,&lcid);CHKERRQ(ierr);
583   maxcols = 0;
584   for (i=0;i<nl;i++) {
585     amax_pos[i] = 0.;
586     amax_neg[i] = 0.;
587     pcontrib[i] = 0.;
588     ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
589     for (j=0;j<ncols;j++) {
590       if (i != icol[j]) {
591         if (PetscRealPart(vcol[j]) > 0.) {
592           if (amax_pos[i] < PetscAbsScalar(vcol[j])) amax_pos[i] = PetscAbsScalar(vcol[j]);
593         } else {
594           if (amax_neg[i] < PetscAbsScalar(vcol[j])) amax_neg[i] = PetscAbsScalar(vcol[j]);
595         }
596       }
597     }
598     ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
599   }
600   /* count the number of unique contributing coarse cells for each fine */
601   for (i=0;i<nl;i++) {
602     if (gidx[i] >= fs && gidx[i] < fe) {
603       li = gidx[i] - fs;
604       lsparse[li] = 0;
605       gsparse[li] = 0;
606       cid = (PetscInt)lcid[i];
607       if (cid >= 0) {
608         lsparse[li] = 1;
609       } else {
610         ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
611         for (j=0;j<ncols;j++) {
612         }
613         for (j=0;j<ncols;j++) {
614           if ((PetscInt)lcid[icol[j]] >= 0) {
615             pcontrib[icol[j]] = 1.;
616           } else {
617             ci = icol[j];
618             ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
619             ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
620             for (k=0;k<ncols;k++) {
621               if ((PetscInt)lcid[icol[k]] >= 0) {
622                 pcontrib[icol[k]] = 1.;
623               }
624             }
625             ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
626             ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
627           }
628         }
629         for (j=0;j<ncols;j++) {
630           if (lcid[icol[j]] >= 0 && pcontrib[icol[j]] != 0.) {
631             /* the neighbor is a coarse node */
632             lni = (PetscInt)lcid[icol[j]];
633             if (lni >= cs && lni < ce) {
634               lsparse[li]++;
635             } else {
636               gsparse[li]++;
637             }
638             pcontrib[icol[j]] = 0.;
639           } else {
640             /* the neighbor is a strongly connected fine node */
641             ci = icol[j];
642             ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
643             ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
644             for (k=0;k<ncols;k++) {
645               if (lcid[icol[k]] >= 0 && pcontrib[icol[k]] != 0.) {
646                 lni = (PetscInt)lcid[icol[k]];
647                 if (lni >= cs && lni < ce) {
648                   lsparse[li]++;
649                 } else {
650                   gsparse[li]++;
651                 }
652                 pcontrib[icol[k]] = 0.;
653               }
654             }
655             ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
656             ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
657           }
658         }
659         if (lsparse[li] + gsparse[li] > maxcols) maxcols = lsparse[li] + gsparse[li];
660         ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
661       }
662     }
663   }
664   ierr = PetscMalloc(sizeof(PetscInt)*maxcols,&picol);CHKERRQ(ierr);
665   ierr = PetscMalloc(sizeof(PetscScalar)*maxcols,&pvcol);CHKERRQ(ierr);
666   ierr = MatCreate(PetscObjectComm((PetscObject)A),P);CHKERRQ(ierr);
667   ierr = MatGetType(A,&mtype);CHKERRQ(ierr);
668   ierr = MatSetType(*P,mtype);CHKERRQ(ierr);
669 
670   ierr = MatSetSizes(*P,fn,cn,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
671   ierr = MatMPIAIJSetPreallocation(*P,0,lsparse,0,gsparse);CHKERRQ(ierr);
672   ierr = MatSeqAIJSetPreallocation(*P,0,lsparse);CHKERRQ(ierr);
673   for (i=0;i<nl;i++) {
674     if (gidx[i] >= fs && gidx[i] < fe) {
675       a_pos = 0.;
676       a_neg = 0.;
677       g_pos = 0.;
678       g_neg = 0.;
679       li = gidx[i] - fs;
680       pncols=0;
681       cid = (PetscInt)lcid[i];
682       if (cid >= 0) {
683         pncols = 1;
684         picol[0] = cid;
685         pvcol[0] = 1.;
686       } else {
687         ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
688         for (j=0;j<ncols;j++) {
689           if (icol[j] == i) {
690             diag = vcol[j];
691           } else {
692             if ((PetscInt)lcid[icol[j]] >= 0 && (PetscRealPart(vcol[j]) > gamg->threshold*amax_pos[i] || PetscRealPart(-vcol[j]) > gamg->threshold*amax_neg[i])) {
693               if (PetscRealPart(vcol[j]) > 0.) {
694                 g_pos += vcol[j];
695               } else {
696                 g_neg += vcol[j];
697               }
698             }
699             if (PetscRealPart(vcol[j]) > 0.) {
700               a_pos += vcol[j];
701             } else {
702               a_neg += vcol[j];
703             }
704           }
705         }
706         if (g_neg == 0.) {
707           alpha = 0.;
708         } else {
709           alpha = a_neg/g_neg;
710         }
711         if (g_pos == 0.) {
712           diag += a_pos;
713           beta = 0.;
714         } else {
715           beta = a_pos/g_pos;
716         }
717         invdiag = 0;
718         if (diag != 0.) {
719           invdiag = 1./diag;
720         }
721         for (j=0;j<ncols;j++) {
722           if (PetscRealPart(vcol[j]) > gamg->threshold*amax_pos[i] || PetscRealPart(-vcol[j]) > gamg->threshold*amax_neg[i]) {
723             if (PetscRealPart(vcol[j]) < 0.) {
724               pentry = -vcol[j]*invdiag*alpha;
725             } else {
726               pentry = -vcol[j]*invdiag*beta;
727             }
728             if ((PetscInt)lcid[icol[j]] >= 0) {
729               /* coarse neighbor */
730               pcontrib[icol[j]] = pentry;
731             } else {
732               /* the neighbor is a strongly connected fine node */
733               ci = icol[j];
734               vi = vcol[j];
735               ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
736               ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
737               jdiag = 0.;
738               invjdiag = 0.;
739               for (k=0;k<ncols;k++) {
740                 if (ci == icol[k]) jdiag = PetscRealPart(vcol[k]);
741               }
742               if (jdiag != 0) {
743                 invjdiag = 1. / jdiag;
744               }
745               for (k=0;k<ncols;k++) {
746                 if ((PetscInt)lcid[icol[k]] >= 0 && (PetscAbsScalar(vcol[k]) > gamg->threshold*amax_pos[ci] || PetscRealPart(vcol[k]) < gamg->threshold*amax_neg[ci])) {
747                   /* pcontrib[icol[k]] += -pentry*vcol[k]*invjdiag; */
748                 }
749               }
750               ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
751               ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
752             }
753           }
754         }
755         for (j=0;j<ncols;j++) {
756           if (lcid[icol[j]] >= 0 && pcontrib[icol[j]] != 0.) {
757             /* the neighbor is a coarse node */
758             lni = (PetscInt)lcid[icol[j]];
759             pvcol[pncols] = pcontrib[icol[j]];
760             picol[pncols] = lni;
761             pcontrib[icol[j]] = 0.;
762             pncols++;
763           } else {
764             /* the neighbor is a strongly connected fine node */
765             ci = icol[j];
766             ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
767             ierr = MatGetRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
768             for (k=0;k<ncols;k++) {
769               if (lcid[icol[k]] >= 0 && pcontrib[icol[k]] != 0.) {
770                 lni = (PetscInt)lcid[icol[k]];
771                 pvcol[pncols] = pcontrib[icol[k]];
772                 picol[pncols] = lni;
773                 pcontrib[icol[k]] = 0.;
774                 pncols++;
775               }
776             }
777             ierr = MatRestoreRow(lA[0],ci,&ncols,&icol,&vcol);CHKERRQ(ierr);
778             ierr = MatGetRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
779           }
780         }
781         ierr = MatRestoreRow(lA[0],i,&ncols,&icol,&vcol);CHKERRQ(ierr);
782       }
783       ci = gidx[i];
784       li = gidx[i] - fs;
785       if (pncols > 0) {
786         ierr = MatSetValues(*P,1,&ci,pncols,picol,pvcol,INSERT_VALUES);CHKERRQ(ierr);
787       }
788     }
789   }
790 
791   ierr = ISRestoreIndices(lis,&gidx);CHKERRQ(ierr);
792   ierr = VecRestoreArray(lv,&lcid);CHKERRQ(ierr);
793 
794   ierr = PetscFree(amax_pos);CHKERRQ(ierr);
795   ierr = PetscFree(amax_neg);CHKERRQ(ierr);
796   ierr = PetscFree(pcontrib);CHKERRQ(ierr);
797   ierr = PetscFree(picol);CHKERRQ(ierr);
798   ierr = PetscFree(pvcol);CHKERRQ(ierr);
799   ierr = PetscFree(lsparse);CHKERRQ(ierr);
800   ierr = PetscFree(gsparse);CHKERRQ(ierr);
801   ierr = ISDestroy(&lis);CHKERRQ(ierr);
802   ierr = MatDestroyMatrices(1,&lA);CHKERRQ(ierr);
803   ierr = VecDestroy(&lv);CHKERRQ(ierr);
804   ierr = VecDestroy(&cv);CHKERRQ(ierr);
805   ierr = VecDestroy(&v);CHKERRQ(ierr);
806   ierr = VecScatterDestroy(&lscat);CHKERRQ(ierr);
807 
808   ierr = MatAssemblyBegin(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
809   ierr = MatAssemblyEnd(*P, MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
810 
811   /*
812   Mat Pold;
813   ierr = PCGAMGProlongator_Classical(pc,A,G,agg_lists,&Pold);CHKERRQ(ierr);
814   ierr = MatView(Pold,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
815   ierr = MatView(*P,PETSC_VIEWER_STDOUT_WORLD);CHKERRQ(ierr);
816   ierr = MatDestroy(&Pold);CHKERRQ(ierr);
817    */
818 
819   PetscFunctionReturn(0);
820 }
821 
822 #undef __FUNCT__
823 #define __FUNCT__ "PCGAMGDestroy_Classical"
824 PetscErrorCode PCGAMGDestroy_Classical(PC pc)
825 {
826   PetscErrorCode ierr;
827   PC_MG          *mg          = (PC_MG*)pc->data;
828   PC_GAMG        *pc_gamg     = (PC_GAMG*)mg->innerctx;
829 
830   PetscFunctionBegin;
831   ierr = PetscFree(pc_gamg->subctx);CHKERRQ(ierr);
832   PetscFunctionReturn(0);
833 }
834 
835 #undef __FUNCT__
836 #define __FUNCT__ "PCGAMGSetFromOptions_Classical"
837 PetscErrorCode PCGAMGSetFromOptions_Classical(PC pc)
838 {
839   PetscFunctionBegin;
840   PetscFunctionReturn(0);
841 }
842 
843 #undef __FUNCT__
844 #define __FUNCT__ "PCGAMGSetData_Classical"
845 PetscErrorCode PCGAMGSetData_Classical(PC pc, Mat A)
846 {
847   PC_MG          *mg      = (PC_MG*)pc->data;
848   PC_GAMG        *pc_gamg = (PC_GAMG*)mg->innerctx;
849 
850   PetscFunctionBegin;
851   /* no data for classical AMG */
852   pc_gamg->data           = NULL;
853   pc_gamg->data_cell_cols = 0;
854   pc_gamg->data_cell_rows = 0;
855   pc_gamg->data_sz = 0;
856   PetscFunctionReturn(0);
857 }
858 
859 /* -------------------------------------------------------------------------- */
860 /*
861    PCCreateGAMG_Classical
862 
863 */
864 #undef __FUNCT__
865 #define __FUNCT__ "PCCreateGAMG_Classical"
866 PetscErrorCode  PCCreateGAMG_Classical(PC pc)
867 {
868   PetscErrorCode ierr;
869   PC_MG             *mg      = (PC_MG*)pc->data;
870   PC_GAMG           *pc_gamg = (PC_GAMG*)mg->innerctx;
871   PC_GAMG_Classical *pc_gamg_classical;
872 
873   PetscFunctionBegin;
874   if (pc_gamg->subctx) {
875     /* call base class */
876     ierr = PCDestroy_GAMG(pc);CHKERRQ(ierr);
877   }
878 
879   /* create sub context for SA */
880   ierr = PetscNewLog(pc, PC_GAMG_Classical, &pc_gamg_classical);CHKERRQ(ierr);
881   pc_gamg->subctx = pc_gamg_classical;
882   pc->ops->setfromoptions = PCGAMGSetFromOptions_Classical;
883   /* reset does not do anything; setup not virtual */
884 
885   /* set internal function pointers */
886   pc_gamg->ops->destroy     = PCGAMGDestroy_Classical;
887   pc_gamg->ops->graph       = PCGAMGGraph_Classical;
888   pc_gamg->ops->coarsen     = PCGAMGCoarsen_Classical;
889   pc_gamg->ops->prolongator = PCGAMGProlongator_Standard_Classical;
890   pc_gamg->ops->optprol     = NULL;
891 
892   pc_gamg->ops->createdefaultdata = PCGAMGSetData_Classical;
893   PetscFunctionReturn(0);
894 }
895