1 static char help[] = "Tests MatCreateDenseCUDA(), MatDenseCUDAPlaceArray(), MatDenseCUDAReplaceArray(), MatDenseCUDAResetArray()\n"; 2 3 #include <petscmat.h> 4 5 static PetscErrorCode MatMult_S(Mat S,Vec x,Vec y) 6 { 7 PetscErrorCode ierr; 8 Mat A; 9 10 PetscFunctionBegin; 11 ierr = MatShellGetContext(S,(void**)&A);CHKERRQ(ierr); 12 ierr = MatMult(A,x,y);CHKERRQ(ierr); 13 PetscFunctionReturn(0); 14 } 15 16 int main(int argc,char **argv) 17 { 18 Mat A,B,C,S; 19 Vec t,v; 20 PetscScalar *vv,*aa; 21 PetscInt n=30,k=6,l=0,i,Istart,Iend,nloc,bs,test=1; 22 PetscBool flg,reset,use_shell = PETSC_FALSE; 23 PetscErrorCode ierr; 24 VecType vtype; 25 26 ierr = PetscInitialize(&argc,&argv,(char*)0,help);if (ierr) return ierr; 27 ierr = PetscOptionsGetInt(NULL,NULL,"-n",&n,NULL);CHKERRQ(ierr); 28 ierr = PetscOptionsGetInt(NULL,NULL,"-k",&k,NULL);CHKERRQ(ierr); 29 ierr = PetscOptionsGetInt(NULL,NULL,"-l",&l,NULL);CHKERRQ(ierr); 30 ierr = PetscOptionsGetInt(NULL,NULL,"-test",&test,NULL);CHKERRQ(ierr); 31 ierr = PetscOptionsGetBool(NULL,NULL,"-use_shell",&use_shell,NULL);CHKERRQ(ierr); 32 if (k < 0) SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_USER,"k %D must be positive",k); 33 if (l < 0) SETERRQ1(PETSC_COMM_WORLD,PETSC_ERR_USER,"l %D must be positive",l); 34 if (l > k) SETERRQ2(PETSC_COMM_WORLD,PETSC_ERR_USER,"l %D must be smaller or equal than k %D\n",l,k); 35 36 /* sparse matrix */ 37 ierr = MatCreate(PETSC_COMM_WORLD,&A);CHKERRQ(ierr); 38 ierr = MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,n,n);CHKERRQ(ierr); 39 ierr = MatSetType(A,MATAIJCUSPARSE);CHKERRQ(ierr); 40 ierr = MatSetOptionsPrefix(A,"A_");CHKERRQ(ierr); 41 ierr = MatSetFromOptions(A);CHKERRQ(ierr); 42 ierr = MatSetUp(A);CHKERRQ(ierr); 43 44 ierr = MatGetOwnershipRange(A,&Istart,&Iend);CHKERRQ(ierr); 45 for (i=Istart;i<Iend;i++) { 46 if (i>0) { ierr = MatSetValue(A,i,i-1,-1.0,INSERT_VALUES);CHKERRQ(ierr); } 47 if (i<n-1) { ierr = MatSetValue(A,i,i+1,-1.0,INSERT_VALUES);CHKERRQ(ierr); } 48 ierr = MatSetValue(A,i,i,2.0,INSERT_VALUES);CHKERRQ(ierr); 49 } 50 ierr = MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); 51 ierr = MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); 52 53 /* template vector */ 54 ierr = MatCreateVecs(A,NULL,&t);CHKERRQ(ierr); 55 ierr = VecGetType(t,&vtype);CHKERRQ(ierr); 56 57 /* long vector, contains the stacked columns of an nxk dense matrix */ 58 ierr = VecGetLocalSize(t,&nloc);CHKERRQ(ierr); 59 ierr = VecGetBlockSize(t,&bs);CHKERRQ(ierr); 60 ierr = VecCreate(PetscObjectComm((PetscObject)t),&v);CHKERRQ(ierr); 61 ierr = VecSetType(v,vtype);CHKERRQ(ierr); 62 ierr = VecSetSizes(v,k*nloc,k*n);CHKERRQ(ierr); 63 ierr = VecSetBlockSize(v,bs);CHKERRQ(ierr); 64 ierr = VecSetRandom(v,NULL);CHKERRQ(ierr); 65 66 /* dense matrix that contains the columns of v */ 67 ierr = VecCUDAGetArray(v,&vv);CHKERRQ(ierr); 68 69 /* test few cases for MatDenseCUDA handling pointers */ 70 switch (test) { 71 case 1: 72 ierr = MatCreateDenseCUDA(PetscObjectComm((PetscObject)v),nloc,PETSC_DECIDE,n,k-l,vv,&B);CHKERRQ(ierr); /* pass a pointer to avoid allocation of storage */ 73 ierr = MatDenseCUDAReplaceArray(B,NULL);CHKERRQ(ierr); /* replace with a null pointer, the value after BVRestoreMat */ 74 ierr = MatDenseCUDAPlaceArray(B,vv+l*nloc);CHKERRQ(ierr); /* set the actual pointer */ 75 reset = PETSC_TRUE; 76 break; 77 case 2: 78 ierr = MatCreateDenseCUDA(PetscObjectComm((PetscObject)v),nloc,PETSC_DECIDE,n,k-l,NULL,&B);CHKERRQ(ierr); 79 ierr = MatDenseCUDAPlaceArray(B,vv+l*nloc);CHKERRQ(ierr); /* set the actual pointer */ 80 reset = PETSC_TRUE; 81 break; 82 default: 83 ierr = MatCreateDenseCUDA(PetscObjectComm((PetscObject)v),nloc,PETSC_DECIDE,n,k-l,vv+l*nloc,&B);CHKERRQ(ierr); 84 reset = PETSC_FALSE; 85 break; 86 } 87 ierr = VecCUDARestoreArray(v,&vv);CHKERRQ(ierr); 88 89 /* Test MatMatMult */ 90 if (use_shell) { 91 ierr = MatCreateShell(PetscObjectComm((PetscObject)v),nloc,nloc,n,n,A,&S);CHKERRQ(ierr); 92 ierr = MatShellSetOperation(S,MATOP_MULT,(void(*)(void))MatMult_S);CHKERRQ(ierr); 93 ierr = MatShellSetVecType(S,vtype);CHKERRQ(ierr); 94 /* we could have called the general convertor also */ 95 /* ierr = MatConvert(A,MATSHELL,MAT_INITIAL_MATRIX,&S);CHKERRQ(ierr); */ 96 } else { 97 ierr = PetscObjectReference((PetscObject)A);CHKERRQ(ierr); 98 S = A; 99 } 100 101 ierr = MatCreateDenseCUDA(PetscObjectComm((PetscObject)v),nloc,PETSC_DECIDE,n,k-l,NULL,&C);CHKERRQ(ierr); 102 ierr = MatAssemblyBegin(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); 103 ierr = MatAssemblyEnd(B,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); 104 ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); 105 ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr); 106 ierr = MatProductCreateWithMat(S,B,NULL,C);CHKERRQ(ierr); 107 ierr = MatProductSetType(C,MATPRODUCT_AB);CHKERRQ(ierr); 108 ierr = MatProductSetFromOptions(C);CHKERRQ(ierr); 109 ierr = MatProductSymbolic(C);CHKERRQ(ierr); 110 ierr = MatProductNumeric(C);CHKERRQ(ierr); 111 112 /* test MatMatMult */ 113 ierr = MatMatMultEqual(S,B,C,10,&flg);CHKERRQ(ierr); 114 if (!flg) { ierr = PetscPrintf(PETSC_COMM_WORLD,"Error MatMatMult\n"); } 115 116 ierr = MatDestroy(&C);CHKERRQ(ierr); 117 ierr = MatDestroy(&S);CHKERRQ(ierr); 118 119 /* finished using B */ 120 ierr = MatDenseCUDAGetArray(B,&aa);CHKERRQ(ierr); 121 if (vv != aa-l*nloc) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Wrong array"); 122 ierr = MatDenseCUDARestoreArray(B,&aa);CHKERRQ(ierr); 123 if (reset) { 124 ierr = MatDenseCUDAResetArray(B);CHKERRQ(ierr); 125 } 126 ierr = VecCUDARestoreArray(v,&vv);CHKERRQ(ierr); 127 128 if (test == 1) { 129 ierr = MatDenseCUDAGetArray(B,&aa);CHKERRQ(ierr); 130 if (aa) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"Expected a null pointer"); 131 ierr = MatDenseCUDARestoreArray(B,&aa);CHKERRQ(ierr); 132 } 133 134 /* free work space */ 135 ierr = MatDestroy(&B);CHKERRQ(ierr); 136 ierr = MatDestroy(&A);CHKERRQ(ierr); 137 ierr = VecDestroy(&t);CHKERRQ(ierr); 138 ierr = VecDestroy(&v);CHKERRQ(ierr); 139 ierr = PetscFinalize(); 140 return ierr; 141 } 142 143 /*TEST 144 145 build: 146 requires: cuda 147 148 test: 149 requires: cuda 150 suffix: 1 151 nsize: {{1 2}} 152 args: -A_mat_type {{aij aijcusparse}} -test {{0 1 2}} -k 6 -l {{0 5}} -use_shell {{0 1}} 153 154 TEST*/ 155