Actual source code: tools.c

petsc-3.3-p6 2013-02-11
  1: /*
  2:  GAMG geometric-algebric multigrid PC - Mark Adams 2011
  3:  */
  4: #include "petsc-private/matimpl.h"    /*I "petscmat.h" I*/
  5: #include <../src/ksp/pc/impls/gamg/gamg.h>           /*I "petscpc.h" I*/
  6: #include <petsc-private/kspimpl.h>

  8: /* -------------------------------------------------------------------------- */
  9: /*
 10:    PCGAMGCreateGraph - create simple scaled scalar graph from matrix
 11:  
 12:  Input Parameter:
 13:  . Amat - matrix
 14:  Output Parameter:
 15:  . a_Gmaat - eoutput scalar graph (symmetric?)
 16:  */
 19: PetscErrorCode PCGAMGCreateGraph( const Mat Amat, Mat *a_Gmat )
 20: {
 22:   PetscInt       Istart,Iend,Ii,jj,ncols,nloc,NN,MM,bs;
 23:   PetscMPIInt    mype, npe;
 24:   MPI_Comm       wcomm = ((PetscObject)Amat)->comm;
 25:   Mat            Gmat;

 28:   MPI_Comm_rank(wcomm,&mype);
 29:   MPI_Comm_size(wcomm,&npe);
 30:   MatGetOwnershipRange( Amat, &Istart, &Iend );
 31:   MatGetSize( Amat, &MM, &NN );
 32:   MatGetBlockSize( Amat, &bs );
 33:   nloc = (Iend-Istart)/bs;
 34: 
 35: #if defined PETSC_GAMG_USE_LOG
 36:   PetscLogEventBegin(petsc_gamg_setup_events[GRAPH],0,0,0,0);
 37: #endif
 38:   if( bs > 1 ) {
 39:     const PetscScalar *vals;
 40:     const PetscInt *idx;
 41:     PetscInt       *d_nnz, *o_nnz;
 42:     /* count nnz, there is sparcity in here so this might not be enough */
 43:     PetscMalloc( nloc*sizeof(PetscInt), &d_nnz );
 44:     PetscMalloc( nloc*sizeof(PetscInt), &o_nnz );
 45:     for ( Ii = Istart, jj = 0 ; Ii < Iend ; Ii += bs, jj++ ) {
 46:       MatGetRow(Amat,Ii,&ncols,0,0);
 47:       d_nnz[jj] = ncols; /* very pessimistic */
 48:       o_nnz[jj] = ncols;
 49:       if( d_nnz[jj] > nloc ) d_nnz[jj] = nloc;
 50:       if( o_nnz[jj] > (NN/bs-nloc) ) o_nnz[jj] = NN/bs-nloc;
 51:       MatRestoreRow(Amat,Ii,&ncols,0,0);
 52:     }

 54:     /* get scalar copy (norms) of matrix -- AIJ specific!!! */
 55:     MatCreateAIJ( wcomm, nloc, nloc,
 56:                             PETSC_DETERMINE, PETSC_DETERMINE,
 57:                             0, d_nnz, 0, o_nnz, &Gmat );

 59:     PetscFree( d_nnz );
 60:     PetscFree( o_nnz );

 62:     for( Ii = Istart; Ii < Iend ; Ii++ ) {
 63:       PetscInt dest_row = Ii/bs;
 64:       MatGetRow(Amat,Ii,&ncols,&idx,&vals);
 65:       for(jj=0;jj<ncols;jj++){
 66:         PetscInt dest_col = idx[jj]/bs;
 67:         PetscScalar sv = PetscAbs(PetscRealPart(vals[jj]));
 68:         MatSetValues(Gmat,1,&dest_row,1,&dest_col,&sv,ADD_VALUES);
 69:       }
 70:       MatRestoreRow(Amat,Ii,&ncols,&idx,&vals);
 71:     }
 72:     MatAssemblyBegin(Gmat,MAT_FINAL_ASSEMBLY);
 73:     MatAssemblyEnd(Gmat,MAT_FINAL_ASSEMBLY);
 74:   }
 75:   else {
 76:     /* just copy scalar matrix - abs() not taken here but scaled later */
 77:     MatDuplicate( Amat, MAT_COPY_VALUES, &Gmat );
 78:   }

 80: #if defined PETSC_GAMG_USE_LOG
 81:   PetscLogEventEnd(petsc_gamg_setup_events[GRAPH],0,0,0,0);
 82: #endif

 84:   *a_Gmat = Gmat;

 86:   return(0);
 87: }

 89: /* -------------------------------------------------------------------------- */
 90: /*
 91:    PCGAMGFilterGraph - filter graph and symetrize if needed
 92:  
 93:  Input Parameter:
 94:  . vfilter - threshold paramter [0,1)
 95:  . symm - symetrize?
 96:  In/Output Parameter:
 97:  . a_Gmat - original graph
 98:  */
101: PetscErrorCode PCGAMGFilterGraph( Mat *a_Gmat, const PetscReal vfilter, const PetscBool symm, const PetscInt verbose )
102: {
104:   PetscInt       Istart,Iend,Ii,jj,ncols,nnz0,nnz1, NN, MM, nloc;
105:   PetscMPIInt    mype, npe;
106:   Mat            Gmat = *a_Gmat, tGmat, matTrans;
107:   MPI_Comm       wcomm = ((PetscObject)Gmat)->comm;
108:   const PetscScalar *vals;
109:   const PetscInt *idx;
110:   PetscInt *d_nnz, *o_nnz;
111:   Vec diag;

114:   MPI_Comm_rank(wcomm,&mype);
115:   MPI_Comm_size(wcomm,&npe);
116:   MatGetOwnershipRange( Gmat, &Istart, &Iend );
117:   nloc = Iend - Istart;
118:   MatGetSize( Gmat, &MM, &NN );
119: #if defined PETSC_GAMG_USE_LOG
120:   PetscLogEventBegin(petsc_gamg_setup_events[GRAPH],0,0,0,0);
121: #endif
122:   /* scale Gmat so filter works */
123:   MatGetVecs( Gmat, &diag, 0 );
124:   MatGetDiagonal( Gmat, diag );
125:   VecReciprocal( diag );
126:   VecSqrtAbs( diag );
127:   MatDiagonalScale( Gmat, diag, diag );
128:   VecDestroy( &diag );

130:   if( symm ) {
131:     MatTranspose( Gmat, MAT_INITIAL_MATRIX, &matTrans );
132:   }

134:   /* filter - dup zeros out matrix */
135:   PetscMalloc( nloc*sizeof(PetscInt), &d_nnz );
136:   PetscMalloc( nloc*sizeof(PetscInt), &o_nnz );
137:   for( Ii = Istart, jj = 0 ; Ii < Iend; Ii++, jj++ ){
138:     MatGetRow(Gmat,Ii,&ncols,PETSC_NULL,PETSC_NULL);
139:     d_nnz[jj] = ncols;
140:     o_nnz[jj] = ncols;
141:     MatRestoreRow(Gmat,Ii,&ncols,PETSC_NULL,PETSC_NULL);
142:     if( symm ) {
143:       MatGetRow(matTrans,Ii,&ncols,PETSC_NULL,PETSC_NULL);
144:       d_nnz[jj] += ncols;
145:       o_nnz[jj] += ncols;
146:       MatRestoreRow(matTrans,Ii,&ncols,PETSC_NULL,PETSC_NULL);
147:     }
148:     if( d_nnz[jj] > nloc ) d_nnz[jj] = nloc;
149:     if( o_nnz[jj] > (MM-nloc) ) o_nnz[jj] = MM - nloc;
150:   }
151:   MatCreateAIJ( wcomm, nloc, nloc, MM, MM, 0, d_nnz, 0, o_nnz, &tGmat );
152: 
153:   PetscFree( d_nnz );
154:   PetscFree( o_nnz );
155:   if( symm ) {
156:     MatDestroy( &matTrans );
157:   }

159:   for( Ii = Istart, nnz0 = nnz1 = 0 ; Ii < Iend; Ii++ ){
160:     MatGetRow(Gmat,Ii,&ncols,&idx,&vals);
161:     for(jj=0;jj<ncols;jj++,nnz0++){
162:       PetscScalar sv = PetscAbs(PetscRealPart(vals[jj]));
163:       if( PetscRealPart(sv) > vfilter ) {
164:         nnz1++;
165:         if( symm ) {
166:           sv *= 0.5;
167:           MatSetValues(tGmat,1,&Ii,1,&idx[jj],&sv,ADD_VALUES);
168:           MatSetValues(tGmat,1,&idx[jj],1,&Ii,&sv,ADD_VALUES);
169:         }
170:         else {
171:           MatSetValues(tGmat,1,&Ii,1,&idx[jj],&sv,ADD_VALUES);
172:         }
173:       }
174:     }
175:     MatRestoreRow(Gmat,Ii,&ncols,&idx,&vals);
176:   }
177:   MatAssemblyBegin(tGmat,MAT_FINAL_ASSEMBLY);
178:   MatAssemblyEnd(tGmat,MAT_FINAL_ASSEMBLY);

180: #if defined PETSC_GAMG_USE_LOG
181:   PetscLogEventEnd(petsc_gamg_setup_events[GRAPH],0,0,0,0);
182: #endif

184:   if( verbose ) {
185:     if( verbose == 1 ) {
186:       PetscPrintf(wcomm,"\t[%d]%s %g%% nnz after filtering, with threshold %g, %g nnz ave. (N=%d)\n",mype,__FUNCT__,
187:                   100.*(double)nnz1/(double)nnz0,vfilter,(double)nnz0/(double)nloc,MM);
188:     }
189:     else {
190:       PetscInt nnz[2],out[2];
191:       nnz[0] = nnz0;
192:       nnz[1] = nnz1;
193:       MPI_Allreduce( nnz, out, 2, MPIU_INT, MPI_SUM, wcomm );
194:       PetscPrintf(wcomm,"\t[%d]%s %g%% nnz after filtering, with threshold %g, %g nnz ave. (N=%d)\n",mype,__FUNCT__,
195:                   100.*(double)out[1]/(double)out[0],vfilter,(double)out[0]/(double)MM,MM);
196:     }
197:   }
198: 
199:   MatDestroy( &Gmat );

201:   *a_Gmat = tGmat;

203:   return(0);
204: }

206: /* -------------------------------------------------------------------------- */
207: /*
208:    PCGAMGGetDataWithGhosts - hacks into Mat MPIAIJ so this must have > 1 pe

210:    Input Parameter:
211:    . Gmat - MPIAIJ matrix for scattters
212:    . data_sz - number of data terms per node (# cols in output)
213:    . data_in[nloc*data_sz] - column oriented data
214:    Output Parameter:
215:    . a_stride - numbrt of rows of output
216:    . a_data_out[stride*data_sz] - output data with ghosts
217: */
220: PetscErrorCode PCGAMGGetDataWithGhosts( const Mat Gmat,
221:                                         const PetscInt data_sz,
222:                                         const PetscReal data_in[],
223:                                         PetscInt *a_stride,
224:                                         PetscReal **a_data_out
225:                                         )
226: {
228:   PetscMPIInt    mype,npe;
229:   MPI_Comm       wcomm = ((PetscObject)Gmat)->comm;
230:   Vec            tmp_crds;
231:   Mat_MPIAIJ    *mpimat = (Mat_MPIAIJ*)Gmat->data;
232:   PetscInt       nnodes,num_ghosts,dir,kk,jj,my0,Iend,nloc;
233:   PetscScalar   *data_arr;
234:   PetscReal     *datas;
235:   PetscBool      isMPIAIJ;

238:   PetscObjectTypeCompare( (PetscObject)Gmat, MATMPIAIJ, &isMPIAIJ );
239:   MPI_Comm_rank(wcomm,&mype);
240:   MPI_Comm_size(wcomm,&npe);
241:   MatGetOwnershipRange( Gmat, &my0, &Iend );
242:   nloc = Iend - my0;
243:   VecGetLocalSize( mpimat->lvec, &num_ghosts );
244:   nnodes = num_ghosts + nloc;
245:   *a_stride = nnodes;
246:   MatGetVecs( Gmat, &tmp_crds, 0 );

248:   PetscMalloc( data_sz*nnodes*sizeof(PetscReal), &datas);
249:   for(dir=0; dir<data_sz; dir++) {
250:     /* set local, and global */
251:     for(kk=0; kk<nloc; kk++) {
252:       PetscInt gid = my0 + kk;
253:       PetscScalar crd = (PetscScalar)data_in[dir*nloc + kk]; /* col oriented */
254:       datas[dir*nnodes + kk] = PetscRealPart(crd);
255:       VecSetValues(tmp_crds, 1, &gid, &crd, INSERT_VALUES );
256:     }
257:     VecAssemblyBegin( tmp_crds );
258:     VecAssemblyEnd( tmp_crds );
259:     /* get ghost datas */
260:     VecScatterBegin(mpimat->Mvctx,tmp_crds,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
261: 
262:     VecScatterEnd(mpimat->Mvctx,tmp_crds,mpimat->lvec,INSERT_VALUES,SCATTER_FORWARD);
263: 
264:     VecGetArray( mpimat->lvec, &data_arr );
265:     for(kk=nloc,jj=0;jj<num_ghosts;kk++,jj++){
266:       datas[dir*nnodes + kk] = PetscRealPart(data_arr[jj]);
267:     }
268:     VecRestoreArray( mpimat->lvec, &data_arr );
269:   }
270:   VecDestroy(&tmp_crds);

272:   *a_data_out = datas;

274:   return(0);
275: }


278: /* hash table stuff - simple, not dymanic, key >= 0, has table
279:  *
280:  *  GAMGTableCreate
281:  */
282: /* avoid overflow */
283: #define GAMG_HASH(key) ((7*key)%a_tab->size)
284: PetscErrorCode GAMGTableCreate( PetscInt a_size, GAMGHashTable *a_tab )
285: {
287:   PetscInt kk;
288:   a_tab->size = a_size;
289:   PetscMalloc(a_size*sizeof(PetscInt), &a_tab->table );
290:   PetscMalloc(a_size*sizeof(PetscInt), &a_tab->data );
291:   for(kk=0;kk<a_size;kk++) a_tab->table[kk] = -1;
292:   return 0;
293: }

295: PetscErrorCode GAMGTableDestroy( GAMGHashTable *a_tab )
296: {
298:   PetscFree( a_tab->table );
299:   PetscFree( a_tab->data );
300:   return 0;
301: }

303: PetscErrorCode GAMGTableAdd( GAMGHashTable *a_tab, PetscInt a_key, PetscInt a_data )
304: {
305:   PetscInt kk,idx;
306:   if(a_key<0)SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"Table size %d too small.",a_tab->size);
307:   for( kk = 0, idx = GAMG_HASH(a_key) ; kk < a_tab->size ; kk++, idx = (idx==(a_tab->size-1)) ? 0 : idx + 1 ){
308:     if( a_tab->table[idx] == a_key ) {
309:       /* exists */
310:       assert(0); /* not used this way now */
311:       a_tab->data[idx] = a_data;
312:       break;
313:     }
314:     else if( a_tab->table[idx] == -1 ) {
315:       /* add */
316:       a_tab->table[idx] = a_key;
317:       a_tab->data[idx] = a_data;
318:       break;
319:     }
320:   }
321:   if(kk==a_tab->size) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"Table size %d too small.",a_tab->size);
322:   return 0;
323: }

325: PetscErrorCode GAMGTableFind( GAMGHashTable *a_tab, PetscInt a_key, PetscInt *a_data )
326: {
327:   PetscInt kk,idx;
328:   if(a_key<0)SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"Table size %d too small.",a_tab->size);
329:   for( kk = 0, idx = GAMG_HASH(a_key) ; kk < a_tab->size ; kk++, idx = (idx==(a_tab->size-1)) ? 0 : idx + 1 ){
330:     if( a_tab->table[idx] == a_key ) {
331:       *a_data = a_tab->data[idx];
332:       break;
333:     }
334:     else if( a_tab->table[idx] == -1 ) {
335:       /* not here */
336:       *a_data = -1;
337:       break;
338:     }
339:   }
340:   if(kk==a_tab->size) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_USER,"Table size %d too small.",a_tab->size);
341:   return 0;
342: }