Actual source code: ex10.c

  2: static char help[] = "Reads a PETSc matrix and vector from a file and solves a linear system.\n\
  3: This version first preloads and solves a small system, then loads \n\
  4: another (larger) system and solves it as well.  This example illustrates\n\
  5: preloading of instructions with the smaller system so that more accurate\n\
  6: performance monitoring can be done with the larger one (that actually\n\
  7: is the system of interest).  See the 'Performance Hints' chapter of the\n\
  8: users manual for a discussion of preloading.  Input parameters include\n\
  9:   -f0 <input_file> : first file to load (small system)\n\
 10:   -f1 <input_file> : second file to load (larger system)\n\n\
 11:   -trans  : solve transpose system instead\n\n";
 12: /*
 13:   This code can be used to test PETSc interface to other packages.\n\
 14:   Examples of command line options:       \n\
 15:    ex10 -f0 <datafile> -ksp_type preonly  \n\
 16:         -help -ksp_view                  \n\
 17:         -num_numfac <num_numfac> -num_rhs <num_rhs> \n\
 18:         -ksp_type preonly -pc_type lu -mat_type aijspooles/superlu/superlu_dist/aijmumps \n\
 19:         -ksp_type preonly -pc_type cholesky -mat_type sbaijspooles/dscpack/sbaijmumps \n\
 20:         -f0 <A> -fB <B> -mat_type sbaijmumps -ksp_type preonly -pc_type cholesky -test_inertia -mat_sigma <sigma> \n\
 21:    mpiexec -np <np> ex10 -f0 <datafile> -ksp_type cg -pc_type asm -pc_asm_type basic -sub_pc_type icc -mat_type sbaij
 22:  \n\n";
 23: */
 24: /*T
 25:    Concepts: KSP^solving a linear system
 26:    Processors: n
 27: T*/

 29: /* 
 30:   Include "petscksp.h" so that we can use KSP solvers.  Note that this file
 31:   automatically includes:
 32:      petsc.h       - base PETSc routines   petscvec.h - vectors
 33:      petscsys.h    - system routines       petscmat.h - matrices
 34:      petscis.h     - index sets            petscksp.h - Krylov subspace methods
 35:      petscviewer.h - viewers               petscpc.h  - preconditioners
 36: */
 37:  #include petscksp.h

 41: int main(int argc,char **args)
 42: {
 43:   KSP            ksp;             /* linear solver context */
 44:   Mat            A,B;            /* matrix */
 45:   Vec            x,b,u;          /* approx solution, RHS, exact solution */
 46:   PetscViewer    fd;               /* viewer */
 47:   char           file[3][PETSC_MAX_PATH_LEN];     /* input file name */
 48:   PetscTruth     table,flg,flgB=PETSC_FALSE,trans=PETSC_FALSE,partition=PETSC_FALSE,initialguess = PETSC_FALSE;
 50:   PetscInt       its,num_numfac,m,n,M;
 51:   PetscReal      norm;
 52:   PetscLogDouble tsetup,tsetup1,tsetup2,tsolve,tsolve1,tsolve2;
 53:   PetscTruth     preload=PETSC_TRUE,diagonalscale,isSymmetric,cknorm=PETSC_FALSE,Test_MatDuplicate=PETSC_FALSE;
 54:   PetscMPIInt    rank;
 55:   PetscScalar    sigma;

 57:   PetscInitialize(&argc,&args,(char *)0,help);
 58:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
 59:   PetscOptionsHasName(PETSC_NULL,"-table",&table);
 60:   PetscOptionsHasName(PETSC_NULL,"-trans",&trans);
 61:   PetscOptionsHasName(PETSC_NULL,"-partition",&partition);
 62:   PetscOptionsHasName(PETSC_NULL,"-initialguess",&initialguess);

 64:   /* 
 65:      Determine files from which we read the two linear systems
 66:      (matrix and right-hand-side vector).
 67:   */
 68:   PetscOptionsGetString(PETSC_NULL,"-f",file[0],PETSC_MAX_PATH_LEN-1,&flg);
 69:   if (flg) {
 70:     PetscStrcpy(file[1],file[0]);
 71:     preload = PETSC_FALSE;
 72:   } else {
 73:     PetscOptionsGetString(PETSC_NULL,"-f0",file[0],PETSC_MAX_PATH_LEN-1,&flg);
 74:     if (!flg) SETERRQ(1,"Must indicate binary file with the -f0 or -f option");
 75:     PetscOptionsGetString(PETSC_NULL,"-f1",file[1],PETSC_MAX_PATH_LEN-1,&flg);
 76:     if (!flg) {preload = PETSC_FALSE;} /* don't bother with second system */
 77:   }

 79:   /* -----------------------------------------------------------
 80:                   Beginning of linear solver loop
 81:      ----------------------------------------------------------- */
 82:   /* 
 83:      Loop through the linear solve 2 times.  
 84:       - The intention here is to preload and solve a small system;
 85:         then load another (larger) system and solve it as well.
 86:         This process preloads the instructions with the smaller
 87:         system so that more accurate performance monitoring (via
 88:         -log_summary) can be done with the larger one (that actually
 89:         is the system of interest). 
 90:   */
 91:   PreLoadBegin(preload,"Load system");

 93:     /* - - - - - - - - - - - New Stage - - - - - - - - - - - - -
 94:                            Load system
 95:      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

 97:     /* 
 98:        Open binary file.  Note that we use FILE_MODE_READ to indicate
 99:        reading from this file.
100:     */
101:     PetscViewerBinaryOpen(PETSC_COMM_WORLD,file[PreLoadIt],FILE_MODE_READ,&fd);
102: 
103:     /*
104:        Load the matrix and vector; then destroy the viewer.
105:     */
106:     MatLoad(fd,MATAIJ,&A);
107: 
108:     if (!preload){
109:       flg = PETSC_FALSE;
110:       PetscOptionsGetString(PETSC_NULL,"-rhs",file[2],PETSC_MAX_PATH_LEN-1,&flg);
111:       if (flg){ /* rhs is stored in a separate file */
112:         PetscViewerDestroy(fd);
113:         PetscViewerBinaryOpen(PETSC_COMM_WORLD,file[2],FILE_MODE_READ,&fd);
114:       }
115:     }
116:     if (rank){
117:         PetscExceptionTry1(VecLoad(fd,PETSC_NULL,&b),PETSC_ERR_FILE_UNEXPECTED);
118:     } else {
119:       PetscExceptionTry1(VecLoad(fd,PETSC_NULL,&b),PETSC_ERR_FILE_READ);
120:     }
121:     if (PetscExceptionCaught(ierr,PETSC_ERR_FILE_UNEXPECTED) || PetscExceptionCaught(ierr,PETSC_ERR_FILE_READ)) { /* if file contains no RHS, then use a vector of all ones */
122:       PetscInt    m;
123:       PetscScalar one = 1.0;
124:       PetscInfo(0,"Using vector of ones for RHS\n");
125:       MatGetLocalSize(A,&m,PETSC_NULL);
126:       VecCreate(PETSC_COMM_WORLD,&b);
127:       VecSetSizes(b,m,PETSC_DECIDE);
128:       VecSetFromOptions(b);
129:       VecSet(b,one);
130:     } else
131:     PetscViewerDestroy(fd);

133:     /* Test MatDuplicate() */
134:     if (Test_MatDuplicate){
135:       MatDuplicate(A,MAT_COPY_VALUES,&B);
136:       MatEqual(A,B,&flg);
137:       if (!flg){
138:         PetscPrintf(PETSC_COMM_WORLD,"  A != B \n");
139:       }
140:       MatDestroy(B);
141:     }

143:     /* Add a shift to A */
144:     PetscOptionsGetScalar(PETSC_NULL,"-mat_sigma",&sigma,&flg);
145:     if (flg) {
146:       PetscOptionsGetString(PETSC_NULL,"-fB",file[2],PETSC_MAX_PATH_LEN-1,&flgB);
147:       if (flgB){
148:         /* load B to get A = A + sigma*B */
149:         PetscViewerBinaryOpen(PETSC_COMM_WORLD,file[2],FILE_MODE_READ,&fd);
150:         MatLoad(fd,MATAIJ,&B);
151:         PetscViewerDestroy(fd);
152:         MatAXPY(A,sigma,B,DIFFERENT_NONZERO_PATTERN); /* A <- sigma*B + A */
153:       } else {
154:         MatShift(A,sigma);
155:       }
156:     }

158:     /* Make A singular for testing zero-pivot of ilu factorization        */
159:     /* Example: ./ex10 -f0 <datafile> -test_zeropivot -set_row_zero -pc_factor_shift_nonzero */
160:     PetscOptionsHasName(PETSC_NULL, "-test_zeropivot", &flg);
161:     if (flg) {
162:       PetscInt          row,ncols;
163:       const PetscInt    *cols;
164:       const PetscScalar *vals;
165:       PetscTruth        flg1=PETSC_FALSE;
166:       PetscScalar       *zeros;
167:       row = 0;
168:       MatGetRow(A,row,&ncols,&cols,&vals);
169:       PetscMalloc(sizeof(PetscScalar)*(ncols+1),&zeros);
170:       PetscMemzero(zeros,(ncols+1)*sizeof(PetscScalar));
171:       PetscOptionsHasName(PETSC_NULL, "-set_row_zero", &flg1);
172:       if (flg1){ /* set entire row as zero */
173:         MatSetValues(A,1,&row,ncols,cols,zeros,INSERT_VALUES);
174:       } else { /* only set (row,row) entry as zero */
175:         MatSetValues(A,1,&row,1,&row,zeros,INSERT_VALUES);
176:       }
177:       MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
178:       MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
179:     }

181:     /* Check whether A is symmetric */
182:     PetscOptionsHasName(PETSC_NULL, "-check_symmetry", &flg);
183:     if (flg) {
184:       Mat Atrans;
185:       MatTranspose(A, &Atrans);
186:       MatEqual(A, Atrans, &isSymmetric);
187:       if (isSymmetric) {
188:         PetscPrintf(PETSC_COMM_WORLD,"A is symmetric \n");
189:       } else {
190:         PetscPrintf(PETSC_COMM_WORLD,"A is non-symmetric \n");
191:       }
192:       MatDestroy(Atrans);
193:     }

195:     /* 
196:        If the loaded matrix is larger than the vector (due to being padded 
197:        to match the block size of the system), then create a new padded vector.
198:     */
199: 
200:     MatGetLocalSize(A,&m,&n);
201:     if (m != n) {
202:       SETERRQ2(PETSC_ERR_ARG_SIZ, "This example is not intended for rectangular matrices (%d, %d)", m, n);
203:     }
204:     MatGetSize(A,&M,PETSC_NULL);
205:     VecGetSize(b,&m);
206:     if (M != m) { /* Create a new vector b by padding the old one */
207:       PetscInt    j,mvec,start,end,indx;
208:       Vec         tmp;
209:       PetscScalar *bold;

211:       VecCreate(PETSC_COMM_WORLD,&tmp);
212:       VecSetSizes(tmp,n,PETSC_DECIDE);
213:       VecSetFromOptions(tmp);
214:       VecGetOwnershipRange(b,&start,&end);
215:       VecGetLocalSize(b,&mvec);
216:       VecGetArray(b,&bold);
217:       for (j=0; j<mvec; j++) {
218:         indx = start+j;
219:         VecSetValues(tmp,1,&indx,bold+j,INSERT_VALUES);
220:       }
221:       VecRestoreArray(b,&bold);
222:       VecDestroy(b);
223:       VecAssemblyBegin(tmp);
224:       VecAssemblyEnd(tmp);
225:       b = tmp;
226:     }
227:     VecDuplicate(b,&x);
228:     VecDuplicate(b,&u);
229:     VecSet(x,0.0);

231:     /* - - - - - - - - - - - New Stage - - - - - - - - - - - - -
232:                       Setup solve for system
233:      - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */


236:     if (partition) {
237:       MatPartitioning mpart;
238:       IS              mis,nis,isn,is;
239:       PetscInt        *count;
240:       PetscMPIInt     size;
241:       Mat             BB;
242:       MPI_Comm_size(PETSC_COMM_WORLD,&size);
243:       MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
244:       PetscMalloc(size*sizeof(PetscInt),&count);
245:       MatPartitioningCreate(PETSC_COMM_WORLD, &mpart);
246:       MatPartitioningSetAdjacency(mpart, A);
247:       /* MatPartitioningSetVertexWeights(mpart, weight); */
248:       MatPartitioningSetFromOptions(mpart);
249:       MatPartitioningApply(mpart, &mis);
250:       MatPartitioningDestroy(mpart);
251:       ISPartitioningToNumbering(mis,&nis);
252:       ISPartitioningCount(mis,count);
253:       ISDestroy(mis);
254:       ISInvertPermutation(nis, count[rank], &is);
255:       PetscFree(count);
256:       ISDestroy(nis);
257:       ISSort(is);
258:       ISAllGather(is,&isn);
259:       MatGetSubMatrix(A,is,isn,PETSC_DECIDE,MAT_INITIAL_MATRIX,&BB);

261:       /* need to move the vector also */
262:       ISDestroy(is);
263:       ISDestroy(isn);
264:       MatDestroy(A);
265:       A    = BB;
266:     }
267: 
268:     /*
269:        Conclude profiling last stage; begin profiling next stage.
270:     */
271:     PreLoadStage("KSPSetUp");

273:     /*
274:        We also explicitly time this stage via PetscGetTime()
275:     */
276:     PetscGetTime(&tsetup1);

278:     /*
279:        Create linear solver; set operators; set runtime options.
280:     */
281:     KSPCreate(PETSC_COMM_WORLD,&ksp);
282:     KSPSetInitialGuessNonzero(ksp,initialguess);
283:     num_numfac = 1;
284:     PetscOptionsGetInt(PETSC_NULL,"-num_numfac",&num_numfac,PETSC_NULL);
285:     while ( num_numfac-- ){
286: 

288:       KSPSetOperators(ksp,A,A,SAME_NONZERO_PATTERN);
289:       KSPSetFromOptions(ksp);

291:       /* 
292:        Here we explicitly call KSPSetUp() and KSPSetUpOnBlocks() to
293:        enable more precise profiling of setting up the preconditioner.
294:        These calls are optional, since both will be called within
295:        KSPSolve() if they haven't been called already.
296:       */
297:       KSPSetUp(ksp);
298:       KSPSetUpOnBlocks(ksp);
299:       PetscGetTime(&tsetup2);
300:       tsetup = tsetup2 - tsetup1;

302:       /*
303:       Test MatGetInertia()
304:       Usage:
305:       ex10 -f0 <mat_binaryfile> -ksp_type preonly -pc_type cholesky -mat_type seqsbaij -test_inertia -mat_sigma <sigma>
306:       */
307:       PetscOptionsHasName(PETSC_NULL,"-test_inertia",&flg);
308:       if (flg){
309:         PC        pc;
310:         PetscInt  nneg, nzero, npos;
311:         Mat       F;
312: 
313:         KSPGetPC(ksp,&pc);
314:         PCFactorGetMatrix(pc,&F);
315:         MatGetInertia(F,&nneg,&nzero,&npos);
316:         PetscPrintf(PETSC_COMM_SELF," MatInertia: nneg: %D, nzero: %D, npos: %D\n",nneg,nzero,npos);
317:       }

319:       /*
320:        Tests "diagonal-scaling of preconditioned residual norm" as used 
321:        by many ODE integrator codes including SUNDIALS. Note this is different
322:        than diagonally scaling the matrix before computing the preconditioner
323:       */
324:       PetscOptionsHasName(PETSC_NULL,"-diagonal_scale",&diagonalscale);
325:       if (diagonalscale) {
326:         PC       pc;
327:         PetscInt j,start,end,n;
328:         Vec      scale;
329: 
330:         KSPGetPC(ksp,&pc);
331:         VecGetSize(x,&n);
332:         VecDuplicate(x,&scale);
333:         VecGetOwnershipRange(scale,&start,&end);
334:         for (j=start; j<end; j++) {
335:           VecSetValue(scale,j,((PetscReal)(j+1))/((PetscReal)n),INSERT_VALUES);
336:         }
337:         VecAssemblyBegin(scale);
338:         VecAssemblyEnd(scale);
339:         PCDiagonalScaleSet(pc,scale);
340:         VecDestroy(scale);
341:       }

343:       /* - - - - - - - - - - - New Stage - - - - - - - - - - - - -
344:                            Solve system
345:         - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

347:       /*
348:        Begin profiling next stage
349:       */
350:       PreLoadStage("KSPSolve");

352:       /*
353:        Solve linear system; we also explicitly time this stage.
354:       */
355:       PetscGetTime(&tsolve1);
356:       if (trans) {
357:         KSPSolveTranspose(ksp,b,x);
358:         KSPGetIterationNumber(ksp,&its);
359:       } else {
360:         PetscInt  num_rhs=1;
361:         PetscOptionsGetInt(PETSC_NULL,"-num_rhs",&num_rhs,PETSC_NULL);
362:         PetscOptionsHasName(PETSC_NULL,"-cknorm",&cknorm);
363:         while ( num_rhs-- ) {
364:           KSPSolve(ksp,b,x);
365:         }
366:         KSPGetIterationNumber(ksp,&its);
367:         if (cknorm){   /* Check error for each rhs */
368:           if (trans) {
369:             MatMultTranspose(A,x,u);
370:           } else {
371:             MatMult(A,x,u);
372:           }
373:           VecAXPY(u,-1.0,b);
374:           VecNorm(u,NORM_2,&norm);
375:           PetscPrintf(PETSC_COMM_WORLD,"  Number of iterations = %3D\n",its);
376:           PetscPrintf(PETSC_COMM_WORLD,"  Residual norm %A\n",norm);
377:         }
378:       } /* while ( num_rhs-- ) */
379:       PetscGetTime(&tsolve2);
380:       tsolve = tsolve2 - tsolve1;

382:       /* 
383:        Conclude profiling this stage
384:       */
385:       PreLoadStage("Cleanup");

387:       /* - - - - - - - - - - - New Stage - - - - - - - - - - - - -
388:             Check error, print output, free data structures.
389:        - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */

391:       /* 
392:          Check error
393:       */
394:       if (trans) {
395:         MatMultTranspose(A,x,u);
396:       } else {
397:         MatMult(A,x,u);
398:       }
399:       VecAXPY(u,-1.0,b);
400:       VecNorm(u,NORM_2,&norm);

402:       /*
403:        Write output (optinally using table for solver details).
404:         - PetscPrintf() handles output for multiprocessor jobs 
405:           by printing from only one processor in the communicator.
406:         - KSPView() prints information about the linear solver.
407:       */
408:       if (table) {
409:         char        *matrixname,kspinfo[120];
410:         PetscViewer viewer;

412:         /*
413:          Open a string viewer; then write info to it.
414:         */
415:         PetscViewerStringOpen(PETSC_COMM_WORLD,kspinfo,120,&viewer);
416:         KSPView(ksp,viewer);
417:         PetscStrrchr(file[PreLoadIt],'/',&matrixname);
418:         PetscPrintf(PETSC_COMM_WORLD,"%-8.8s %3D %2.0e %2.1e %2.1e %2.1e %s \n",
419:                 matrixname,its,norm,tsetup+tsolve,tsetup,tsolve,kspinfo);

421:         /*
422:           Destroy the viewer
423:         */
424:         PetscViewerDestroy(viewer);
425:       } else {
426:         PetscPrintf(PETSC_COMM_WORLD,"Number of iterations = %3D\n",its);
427:         PetscPrintf(PETSC_COMM_WORLD,"Residual norm %A\n",norm);
428:       }

430:       PetscOptionsHasName(PETSC_NULL, "-ksp_reason", &flg);
431:       if (flg){
432:         KSPConvergedReason reason;
433:         KSPGetConvergedReason(ksp,&reason);
434:         PetscPrintf(PETSC_COMM_WORLD,"KSPConvergedReason: %D\n", reason);
435:       }
436: 
437:     } /* while ( num_numfac-- ) */

439:     /* 
440:        Free work space.  All PETSc objects should be destroyed when they
441:        are no longer needed.
442:     */
443:     MatDestroy(A); VecDestroy(b);
444:     VecDestroy(u); VecDestroy(x);
445:     KSPDestroy(ksp);
446:     if (flgB) { MatDestroy(B); }
447:   PreLoadEnd();
448:   /* -----------------------------------------------------------
449:                       End of linear solver loop
450:      ----------------------------------------------------------- */

452:   PetscFinalize();
453:   return 0;
454: }