Actual source code: ispai.c

  1: /* $Id: ispai.c,v 1.28 2001/08/07 03:03:40 balay Exp $*/

  3: /* 
  4:    3/99 Modified by Stephen Barnard to support SPAI version 3.0 
  5: */

  7: /*
  8:       Provides an interface to the SPAI Sparse Approximate Inverse Preconditioner
  9:    Code written by Stephen Barnard.

 11:       Note: there is some BAD memory bleeding below!

 13:       This code needs work

 15:    1) get rid of all memory bleeding
 16:    2) fix PETSc/interface so that it gets if the matrix is symmetric from the matrix
 17:       rather than having the sp flag for PC_SPAI

 19: */

 21:  #include src/sles/pc/pcimpl.h

 23: /*
 24:     These are the SPAI include files
 25: */
 26: EXTERN_C_BEGIN
 27: #include "spai.h"
 28: #include "matrix.h"
 29: #include "read_mm_matrix.h"
 30: EXTERN_C_END

 32: EXTERN int ConvertMatToMatrix(MPI_Comm,Mat,Mat,matrix**);
 33: EXTERN int ConvertMatrixToMat(MPI_Comm,matrix *,Mat *);
 34: EXTERN int ConvertVectorToVec(MPI_Comm,vector *v,Vec *Pv);
 35: EXTERN int MM_to_PETSC(char *,char *,char *);

 37: typedef struct {

 39:   matrix *B;              /* matrix in SPAI format */
 40:   matrix *BT;             /* transpose of matrix in SPAI format */
 41:   matrix *M;              /* the approximate inverse in SPAI format */

 43:   Mat    PM;              /* the approximate inverse PETSc format */

 45:   double epsilon;         /* tolerance */
 46:   int    nbsteps;         /* max number of "improvement" steps per line */
 47:   int    max;             /* max dimensions of is_I, q, etc. */
 48:   int    maxnew;          /* max number of new entries per step */
 49:   int    block_size;      /* constant block size */
 50:   int    cache_size;      /* one of (1,2,3,4,5,6) indicting size of cache */
 51:   int    verbose;         /* SPAI prints timing and statistics */

 53:   int    sp;              /* symmetric nonzero pattern */
 54:   MPI_Comm comm_spai;     /* communicator to be used with spai */
 55: } PC_SPAI;

 57: /**********************************************************************/

 61: static int PCSetUp_SPAI(PC pc)
 62: {
 63:   PC_SPAI *ispai = (PC_SPAI*)pc->data;
 64:   int      ierr;
 65:   Mat      AT;


 69:   init_SPAI();

 71:   if (ispai->sp) {
 72:     ConvertMatToMatrix(ispai->comm_spai,pc->pmat,pc->pmat,&ispai->B);
 73:   } else {
 74:     /* Use the transpose to get the column nonzero structure. */
 75:     MatTranspose(pc->pmat,&AT);
 76:     ConvertMatToMatrix(ispai->comm_spai,pc->pmat,AT,&ispai->B);
 77:     MatDestroy(AT);
 78:   }

 80:   /* Destroy the transpose */
 81:   /* Don't know how to do it. PETSc developers? */
 82: 
 83:   /* construct SPAI preconditioner */
 84:   /* FILE *messages */     /* file for warning messages */
 85:   /* double epsilon */     /* tolerance */
 86:   /* int nbsteps */        /* max number of "improvement" steps per line */
 87:   /* int max */            /* max dimensions of is_I, q, etc. */
 88:   /* int maxnew */         /* max number of new entries per step */
 89:   /* int block_size */     /* block_size == 1 specifies scalar elments
 90:                               block_size == n specifies nxn constant-block elements
 91:                               block_size == 0 specifies variable-block elements */
 92:   /* int cache_size */     /* one of (1,2,3,4,5,6) indicting size of cache */
 93:                            /* cache_size == 0 indicates no caching */
 94:   /* int    verbose    */  /* verbose == 0 specifies that SPAI is silent
 95:                               verbose == 1 prints timing and matrix statistics */

 97:   ispai->M = bspai(ispai->B,
 98:                    stdout,
 99:                    ispai->epsilon,
100:                    ispai->nbsteps,
101:                    ispai->max,
102:                    ispai->maxnew,
103:                    ispai->block_size,
104:                    ispai->cache_size,
105:                    ispai->verbose);

107:   if (!ispai->M) SETERRQ(1,"Unable to create SPAI preconditioner");

109:   ConvertMatrixToMat(pc->comm,ispai->M,&ispai->PM);

111:   /* free the SPAI matrices */
112:   sp_free_matrix(ispai->B);
113:   sp_free_matrix(ispai->M);

115:   return(0);
116: }

118: /**********************************************************************/

122: static int PCApply_SPAI(PC pc,Vec xx,Vec y)
123: {
124:   PC_SPAI *ispai = (PC_SPAI*)pc->data;
125:   int      ierr;

128:   /* Now using PETSc's multiply */
129:   MatMult(ispai->PM,xx,y);
130:   return(0);
131: }

133: /**********************************************************************/

137: static int PCDestroy_SPAI(PC pc)
138: {
139:   int     ierr;
140:   PC_SPAI *ispai = (PC_SPAI*)pc->data;

143:   if (ispai->PM) {MatDestroy(ispai->PM);}
144:   MPI_Comm_free(&(ispai->comm_spai));
145:   PetscFree(ispai);
146:   return(0);
147: }

149: /**********************************************************************/

153: static int PCView_SPAI(PC pc,PetscViewer viewer)
154: {
155:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
156:   int        ierr;
157:   PetscTruth isascii;

160:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
161:   if (isascii) {
162:     PetscViewerASCIIPrintf(viewer,"    SPAI preconditioner\n");
163:     PetscViewerASCIIPrintf(viewer,"    epsilon %g\n",   ispai->epsilon);
164:     PetscViewerASCIIPrintf(viewer,"    nbsteps %d\n",   ispai->nbsteps);
165:     PetscViewerASCIIPrintf(viewer,"    max %d\n",       ispai->max);
166:     PetscViewerASCIIPrintf(viewer,"    maxnew %d\n",    ispai->maxnew);
167:     PetscViewerASCIIPrintf(viewer,"    block_size %d\n",ispai->block_size);
168:     PetscViewerASCIIPrintf(viewer,"    cache_size %d\n",ispai->cache_size);
169:     PetscViewerASCIIPrintf(viewer,"    verbose %d\n",   ispai->verbose);
170:     PetscViewerASCIIPrintf(viewer,"    sp %d\n",        ispai->sp);

172:   }
173:   return(0);
174: }

176: EXTERN_C_BEGIN
179: int PCSPAISetEpsilon_SPAI(PC pc,double epsilon1)
180: {
181:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
183:   ispai->epsilon = epsilon1;
184:   return(0);
185: }
186: EXTERN_C_END
187: 
188: /**********************************************************************/

190: EXTERN_C_BEGIN
193: int PCSPAISetNBSteps_SPAI(PC pc,int nbsteps1)
194: {
195:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
197:   ispai->nbsteps = nbsteps1;
198:   return(0);
199: }
200: EXTERN_C_END

202: /**********************************************************************/

204: /* added 1/7/99 g.h. */
205: EXTERN_C_BEGIN
208: int PCSPAISetMax_SPAI(PC pc,int max1)
209: {
210:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
212:   ispai->max = max1;
213:   return(0);
214: }
215: EXTERN_C_END

217: /**********************************************************************/

219: EXTERN_C_BEGIN
222: int PCSPAISetMaxNew_SPAI(PC pc,int maxnew1)
223: {
224:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
226:   ispai->maxnew = maxnew1;
227:   return(0);
228: }
229: EXTERN_C_END

231: /**********************************************************************/

233: EXTERN_C_BEGIN
236: int PCSPAISetBlockSize_SPAI(PC pc,int block_size1)
237: {
238:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
240:   ispai->block_size = block_size1;
241:   return(0);
242: }
243: EXTERN_C_END

245: /**********************************************************************/

247: EXTERN_C_BEGIN
250: int PCSPAISetCacheSize_SPAI(PC pc,int cache_size)
251: {
252:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
254:   ispai->cache_size = cache_size;
255:   return(0);
256: }
257: EXTERN_C_END

259: /**********************************************************************/

261: EXTERN_C_BEGIN
264: int PCSPAISetVerbose_SPAI(PC pc,int verbose)
265: {
266:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
268:   ispai->verbose = verbose;
269:   return(0);
270: }
271: EXTERN_C_END

273: /**********************************************************************/

275: EXTERN_C_BEGIN
278: int PCSPAISetSp_SPAI(PC pc,int sp)
279: {
280:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
282:   ispai->sp = sp;
283:   return(0);
284: }
285: EXTERN_C_END

287: /* -------------------------------------------------------------------*/

291: /*@
292:   PCSPAISetEpsilon -- Set the tolerance for the SPAI preconditioner

294:   Input Parameters:
295: + pc - the preconditioner
296: - eps - epsilon (default .4)

298:   Notes:  Espilon must be between 0 and 1. It controls the
299:                  quality of the approximation of M to the inverse of
300:                  A. Higher values of epsilon lead to more work, more
301:                  fill, and usually better preconditioners. In many
302:                  cases the best choice of epsilon is the one that
303:                  divides the total solution time equally between the
304:                  preconditioner and the solver.
305:   
306:   Level: intermediate

308: .seealso: PCSPAI, PCSetType()
309:   @*/
310: int PCSPAISetEpsilon(PC pc,double epsilon1)
311: {
312:   int ierr,(*f)(PC,double);
314:   PetscObjectQueryFunction((PetscObject)pc,"PCSPAISetEpsilon_C",(void (**)(void))&f);
315:   if (f) {
316:     (*f)(pc,epsilon1);
317:   }
318:   return(0);
319: }
320: 
321: /**********************************************************************/

325: /*@
326:   PCSPAISetNBSteps - set maximum number of improvement steps per row in 
327:         the SPAI preconditioner

329:   Input Parameters:
330: + pc - the preconditioner
331: - n - number of steps (default 5)

333:   Notes:  SPAI constructs to approximation to every column of
334:                  the exact inverse of A in a series of improvement
335:                  steps. The quality of the approximation is determined
336:                  by epsilon. If an approximation achieving an accuracy
337:                  of epsilon is not obtained after ns steps, SPAI simply
338:                  uses the best approximation constructed so far.

340:   Level: intermediate

342: .seealso: PCSPAI, PCSetType(), PCSPAISetMaxNew()
343: @*/
344: int PCSPAISetNBSteps(PC pc,int nbsteps1)
345: {
346:   int ierr,(*f)(PC,int);
348:   PetscObjectQueryFunction((PetscObject)pc,"PCSPAISetNBSteps_C",(void (**)(void))&f);
349:   if (f) {
350:     (*f)(pc,nbsteps1);
351:   }
352:   return(0);
353: }

355: /**********************************************************************/

357: /* added 1/7/99 g.h. */
360: /*@
361:   PCSPAISetMax - set the size of various working buffers in 
362:         the SPAI preconditioner

364:   Input Parameters:
365: + pc - the preconditioner
366: - n - size (default is 5000)

368:   Level: intermediate

370: .seealso: PCSPAI, PCSetType()
371: @*/
372: int PCSPAISetMax(PC pc,int max1)
373: {
374:   int ierr,(*f)(PC,int);
376:   PetscObjectQueryFunction((PetscObject)pc,"PCSPAISetMax_C",(void (**)(void))&f);
377:   if (f) {
378:     (*f)(pc,max1);
379:   }
380:   return(0);
381: }

383: /**********************************************************************/

387: /*@
388:   PCSPAISetMaxNew - set maximum number of new nonzero candidates per step
389:    in SPAI preconditioner

391:   Input Parameters:
392: + pc - the preconditioner
393: - n - maximum number (default 5)

395:   Level: intermediate

397: .seealso: PCSPAI, PCSetType(), PCSPAISetNBSteps()
398: @*/
399: int PCSPAISetMaxNew(PC pc,int maxnew1)
400: {
401:   int ierr,(*f)(PC,int);
403:   PetscObjectQueryFunction((PetscObject)pc,"PCSPAISetMaxNew_C",(void (**)(void))&f);
404:   if (f) {
405:     (*f)(pc,maxnew1);
406:   }
407:   return(0);
408: }

410: /**********************************************************************/

414: /*@
415:   PCSPAISetBlockSize - set the block size for the SPAI preconditioner

417:   Input Parameters:
418: + pc - the preconditioner
419: - n - block size (default 1)

421:   Notes: A block
422:                  size of 1 treats A as a matrix of scalar elements. A
423:                  block size of s > 1 treats A as a matrix of sxs
424:                  blocks. A block size of 0 treats A as a matrix with
425:                  variable sized blocks, which are determined by
426:                  searching for dense square diagonal blocks in A.
427:                  This can be very effective for finite-element
428:                  matrices.

430:                  SPAI will convert A to block form, use a block
431:                  version of the preconditioner algorithm, and then
432:                  convert the result back to scalar form.

434:                  In many cases the a block-size parameter other than 1
435:                  can lead to very significant improvement in
436:                  performance.


439:   Level: intermediate

441: .seealso: PCSPAI, PCSetType()
442: @*/
443: int PCSPAISetBlockSize(PC pc,int block_size1)
444: {
445:   int ierr,(*f)(PC,int);
447:   PetscObjectQueryFunction((PetscObject)pc,"PCSPAISetBlockSize_C",(void (**)(void))&f);
448:   if (f) {
449:     (*f)(pc,block_size1);
450:   }
451:   return(0);
452: }

454: /**********************************************************************/

458: /*@
459:   PCSPAISetCacheSize - specify cache size in the SPAI preconditioner

461:   Input Parameters:
462: + pc - the preconditioner
463: - n -  cache size {0,1,2,3,4,5} (default 5)

465:   Notes:    SPAI uses a hash table to cache messages and avoid
466:                  redundant communication. If suggest always using
467:                  5. This parameter is irrelevant in the serial
468:                  version.

470:   Level: intermediate

472: .seealso: PCSPAI, PCSetType()
473: @*/
474: int PCSPAISetCacheSize(PC pc,int cache_size)
475: {
476:   int ierr,(*f)(PC,int);
478:   PetscObjectQueryFunction((PetscObject)pc,"PCSPAISetCacheSize_C",(void (**)(void))&f);
479:   if (f) {
480:     (*f)(pc,cache_size);
481:   }
482:   return(0);
483: }

485: /**********************************************************************/

489: /*@
490:   PCSPAISetVerbose - verbosity level for the SPAI preconditioner

492:   Input Parameters:
493: + pc - the preconditioner
494: - n - level (default 1)

496:   Notes: print parameters, timings and matrix statistics

498:   Level: intermediate

500: .seealso: PCSPAI, PCSetType()
501: @*/
502: int PCSPAISetVerbose(PC pc,int verbose)
503: {
504:   int ierr,(*f)(PC,int);
506:   PetscObjectQueryFunction((PetscObject)pc,"PCSPAISetVerbose_C",(void (**)(void))&f);
507:   if (f) {
508:     (*f)(pc,verbose);
509:   }
510:   return(0);
511: }

513: /**********************************************************************/

517: /*@
518:   PCSPAISetSp - specify a symmetric matrix sparsity pattern in the SPAI preconditioner

520:   Input Parameters:
521: + pc - the preconditioner
522: - n - 0 or 1

524:   Notes: If A has a symmetric nonzero pattern use -sp 1 to
525:                  improve performance by eliminating some communication
526:                  in the parallel version. Even if A does not have a
527:                  symmetric nonzero pattern -sp 1 may well lead to good
528:                  results, but the code will not follow the published
529:                  SPAI algorithm exactly.


532:   Level: intermediate

534: .seealso: PCSPAI, PCSetType()
535: @*/
536: int PCSPAISetSp(PC pc,int sp)
537: {
538:   int ierr,(*f)(PC,int);
540:   PetscObjectQueryFunction((PetscObject)pc,"PCSPAISetSp_C",(void (**)(void))&f);
541:   if (f) {
542:     (*f)(pc,sp);
543:   }
544:   return(0);
545: }

547: /**********************************************************************/

549: /**********************************************************************/

553: static int PCSetFromOptions_SPAI(PC pc)
554: {
555:   PC_SPAI    *ispai = (PC_SPAI*)pc->data;
556:   int        ierr,nbsteps1,max1,maxnew1,block_size1,cache_size,verbose,sp;
557:   double     epsilon1;
558:   PetscTruth flg;

561:   PetscOptionsHead("SPAI options");
562:     PetscOptionsReal("-pc_spai_epsilon","","PCSPAISetEpsilon",ispai->epsilon,&epsilon1,&flg);
563:     if (flg) {
564:       PCSPAISetEpsilon(pc,epsilon1);
565:     }
566:     PetscOptionsInt("-pc_spai_nbsteps","","PCSPAISetNBSteps",ispai->nbsteps,&nbsteps1,&flg);
567:     if (flg) {
568:       PCSPAISetNBSteps(pc,nbsteps1);
569:     }
570:     /* added 1/7/99 g.h. */
571:     PetscOptionsInt("-pc_spai_max","","PCSPAISetMax",ispai->max,&max1,&flg);
572:     if (flg) {
573:       PCSPAISetMax(pc,max1);
574:     }
575:     PetscOptionsInt("-pc_spai_maxnew","","PCSPAISetMaxNew",ispai->maxnew,&maxnew1,&flg);
576:     if (flg) {
577:       PCSPAISetMaxNew(pc,maxnew1);
578:     }
579:     PetscOptionsInt("-pc_spai_block_size","","PCSPAISetBlockSize",ispai->block_size,&block_size1,&flg);
580:     if (flg) {
581:       PCSPAISetBlockSize(pc,block_size1);
582:     }
583:     PetscOptionsInt("-pc_spai_cache_size","","PCSPAISetCacheSize",ispai->cache_size,&cache_size,&flg);
584:     if (flg) {
585:       PCSPAISetCacheSize(pc,cache_size);
586:     }
587:     PetscOptionsInt("-pc_spai_verbose","","PCSPAISetVerbose",ispai->verbose,&verbose,&flg);
588:     if (flg) {
589:       PCSPAISetVerbose(pc,verbose);
590:     }
591:     PetscOptionsInt("-pc_spai_sp","","PCSPAISetSp",ispai->sp,&sp,&flg);
592:     if (flg) {
593:       PCSPAISetSp(pc,sp);
594:     }
595:   PetscOptionsTail();
596:   return(0);
597: }

599: /**********************************************************************/

601: /*MC
602:    PCSPAI - Use the Sparse Approximate Inverse method of Grote and Barnard
603:      as a preconditioner (SIAM J. Sci. Comput.; vol 18, nr 3)

605:    Options Database Keys:
606: +  -pc_spai_set_epsilon <eps> - set tolerance
607: .  -pc_spai_nbstep <n> - set nbsteps
608: .  -pc_spai_max <m> - set max
609: .  -pc_spai_max_new <m> - set maxnew
610: .  -pc_spai_block_size <n> - set block size
611: .  -pc_spai_cache_size <n> - set cache size
612: .  -pc_spai_sp <m> - set sp
613: -  -pc_spai_set_verbose <true,false> - verbose output

615:    Notes: This only works with SeqAIJ matrices.

617:    Level: beginner

619:    Concepts: approximate inverse

621: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC,
622:     PCSPAISetEpsilon(), PCSPAISetMax(), PCSPAISetMaxNew(), PCSPAISetBlockSize(),
623:     PCSPAISetVerbose(), PCSPAISetSp()
624: M*/

626: EXTERN_C_BEGIN
627: /*
628:    PCCreate_SPAI - Creates the preconditioner context for the SPAI 
629:                    preconditioner written by Stephen Barnard.

631: */
634: int PCCreate_SPAI(PC pc)
635: {
636:   PC_SPAI *ispai;
637:   int     ierr;

640:   PetscNew(PC_SPAI,&ispai);
641:   pc->data           = (void*)ispai;

643:   pc->ops->destroy         = PCDestroy_SPAI;
644:   pc->ops->apply           = PCApply_SPAI;
645:   pc->ops->applyrichardson = 0;
646:   pc->ops->setup           = PCSetUp_SPAI;
647:   pc->ops->view            = PCView_SPAI;
648:   pc->ops->setfromoptions  = PCSetFromOptions_SPAI;

650:   pc->name          = 0;
651:   ispai->epsilon    = .4;
652:   ispai->nbsteps    = 5;
653:   ispai->max        = 5000;
654:   ispai->maxnew     = 5;
655:   ispai->block_size = 1;
656:   ispai->cache_size = 5;
657:   ispai->verbose    = 0;

659:   ispai->sp         = 1;
660:   MPI_Comm_dup(pc->comm,&(ispai->comm_spai));

662:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCSPAISetEpsilon_C",
663:                     "PCSPAISetEpsilon_SPAI",
664:                      PCSPAISetEpsilon_SPAI);
665:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCSPAISetNBSteps_C",
666:                     "PCSPAISetNBSteps_SPAI",
667:                      PCSPAISetNBSteps_SPAI);
668:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCSPAISetMax_C",
669:                     "PCSPAISetMax_SPAI",
670:                      PCSPAISetMax_SPAI);
671:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCSPAISetMaxNew_CC",
672:                     "PCSPAISetMaxNew_SPAI",
673:                      PCSPAISetMaxNew_SPAI);
674:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCSPAISetBlockSize_C",
675:                     "PCSPAISetBlockSize_SPAI",
676:                      PCSPAISetBlockSize_SPAI);
677:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCSPAISetCacheSize_C",
678:                     "PCSPAISetCacheSize_SPAI",
679:                      PCSPAISetCacheSize_SPAI);
680:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCSPAISetVerbose_C",
681:                     "PCSPAISetVerbose_SPAI",
682:                      PCSPAISetVerbose_SPAI);
683:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCSPAISetSp_C",
684:                     "PCSPAISetSp_SPAI",
685:                      PCSPAISetSp_SPAI);

687:   return(0);
688: }
689: EXTERN_C_END

691: /**********************************************************************/

693: /*
694:    Converts from a PETSc matrix to an SPAI matrix 
695: */
698: int ConvertMatToMatrix(MPI_Comm comm, Mat A,Mat AT,matrix **B)
699: {
700:   matrix   *M;
701:   int      i,j,col;
702:   int      row_indx;
703:   int      len,pe,local_indx,start_indx;
704:   int      *mapping;
705:   int      ierr,*cols;
706:   double   *vals;
707:   int      *num_ptr,n,mnl,nnl,rank,size,nz,rstart,rend;
708:   struct   compressed_lines *rows;

711: 
712:   MPI_Comm_size(comm,&size);
713:   MPI_Comm_rank(comm,&rank);
714:   MatGetSize(A,&n,&n);
715:   MatGetLocalSize(A,&mnl,&nnl);

717:   /*
718:     not sure why a barrier is required. commenting out
719:   MPI_Barrier(comm);
720:   */

722:   M = new_matrix((void *)comm);
723: 
724:   M->n = n;
725:   M->bs = 1;
726:   M->max_block_size = 1;

728:   M->mnls          = (int*)malloc(sizeof(int)*size);
729:   M->start_indices = (int*)malloc(sizeof(int)*size);
730:   M->pe            = (int*)malloc(sizeof(int)*n);
731:   M->block_sizes   = (int*)malloc(sizeof(int)*n);
732:   for (i=0; i<n; i++) M->block_sizes[i] = 1;

734:   MPI_Allgather(&mnl,1,MPI_INT,M->mnls,1,MPI_INT,comm);

736:   M->start_indices[0] = 0;
737:   for (i=1; i<size; i++) {
738:     M->start_indices[i] = M->start_indices[i-1] + M->mnls[i-1];
739:   }

741:   M->mnl = M->mnls[M->myid];
742:   M->my_start_index = M->start_indices[M->myid];

744:   for (i=0; i<size; i++) {
745:     start_indx = M->start_indices[i];
746:     for (j=0; j<M->mnls[i]; j++)
747:       M->pe[start_indx+j] = i;
748:   }

750:   if (AT) {
751:     M->lines = new_compressed_lines(M->mnls[rank],1);
752:   } else {
753:     M->lines = new_compressed_lines(M->mnls[rank],0);
754:   }

756:   rows     = M->lines;

758:   /* Determine the mapping from global indices to pointers */
759:   PetscMalloc(M->n*sizeof(int),&mapping);
760:   pe         = 0;
761:   local_indx = 0;
762:   for (i=0; i<M->n; i++) {
763:     if (local_indx >= M->mnls[pe]) {
764:       pe++;
765:       local_indx = 0;
766:     }
767:     mapping[i] = local_indx + M->start_indices[pe];
768:     local_indx++;
769:   }


772:   PetscMalloc(mnl*sizeof(int),&num_ptr);

774:   /*********************************************************/
775:   /************** Set up the row structure *****************/
776:   /*********************************************************/

778:   /* count number of nonzeros in every row */
779:   MatGetOwnershipRange(A,&rstart,&rend);
780:   for (i=rstart; i<rend; i++) {
781:     MatGetRow(A,i,&num_ptr[i-rstart],PETSC_NULL,PETSC_NULL);
782:     MatRestoreRow(A,i,&num_ptr[i-rstart],PETSC_NULL,PETSC_NULL);
783:   }

785:   /* allocate buffers */
786:   len = 0;
787:   for (i=0; i<mnl; i++) {
788:     if (len < num_ptr[i]) len = num_ptr[i];
789:   }

791:   for (i=rstart; i<rend; i++) {
792:     row_indx             = i-rstart;
793:     len                  = num_ptr[row_indx];
794:     rows->ptrs[row_indx] = (int*)malloc(len*sizeof(int));
795:     rows->A[row_indx]    = (double*)malloc(len*sizeof(double));
796:   }

798:   /* copy the matrix */
799:   for (i=rstart; i<rend; i++) {
800:     row_indx = i - rstart;
801:     MatGetRow(A,i,&nz,&cols,&vals);
802:     for (j=0; j<nz; j++) {
803:       col = cols[j];
804:       len = rows->len[row_indx]++;
805:       rows->ptrs[row_indx][len] = mapping[col];
806:       rows->A[row_indx][len]    = vals[j];
807:     }
808:     rows->slen[row_indx] = rows->len[row_indx];
809:     MatRestoreRow(A,i,&nz,&cols,&vals);
810:   }


813:   /************************************************************/
814:   /************** Set up the column structure *****************/
815:   /*********************************************************/

817:   if (AT) {

819:     /* count number of nonzeros in every column */
820:     for (i=rstart; i<rend; i++) {
821:       MatGetRow(AT,i,&num_ptr[i-rstart],PETSC_NULL,PETSC_NULL);
822:       MatRestoreRow(AT,i,&num_ptr[i-rstart],PETSC_NULL,PETSC_NULL);
823:     }

825:     /* allocate buffers */
826:     len = 0;
827:     for (i=0; i<mnl; i++) {
828:       if (len < num_ptr[i]) len = num_ptr[i];
829:     }

831:     for (i=rstart; i<rend; i++) {
832:       row_indx = i-rstart;
833:       len      = num_ptr[row_indx];
834:       rows->rptrs[row_indx]     = (int*)malloc(len*sizeof(int));
835:     }

837:     /* copy the matrix (i.e., the structure) */
838:     for (i=rstart; i<rend; i++) {
839:       row_indx = i - rstart;
840:       MatGetRow(AT,i,&nz,&cols,&vals);
841:       for (j=0; j<nz; j++) {
842:         col = cols[j];
843:         len = rows->rlen[row_indx]++;
844:         rows->rptrs[row_indx][len] = mapping[col];
845:       }
846:       MatRestoreRow(AT,i,&nz,&cols,&vals);
847:     }
848:   }

850:   PetscFree(num_ptr);
851:   PetscFree(mapping);

853:   order_pointers(M);
854:   M->maxnz = calc_maxnz(M);

856:   *B = M;

858:   return(0);
859: }

861: /**********************************************************************/

863: /*
864:    Converts from an SPAI matrix B  to a PETSc matrix PB.
865:    This assumes that the the SPAI matrix B is stored in
866:    COMPRESSED-ROW format.
867: */
870: int ConvertMatrixToMat(MPI_Comm comm,matrix *B,Mat *PB)
871: {
872:   int         size,rank;
873:   int         ierr;
874:   int         m,n,M,N;
875:   int         d_nz,o_nz;
876:   int         *d_nnz,*o_nnz;
877:   int         i,k,global_row,global_col,first_diag_col,last_diag_col;
878:   PetscScalar val;

881:   MPI_Comm_size(comm,&size);
882:   MPI_Comm_rank(comm,&rank);
883: 
884:   m = n = B->mnls[rank];
885:   d_nz = o_nz = 0;

887:   /* Determine preallocation for MatCreateMPIAIJ */
888:   PetscMalloc(m*sizeof(int),&d_nnz);
889:   PetscMalloc(m*sizeof(int),&o_nnz);
890:   for (i=0; i<m; i++) d_nnz[i] = o_nnz[i] = 0;
891:   first_diag_col = B->start_indices[rank];
892:   last_diag_col = first_diag_col + B->mnls[rank];
893:   for (i=0; i<B->mnls[rank]; i++) {
894:     for (k=0; k<B->lines->len[i]; k++) {
895:       global_col = B->lines->ptrs[i][k];
896:       if ((global_col >= first_diag_col) && (global_col <= last_diag_col))
897:         d_nnz[i]++;
898:       else
899:         o_nnz[i]++;
900:     }
901:   }

903:   M = N = B->n;
904:   MatCreateMPIAIJ(comm,m,n,M,N,d_nz,d_nnz,o_nz,o_nnz,PB);

906:   for (i=0; i<B->mnls[rank]; i++) {
907:     global_row = B->start_indices[rank]+i;
908:     for (k=0; k<B->lines->len[i]; k++) {
909:       global_col = B->lines->ptrs[i][k];
910:       val = B->lines->A[i][k];
911:       MatSetValues(*PB,1,&global_row,1,&global_col,&val,ADD_VALUES);
912:     }
913:   }

915:   PetscFree(d_nnz);
916:   PetscFree(o_nnz);

918:   MatAssemblyBegin(*PB,MAT_FINAL_ASSEMBLY);
919:   MatAssemblyEnd(*PB,MAT_FINAL_ASSEMBLY);

921:   return(0);
922: }

924: /**********************************************************************/

926: /*
927:    Converts from an SPAI vector v  to a PETSc vec Pv.
928: */
931: int ConvertVectorToVec(MPI_Comm comm,vector *v,Vec *Pv)
932: {
933:   int size,rank,ierr,m,M,i,*mnls,*start_indices,*global_indices;
934: 
936:   MPI_Comm_size(comm,&size);
937:   MPI_Comm_rank(comm,&rank);
938: 
939:   m = v->mnl;
940:   M = v->n;
941: 
942: 
943:   VecCreateMPI(comm,m,M,Pv);

945:   PetscMalloc(size*sizeof(int),&mnls);
946:   MPI_Allgather((void*)&v->mnl,1,MPI_INT,(void*)mnls,1,MPI_INT,comm);
947: 
948:   PetscMalloc(size*sizeof(int),&start_indices);
949:   start_indices[0] = 0;
950:   for (i=1; i<size; i++)
951:     start_indices[i] = start_indices[i-1] +mnls[i-1];
952: 
953:   PetscMalloc(v->mnl*sizeof(int),&global_indices);
954:   for (i=0; i<v->mnl; i++)
955:     global_indices[i] = start_indices[rank] + i;

957:   PetscFree(mnls);
958:   PetscFree(start_indices);
959: 
960:   VecSetValues(*Pv,v->mnl,global_indices,v->v,INSERT_VALUES);

962:   PetscFree(global_indices);

964:   VecAssemblyBegin(*Pv);
965:   VecAssemblyEnd(*Pv);
966: 
967:   return(0);
968: }

970: /**********************************************************************/

972: /*

974:   Reads a matrix and a RHS vector in Matrix Market format and writes them
975:   in PETSc binary format.

977:   !!!! Warning!!!!! PETSc supports only serial execution of this routine.
978:   
979:   f0 <input_file> : matrix in Matrix Market format
980:   f1 <input_file> : vector in Matrix Market array format
981:   f2 <output_file> : matrix and vector in PETSc format

983: */
986: int MM_to_PETSC(char *f0,char *f1,char *f2)
987: {
988:   Mat         A_PETSC;          /* matrix */
989:   Vec         b_PETSC;          /* RHS */
990:   PetscViewer fd;               /* viewer */
991:   int         ierr;
992:   matrix      *A_spai;
993:   vector      *b_spai;


997:   A_spai = read_mm_matrix(f0,1,1,1,0,0,0,NULL);
998:   b_spai = read_rhs_for_matrix(f1,A_spai);

1000:   ConvertMatrixToMat(PETSC_COMM_SELF,A_spai,&A_PETSC);
1001:   ConvertVectorToVec(PETSC_COMM_SELF,b_spai,&b_PETSC);

1003:   PetscViewerBinaryOpen(PETSC_COMM_SELF,f1,PETSC_BINARY_CREATE,&fd);
1004:   MatView(A_PETSC,fd);
1005:   VecView(b_PETSC,fd);

1007:   return(0);
1008: }