Actual source code: pinit.c

  1: /*$Id: pinit.c,v 1.49 2001/04/13 15:09:31 bsmith Exp $*/
  2: /*
  3:    This file defines the initialization of PETSc, including PetscInitialize()
  4: */

 6:  #include petsc.h
 7:  #include petscsys.h

  9: /* -----------------------------------------------------------------------------------------*/

 11: extern FILE *petsc_history;

 13: EXTERN int PetscInitialize_DynamicLibraries(void);
 14: EXTERN int PetscFinalize_DynamicLibraries(void);
 15: EXTERN int PetscFListDestroyAll(void);
 16: EXTERN int PetscLogEventRegisterDestroy_Private(void);
 17: EXTERN int PetscLogStageDestroy_Private(void);
 18: EXTERN int PetscSequentialPhaseBegin_Private(MPI_Comm,int);
 19: EXTERN int PetscSequentialPhaseEnd_Private(MPI_Comm,int);
 20: EXTERN int PetscLogCloseHistoryFile(FILE **);

 22:  #include petscsnes.h

 24: /* this is used by the _, __, and ___ macros (see include/petscerror.h) */
 25: int __g0;

 27: /*
 28:        Checks the options database for initializations related to the 
 29:     PETSc components
 30: */
 31: int PetscOptionsCheckInitial_Components(void)
 32: {
 33:   MPI_Comm   comm = PETSC_COMM_WORLD;
 34:   PetscTruth flg1;
 35:   int        ierr;
 36:   char       *f,mname[256];

 39:   /*
 40:      Publishing to the AMS
 41:   */
 42: #if defined(PETSC_HAVE_AMS)
 43:   PetscOptionsHasName(PETSC_NULL,"-ams_publish_objects",&flg1);
 44:   if (flg1) {
 45:     PetscAMSPublishAll = PETSC_TRUE;
 46:   }
 47:   PetscOptionsHasName(PETSC_NULL,"-ams_publish_stack",&flg1);
 48:   if (flg1) {
 49:     PetscStackPublish();
 50:   }
 51: #endif

 53:   PetscOptionsGetString(PETSC_NULL,"-log_info_exclude",mname,256,&flg1);
 54:   if (flg1) {
 55:     PetscStrstr(mname,"vec",&f);
 56:     if (f) {
 57:       PetscLogInfoDeactivateClass(VEC_COOKIE);
 58:     }
 59:     PetscStrstr(mname,"mat",&f);
 60:     if (f) {
 61:       PetscLogInfoDeactivateClass(MAT_COOKIE);
 62:     }
 63:     PetscStrstr(mname,"sles",&f);
 64:     if (f) {
 65:       PetscLogInfoDeactivateClass(SLES_COOKIE);
 66:     }
 67:     PetscStrstr(mname,"snes",&f);
 68:     if (f) {
 69:       PetscLogInfoDeactivateClass(SNES_COOKIE);
 70:     }
 71:   }
 72:   PetscOptionsGetString(PETSC_NULL,"-log_summary_exclude",mname,256,&flg1);
 73:   if (flg1) {
 74:     PetscStrstr(mname,"vec",&f);
 75:     if (f) {
 76:       PetscLogEventDeactivateClass(VEC_COOKIE);
 77:     }
 78:     PetscStrstr(mname,"mat",&f);
 79:     if (f) {
 80:       PetscLogEventDeactivateClass(MAT_COOKIE);
 81:     }
 82:     PetscStrstr(mname,"sles",&f);
 83:     if (f) {
 84:       PetscLogEventDeactivateClass(SLES_COOKIE);
 85:     }
 86:     PetscStrstr(mname,"snes",&f);
 87:     if (f) {
 88:       PetscLogEventDeactivateClass(SNES_COOKIE);
 89:     }
 90:   }
 91: 
 92:   PetscOptionsHasName(PETSC_NULL,"-log_sync",&flg1);
 93:   if (flg1) {
 94:     PetscLogEventActivate(VEC_ScatterBarrier);
 95:     PetscLogEventActivate(VEC_NormBarrier);
 96:     PetscLogEventActivate(VEC_DotBarrier);
 97:     PetscLogEventActivate(VEC_MDotBarrier);
 98:     PetscLogEventActivate(VEC_ReduceBarrier);
 99:   }

101:   PetscOptionsHasName(PETSC_NULL,"-help",&flg1);
102:   if (flg1) {
103: #if defined (PETSC_USE_LOG)
104:     (*PetscHelpPrintf)(comm,"------Additional PETSc component options--------n");
105:     (*PetscHelpPrintf)(comm," -log_summary_exclude: <vec,mat,sles,snes>n");
106:     (*PetscHelpPrintf)(comm," -log_info_exclude: <null,vec,mat,sles,snes,ts>n");
107:     (*PetscHelpPrintf)(comm,"-----------------------------------------------n");
108: #endif
109:   }
110:   return(0);
111: }

113: /*@C
114:       PetscInitializeNoArguments - Calls PetscInitialize() from C/C++ without
115:         the command line arguments.

117:    Collective
118:   
119:    Level: advanced

121: .seealso: PetscInitialize(), PetscInitializeFortran()
122: @*/
123: int PetscInitializeNoArguments(void)
124: {
125:   int ierr,argc = 0;
126:   char **args = 0;

129:   PetscInitialize(&argc,&args,PETSC_NULL,PETSC_NULL);
130:   PetscFunctionReturn(ierr);
131: }

133: EXTERN int        PetscOptionsCheckInitial(void);
134: extern PetscTruth PetscBeganMPI;

136: /*
137:        This function is the MPI reduction operation used to compute the sum of the 
138:    first half of the entries and the max of the second half.
139: */
140: MPI_Op PetscMaxSum_Op = 0;

142: EXTERN_C_BEGIN
143: void PetscMaxSum_Local(void *in,void *out,int *cnt,MPI_Datatype *datatype)
144: {
145:   int *xin = (int *)in,*xout = (int*)out,i,count = *cnt;

148:   if (*datatype != MPI_INT) {
149:     (*PetscErrorPrintf)("Can only handle MPI_INT data types");
150:     MPI_Abort(MPI_COMM_WORLD,1);
151:   }
152:   if (count % 2) {
153:     (*PetscErrorPrintf)("Count must be divisible by 2");
154:     MPI_Abort(MPI_COMM_WORLD,1);
155:   }

157:   count = count/2;
158:   for (i=0; i<count; i++) {
159:     xout[i] = PetscMax(xout[i],xin[i]);
160:   }
161:   for (i=count; i<2*count; i++) {
162:     xout[i] += xin[i];
163:   }

165:   PetscStackPop;
166:   return;
167: }
168: EXTERN_C_END

170: #if defined(PETSC_USE_COMPLEX)
171: MPI_Op PetscSum_Op = 0;

173: EXTERN_C_BEGIN
174: void PetscSum_Local(void *in,void *out,int *cnt,MPI_Datatype *datatype)
175: {
176:   Scalar *xin = (Scalar *)in,*xout = (Scalar*)out;
177:   int    i,count = *cnt;

180:   if (*datatype != MPIU_SCALAR) {
181:     (*PetscErrorPrintf)("Can only handle MPIU_SCALAR data (i.e. double or complex) types");
182:     MPI_Abort(MPI_COMM_WORLD,1);
183:   }

185:   for (i=0; i<count; i++) {
186:     xout[i] += xin[i];
187:   }

189:   PetscStackPop;
190:   return;
191: }
192: EXTERN_C_END
193: #endif

195: /*@C
196:    PetscInitialize - Initializes the PETSc database and MPI. 
197:    PetscInitialize() calls MPI_Init() if that has yet to be called,
198:    so this routine should always be called near the beginning of 
199:    your program -- usually the very first line! 

201:    Collective on MPI_COMM_WORLD or PETSC_COMM_WORLD if it has been set

203:    Input Parameters:
204: +  argc - count of number of command line arguments
205: .  args - the command line arguments
206: .  file - [optional] PETSc database file, defaults to ~username/.petscrc
207:           (use PETSC_NULL for default)
208: -  help - [optional] Help message to print, use PETSC_NULL for no message

210:    Options Database Keys:
211: +  -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger
212: .  -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected
213: .  -on_error_emacs <machinename> causes emacsclient to jump to error file
214: .  -debugger_nodes [node1,node2,...] - Indicates nodes to start in debugger
215: .  -debugger_pause [sleeptime] (in seconds) - Pauses debugger
216: .  -stop_for_debugger - Print message on how to attach debugger manually to 
217:                         process and wait (-debugger_pause) seconds for attachment
218: .  -trmalloc - Indicates use of PETSc error-checking malloc
219: .  -trmalloc_off - Indicates not to use error-checking malloc
220: .  -fp_trap - Stops on floating point exceptions (Note that on the
221:               IBM RS6000 this slows code by at least a factor of 10.)
222: .  -no_signal_handler - Indicates not to trap error signals
223: .  -shared_tmp - indicates /tmp directory is shared by all processors
224: .  -not_shared_tmp - each processor has own /tmp
225: .  -tmp - alternative name of /tmp directory
226: .  -get_total_flops - returns total flops done by all processors
227: -  -get_resident_set_size - Print memory usage at end of run

229:    Options Database Keys for Profiling:
230:    See the Profiling chapter of the users manual for details.
231: +  -log_trace [filename] - Print traces of all PETSc calls
232:         to the screen (useful to determine where a program
233:         hangs without running in the debugger).  See PetscLogTraceBegin().
234: .  -log_info <optional filename> - Prints verbose information to the screen
235: -  -log_info_exclude <null,vec,mat,sles,snes,ts> - Excludes some of the verbose messages

237:    Environmental Variables:
238: +   PETSC_TMP - alternative tmp directory
239: .   PETSC_SHARED_TMP - tmp is shared by all processes
240: .   PETSC_NOT_SHARED_TMP - each process has its own private tmp
241: .   PETSC_VIEWER_SOCKET_PORT - socket number to use for socket viewer
242: -   PETSC_VIEWER_SOCKET_MACHINE - machine to use for socket viewer to connect to


245:    Level: beginner

247:    Notes:
248:    If for some reason you must call MPI_Init() separately, call
249:    it before PetscInitialize().

251:    Fortran Version:
252:    In Fortran this routine has the format
253: $       call PetscInitialize(file,ierr)

255: +   ierr - error return code
256: -   file - [optional] PETSc database file name, defaults to 
257:            ~username/.petscrc (use PETSC_NULL_CHARACTER for default)
258:            
259:    Important Fortran Note:
260:    In Fortran, you MUST use PETSC_NULL_CHARACTER to indicate a
261:    null character string; you CANNOT just use PETSC_NULL as 
262:    in the C version.  See the users manual for details.


265:    Concepts: initializing PETSc
266:    
267: .seealso: PetscFinalize(), PetscInitializeFortran()
268: @*/
269: int PetscInitialize(int *argc,char ***args,char file[],const char help[])
270: {
271:   int        ierr,flag,dummy_tag,size;
272:   PetscTruth flg;
273:   char       hostname[64];

276:   if (PetscInitializeCalled) return(0);

278:   PetscOptionsCreate();

280:   /*
281:      We initialize the program name here (before MPI_Init()) because MPICH has a bug in 
282:      it that it sets args[0] on all processors to be args[0] on the first processor.
283:   */
284:   if (argc && *argc) {
285:     PetscSetProgramName(**args);
286:   } else {
287:     PetscSetProgramName("Unknown Name");
288:   }

290:   /* Also initialize the initial datestamp */
291:   PetscSetInitialDate();

293:   MPI_Initialized(&flag);
294:   if (!flag) {
295:     ierr          = MPI_Init(argc,args);
296:     PetscBeganMPI = PETSC_TRUE;
297:   }
298:   PetscInitializeCalled = PETSC_TRUE;

300:   if (!PETSC_COMM_WORLD) {
301:     PETSC_COMM_WORLD = MPI_COMM_WORLD;
302:   }

304:   MPI_Comm_rank(MPI_COMM_WORLD,&PetscGlobalRank);
305:   MPI_Comm_size(MPI_COMM_WORLD,&PetscGlobalSize);

307: #if defined(PETSC_USE_COMPLEX)
308:   /* 
309:      Initialized the global complex variable; this is because with 
310:      shared libraries the constructors for global variables
311:      are not called; at least on IRIX.
312:   */
313:   {
314:     Scalar ic(0.0,1.0);
315:     PETSC_i = ic;
316:   }
317:   MPI_Type_contiguous(2,MPI_DOUBLE,&MPIU_COMPLEX);
318:   MPI_Type_commit(&MPIU_COMPLEX);
319:   MPI_Op_create(PetscSum_Local,1,&PetscSum_Op);
320: #endif

322:   /*
323:        Create the PETSc MPI reduction operator that sums of the first
324:      half of the entries and maxes the second half.
325:   */
326:   MPI_Op_create(PetscMaxSum_Local,1,&PetscMaxSum_Op);

328:   /*
329:         Build the options database and check for user setup requests
330:   */
331:   PetscOptionsInsert(argc,args,file);
332:   /*
333:       Print main application help message
334:   */
335:   PetscOptionsHasName(PETSC_NULL,"-help",&flg);
336:   if (help && flg) {
337:     PetscPrintf(PETSC_COMM_WORLD,help);
338:   }
339:   PetscOptionsCheckInitial();

341:   /*
342:        Initialize PETSC_COMM_SELF and WORLD as a MPI_Comm with the PETSc attribute.
343:     
344:        We delay until here to do it, since PetscMalloc() may not have been
345:      setup before this.
346:   */
347:   PetscCommDuplicate_Private(MPI_COMM_SELF,&PETSC_COMM_SELF,&dummy_tag);
348:   PetscCommDuplicate_Private(PETSC_COMM_WORLD,&PETSC_COMM_WORLD,&dummy_tag);

350:   /*
351:       Load the dynamic libraries (on machines that support them), this registers all
352:     the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes)
353:   */
354:   PetscInitialize_DynamicLibraries();

356:   /*
357:       Initialize all the default viewers
358:   */
359:   MPI_Comm_size(PETSC_COMM_WORLD,&size);
360:   PetscLogInfo(0,"PetscInitialize:PETSc successfully started: number of processors = %dn",size);
361:   PetscGetHostName(hostname,64);
362:   PetscLogInfo(0,"PetscInitialize:Running on machine: %sn",hostname);

364:   PetscOptionsCheckInitial_Components();

366:   PetscFunctionReturn(ierr);
367: }

369: /*@C 
370:    PetscFinalize - Checks for options to be called at the conclusion
371:    of the program and calls MPI_Finalize().

373:    Collective on PETSC_COMM_WORLD

375:    Options Database Keys:
376: +  -options_table - Calls OptionsPrint()
377: .  -options_left - Prints unused options that remain in the database
378: .  -options_left no - Does not print unused options that remain in the database
379: .  -mpidump - Calls PetscMPIDump()
380: .  -trdump - Calls PetscTrDump()
381: .  -trinfo - Prints total memory usage
382: .  -trdebug - Calls malloc_debug(2) to activate memory
383:         allocation diagnostics (used by PETSC_ARCH=sun4, 
384:         BOPT=[g,g_c++,g_complex] only!)
385: -  -trmalloc_log - Prints summary of memory usage

387:    Options Database Keys for Profiling:
388:    See the Profiling chapter of the users manual for details.
389: +  -log_summary [filename] - Prints summary of flop and timing
390:         information to screen. If the filename is specified the
391:         summary is written to the file. (for code compiled with 
392:         PETSC_USE_LOG).  See PetscLogPrintSummary().
393: .  -log_all [filename] - Logs extensive profiling information
394:         (for code compiled with PETSC_USE_LOG). See PetscLogDump(). 
395: .  -log [filename] - Logs basic profiline information (for
396:         code compiled with PETSC_USE_LOG).  See PetscLogDump().
397: .  -log_sync - Log the synchronization in scatters, inner products
398:         and norms
399: -  -log_mpe [filename] - Creates a logfile viewable by the 
400:       utility Upshot/Nupshot (in MPICH distribution)

402:    Level: beginner

404:    Note:
405:    See PetscInitialize() for more general runtime options.

407: .seealso: PetscInitialize(), PetscOptionsPrint(), PetscTrDump(), PetscMPIDump()
408: @*/
409: int PetscFinalize(void)
410: {
411:   int            ierr,rank,nopt;
412:   PetscLogDouble rss;
413:   PetscTruth     flg1,flg2,flg3;
414: 

417:   if (!PetscInitializeCalled) {
418:     (*PetscErrorPrintf)("PETSc ERROR: PetscInitialize() must be called before PetscFinalize()n");
419:     return(0);
420:   }
421:   /*
422:      Destroy all the function registration lists created
423:   */
424:   PetscFinalize_DynamicLibraries();


427:   PetscOptionsHasName(PETSC_NULL,"-get_resident_set_size",&flg1);
428:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
429:   if (flg1) {
430:     PetscGetResidentSetSize(&rss);
431:     PetscPrintf(PETSC_COMM_SELF,"[%d] Size of entire process memory %dn",rank,(int)rss);
432:   }

434: #if defined(PETSC_USE_LOG)
435:   PetscOptionsHasName(PETSC_NULL,"-get_total_flops",&flg1);
436:   if (flg1) {
437:     PetscLogDouble flops = 0;
438:     MPI_Reduce(&_TotalFlops,&flops,1,MPI_DOUBLE,MPI_SUM,0,PETSC_COMM_WORLD);
439:     PetscPrintf(PETSC_COMM_WORLD,"Total flops over all processors %gn",flops);
440:   }
441: #endif

443:   /*
444:      Free all objects registered with PetscObjectRegisterDestroy() such ast
445:     PETSC_VIEWER_XXX_().
446:   */
447:   PetscObjectRegisterDestroyAll();

449: #if defined(PETSC_USE_STACK)
450:   if (PetscStackActive) {
451:     PetscStackDestroy();
452:   }
453: #endif

455: #if defined(PETSC_USE_LOG)
456:   {
457:     char mname[64];
458: #if defined(PETSC_HAVE_MPE)
459:     mname[0] = 0;
460:     PetscOptionsGetString(PETSC_NULL,"-log_mpe",mname,64,&flg1);
461:     if (flg1){
462:       if (mname[0]) {PetscLogMPEDump(mname);}
463:       else          {PetscLogMPEDump(0);}
464:     }
465: #endif
466:     mname[0] = 0;
467:     PetscOptionsGetString(PETSC_NULL,"-log_summary",mname,64,&flg1);
468:     if (flg1) {
469:       if (mname[0])  {PetscLogPrintSummary(PETSC_COMM_WORLD,mname);}
470:       else           {PetscLogPrintSummary(PETSC_COMM_WORLD,0);}
471:     }

473:     mname[0] = 0;
474:     PetscOptionsGetString(PETSC_NULL,"-log_all",mname,64,&flg1);
475:     PetscOptionsGetString(PETSC_NULL,"-log",mname,64,&flg2);
476:     if (flg1 || flg2){
477:       if (mname[0]) PetscLogDump(mname);
478:       else          PetscLogDump(0);
479:     }
480:     PetscLogDestroy();
481:   }
482: #endif
483:   PetscOptionsHasName(PETSC_NULL,"-no_signal_handler",&flg1);
484:   if (!flg1) { PetscPopSignalHandler();}
485:   PetscOptionsHasName(PETSC_NULL,"-mpidump",&flg1);
486:   if (flg1) {
487:     PetscMPIDump(stdout);
488:   }
489:   PetscOptionsHasName(PETSC_NULL,"-trdump",&flg1);
490:   PetscOptionsHasName(PETSC_NULL,"-optionstable",&flg1);
491:   PetscOptionsHasName(PETSC_NULL,"-options_table",&flg2);
492:   if (flg1 && flg2) {
493:     if (!rank) {PetscOptionsPrint(stdout);}
494:   }

496:   /* to prevent PETSc -options_left from warning */
497:   PetscOptionsHasName(PETSC_NULL,"-nox_warning",&flg1);CHKERRQ(ierr)
498:   PetscOptionsHasName(PETSC_NULL,"-error_output_stderr",&flg1);

500:   PetscOptionsGetLogical(PETSC_NULL,"-options_left",&flg2,&flg1);
501:   PetscOptionsAllUsed(&nopt);
502:   if (flg2) {
503:     PetscOptionsPrint(stdout);
504:     if (!nopt) {
505:       PetscPrintf(PETSC_COMM_WORLD,"There are no unused options.n");
506:     } else if (nopt == 1) {
507:       PetscPrintf(PETSC_COMM_WORLD,"There is one unused database option. It is:n");
508:     } else {
509:       PetscPrintf(PETSC_COMM_WORLD,"There are %d unused database options. They are:n",nopt);
510:     }
511:   }
512: #if defined(PETSC_USE_BOPT_g)
513:   if (nopt && !flg1 && !flg2) {
514:     PetscPrintf(PETSC_COMM_WORLD,"WARNING! There are options you set that were not used!n");
515:     PetscPrintf(PETSC_COMM_WORLD,"WARNING! could be spelling mistake, etc!n");
516: #else 
517:   if (nopt || flg2) {
518: #endif
519:     PetscOptionsLeft();
520:   }

522:   PetscOptionsHasName(PETSC_NULL,"-log_history",&flg1);
523:   if (flg1) {
524:     PetscLogCloseHistoryFile(&petsc_history);
525:     petsc_history = 0;
526:   }


529:   /*
530:        Destroy PETSC_COMM_SELF/WORLD as a MPI_Comm with the PETSc 
531:      attribute.
532:   */
533:   PetscCommDestroy_Private(&PETSC_COMM_SELF);
534:   PetscCommDestroy_Private(&PETSC_COMM_WORLD);

536:   /*
537:        Free all the registered create functions, such as KSPList, VecList, SNESList, etc
538:   */
539:   PetscFListDestroyAll();

541: #if defined(PETSC_USE_LOG)
542:   PetscLogEventRegisterDestroy_Private();
543:   PetscLogStageDestroy_Private();
544: #endif

546:   PetscOptionsHasName(PETSC_NULL,"-trdump",&flg1);
547:   PetscOptionsHasName(PETSC_NULL,"-trinfo",&flg2);
548:   PetscOptionsHasName(PETSC_NULL,"-trmalloc_log",&flg3);
549:   if (flg1) {
550:     char fname[256];
551:     FILE *fd;
552: 
553:     fname[0] = 0;
554:     PetscOptionsGetString(PETSC_NULL,"-trdump",fname,250,&flg1);
555:     if (flg1 && fname[0]) {
556:       char sname[256];

558:       sprintf(sname,"%s_%d",fname,rank);
559:       fd   = fopen(sname,"w"); if (!fd) SETERRQ1(1,"Cannot open log file: %s",sname);
560:       PetscTrDump(fd);
561:       fclose(fd);
562:     } else {
563:       MPI_Comm local_comm;

565:       MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);
566:       PetscSequentialPhaseBegin_Private(local_comm,1);
567:         PetscTrDump(stdout);
568:       PetscSequentialPhaseEnd_Private(local_comm,1);
569:       MPI_Comm_free(&local_comm);
570:     }
571:   } else if (flg2) {
572:     MPI_Comm       local_comm;
573:     PetscLogDouble maxm;

575:     MPI_Comm_dup(MPI_COMM_WORLD,&local_comm);
576:     PetscTrSpace(PETSC_NULL,PETSC_NULL,&maxm);
577:     PetscSequentialPhaseBegin_Private(local_comm,1);
578:       printf("[%d] Maximum memory used %gn",rank,maxm);
579:     PetscSequentialPhaseEnd_Private(local_comm,1);
580:     MPI_Comm_free(&local_comm);
581:   }
582:   if (flg3) {
583:     char fname[256];
584:     FILE *fd;
585: 
586:     fname[0] = 0;
587:     PetscOptionsGetString(PETSC_NULL,"-trmalloc_log",fname,250,&flg1);
588:     if (flg1 && fname[0]) {
589:       char sname[256];

591:       sprintf(sname,"%s_%d",fname,rank);
592:       fd   = fopen(sname,"w"); if (!fd) SETERRQ1(1,"Cannot open log file: %s",sname);
593:       PetscTrLogDump(fd);
594:       fclose(fd);
595:     } else {
596:       PetscTrLogDump(stdout);
597:     }
598:   }
599:   /* Can be destroyed only after all the options are used */
600:   PetscOptionsDestroy();

602:   PetscLogInfo(0,"PetscFinalize:PETSc successfully ended!n");
603:   if (PetscBeganMPI) {
604:     MPI_Finalize();
605:   }

607: /*

609:      Note: In certain cases PETSC_COMM_WORLD is never MPI_Comm_free()ed because 
610:    the communicator has some outstanding requests on it. Specifically if the 
611:    flag PETSC_HAVE_BROKEN_REQUEST_FREE is set (for IBM MPI implementation). See 
612:    src/vec/utils/vpscat.c. Due to this the memory allocated in PetscCommDuplicate_Private()
613:    is never freed as it should be. Thus one may obtain messages of the form
614:    [ 1] 8 bytes PetscCommDuplicate_Private() line 645 in src/sys/src/mpiu.c indicating the
615:    memory was not freed.

617: */
618:   PetscClearMalloc();
619:   PetscInitializeCalled = PETSC_FALSE;
620:   PetscFunctionReturn(ierr);
621: }