Actual source code: init.c

petsc-master 2020-08-25
Report Typos and Errors
  1: /*

  3:    This file defines part of the initialization of PETSc

  5:   This file uses regular malloc and free because it cannot be known
  6:   what malloc is being used until it has already processed the input.
  7: */

  9:  #include <petscsys.h>
 10:  #include <petsc/private/petscimpl.h>
 11:  #include <petscvalgrind.h>
 12:  #include <petscviewer.h>
 13: #if defined(PETSC_USE_LOG)
 14: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
 15: #endif

 17: #if defined(PETSC_HAVE_SYS_SYSINFO_H)
 18: #include <sys/sysinfo.h>
 19: #endif
 20: #if defined(PETSC_HAVE_UNISTD_H)
 21: #include <unistd.h>
 22: #endif

 24: #if defined(PETSC_HAVE_CUDA)
 25:   #include <cuda_runtime.h>
 26:  #include <petsccublas.h>
 27: #endif

 29: #if defined(PETSC_HAVE_HIP)
 30:   #include <hip/hip_runtime.h>
 31: #endif

 33: #if defined(PETSC_HAVE_DEVICE)
 34:   #if defined(PETSC_HAVE_OMPI_MAJOR_VERSION)
 35:     #include "mpi-ext.h" /* Needed for OpenMPI CUDA-aware check */
 36:   #endif
 37: #endif

 39: #if defined(PETSC_HAVE_VIENNACL)
 40: PETSC_EXTERN PetscErrorCode PetscViennaCLInit();
 41: #endif


 44: /* ------------------------Nasty global variables -------------------------------*/
 45: /*
 46:      Indicates if PETSc started up MPI, or it was
 47:    already started before PETSc was initialized.
 48: */
 49: PetscBool   PetscBeganMPI                 = PETSC_FALSE;
 50: PetscBool   PetscErrorHandlingInitialized = PETSC_FALSE;
 51: PetscBool   PetscInitializeCalled         = PETSC_FALSE;
 52: PetscBool   PetscFinalizeCalled           = PETSC_FALSE;

 54: PetscMPIInt PetscGlobalRank               = -1;
 55: PetscMPIInt PetscGlobalSize               = -1;

 57: #if defined(PETSC_HAVE_KOKKOS)
 58: PetscBool   PetscBeganKokkos              = PETSC_FALSE;
 59: #endif

 61: PetscBool   use_gpu_aware_mpi             = PETSC_TRUE;

 63: #if defined(PETSC_HAVE_COMPLEX)
 64: #if defined(PETSC_COMPLEX_INSTANTIATE)
 65: template <> class std::complex<double>; /* instantiate complex template class */
 66: #endif
 67: #if !defined(PETSC_HAVE_MPI_C_DOUBLE_COMPLEX)
 68: MPI_Datatype MPIU_C_DOUBLE_COMPLEX;
 69: MPI_Datatype MPIU_C_COMPLEX;
 70: #endif

 72: /*MC
 73:    PETSC_i - the imaginary number i

 75:    Synopsis:
 76:  #include <petscsys.h>
 77:    PetscComplex PETSC_i;

 79:    Level: beginner

 81:    Note:
 82:    Complex numbers are automatically available if PETSc located a working complex implementation

 84: .seealso: PetscRealPart(), PetscImaginaryPart(), PetscRealPartComplex(), PetscImaginaryPartComplex()
 85: M*/
 86: PetscComplex PETSC_i;
 87: #endif
 88: #if defined(PETSC_USE_REAL___FLOAT128)
 89: MPI_Datatype MPIU___FLOAT128 = 0;
 90: #if defined(PETSC_HAVE_COMPLEX)
 91: MPI_Datatype MPIU___COMPLEX128 = 0;
 92: #endif
 93: #elif defined(PETSC_USE_REAL___FP16)
 94: MPI_Datatype MPIU___FP16 = 0;
 95: #endif
 96: MPI_Datatype MPIU_2SCALAR = 0;
 97: #if defined(PETSC_USE_64BIT_INDICES)
 98: MPI_Datatype MPIU_2INT = 0;
 99: #endif
100: MPI_Datatype MPIU_BOOL;
101: MPI_Datatype MPIU_ENUM;
102: MPI_Datatype MPIU_FORTRANADDR;
103: MPI_Datatype MPIU_SIZE_T;

105: /*
106:        Function that is called to display all error messages
107: */
108: PetscErrorCode (*PetscErrorPrintf)(const char [],...)          = PetscErrorPrintfDefault;
109: PetscErrorCode (*PetscHelpPrintf)(MPI_Comm,const char [],...)  = PetscHelpPrintfDefault;
110: PetscErrorCode (*PetscVFPrintf)(FILE*,const char[],va_list)    = PetscVFPrintfDefault;
111: /*
112:   This is needed to turn on/off GPU synchronization
113: */
114: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;

116: /* ------------------------------------------------------------------------------*/
117: /*
118:    Optional file where all PETSc output from various prints is saved
119: */
120: PETSC_INTERN FILE *petsc_history;
121: FILE *petsc_history = NULL;

123: PetscErrorCode  PetscOpenHistoryFile(const char filename[],FILE **fd)
124: {
126:   PetscMPIInt    rank,size;
127:   char           pfile[PETSC_MAX_PATH_LEN],pname[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN],date[64];
128:   char           version[256];

131:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
132:   if (!rank) {
133:     char        arch[10];
134:     int         err;

136:     PetscGetArchType(arch,10);
137:     PetscGetDate(date,64);
138:     PetscGetVersion(version,256);
139:     MPI_Comm_size(PETSC_COMM_WORLD,&size);
140:     if (filename) {
141:       PetscFixFilename(filename,fname);
142:     } else {
143:       PetscGetHomeDirectory(pfile,sizeof(pfile));
144:       PetscStrlcat(pfile,"/.petschistory",sizeof(pfile));
145:       PetscFixFilename(pfile,fname);
146:     }

148:     *fd = fopen(fname,"a");
149:     if (!fd) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Cannot open file: %s",fname);

151:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
152:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s %s\n",version,date);
153:     PetscGetProgramName(pname,sizeof(pname));
154:     PetscFPrintf(PETSC_COMM_SELF,*fd,"%s on a %s, %d proc. with options:\n",pname,arch,size);
155:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");

157:     err = fflush(*fd);
158:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
159:   }
160:   return(0);
161: }

163: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **fd)
164: {
166:   PetscMPIInt    rank;
167:   char           date[64];
168:   int            err;

171:   MPI_Comm_rank(PETSC_COMM_WORLD,&rank);
172:   if (!rank) {
173:     PetscGetDate(date,64);
174:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
175:     PetscFPrintf(PETSC_COMM_SELF,*fd,"Finished at %s\n",date);
176:     PetscFPrintf(PETSC_COMM_SELF,*fd,"----------------------------------------\n");
177:     err  = fflush(*fd);
178:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fflush() failed on file");
179:     err = fclose(*fd);
180:     if (err) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_SYS,"fclose() failed on file");
181:   }
182:   return(0);
183: }

185: /* ------------------------------------------------------------------------------*/

187: /*
188:    This is ugly and probably belongs somewhere else, but I want to
189:   be able to put a true MPI abort error handler with command line args.

191:     This is so MPI errors in the debugger will leave all the stack
192:   frames. The default MP_Abort() cleans up and exits thus providing no useful information
193:   in the debugger hence we call abort() instead of MPI_Abort().
194: */

196: void Petsc_MPI_AbortOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
197: {
199:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
200:   abort();
201: }

203: void Petsc_MPI_DebuggerOnError(MPI_Comm *comm,PetscMPIInt *flag,...)
204: {

208:   (*PetscErrorPrintf)("MPI error %d\n",*flag);
209:   PetscAttachDebugger();
210:   if (ierr) PETSCABORT(*comm,*flag); /* hopeless so get out */
211: }

213: /*@C
214:    PetscEnd - Calls PetscFinalize() and then ends the program. This is useful if one
215:      wishes a clean exit somewhere deep in the program.

217:    Collective on PETSC_COMM_WORLD

219:    Options Database Keys are the same as for PetscFinalize()

221:    Level: advanced

223:    Note:
224:    See PetscInitialize() for more general runtime options.

226: .seealso: PetscInitialize(), PetscOptionsView(), PetscMallocDump(), PetscMPIDump(), PetscFinalize()
227: @*/
228: PetscErrorCode  PetscEnd(void)
229: {
231:   PetscFinalize();
232:   exit(0);
233:   return 0;
234: }

236: PetscBool PetscOptionsPublish = PETSC_FALSE;
237: PETSC_INTERN PetscErrorCode PetscSetUseHBWMalloc_Private(void);
238: PETSC_INTERN PetscBool      petscsetmallocvisited;
239: static       char           emacsmachinename[256];

241: PetscErrorCode (*PetscExternalVersionFunction)(MPI_Comm) = NULL;
242: PetscErrorCode (*PetscExternalHelpFunction)(MPI_Comm)    = NULL;

244: /*@C
245:    PetscSetHelpVersionFunctions - Sets functions that print help and version information
246:    before the PETSc help and version information is printed. Must call BEFORE PetscInitialize().
247:    This routine enables a "higher-level" package that uses PETSc to print its messages first.

249:    Input Parameter:
250: +  help - the help function (may be NULL)
251: -  version - the version function (may be NULL)

253:    Level: developer

255: @*/
256: PetscErrorCode  PetscSetHelpVersionFunctions(PetscErrorCode (*help)(MPI_Comm),PetscErrorCode (*version)(MPI_Comm))
257: {
259:   PetscExternalHelpFunction    = help;
260:   PetscExternalVersionFunction = version;
261:   return(0);
262: }

264: #if defined(PETSC_USE_LOG)
265: PETSC_INTERN PetscBool   PetscObjectsLog;
266: #endif

268: void PetscMPI_Comm_eh(MPI_Comm *comm, PetscMPIInt *err, ...)
269: {
270:   if (PetscUnlikely(*err)) {
271:     PetscMPIInt len;
272:     char        errstring[MPI_MAX_ERROR_STRING];

274:     MPI_Error_string(*err,errstring,&len);
275:     PetscError(MPI_COMM_SELF,__LINE__,PETSC_FUNCTION_NAME,__FILE__,PETSC_MPI_ERROR_CODE,PETSC_ERROR_INITIAL,"Internal error in MPI: %s",errstring);
276:   }
277:   return;
278: }

280: /* CUPM stands for 'CUDA Programming Model', which is implemented in either CUDA or HIP.
281:    Use the following macros to define CUDA/HIP initialization related vars/routines.
282:  */
283: #if defined(PETSC_HAVE_CUDA)
284:   typedef cudaError_t                             cupmError_t;
285:   typedef struct cudaDeviceProp                   cupmDeviceProp;
286:   #define cupmGetDeviceCount(x)                   cudaGetDeviceCount(x)
287:   #define cupmGetDevice(x)                        cudaGetDevice(x)
288:   #define cupmSetDevice(x)                        cudaSetDevice(x)
289:   #define cupmSetDeviceFlags(x)                   cudaSetDeviceFlags(x)
290:   #define cupmGetDeviceProperties(x,y)            cudaGetDeviceProperties(x,y)
291:   #define cupmGetLastError()                      cudaGetLastError()
292:   #define cupmDeviceMapHost                       cudaDeviceMapHost
293:   #define cupmSuccess                             cudaSuccess
294:   #define cupmErrorSetOnActiveProcess             cudaErrorSetOnActiveProcess
295:   #define CHKERRCUPM(x)                           CHKERRCUDA(x)
296:   #define PetscCUPMBLASInitializeHandle()         PetscCUBLASInitializeHandle()
297:   #define PetscCUPMSOLVERDnInitializeHandle()     PetscCUSOLVERDnInitializeHandle()
298:   #define PetscCUPMInitialize                     PetscCUDAInitialize
299:   #define PetscCUPMInitialized                    PetscCUDAInitialized
300:   #define PetscCUPMInitializeCheck                PetscCUDAInitializeCheck
301:   #define PetscCUPMInitializeAndView              PetscCUDAInitializeAndView
302:   #define PetscCUPMSynchronize                    PetscCUDASynchronize
303:   #define PetscNotUseCUPM                         PetscNotUseCUDA
304:   #define cupmOptionsStr                          "CUDA options"
305:   #define cupmSetDeviceStr                        "-cuda_set_device"
306:   #define cupmViewStr                             "-cuda_view"
307:   #define cupmSynchronizeStr                      "-cuda_synchronize"
308:   #define PetscCUPMInitializeStr                  "PetscCUDAInitialize"
309:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckCUDA
310:   #define PetscMPICUPMAwarenessCheck              PetscMPICUDAAwarenessCheck
311:   #include "cupminit.inc"
312: #endif

314: #if defined(PETSC_HAVE_HIP)
315:   typedef hipError_t                              cupmError_t;
316:   typedef hipDeviceProp_t                         cupmDeviceProp;
317:   #define cupmGetDeviceCount(x)                   hipGetDeviceCount(x)
318:   #define cupmGetDevice(x)                        hipGetDevice(x)
319:   #define cupmSetDevice(x)                        hipSetDevice(x)
320:   #define cupmSetDeviceFlags(x)                   hipSetDeviceFlags(x)
321:   #define cupmGetDeviceProperties(x,y)            hipGetDeviceProperties(x,y)
322:   #define cupmGetLastError()                      hipGetLastError()
323:   #define cupmDeviceMapHost                       hipDeviceMapHost
324:   #define cupmSuccess                             hipSuccess
325:   #define cupmErrorSetOnActiveProcess             hipErrorSetOnActiveProcess
326:   #define CHKERRCUPM(x)                           CHKERRQ((x)==hipSuccess? 0:PETSC_ERR_LIB)
327:   #define PetscCUPMBLASInitializeHandle()         0
328:   #define PetscCUPMSOLVERDnInitializeHandle()     0
329:   #define PetscCUPMInitialize                     PetscHIPInitialize
330:   #define PetscCUPMInitialized                    PetscHIPInitialized
331:   #define PetscCUPMInitializeCheck                PetscHIPInitializeCheck
332:   #define PetscCUPMInitializeAndView              PetscHIPInitializeAndView
333:   #define PetscCUPMSynchronize                    PetscHIPSynchronize
334:   #define PetscNotUseCUPM                         PetscNotUseHIP
335:   #define cupmOptionsStr                          "HIP options"
336:   #define cupmSetDeviceStr                        "-hip_set_device"
337:   #define cupmViewStr                             "-hip_view"
338:   #define cupmSynchronizeStr                      "-hip_synchronize"
339:   #define PetscCUPMInitializeStr                  "PetscHIPInitialize"
340:   #define PetscOptionsCheckCUPM                   PetscOptionsCheckHIP
341:   #define PetscMPICUPMAwarenessCheck              PetscMPIHIPAwarenessCheck
342:   #include "cupminit.inc"
343: #endif

345: PETSC_INTERN PetscErrorCode  PetscOptionsCheckInitial_Private(const char help[])
346: {
347:   char              string[64];
348:   MPI_Comm          comm = PETSC_COMM_WORLD;
349:   PetscBool         flg1 = PETSC_FALSE,flg2 = PETSC_FALSE,flg3 = PETSC_FALSE,flag,hasHelp,logView;
350:   PetscErrorCode    ierr;
351:   PetscReal         si;
352:   PetscInt          intensity;
353:   int               i;
354:   PetscMPIInt       rank;
355:   char              version[256];
356: #if defined(PETSC_USE_LOG)
357:   char              mname[PETSC_MAX_PATH_LEN];
358:   PetscViewerFormat format;
359:   PetscBool         flg4 = PETSC_FALSE;
360: #endif

363:   MPI_Comm_rank(comm,&rank);

365: #if !defined(PETSC_HAVE_THREADSAFETY)
366:   if (!(PETSC_RUNNING_ON_VALGRIND)) {
367:     /*
368:       Setup the memory management; support for tracing malloc() usage
369:     */
370:     PetscBool         mdebug = PETSC_FALSE, eachcall = PETSC_FALSE, initializenan = PETSC_FALSE, mlog = PETSC_FALSE;

372:     if (PetscDefined(USE_DEBUG)) {
373:       mdebug        = PETSC_TRUE;
374:       initializenan = PETSC_TRUE;
375:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
376:     } else {
377:       /* don't warn about unused option */
378:       PetscOptionsHasName(NULL,NULL,"-malloc_test",&flg1);
379:       flg1 = PETSC_FALSE;
380:     }
381:     PetscOptionsGetBool(NULL,NULL,"-malloc_debug",&flg2,&flg3);
382:     if (flg1 || flg2) {
383:       mdebug        = PETSC_TRUE;
384:       eachcall      = PETSC_TRUE;
385:       initializenan = PETSC_TRUE;
386:     } else if (flg3 && !flg2) {
387:       mdebug        = PETSC_FALSE;
388:       eachcall      = PETSC_FALSE;
389:       initializenan = PETSC_FALSE;
390:     }

392:     PetscOptionsHasName(NULL,NULL,"-malloc_view",&mlog);
393:     if (mlog) {
394:       mdebug = PETSC_TRUE;
395:     }
396:     /* the next line is deprecated */
397:     PetscOptionsGetBool(NULL,NULL,"-malloc",&mdebug,NULL);
398:     PetscOptionsGetBool(NULL,NULL,"-malloc_dump",&mdebug,NULL);
399:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&mdebug,NULL);
400:     if (mdebug) {
401:       PetscMallocSetDebug(eachcall,initializenan);
402:     }
403:     if (mlog) {
404:       PetscReal logthreshold = 0;
405:       PetscOptionsGetReal(NULL,NULL,"-malloc_view_threshold",&logthreshold,NULL);
406:       PetscMallocViewSet(logthreshold);
407:     }
408: #if defined(PETSC_USE_LOG)
409:     PetscOptionsGetBool(NULL,NULL,"-log_view_memory",&PetscLogMemory,NULL);
410: #endif
411:   }

413:   PetscOptionsGetBool(NULL,NULL,"-malloc_coalesce",&flg1,&flg2);
414:   if (flg2) {PetscMallocSetCoalesce(flg1);}
415:   flg1 = PETSC_FALSE;
416:   PetscOptionsGetBool(NULL,NULL,"-malloc_hbw",&flg1,NULL);
417:   /* ignore this option if malloc is already set */
418:   if (flg1 && !petscsetmallocvisited) {PetscSetUseHBWMalloc_Private();}

420:   flg1 = PETSC_FALSE;
421:   PetscOptionsGetBool(NULL,NULL,"-malloc_info",&flg1,NULL);
422:   if (!flg1) {
423:     flg1 = PETSC_FALSE;
424:     PetscOptionsGetBool(NULL,NULL,"-memory_view",&flg1,NULL);
425:   }
426:   if (flg1) {
427:     PetscMemorySetGetMaximumUsage();
428:   }
429: #endif

431: #if defined(PETSC_USE_LOG)
432:   PetscOptionsHasName(NULL,NULL,"-objects_dump",&PetscObjectsLog);
433: #endif

435:   /*
436:       Set the display variable for graphics
437:   */
438:   PetscSetDisplay();

440:   /*
441:      Print main Section 1.5 Writing Application Codes with PETSc help message
442:   */
443:   PetscOptionsHasHelp(NULL,&hasHelp);
444:   if (help && hasHelp) {
445:     PetscPrintf(comm,help);
446:     PetscPrintf(comm,"----------------------------------------\n");
447:   }

449:   /*
450:       Print the PETSc version information
451:   */
452:   PetscOptionsHasName(NULL,NULL,"-version",&flg1);
453:   if (flg1 || hasHelp) {
454:     /*
455:        Print "higher-level" package version message
456:     */
457:     if (PetscExternalVersionFunction) {
458:       (*PetscExternalVersionFunction)(comm);
459:     }

461:     PetscGetVersion(version,256);
462:     (*PetscHelpPrintf)(comm,"%s\n",version);
463:     (*PetscHelpPrintf)(comm,"%s",PETSC_AUTHOR_INFO);
464:     (*PetscHelpPrintf)(comm,"See docs/changes/index.html for recent updates.\n");
465:     (*PetscHelpPrintf)(comm,"See docs/faq.html for problems.\n");
466:     (*PetscHelpPrintf)(comm,"See docs/manualpages/index.html for help. \n");
467:     (*PetscHelpPrintf)(comm,"Libraries linked from %s\n",PETSC_LIB_DIR);
468:     (*PetscHelpPrintf)(comm,"----------------------------------------\n");
469:   }

471:   /*
472:        Print "higher-level" package help message
473:   */
474:   if (hasHelp) {
475:     PetscBool hasHelpIntro;

477:     if (PetscExternalHelpFunction) {
478:       (*PetscExternalHelpFunction)(comm);
479:     }
480:     PetscOptionsHasHelpIntro_Internal(NULL,&hasHelpIntro);
481:     if (hasHelpIntro) {
482:       PetscOptionsDestroyDefault();
483:       PetscFreeMPIResources();
484:       MPI_Finalize();
485:       exit(0);
486:     }
487:   }

489:   /*
490:       Setup the error handling
491:   */
492:   flg1 = PETSC_FALSE;
493:   PetscOptionsGetBool(NULL,NULL,"-on_error_abort",&flg1,NULL);
494:   if (flg1) {
495:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_ARE_FATAL);
496:     PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
497:   }
498:   flg1 = PETSC_FALSE;
499:   PetscOptionsGetBool(NULL,NULL,"-on_error_mpiabort",&flg1,NULL);
500:   if (flg1) { PetscPushErrorHandler(PetscMPIAbortErrorHandler,NULL);}
501:   flg1 = PETSC_FALSE;
502:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_on_error",&flg1,NULL);
503:   if (flg1) {
504:     MPI_Comm_set_errhandler(comm,MPI_ERRORS_RETURN);
505:   }
506:   /* experimental */
507:   flg1 = PETSC_FALSE;
508:   PetscOptionsGetBool(NULL,NULL,"-mpi_return_error_string",&flg1,NULL);
509:   if (flg1) {
510:     MPI_Errhandler eh;

512:     MPI_Comm_create_errhandler(PetscMPI_Comm_eh,&eh);
513:     MPI_Comm_set_errhandler(comm,eh);
514:     MPI_Errhandler_free(&eh);
515:   }
516:   flg1 = PETSC_FALSE;
517:   PetscOptionsGetBool(NULL,NULL,"-no_signal_handler",&flg1,NULL);
518:   if (!flg1) {PetscPushSignalHandler(PetscSignalHandlerDefault,(void*)0);}

520:   /*
521:       Setup debugger information
522:   */
523:   PetscSetDefaultDebugger();
524:   PetscOptionsGetString(NULL,NULL,"-on_error_attach_debugger",string,sizeof(string),&flg1);
525:   if (flg1) {
526:     MPI_Errhandler err_handler;

528:     PetscSetDebuggerFromString(string);
529:     MPI_Comm_create_errhandler(Petsc_MPI_DebuggerOnError,&err_handler);
530:     MPI_Comm_set_errhandler(comm,err_handler);
531:     PetscPushErrorHandler(PetscAttachDebuggerErrorHandler,NULL);
532:   }
533:   PetscOptionsGetString(NULL,NULL,"-debug_terminal",string,sizeof(string),&flg1);
534:   if (flg1) { PetscSetDebugTerminal(string); }
535:   PetscOptionsGetString(NULL,NULL,"-start_in_debugger",string,sizeof(string),&flg1);
536:   PetscOptionsGetString(NULL,NULL,"-stop_for_debugger",string,sizeof(string),&flg2);
537:   if (flg1 || flg2) {
538:     PetscMPIInt    size;
539:     PetscInt       lsize,*nodes;
540:     MPI_Errhandler err_handler;
541:     /*
542:        we have to make sure that all processors have opened
543:        connections to all other processors, otherwise once the
544:        debugger has stated it is likely to receive a SIGUSR1
545:        and kill the program.
546:     */
547:     MPI_Comm_size(comm,&size);
548:     if (size > 2) {
549:       PetscMPIInt dummy = 0;
550:       MPI_Status  status;
551:       for (i=0; i<size; i++) {
552:         if (rank != i) {
553:           MPI_Send(&dummy,1,MPI_INT,i,109,comm);
554:         }
555:       }
556:       for (i=0; i<size; i++) {
557:         if (rank != i) {
558:           MPI_Recv(&dummy,1,MPI_INT,i,109,comm,&status);
559:         }
560:       }
561:     }
562:     /* check if this processor node should be in debugger */
563:     PetscMalloc1(size,&nodes);
564:     lsize = size;
565:     PetscOptionsGetIntArray(NULL,NULL,"-debugger_nodes",nodes,&lsize,&flag);
566:     if (flag) {
567:       for (i=0; i<lsize; i++) {
568:         if (nodes[i] == rank) { flag = PETSC_FALSE; break; }
569:       }
570:     }
571:     if (!flag) {
572:       PetscSetDebuggerFromString(string);
573:       PetscPushErrorHandler(PetscAbortErrorHandler,NULL);
574:       if (flg1) {
575:         PetscAttachDebugger();
576:       } else {
577:         PetscStopForDebugger();
578:       }
579:       MPI_Comm_create_errhandler(Petsc_MPI_AbortOnError,&err_handler);
580:       MPI_Comm_set_errhandler(comm,err_handler);
581:     }
582:     PetscFree(nodes);
583:   }

585:   PetscOptionsGetString(NULL,NULL,"-on_error_emacs",emacsmachinename,sizeof(emacsmachinename),&flg1);
586:   if (flg1 && !rank) {PetscPushErrorHandler(PetscEmacsClientErrorHandler,emacsmachinename);}

588:   /*
589:         Setup profiling and logging
590:   */
591: #if defined(PETSC_USE_INFO)
592:   {
593:     PetscInfoSetFromOptions(NULL);
594:   }
595: #endif
596:   PetscDetermineInitialFPTrap();
597:   flg1 = PETSC_FALSE;
598:   PetscOptionsGetBool(NULL,NULL,"-fp_trap",&flg1,&flag);
599:   if (flag) {PetscSetFPTrap((PetscFPTrap)flg1);}
600:   PetscOptionsGetInt(NULL,NULL,"-check_pointer_intensity",&intensity,&flag);
602: #if defined(PETSC_USE_LOG)
603:   mname[0] = 0;
604:   PetscOptionsGetString(NULL,NULL,"-history",mname,sizeof(mname),&flg1);
605:   if (flg1) {
606:     if (mname[0]) {
607:       PetscOpenHistoryFile(mname,&petsc_history);
608:     } else {
609:       PetscOpenHistoryFile(NULL,&petsc_history);
610:     }
611:   }

613:   PetscOptionsGetBool(NULL,NULL,"-log_sync",&PetscLogSyncOn,NULL);

615: #if defined(PETSC_HAVE_MPE)
616:   flg1 = PETSC_FALSE;
617:   PetscOptionsHasName(NULL,NULL,"-log_mpe",&flg1);
618:   if (flg1) {PetscLogMPEBegin();}
619: #endif
620:   flg1 = PETSC_FALSE;
621:   flg3 = PETSC_FALSE;
622:   PetscOptionsGetBool(NULL,NULL,"-log_all",&flg1,NULL);
623:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
624:   if (flg1)                      { PetscLogAllBegin(); }
625:   else if (flg3)                 { PetscLogDefaultBegin();}

627:   PetscOptionsGetString(NULL,NULL,"-log_trace",mname,sizeof(mname),&flg1);
628:   if (flg1) {
629:     char name[PETSC_MAX_PATH_LEN],fname[PETSC_MAX_PATH_LEN];
630:     FILE *file;
631:     if (mname[0]) {
632:       PetscSNPrintf(name,PETSC_MAX_PATH_LEN,"%s.%d",mname,rank);
633:       PetscFixFilename(name,fname);
634:       file = fopen(fname,"w");
635:       if (!file) SETERRQ1(PETSC_COMM_SELF,PETSC_ERR_FILE_OPEN,"Unable to open trace file: %s",fname);
636:     } else file = PETSC_STDOUT;
637:     PetscLogTraceBegin(file);
638:   }

640:   PetscOptionsGetViewer(comm,NULL,NULL,"-log_view",NULL,&format,&flg4);
641:   if (flg4) {
642:     if (format == PETSC_VIEWER_ASCII_XML) {
643:       PetscLogNestedBegin();
644:     } else {
645:       PetscLogDefaultBegin();
646:     }
647:   }
648:   if (flg4 && format == PETSC_VIEWER_ASCII_XML) {
649:     PetscReal threshold = PetscRealConstant(0.01);
650:     PetscOptionsGetReal(NULL,NULL,"-log_threshold",&threshold,&flg1);
651:     if (flg1) {PetscLogSetThreshold((PetscLogDouble)threshold,NULL);}
652:   }
653: #endif

655:   PetscOptionsGetBool(NULL,NULL,"-saws_options",&PetscOptionsPublish,NULL);
656:   PetscOptionsGetBool(NULL,NULL,"-use_gpu_aware_mpi",&use_gpu_aware_mpi,NULL);
657:   /*
658:     If collecting logging information, by default, wait for device to complete its operations
659:     before returning to the CPU in order to get accurate timings of each event
660:   */
661:   PetscOptionsHasName(NULL,NULL,"-log_summary",&logView);
662:   if (!logView) {PetscOptionsHasName(NULL,NULL,"-log_view",&logView);}

664: #if defined(PETSC_HAVE_CUDA)
665:   PetscOptionsCheckCUDA(logView);
666: #endif

668: #if defined(PETSC_HAVE_HIP)
669:   PetscOptionsCheckHIP(logView);
670: #endif

672:   /*
673:        Print basic help message
674:   */
675:   if (hasHelp) {
676:     (*PetscHelpPrintf)(comm,"Options for all PETSc programs:\n");
677:     (*PetscHelpPrintf)(comm," -version: prints PETSc version\n");
678:     (*PetscHelpPrintf)(comm," -help intro: prints example description and PETSc version, and exits\n");
679:     (*PetscHelpPrintf)(comm," -help: prints example description, PETSc version, and available options for used routines\n");
680:     (*PetscHelpPrintf)(comm," -on_error_abort: cause an abort when an error is detected. Useful \n ");
681:     (*PetscHelpPrintf)(comm,"       only when run in the debugger\n");
682:     (*PetscHelpPrintf)(comm," -on_error_attach_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
683:     (*PetscHelpPrintf)(comm,"       start the debugger in new xterm\n");
684:     (*PetscHelpPrintf)(comm,"       unless noxterm is given\n");
685:     (*PetscHelpPrintf)(comm," -start_in_debugger [gdb,dbx,xxgdb,ups,noxterm]\n");
686:     (*PetscHelpPrintf)(comm,"       start all processes in the debugger\n");
687:     (*PetscHelpPrintf)(comm," -on_error_emacs <machinename>\n");
688:     (*PetscHelpPrintf)(comm,"    emacs jumps to error file\n");
689:     (*PetscHelpPrintf)(comm," -debugger_nodes [n1,n2,..] Nodes to start in debugger\n");
690:     (*PetscHelpPrintf)(comm," -debugger_pause [m] : delay (in seconds) to attach debugger\n");
691:     (*PetscHelpPrintf)(comm," -stop_for_debugger : prints message on how to attach debugger manually\n");
692:     (*PetscHelpPrintf)(comm,"                      waits the delay for you to attach\n");
693:     (*PetscHelpPrintf)(comm," -display display: Location where X window graphics and debuggers are displayed\n");
694:     (*PetscHelpPrintf)(comm," -no_signal_handler: do not trap error signals\n");
695:     (*PetscHelpPrintf)(comm," -mpi_return_on_error: MPI returns error code, rather than abort on internal error\n");
696:     (*PetscHelpPrintf)(comm," -fp_trap: stop on floating point exceptions\n");
697:     (*PetscHelpPrintf)(comm,"           note on IBM RS6000 this slows run greatly\n");
698:     (*PetscHelpPrintf)(comm," -malloc_dump <optional filename>: dump list of unfreed memory at conclusion\n");
699:     (*PetscHelpPrintf)(comm," -malloc: use PETSc error checking malloc (deprecated, use -malloc_debug)\n");
700:     (*PetscHelpPrintf)(comm," -malloc no: don't use PETSc error checking malloc (deprecated, use -malloc_debug no)\n");
701:     (*PetscHelpPrintf)(comm," -malloc_info: prints total memory usage\n");
702:     (*PetscHelpPrintf)(comm," -malloc_view <optional filename>: keeps log of all memory allocations, displays in PetscFinalize()\n");
703:     (*PetscHelpPrintf)(comm," -malloc_debug <true or false>: enables or disables extended checking for memory corruption\n");
704:     (*PetscHelpPrintf)(comm," -options_view: dump list of options inputted\n");
705:     (*PetscHelpPrintf)(comm," -options_left: dump list of unused options\n");
706:     (*PetscHelpPrintf)(comm," -options_left no: don't dump list of unused options\n");
707:     (*PetscHelpPrintf)(comm," -tmp tmpdir: alternative /tmp directory\n");
708:     (*PetscHelpPrintf)(comm," -shared_tmp: tmp directory is shared by all processors\n");
709:     (*PetscHelpPrintf)(comm," -not_shared_tmp: each processor has separate tmp directory\n");
710:     (*PetscHelpPrintf)(comm," -memory_view: print memory usage at end of run\n");
711: #if defined(PETSC_USE_LOG)
712:     (*PetscHelpPrintf)(comm," -get_total_flops: total flops over all processors\n");
713:     (*PetscHelpPrintf)(comm," -log_view [:filename:[format]]: logging objects and events\n");
714:     (*PetscHelpPrintf)(comm," -log_trace [filename]: prints trace of all PETSc calls\n");
715:     (*PetscHelpPrintf)(comm," -log_exclude <list,of,classnames>: exclude given classes from logging\n");
716: #if defined(PETSC_HAVE_MPE)
717:     (*PetscHelpPrintf)(comm," -log_mpe: Also create logfile viewable through Jumpshot\n");
718: #endif
719: #endif
720: #if defined(PETSC_USE_INFO)
721:     (*PetscHelpPrintf)(comm," -info [filename][:[~]<list,of,classnames>[:[~]self]]: print verbose information\n");
722: #endif
723:     (*PetscHelpPrintf)(comm," -options_file <file>: reads options from file\n");
724:     (*PetscHelpPrintf)(comm," -options_monitor: monitor options to standard output, including that set previously e.g. in option files\n");
725:     (*PetscHelpPrintf)(comm," -options_monitor_cancel: cancels all hardwired option monitors\n");
726:     (*PetscHelpPrintf)(comm," -petsc_sleep n: sleeps n seconds before running program\n");
727:   }

729: #if defined(PETSC_HAVE_POPEN)
730:   {
731:   char machine[128];
732:   PetscOptionsGetString(NULL,NULL,"-popen_machine",machine,sizeof(machine),&flg1);
733:   if (flg1) {
734:     PetscPOpenSetMachine(machine);
735:   }
736:   }
737: #endif

739:   PetscOptionsGetReal(NULL,NULL,"-petsc_sleep",&si,&flg1);
740:   if (flg1) {
741:     PetscSleep(si);
742:   }

744: #if defined(PETSC_HAVE_VIENNACL)
745:   PetscOptionsHasName(NULL,NULL,"-log_summary",&flg3);
746:   if (!flg3) {
747:     PetscOptionsHasName(NULL,NULL,"-log_view",&flg3);
748:   }
749:   PetscOptionsGetBool(NULL,NULL,"-viennacl_synchronize",&flg3,NULL);
750:   PetscViennaCLSynchronize = flg3;
751:   PetscViennaCLInit();
752: #endif

754:   /*
755:      Creates the logging data structures; this is enabled even if logging is not turned on
756:      This is the last thing we do before returning to the user code to prevent having the
757:      logging numbers contaminated by any startup time associated with MPI and the GPUs
758:   */
759: #if defined(PETSC_USE_LOG)
760:   PetscLogInitialize();
761: #endif

763:   return(0);
764: }