Actual source code: fretrieve.c

  1: /*$Id: fretrieve.c,v 1.38 2001/03/23 23:20:30 balay Exp $*/
  2: /*
  3:       Code for opening and closing files.
  4: */
  5: #include "petsc.h"
  6: #include "petscsys.h"
  7: #include "petscfix.h"
  8: #if defined(PETSC_HAVE_PWD_H)
  9: #include <pwd.h>
 10: #endif
 11: #include <ctype.h>
 12: #include <sys/types.h>
 13: #include <sys/stat.h>
 14: #if defined(PETSC_HAVE_UNISTD_H)
 15: #include <unistd.h>
 16: #endif
 17: #if defined(PETSC_HAVE_STDLIB_H)
 18: #include <stdlib.h>
 19: #endif
 20: #if !defined(PARCH_win32)
 21: #include <sys/utsname.h>
 22: #endif
 23: #if defined(PARCH_win32)
 24: #include <windows.h>
 25: #include <io.h>
 26: #include <direct.h>
 27: #endif
 28: #if defined (PARCH_win32_gnu)
 29: #include <windows.h>
 30: #endif
 31: #include <fcntl.h>
 32: #include <time.h>  
 33: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
 34: #include <sys/systeminfo.h>
 35: #endif
 36: #include "petscfix.h"

 38: EXTERN_C_BEGIN
 39: EXTERN int Petsc_DelTag(MPI_Comm,int,void*,void*);
 40: EXTERN_C_END

 42: /*@C
 43:    PetscGetTmp - Gets the name of the tmp directory

 45:    Collective on MPI_Comm

 47:    Input Parameters:
 48: +  comm - MPI_Communicator that may share /tmp
 49: -  len - length of string to hold name

 51:    Output Parameters:
 52: .  dir - directory name

 54:    Options Database Keys:
 55: +    -shared_tmp 
 56: .    -not_shared_tmp
 57: -    -tmp tmpdir

 59:    Environmental Variables:
 60: +     PETSC_SHARED_TMP
 61: .     PETSC_NOT_SHARED_TMP
 62: -     PETSC_TMP

 64:    Level: developer

 66:    
 67:    If the environmental variable PETSC_TMP is set it will use this directory
 68:   as the "/tmp" directory.

 70: @*/
 71: int PetscGetTmp(MPI_Comm comm,char *dir,int len)
 72: {
 73:   int        ierr;
 74:   PetscTruth flg;

 77:   PetscOptionsGetenv(comm,"PETSC_TMP",dir,len,&flg);
 78:   if (!flg) {
 79:     PetscStrncpy(dir,"/tmp",len);
 80:   }
 81:   return(0);
 82: }

 84: /*@C
 85:    PetscSharedTmp - Determines if all processors in a communicator share a
 86:          /tmp or have different ones.

 88:    Collective on MPI_Comm

 90:    Input Parameters:
 91: .  comm - MPI_Communicator that may share /tmp

 93:    Output Parameters:
 94: .  shared - PETSC_TRUE or PETSC_FALSE

 96:    Options Database Keys:
 97: +    -shared_tmp 
 98: .    -not_shared_tmp
 99: -    -tmp tmpdir

101:    Environmental Variables:
102: +     PETSC_SHARED_TMP
103: .     PETSC_NOT_SHARED_TMP
104: -     PETSC_TMP

106:    Level: developer

108:    Notes:
109:    Stores the status as a MPI attribute so it does not have
110:     to be redetermined each time.

112:       Assumes that all processors in a communicator either
113:        1) have a common /tmp or
114:        2) each has a seperate /tmp
115:       eventually we can write a fancier one that determines which processors
116:       share a common /tmp.

118:    This will be very slow on runs with a large number of processors since
119:    it requires O(p*p) file opens.

121:    If the environmental variable PETSC_TMP is set it will use this directory
122:   as the "/tmp" directory.

124: @*/
125: int PetscSharedTmp(MPI_Comm comm,PetscTruth *shared)
126: {
127:   int        ierr,size,rank,*tagvalp,sum,cnt,i;
128:   PetscTruth flg,iflg;
129:   FILE       *fd;
130:   static int Petsc_Tmp_keyval = MPI_KEYVAL_INVALID;

133:   MPI_Comm_size(comm,&size);
134:   if (size == 1) {
135:     *shared = PETSC_TRUE;
136:     return(0);
137:   }

139:   PetscOptionsGetenv(comm,"PETSC_SHARED_TMP",PETSC_NULL,0,&flg);
140:   if (flg) {
141:     *shared = PETSC_TRUE;
142:     return(0);
143:   }

145:   PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_TMP",PETSC_NULL,0,&flg);
146:   if (flg) {
147:     *shared = PETSC_FALSE;
148:     return(0);
149:   }

151:   if (Petsc_Tmp_keyval == MPI_KEYVAL_INVALID) {
152:     MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelTag,&Petsc_Tmp_keyval,0);
153:   }

155:   MPI_Attr_get(comm,Petsc_Tmp_keyval,(void**)&tagvalp,(int*)&iflg);
156:   if (!iflg) {
157:     char       filename[256],tmpname[256];
158: ;

160:     /* This communicator does not yet have a shared tmp attribute */
161:     PetscMalloc(sizeof(int),&tagvalp);
162:     MPI_Attr_put(comm,Petsc_Tmp_keyval,tagvalp);

164:     PetscOptionsGetenv(comm,"PETSC_TMP",tmpname,238,&iflg);
165:     if (!iflg) {
166:       PetscStrcpy(filename,"/tmp");
167:     } else {
168:       PetscStrcpy(filename,tmpname);
169:     }

171:     PetscStrcat(filename,"/petsctestshared");
172:     MPI_Comm_rank(comm,&rank);
173: 
174:     /* each processor creates a /tmp file and all the later ones check */
175:     /* this makes sure no subset of processors is shared */
176:     *shared = PETSC_FALSE;
177:     for (i=0; i<size-1; i++) {
178:       if (rank == i) {
179:         fd = fopen(filename,"w");
180:         if (!fd) {
181:           SETERRQ1(1,"Unable to open test file %s",filename);
182:         }
183:         fclose(fd);
184:       }
185:       MPI_Barrier(comm);
186:       if (rank >= i) {
187:         fd = fopen(filename,"r");
188:         if (fd) cnt = 1; else cnt = 0;
189:         if (fd) {
190:           fclose(fd);
191:         }
192:       } else {
193:         cnt = 0;
194:       }
195:       MPI_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
196:       if (rank == i) {
197:         unlink(filename);
198:       }

200:       if (sum == size) {
201:         *shared = PETSC_TRUE;
202:         break;
203:       } else if (sum != 1) {
204:         SETERRQ(1,"Subset of processes share /tmp cannot load remote or compressed file");
205:       }
206:     }
207:     *tagvalp = (int)*shared;
208:     PetscLogInfo(0,"PetscSharedTmp: processors %s %sn",(*shared == PETSC_TRUE) ? "share":"do NOT share",(iflg ? tmpname:"/tmp"));
209:   } else {
210:     *shared = (PetscTruth) *tagvalp;
211:   }
212:   return(0);
213: }

215: /*@C
216:    PetscSharedWorkingDirectory - Determines if all processors in a communicator share a
217:          working directory or have different ones.

219:    Collective on MPI_Comm

221:    Input Parameters:
222: .  comm - MPI_Communicator that may share working directory

224:    Output Parameters:
225: .  shared - PETSC_TRUE or PETSC_FALSE

227:    Options Database Keys:
228: +    -shared_working_directory 
229: .    -not_shared_working_directory

231:    Environmental Variables:
232: +     PETSC_SHARED_WORKING_DIRECTORY
233: .     PETSC_NOT_SHARED_WORKING_DIRECTORY

235:    Level: developer

237:    Notes:
238:    Stores the status as a MPI attribute so it does not have
239:     to be redetermined each time.

241:       Assumes that all processors in a communicator either
242:        1) have a common working directory or
243:        2) each has a seperate working directory
244:       eventually we can write a fancier one that determines which processors
245:       share a common working directory.

247:    This will be very slow on runs with a large number of processors since
248:    it requires O(p*p) file opens.

250: @*/
251: int PetscSharedWorkingDirectory(MPI_Comm comm,PetscTruth *shared)
252: {
253:   int        ierr,size,rank,*tagvalp,sum,cnt,i;
254:   PetscTruth flg,iflg;
255:   FILE       *fd;
256:   static int Petsc_WD_keyval = MPI_KEYVAL_INVALID;

259:   MPI_Comm_size(comm,&size);
260:   if (size == 1) {
261:     *shared = PETSC_TRUE;
262:     return(0);
263:   }

265:   PetscOptionsGetenv(comm,"PETSC_SHARED_WORKING_DIRECTORY",PETSC_NULL,0,&flg);
266:   if (flg) {
267:     *shared = PETSC_TRUE;
268:     return(0);
269:   }

271:   PetscOptionsGetenv(comm,"PETSC_NOT_SHARED_WORKING_DIRECTORY",PETSC_NULL,0,&flg);
272:   if (flg) {
273:     *shared = PETSC_FALSE;
274:     return(0);
275:   }

277:   if (Petsc_WD_keyval == MPI_KEYVAL_INVALID) {
278:     MPI_Keyval_create(MPI_NULL_COPY_FN,Petsc_DelTag,&Petsc_WD_keyval,0);
279:   }

281:   MPI_Attr_get(comm,Petsc_WD_keyval,(void**)&tagvalp,(int*)&iflg);
282:   if (!iflg) {
283:     char       filename[256];

285:     /* This communicator does not yet have a shared  attribute */
286:     PetscMalloc(sizeof(int),&tagvalp);
287:     MPI_Attr_put(comm,Petsc_WD_keyval,tagvalp);

289:     PetscGetWorkingDirectory(filename,240);
290:     PetscStrcat(filename,"/petsctestshared");
291:     MPI_Comm_rank(comm,&rank);
292: 
293:     /* each processor creates a  file and all the later ones check */
294:     /* this makes sure no subset of processors is shared */
295:     *shared = PETSC_FALSE;
296:     for (i=0; i<size-1; i++) {
297:       if (rank == i) {
298:         fd = fopen(filename,"w");
299:         if (!fd) {
300:           SETERRQ1(1,"Unable to open test file %s",filename);
301:         }
302:         fclose(fd);
303:       }
304:       MPI_Barrier(comm);
305:       if (rank >= i) {
306:         fd = fopen(filename,"r");
307:         if (fd) cnt = 1; else cnt = 0;
308:         if (fd) {
309:           fclose(fd);
310:         }
311:       } else {
312:         cnt = 0;
313:       }
314:       MPI_Allreduce(&cnt,&sum,1,MPI_INT,MPI_SUM,comm);
315:       if (rank == i) {
316:         unlink(filename);
317:       }

319:       if (sum == size) {
320:         *shared = PETSC_TRUE;
321:         break;
322:       } else if (sum != 1) {
323:         SETERRQ(1,"Subset of processes share working directory");
324:       }
325:     }
326:     *tagvalp = (int)*shared;
327:   } else {
328:     *shared = (PetscTruth) *tagvalp;
329:   }
330:   PetscLogInfo(0,"PetscSharedWorkingDirectory: processors %s working directoryn",(*shared == PETSC_TRUE) ? "shared" : "do NOT share");
331:   return(0);
332: }


335: /*@C
336:     PetscFileRetrieve - Obtains a library from a URL or compressed 
337:         and copies into local disk space as uncompressed.

339:     Collective on MPI_Comm

341:     Input Parameter:
342: +   comm     - processors accessing the library
343: .   libname  - name of library, including entire URL (with or without .gz)
344: -   llen     - length of llibname

346:     Output Parameter:
347: +   llibname - name of local copy of library
348: -   found - if found and retrieved the file

350:     Level: developer

352: @*/
353: int PetscFileRetrieve(MPI_Comm comm,const char *libname,char *llibname,int llen,PetscTruth *found)
354: {
355:   char              *par,buf[1024],tmpdir[256],urlget[256];
356:   FILE              *fp;
357:   int               i,rank,ierr,len = 0;
358:   PetscTruth        flg1,flg2,sharedtmp,exists;

361:   *found = PETSC_FALSE;

363:   /* if file does not have an ftp:// or http:// or .gz then need not process file */
364:   PetscStrstr(libname,".gz",&par);
365:   if (par) {PetscStrlen(par,&len);}

367:   PetscStrncmp(libname,"ftp://",6,&flg1);
368:   PetscStrncmp(libname,"http://",7,&flg2);
369:   if (!flg1 && !flg2 && (!par || len != 3)) {
370:     PetscStrncpy(llibname,libname,llen);
371:     PetscTestFile(libname,'r',found);
372:     return(0);
373:   }

375:   /* Determine if all processors share a common /tmp */
376:   PetscSharedTmp(comm,&sharedtmp);
377:   PetscOptionsGetenv(comm,"PETSC_TMP",tmpdir,256,&flg1);

379:   MPI_Comm_rank(comm,&rank);
380:   if (!rank || !sharedtmp) {
381: 
382:     /* Construct the Python script to get URL file */
383:     PetscMalloc(1024*sizeof(char),&par);
384: #if !defined(PETSC_PYTHON)
385: #define PETSC_PYTHON "python"
386: #endif
387:     PetscStrcpy(par,PETSC_PYTHON);
388:     PetscStrcat(par," ");

390:     PetscStrcpy(urlget,PETSC_DIR);
391:     PetscStrcat(urlget,"/bin/urlget.py");
392:     PetscTestFile(urlget,'r',&exists);
393:     if (!exists) {
394:       PetscTestFile("urlget.py",'r',&exists);
395:       if (!exists) {
396:         SETERRQ1(1,"Cannot locate PETSc python script urlget.py in %s or current directory",urlget);
397:       }
398:       PetscStrcpy(urlget,"urlget.py");
399:     }
400:     PetscStrcat(par,urlget);
401:     PetscStrcat(par," ");

403:     /* are we using an alternative /tmp? */
404:     if (flg1) {
405:       PetscStrcat(par,"-tmp ");
406:       PetscStrcat(par,tmpdir);
407:       PetscStrcat(par," ");
408:     }

410:     PetscStrcat(par,libname);
411:     PetscStrcat(par," 2>&1 ");

413:     PetscPOpen(PETSC_COMM_SELF,PETSC_NULL,par,"r",&fp);
414:     if (!fgets(buf,1024,fp)) {
415:       SETERRQ1(1,"No output from ${PETSC_DIR}/bin/urlget.py in getting file %s",libname);
416:     }
417:     PetscLogInfo(0,"PetscFileRetrieve:Message back from Python: %sn",buf);

419:     PetscStrncmp(buf,"Error",5,&flg1);
420:     PetscStrncmp(buf,"Traceback",9,&flg2);
421:     PetscPClose(PETSC_COMM_SELF,fp);
422:     if (flg1 || flg2) {
423:       *found = PETSC_FALSE;
424:     } else {
425:       *found = PETSC_TRUE;
426: 
427:       /* Check for n and make it 0 */
428:       for (i=0; i<1024; i++) {
429:         if (buf[i] == 'n') {
430:           buf[i] = 0;
431:           break;
432:         }
433:       }
434:       PetscStrncpy(llibname,buf,llen);
435:     }
436:     PetscFree(par);
437:   }
438:   if (sharedtmp) { /* send library name to all processors */
439:     MPI_Bcast(found,1,MPI_INT,0,comm);
440:     if (*found) {
441:       MPI_Bcast(llibname,llen,MPI_CHAR,0,comm);
442:       MPI_Bcast(found,1,MPI_INT,0,comm);
443:     }
444:   }

446:   return(0);
447: }