Actual source code: fretrieve.c


  2: /*
  3:       Code for opening and closing files.
  4: */
  5: #include <petscsys.h>
  6: #if defined(PETSC_HAVE_PWD_H)
  7:   #include <pwd.h>
  8: #endif
  9: #include <ctype.h>
 10: #include <sys/stat.h>
 11: #if defined(PETSC_HAVE_UNISTD_H)
 12:   #include <unistd.h>
 13: #endif
 14: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
 15:   #include <sys/utsname.h>
 16: #endif
 17: #include <fcntl.h>
 18: #include <time.h>
 19: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
 20:   #include <sys/systeminfo.h>
 21: #endif
 22: #include <petsc/private/petscimpl.h>

 24: /*
 25:    Private routine to delete tmp/shared storage

 27:    This is called by MPI, not by users.

 29:    Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()

 31: */
 32: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm, PetscMPIInt keyval, void *count_val, void *extra_state)
 33: {
 34:   PetscFunctionBegin;
 35:   PetscCallMPI(PetscInfo(NULL, "Deleting tmp/shared data in an MPI_Comm %ld\n", (long)comm));
 36:   PetscCallMPI(PetscFree(count_val));
 37:   PetscFunctionReturn(MPI_SUCCESS);
 38: }

 40: /*@C
 41:    PetscGetTmp - Gets the name of the tmp directory

 43:    Collective

 45:    Input Parameters:
 46: +  comm - MPI_Communicator that may share /tmp
 47: -  len - length of string to hold name

 49:    Output Parameter:
 50: .  dir - directory name

 52:    Options Database Keys:
 53: +    -shared_tmp  - indicates the directory is shared among the MPI ranks
 54: .    -not_shared_tmp - indicates the directory is not shared among the MPI ranks
 55: -    -tmp tmpdir - name of the directory you wish to use as /tmp

 57:    Environmental Variables:
 58: +     `PETSC_SHARED_TMP` - indicates the directory is shared among the MPI ranks
 59: .     `PETSC_NOT_SHARED_TMP` - indicates the directory is not shared among the MPI ranks
 60: -     `PETSC_TMP` - name of the directory you wish to use as /tmp

 62:    Level: developer

 64: .seealso: `PetscSharedTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
 65: @*/
 66: PetscErrorCode PetscGetTmp(MPI_Comm comm, char dir[], size_t len)
 67: {
 68:   PetscBool flg;

 70:   PetscFunctionBegin;
 71:   PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", dir, len, &flg));
 72:   if (!flg) PetscCall(PetscStrncpy(dir, "/tmp", len));
 73:   PetscFunctionReturn(PETSC_SUCCESS);
 74: }

 76: /*@C
 77:    PetscSharedTmp - Determines if all processors in a communicator share a
 78:          /tmp or have different ones.

 80:    Collective

 82:    Input Parameter:
 83: .  comm - MPI_Communicator that may share /tmp

 85:    Output Parameter:
 86: .  shared - `PETSC_TRUE` or `PETSC_FALSE`

 88:    Options Database Keys:
 89: +    -shared_tmp  - indicates the directory is shared among the MPI ranks
 90: .    -not_shared_tmp - indicates the directory is not shared among the MPI ranks
 91: -    -tmp tmpdir - name of the directory you wish to use as /tmp

 93:    Environmental Variables:
 94: +     `PETSC_SHARED_TMP`  - indicates the directory is shared among the MPI ranks
 95: .     `PETSC_NOT_SHARED_TMP` - indicates the directory is not shared among the MPI ranks
 96: -     `PETSC_TMP` - name of the directory you wish to use as /tmp

 98:    Level: developer

100:    Notes:
101:    Stores the status as a MPI attribute so it does not have
102:     to be redetermined each time.

104:       Assumes that all processors in a communicator either
105:        1) have a common /tmp or
106:        2) each has a separate /tmp
107:       eventually we can write a fancier one that determines which processors
108:       share a common /tmp.

110:    This will be very slow on runs with a large number of processors since
111:    it requires O(p*p) file opens.

113:    If the environmental variable PETSC_TMP is set it will use this directory
114:   as the "/tmp" directory.

116: .seealso: `PetscGetTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
117: @*/
118: PetscErrorCode PetscSharedTmp(MPI_Comm comm, PetscBool *shared)
119: {
120:   PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
121:   PetscBool   flg, iflg;
122:   FILE       *fd;
123:   int         err;

125:   PetscFunctionBegin;
126:   PetscCallMPI(MPI_Comm_size(comm, &size));
127:   if (size == 1) {
128:     *shared = PETSC_TRUE;
129:     PetscFunctionReturn(PETSC_SUCCESS);
130:   }

132:   PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_TMP", NULL, 0, &flg));
133:   if (flg) {
134:     *shared = PETSC_TRUE;
135:     PetscFunctionReturn(PETSC_SUCCESS);
136:   }

138:   PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_TMP", NULL, 0, &flg));
139:   if (flg) {
140:     *shared = PETSC_FALSE;
141:     PetscFunctionReturn(PETSC_SUCCESS);
142:   }

144:   if (Petsc_SharedTmp_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedTmp_keyval, NULL));

146:   PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedTmp_keyval, (void **)&tagvalp, (int *)&iflg));
147:   if (!iflg) {
148:     char filename[PETSC_MAX_PATH_LEN], tmpname[PETSC_MAX_PATH_LEN];

150:     /* This communicator does not yet have a shared tmp attribute */
151:     PetscCall(PetscMalloc1(1, &tagvalp));
152:     PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedTmp_keyval, tagvalp));

154:     PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", tmpname, 238, &iflg));
155:     if (!iflg) {
156:       PetscCall(PetscStrncpy(filename, "/tmp", sizeof(filename)));
157:     } else {
158:       PetscCall(PetscStrncpy(filename, tmpname, sizeof(filename)));
159:     }

161:     PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
162:     PetscCallMPI(MPI_Comm_rank(comm, &rank));

164:     /* each processor creates a /tmp file and all the later ones check */
165:     /* this makes sure no subset of processors is shared */
166:     *shared = PETSC_FALSE;
167:     for (i = 0; i < size - 1; i++) {
168:       if (rank == i) {
169:         fd = fopen(filename, "w");
170:         PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
171:         err = fclose(fd);
172:         PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
173:       }
174:       PetscCallMPI(MPI_Barrier(comm));
175:       if (rank >= i) {
176:         fd = fopen(filename, "r");
177:         if (fd) cnt = 1;
178:         else cnt = 0;
179:         if (fd) {
180:           err = fclose(fd);
181:           PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
182:         }
183:       } else cnt = 0;

185:       PetscCall(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
186:       if (rank == i) unlink(filename);

188:       if (sum == size) {
189:         *shared = PETSC_TRUE;
190:         break;
191:       } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share /tmp ");
192:     }
193:     *tagvalp = (int)*shared;
194:     PetscCall(PetscInfo(NULL, "processors %s %s\n", (*shared) ? "share" : "do NOT share", (iflg ? tmpname : "/tmp")));
195:   } else *shared = (PetscBool)*tagvalp;
196:   PetscFunctionReturn(PETSC_SUCCESS);
197: }

199: /*@C
200:   PetscSharedWorkingDirectory - Determines if all processors in a communicator share a working directory or have different ones.

202:   Collective

204:   Input Parameter:
205: . comm - MPI_Communicator that may share working directory

207:   Output Parameter:
208: . shared - `PETSC_TRUE` or `PETSC_FALSE`

210:   Options Database Keys:
211: + -shared_working_directory - indicates the directory is shared among the MPI ranks
212: - -not_shared_working_directory - indicates the directory is shared among the MPI ranks

214:   Environmental Variables:
215: + `PETSC_SHARED_WORKING_DIRECTORY` - indicates the directory is shared among the MPI ranks
216: - `PETSC_NOT_SHARED_WORKING_DIRECTORY` - indicates the directory is shared among the MPI ranks

218:   Level: developer

220:   Notes:
221:   Stores the status as a MPI attribute so it does not have to be redetermined each time.

223:   Assumes that all processors in a communicator either
224: $   1) have a common working directory or
225: $   2) each has a separate working directory
226:   eventually we can write a fancier one that determines which processors share a common working directory.

228:   This will be very slow on runs with a large number of processors since it requires O(p*p) file opens.

230: .seealso: `PetscGetTmp()`, `PetscSharedTmp()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
231: @*/
232: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm, PetscBool *shared)
233: {
234:   PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
235:   PetscBool   flg, iflg;
236:   FILE       *fd;
237:   int         err;

239:   PetscFunctionBegin;
240:   PetscCallMPI(MPI_Comm_size(comm, &size));
241:   if (size == 1) {
242:     *shared = PETSC_TRUE;
243:     PetscFunctionReturn(PETSC_SUCCESS);
244:   }

246:   PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
247:   if (flg) {
248:     *shared = PETSC_TRUE;
249:     PetscFunctionReturn(PETSC_SUCCESS);
250:   }

252:   PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
253:   if (flg) {
254:     *shared = PETSC_FALSE;
255:     PetscFunctionReturn(PETSC_SUCCESS);
256:   }

258:   if (Petsc_SharedWD_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedWD_keyval, NULL));

260:   PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedWD_keyval, (void **)&tagvalp, (int *)&iflg));
261:   if (!iflg) {
262:     char filename[PETSC_MAX_PATH_LEN];

264:     /* This communicator does not yet have a shared  attribute */
265:     PetscCall(PetscMalloc1(1, &tagvalp));
266:     PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedWD_keyval, tagvalp));

268:     PetscCall(PetscGetWorkingDirectory(filename, 240));
269:     PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
270:     PetscCallMPI(MPI_Comm_rank(comm, &rank));

272:     /* each processor creates a  file and all the later ones check */
273:     /* this makes sure no subset of processors is shared */
274:     *shared = PETSC_FALSE;
275:     for (i = 0; i < size - 1; i++) {
276:       if (rank == i) {
277:         fd = fopen(filename, "w");
278:         PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
279:         err = fclose(fd);
280:         PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
281:       }
282:       PetscCallMPI(MPI_Barrier(comm));
283:       if (rank >= i) {
284:         fd = fopen(filename, "r");
285:         if (fd) cnt = 1;
286:         else cnt = 0;
287:         if (fd) {
288:           err = fclose(fd);
289:           PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
290:         }
291:       } else cnt = 0;

293:       PetscCall(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
294:       if (rank == i) unlink(filename);

296:       if (sum == size) {
297:         *shared = PETSC_TRUE;
298:         break;
299:       } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share working directory");
300:     }
301:     *tagvalp = (int)*shared;
302:   } else *shared = (PetscBool)*tagvalp;
303:   PetscCall(PetscInfo(NULL, "processors %s working directory\n", (*shared) ? "shared" : "do NOT share"));
304:   PetscFunctionReturn(PETSC_SUCCESS);
305: }

307: /*@C
308:     PetscFileRetrieve - Obtains a file from a URL or compressed
309:         and copies into local disk space as uncompressed.

311:     Collective

313:     Input Parameters:
314: +   comm     - processors accessing the file
315: .   url      - name of file, including entire URL (with or without .gz)
316: -   llen     - length of localname

318:     Output Parameters:
319: +   localname - name of local copy of file - valid on only process zero
320: -   found - if found or retrieved the file - valid on all processes

322:     Level: intermediate

324:     Note:
325:     if the file already exists local this function just returns without downloading it.

327: @*/
328: PetscErrorCode PetscFileRetrieve(MPI_Comm comm, const char url[], char localname[], size_t llen, PetscBool *found)
329: {
330:   char        buffer[PETSC_MAX_PATH_LEN], *par = NULL, *tlocalname = NULL, name[PETSC_MAX_PATH_LEN];
331:   FILE       *fp;
332:   PetscMPIInt rank;
333:   size_t      len = 0;
334:   PetscBool   flg1, flg2, flg3, flg4, download, compressed = PETSC_FALSE;

336:   PetscFunctionBegin;
337:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
338:   if (rank == 0) {
339:     *found = PETSC_FALSE;

341:     PetscCall(PetscStrstr(url, ".gz", &par));
342:     if (par) {
343:       PetscCall(PetscStrlen(par, &len));
344:       if (len == 3) compressed = PETSC_TRUE;
345:     }

347:     PetscCall(PetscStrncmp(url, "ftp://", 6, &flg1));
348:     PetscCall(PetscStrncmp(url, "http://", 7, &flg2));
349:     PetscCall(PetscStrncmp(url, "file://", 7, &flg3));
350:     PetscCall(PetscStrncmp(url, "https://", 8, &flg4));
351:     download = (PetscBool)(flg1 || flg2 || flg3 || flg4);

353:     if (!download && !compressed) {
354:       PetscCall(PetscStrncpy(localname, url, llen));
355:       PetscCall(PetscTestFile(url, 'r', found));
356:       if (*found) {
357:         PetscCall(PetscInfo(NULL, "Found file %s\n", url));
358:       } else {
359:         PetscCall(PetscInfo(NULL, "Did not find file %s\n", url));
360:       }
361:       goto done;
362:     }

364:     /* look for uncompressed file in requested directory */
365:     if (compressed) {
366:       PetscCall(PetscStrncpy(localname, url, llen));
367:       PetscCall(PetscStrstr(localname, ".gz", &par));
368:       *par = 0; /* remove .gz extension */
369:       PetscCall(PetscTestFile(localname, 'r', found));
370:       if (*found) goto done;
371:     }

373:     /* look for file in current directory */
374:     PetscCall(PetscStrrchr(url, '/', &tlocalname));
375:     PetscCall(PetscStrncpy(localname, tlocalname, llen));
376:     if (compressed) {
377:       PetscCall(PetscStrstr(localname, ".gz", &par));
378:       *par = 0; /* remove .gz extension */
379:     }
380:     PetscCall(PetscTestFile(localname, 'r', found));
381:     if (*found) goto done;

383:     if (download) {
384:       /* local file is not already here so use curl to get it */
385:       PetscCall(PetscStrncpy(localname, tlocalname, llen));
386:       PetscCall(PetscStrncpy(buffer, "curl --fail --silent --show-error ", sizeof(buffer)));
387:       PetscCall(PetscStrlcat(buffer, url, sizeof(buffer)));
388:       PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
389:       PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
390: #if defined(PETSC_HAVE_POPEN)
391:       PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
392:       PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
393: #else
394:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
395: #endif
396:       PetscCall(PetscTestFile(localname, 'r', found));
397:       if (*found) {
398:         FILE *fd;
399:         char  buf[1024], *str, *substring;

401:         /* check if the file didn't exist so it downloaded an HTML message instead */
402:         fd = fopen(localname, "r");
403:         PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscTestFile() indicates %s exists but fopen() cannot open it", localname);
404:         str = fgets(buf, sizeof(buf) - 1, fd);
405:         while (str) {
406:           PetscCall(PetscStrstr(buf, "<!DOCTYPE html>", &substring));
407:           PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
408:           PetscCall(PetscStrstr(buf, "Not Found", &substring));
409:           PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
410:           str = fgets(buf, sizeof(buf) - 1, fd);
411:         }
412:         fclose(fd);
413:       }
414:     } else if (compressed) {
415:       PetscCall(PetscTestFile(url, 'r', found));
416:       if (!*found) goto done;
417:       PetscCall(PetscStrncpy(localname, url, llen));
418:     }
419:     if (compressed) {
420:       PetscCall(PetscStrrchr(localname, '/', &tlocalname));
421:       PetscCall(PetscStrncpy(name, tlocalname, PETSC_MAX_PATH_LEN));
422:       PetscCall(PetscStrstr(name, ".gz", &par));
423:       *par = 0; /* remove .gz extension */
424:       /* uncompress file */
425:       PetscCall(PetscStrncpy(buffer, "gzip -c -d ", sizeof(buffer)));
426:       PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
427:       PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
428:       PetscCall(PetscStrlcat(buffer, name, sizeof(buffer)));
429: #if defined(PETSC_HAVE_POPEN)
430:       PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
431:       PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
432: #else
433:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
434: #endif
435:       PetscCall(PetscStrncpy(localname, name, llen));
436:       PetscCall(PetscTestFile(localname, 'r', found));
437:     }
438:   }
439: done:
440:   PetscCallMPI(MPI_Bcast(found, 1, MPIU_BOOL, 0, comm));
441:   PetscCallMPI(MPI_Bcast(localname, llen, MPI_CHAR, 0, comm));
442:   PetscFunctionReturn(PETSC_SUCCESS);
443: }