Actual source code: fretrieve.c
2: /*
3: Code for opening and closing files.
4: */
5: #include <petscsys.h>
6: #if defined(PETSC_HAVE_PWD_H)
7: #include <pwd.h>
8: #endif
9: #include <ctype.h>
10: #include <sys/stat.h>
11: #if defined(PETSC_HAVE_UNISTD_H)
12: #include <unistd.h>
13: #endif
14: #if defined(PETSC_HAVE_SYS_UTSNAME_H)
15: #include <sys/utsname.h>
16: #endif
17: #include <fcntl.h>
18: #include <time.h>
19: #if defined(PETSC_HAVE_SYS_SYSTEMINFO_H)
20: #include <sys/systeminfo.h>
21: #endif
22: #include <petsc/private/petscimpl.h>
24: /*
25: Private routine to delete tmp/shared storage
27: This is called by MPI, not by users.
29: Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
31: */
32: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_DelTmpShared(MPI_Comm comm, PetscMPIInt keyval, void *count_val, void *extra_state)
33: {
34: PetscFunctionBegin;
35: PetscCallMPI(PetscInfo(NULL, "Deleting tmp/shared data in an MPI_Comm %ld\n", (long)comm));
36: PetscCallMPI(PetscFree(count_val));
37: PetscFunctionReturn(MPI_SUCCESS);
38: }
40: /*@C
41: PetscGetTmp - Gets the name of the tmp directory
43: Collective
45: Input Parameters:
46: + comm - MPI_Communicator that may share /tmp
47: - len - length of string to hold name
49: Output Parameter:
50: . dir - directory name
52: Options Database Keys:
53: + -shared_tmp - indicates the directory is shared among the MPI ranks
54: . -not_shared_tmp - indicates the directory is not shared among the MPI ranks
55: - -tmp tmpdir - name of the directory you wish to use as /tmp
57: Environmental Variables:
58: + `PETSC_SHARED_TMP` - indicates the directory is shared among the MPI ranks
59: . `PETSC_NOT_SHARED_TMP` - indicates the directory is not shared among the MPI ranks
60: - `PETSC_TMP` - name of the directory you wish to use as /tmp
62: Level: developer
64: .seealso: `PetscSharedTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
65: @*/
66: PetscErrorCode PetscGetTmp(MPI_Comm comm, char dir[], size_t len)
67: {
68: PetscBool flg;
70: PetscFunctionBegin;
71: PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", dir, len, &flg));
72: if (!flg) PetscCall(PetscStrncpy(dir, "/tmp", len));
73: PetscFunctionReturn(PETSC_SUCCESS);
74: }
76: /*@C
77: PetscSharedTmp - Determines if all processors in a communicator share a
78: /tmp or have different ones.
80: Collective
82: Input Parameter:
83: . comm - MPI_Communicator that may share /tmp
85: Output Parameter:
86: . shared - `PETSC_TRUE` or `PETSC_FALSE`
88: Options Database Keys:
89: + -shared_tmp - indicates the directory is shared among the MPI ranks
90: . -not_shared_tmp - indicates the directory is not shared among the MPI ranks
91: - -tmp tmpdir - name of the directory you wish to use as /tmp
93: Environmental Variables:
94: + `PETSC_SHARED_TMP` - indicates the directory is shared among the MPI ranks
95: . `PETSC_NOT_SHARED_TMP` - indicates the directory is not shared among the MPI ranks
96: - `PETSC_TMP` - name of the directory you wish to use as /tmp
98: Level: developer
100: Notes:
101: Stores the status as a MPI attribute so it does not have
102: to be redetermined each time.
104: Assumes that all processors in a communicator either
105: 1) have a common /tmp or
106: 2) each has a separate /tmp
107: eventually we can write a fancier one that determines which processors
108: share a common /tmp.
110: This will be very slow on runs with a large number of processors since
111: it requires O(p*p) file opens.
113: If the environmental variable PETSC_TMP is set it will use this directory
114: as the "/tmp" directory.
116: .seealso: `PetscGetTmp()`, `PetscSharedWorkingDirectory()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
117: @*/
118: PetscErrorCode PetscSharedTmp(MPI_Comm comm, PetscBool *shared)
119: {
120: PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
121: PetscBool flg, iflg;
122: FILE *fd;
123: int err;
125: PetscFunctionBegin;
126: PetscCallMPI(MPI_Comm_size(comm, &size));
127: if (size == 1) {
128: *shared = PETSC_TRUE;
129: PetscFunctionReturn(PETSC_SUCCESS);
130: }
132: PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_TMP", NULL, 0, &flg));
133: if (flg) {
134: *shared = PETSC_TRUE;
135: PetscFunctionReturn(PETSC_SUCCESS);
136: }
138: PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_TMP", NULL, 0, &flg));
139: if (flg) {
140: *shared = PETSC_FALSE;
141: PetscFunctionReturn(PETSC_SUCCESS);
142: }
144: if (Petsc_SharedTmp_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedTmp_keyval, NULL));
146: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedTmp_keyval, (void **)&tagvalp, (int *)&iflg));
147: if (!iflg) {
148: char filename[PETSC_MAX_PATH_LEN], tmpname[PETSC_MAX_PATH_LEN];
150: /* This communicator does not yet have a shared tmp attribute */
151: PetscCall(PetscMalloc1(1, &tagvalp));
152: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedTmp_keyval, tagvalp));
154: PetscCall(PetscOptionsGetenv(comm, "PETSC_TMP", tmpname, 238, &iflg));
155: if (!iflg) {
156: PetscCall(PetscStrncpy(filename, "/tmp", sizeof(filename)));
157: } else {
158: PetscCall(PetscStrncpy(filename, tmpname, sizeof(filename)));
159: }
161: PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
162: PetscCallMPI(MPI_Comm_rank(comm, &rank));
164: /* each processor creates a /tmp file and all the later ones check */
165: /* this makes sure no subset of processors is shared */
166: *shared = PETSC_FALSE;
167: for (i = 0; i < size - 1; i++) {
168: if (rank == i) {
169: fd = fopen(filename, "w");
170: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
171: err = fclose(fd);
172: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
173: }
174: PetscCallMPI(MPI_Barrier(comm));
175: if (rank >= i) {
176: fd = fopen(filename, "r");
177: if (fd) cnt = 1;
178: else cnt = 0;
179: if (fd) {
180: err = fclose(fd);
181: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
182: }
183: } else cnt = 0;
185: PetscCall(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
186: if (rank == i) unlink(filename);
188: if (sum == size) {
189: *shared = PETSC_TRUE;
190: break;
191: } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share /tmp ");
192: }
193: *tagvalp = (int)*shared;
194: PetscCall(PetscInfo(NULL, "processors %s %s\n", (*shared) ? "share" : "do NOT share", (iflg ? tmpname : "/tmp")));
195: } else *shared = (PetscBool)*tagvalp;
196: PetscFunctionReturn(PETSC_SUCCESS);
197: }
199: /*@C
200: PetscSharedWorkingDirectory - Determines if all processors in a communicator share a working directory or have different ones.
202: Collective
204: Input Parameter:
205: . comm - MPI_Communicator that may share working directory
207: Output Parameter:
208: . shared - `PETSC_TRUE` or `PETSC_FALSE`
210: Options Database Keys:
211: + -shared_working_directory - indicates the directory is shared among the MPI ranks
212: - -not_shared_working_directory - indicates the directory is shared among the MPI ranks
214: Environmental Variables:
215: + `PETSC_SHARED_WORKING_DIRECTORY` - indicates the directory is shared among the MPI ranks
216: - `PETSC_NOT_SHARED_WORKING_DIRECTORY` - indicates the directory is shared among the MPI ranks
218: Level: developer
220: Notes:
221: Stores the status as a MPI attribute so it does not have to be redetermined each time.
223: Assumes that all processors in a communicator either
224: $ 1) have a common working directory or
225: $ 2) each has a separate working directory
226: eventually we can write a fancier one that determines which processors share a common working directory.
228: This will be very slow on runs with a large number of processors since it requires O(p*p) file opens.
230: .seealso: `PetscGetTmp()`, `PetscSharedTmp()`, `PetscGetWorkingDirectory()`, `PetscGetHomeDirectory()`
231: @*/
232: PetscErrorCode PetscSharedWorkingDirectory(MPI_Comm comm, PetscBool *shared)
233: {
234: PetscMPIInt size, rank, *tagvalp, sum, cnt, i;
235: PetscBool flg, iflg;
236: FILE *fd;
237: int err;
239: PetscFunctionBegin;
240: PetscCallMPI(MPI_Comm_size(comm, &size));
241: if (size == 1) {
242: *shared = PETSC_TRUE;
243: PetscFunctionReturn(PETSC_SUCCESS);
244: }
246: PetscCall(PetscOptionsGetenv(comm, "PETSC_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
247: if (flg) {
248: *shared = PETSC_TRUE;
249: PetscFunctionReturn(PETSC_SUCCESS);
250: }
252: PetscCall(PetscOptionsGetenv(comm, "PETSC_NOT_SHARED_WORKING_DIRECTORY", NULL, 0, &flg));
253: if (flg) {
254: *shared = PETSC_FALSE;
255: PetscFunctionReturn(PETSC_SUCCESS);
256: }
258: if (Petsc_SharedWD_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_DelTmpShared, &Petsc_SharedWD_keyval, NULL));
260: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_SharedWD_keyval, (void **)&tagvalp, (int *)&iflg));
261: if (!iflg) {
262: char filename[PETSC_MAX_PATH_LEN];
264: /* This communicator does not yet have a shared attribute */
265: PetscCall(PetscMalloc1(1, &tagvalp));
266: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_SharedWD_keyval, tagvalp));
268: PetscCall(PetscGetWorkingDirectory(filename, 240));
269: PetscCall(PetscStrlcat(filename, "/petsctestshared", sizeof(filename)));
270: PetscCallMPI(MPI_Comm_rank(comm, &rank));
272: /* each processor creates a file and all the later ones check */
273: /* this makes sure no subset of processors is shared */
274: *shared = PETSC_FALSE;
275: for (i = 0; i < size - 1; i++) {
276: if (rank == i) {
277: fd = fopen(filename, "w");
278: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Unable to open test file %s", filename);
279: err = fclose(fd);
280: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
281: }
282: PetscCallMPI(MPI_Barrier(comm));
283: if (rank >= i) {
284: fd = fopen(filename, "r");
285: if (fd) cnt = 1;
286: else cnt = 0;
287: if (fd) {
288: err = fclose(fd);
289: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
290: }
291: } else cnt = 0;
293: PetscCall(MPIU_Allreduce(&cnt, &sum, 1, MPI_INT, MPI_SUM, comm));
294: if (rank == i) unlink(filename);
296: if (sum == size) {
297: *shared = PETSC_TRUE;
298: break;
299: } else PetscCheck(sum == 1, PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Subset of processes share working directory");
300: }
301: *tagvalp = (int)*shared;
302: } else *shared = (PetscBool)*tagvalp;
303: PetscCall(PetscInfo(NULL, "processors %s working directory\n", (*shared) ? "shared" : "do NOT share"));
304: PetscFunctionReturn(PETSC_SUCCESS);
305: }
307: /*@C
308: PetscFileRetrieve - Obtains a file from a URL or compressed
309: and copies into local disk space as uncompressed.
311: Collective
313: Input Parameters:
314: + comm - processors accessing the file
315: . url - name of file, including entire URL (with or without .gz)
316: - llen - length of localname
318: Output Parameters:
319: + localname - name of local copy of file - valid on only process zero
320: - found - if found or retrieved the file - valid on all processes
322: Level: intermediate
324: Note:
325: if the file already exists local this function just returns without downloading it.
327: @*/
328: PetscErrorCode PetscFileRetrieve(MPI_Comm comm, const char url[], char localname[], size_t llen, PetscBool *found)
329: {
330: char buffer[PETSC_MAX_PATH_LEN], *par = NULL, *tlocalname = NULL, name[PETSC_MAX_PATH_LEN];
331: FILE *fp;
332: PetscMPIInt rank;
333: size_t len = 0;
334: PetscBool flg1, flg2, flg3, flg4, download, compressed = PETSC_FALSE;
336: PetscFunctionBegin;
337: PetscCallMPI(MPI_Comm_rank(comm, &rank));
338: if (rank == 0) {
339: *found = PETSC_FALSE;
341: PetscCall(PetscStrstr(url, ".gz", &par));
342: if (par) {
343: PetscCall(PetscStrlen(par, &len));
344: if (len == 3) compressed = PETSC_TRUE;
345: }
347: PetscCall(PetscStrncmp(url, "ftp://", 6, &flg1));
348: PetscCall(PetscStrncmp(url, "http://", 7, &flg2));
349: PetscCall(PetscStrncmp(url, "file://", 7, &flg3));
350: PetscCall(PetscStrncmp(url, "https://", 8, &flg4));
351: download = (PetscBool)(flg1 || flg2 || flg3 || flg4);
353: if (!download && !compressed) {
354: PetscCall(PetscStrncpy(localname, url, llen));
355: PetscCall(PetscTestFile(url, 'r', found));
356: if (*found) {
357: PetscCall(PetscInfo(NULL, "Found file %s\n", url));
358: } else {
359: PetscCall(PetscInfo(NULL, "Did not find file %s\n", url));
360: }
361: goto done;
362: }
364: /* look for uncompressed file in requested directory */
365: if (compressed) {
366: PetscCall(PetscStrncpy(localname, url, llen));
367: PetscCall(PetscStrstr(localname, ".gz", &par));
368: *par = 0; /* remove .gz extension */
369: PetscCall(PetscTestFile(localname, 'r', found));
370: if (*found) goto done;
371: }
373: /* look for file in current directory */
374: PetscCall(PetscStrrchr(url, '/', &tlocalname));
375: PetscCall(PetscStrncpy(localname, tlocalname, llen));
376: if (compressed) {
377: PetscCall(PetscStrstr(localname, ".gz", &par));
378: *par = 0; /* remove .gz extension */
379: }
380: PetscCall(PetscTestFile(localname, 'r', found));
381: if (*found) goto done;
383: if (download) {
384: /* local file is not already here so use curl to get it */
385: PetscCall(PetscStrncpy(localname, tlocalname, llen));
386: PetscCall(PetscStrncpy(buffer, "curl --fail --silent --show-error ", sizeof(buffer)));
387: PetscCall(PetscStrlcat(buffer, url, sizeof(buffer)));
388: PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
389: PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
390: #if defined(PETSC_HAVE_POPEN)
391: PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
392: PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
393: #else
394: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
395: #endif
396: PetscCall(PetscTestFile(localname, 'r', found));
397: if (*found) {
398: FILE *fd;
399: char buf[1024], *str, *substring;
401: /* check if the file didn't exist so it downloaded an HTML message instead */
402: fd = fopen(localname, "r");
403: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscTestFile() indicates %s exists but fopen() cannot open it", localname);
404: str = fgets(buf, sizeof(buf) - 1, fd);
405: while (str) {
406: PetscCall(PetscStrstr(buf, "<!DOCTYPE html>", &substring));
407: PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
408: PetscCall(PetscStrstr(buf, "Not Found", &substring));
409: PetscCheck(!substring, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Unable to download %s it does not appear to exist at this URL, dummy HTML file was downloaded", url);
410: str = fgets(buf, sizeof(buf) - 1, fd);
411: }
412: fclose(fd);
413: }
414: } else if (compressed) {
415: PetscCall(PetscTestFile(url, 'r', found));
416: if (!*found) goto done;
417: PetscCall(PetscStrncpy(localname, url, llen));
418: }
419: if (compressed) {
420: PetscCall(PetscStrrchr(localname, '/', &tlocalname));
421: PetscCall(PetscStrncpy(name, tlocalname, PETSC_MAX_PATH_LEN));
422: PetscCall(PetscStrstr(name, ".gz", &par));
423: *par = 0; /* remove .gz extension */
424: /* uncompress file */
425: PetscCall(PetscStrncpy(buffer, "gzip -c -d ", sizeof(buffer)));
426: PetscCall(PetscStrlcat(buffer, localname, sizeof(buffer)));
427: PetscCall(PetscStrlcat(buffer, " > ", sizeof(buffer)));
428: PetscCall(PetscStrlcat(buffer, name, sizeof(buffer)));
429: #if defined(PETSC_HAVE_POPEN)
430: PetscCall(PetscPOpen(PETSC_COMM_SELF, NULL, buffer, "r", &fp));
431: PetscCall(PetscPClose(PETSC_COMM_SELF, fp));
432: #else
433: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "Cannot run external programs on this machine");
434: #endif
435: PetscCall(PetscStrncpy(localname, name, llen));
436: PetscCall(PetscTestFile(localname, 'r', found));
437: }
438: }
439: done:
440: PetscCallMPI(MPI_Bcast(found, 1, MPIU_BOOL, 0, comm));
441: PetscCallMPI(MPI_Bcast(localname, llen, MPI_CHAR, 0, comm));
442: PetscFunctionReturn(PETSC_SUCCESS);
443: }