Actual source code: ex1.c
1: static char help[] = "This example demonstrates the use of DMNetwork interface with subnetworks for solving a coupled nonlinear \n\
2: electric power grid and water pipe problem.\n\
3: The available solver options are in the ex1options file \n\
4: and the data files are in the datafiles of subdirectories.\n\
5: This example shows the use of subnetwork feature in DMNetwork. \n\
6: Run this program: mpiexec -n <n> ./ex1 \n\\n";
8: #include "power/power.h"
9: #include "water/water.h"
11: typedef struct {
12: UserCtx_Power appctx_power;
13: AppCtx_Water appctx_water;
14: PetscInt subsnes_id; /* snes solver id */
15: PetscInt it; /* iteration number */
16: Vec localXold; /* store previous solution, used by FormFunction_Dummy() */
17: } UserCtx;
19: PetscErrorCode UserMonitor(SNES snes, PetscInt its, PetscReal fnorm, void *appctx)
20: {
21: UserCtx *user = (UserCtx *)appctx;
22: Vec X, localXold = user->localXold;
23: DM networkdm;
24: PetscMPIInt rank;
25: MPI_Comm comm;
27: PetscFunctionBegin;
28: PetscCall(PetscObjectGetComm((PetscObject)snes, &comm));
29: PetscCallMPI(MPI_Comm_rank(comm, &rank));
30: #if 0
31: if (rank == 0) {
32: PetscInt subsnes_id = user->subsnes_id;
33: if (subsnes_id == 2) {
34: PetscCall(PetscPrintf(PETSC_COMM_SELF," it %" PetscInt_FMT ", subsnes_id %" PetscInt_FMT ", fnorm %g\n",user->it,user->subsnes_id,(double)fnorm));
35: } else {
36: PetscCall(PetscPrintf(PETSC_COMM_SELF," subsnes_id %" PetscInt_FMT ", fnorm %g\n",user->subsnes_id,(double)fnorm));
37: }
38: }
39: #endif
40: PetscCall(SNESGetSolution(snes, &X));
41: PetscCall(SNESGetDM(snes, &networkdm));
42: PetscCall(DMGlobalToLocalBegin(networkdm, X, INSERT_VALUES, localXold));
43: PetscCall(DMGlobalToLocalEnd(networkdm, X, INSERT_VALUES, localXold));
44: PetscFunctionReturn(PETSC_SUCCESS);
45: }
47: PetscErrorCode FormJacobian_subPower(SNES snes, Vec X, Mat J, Mat Jpre, void *appctx)
48: {
49: DM networkdm;
50: Vec localX;
51: PetscInt nv, ne, i, j, offset, nvar, row;
52: const PetscInt *vtx, *edges;
53: PetscBool ghostvtex;
54: PetscScalar one = 1.0;
55: PetscMPIInt rank;
56: MPI_Comm comm;
58: PetscFunctionBegin;
59: PetscCall(SNESGetDM(snes, &networkdm));
60: PetscCall(DMGetLocalVector(networkdm, &localX));
62: PetscCall(PetscObjectGetComm((PetscObject)networkdm, &comm));
63: PetscCallMPI(MPI_Comm_rank(comm, &rank));
65: PetscCall(DMGlobalToLocalBegin(networkdm, X, INSERT_VALUES, localX));
66: PetscCall(DMGlobalToLocalEnd(networkdm, X, INSERT_VALUES, localX));
68: PetscCall(MatZeroEntries(J));
70: /* Power subnetwork: copied from power/FormJacobian_Power() */
71: PetscCall(DMNetworkGetSubnetwork(networkdm, 0, &nv, &ne, &vtx, &edges));
72: PetscCall(FormJacobian_Power_private(networkdm, localX, J, nv, ne, vtx, edges, appctx));
74: /* Water subnetwork: Identity */
75: PetscCall(DMNetworkGetSubnetwork(networkdm, 1, &nv, &ne, &vtx, &edges));
76: for (i = 0; i < nv; i++) {
77: PetscCall(DMNetworkIsGhostVertex(networkdm, vtx[i], &ghostvtex));
78: if (ghostvtex) continue;
80: PetscCall(DMNetworkGetGlobalVecOffset(networkdm, vtx[i], ALL_COMPONENTS, &offset));
81: PetscCall(DMNetworkGetComponent(networkdm, vtx[i], ALL_COMPONENTS, NULL, NULL, &nvar));
82: for (j = 0; j < nvar; j++) {
83: row = offset + j;
84: PetscCall(MatSetValues(J, 1, &row, 1, &row, &one, ADD_VALUES));
85: }
86: }
87: PetscCall(MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY));
88: PetscCall(MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY));
90: PetscCall(DMRestoreLocalVector(networkdm, &localX));
91: PetscFunctionReturn(PETSC_SUCCESS);
92: }
94: /* Dummy equation localF(X) = localX - localXold */
95: PetscErrorCode FormFunction_Dummy(DM networkdm, Vec localX, Vec localF, PetscInt nv, PetscInt ne, const PetscInt *vtx, const PetscInt *edges, void *appctx)
96: {
97: const PetscScalar *xarr, *xoldarr;
98: PetscScalar *farr;
99: PetscInt i, j, offset, nvar;
100: PetscBool ghostvtex;
101: UserCtx *user = (UserCtx *)appctx;
102: Vec localXold = user->localXold;
104: PetscFunctionBegin;
105: PetscCall(VecGetArrayRead(localX, &xarr));
106: PetscCall(VecGetArrayRead(localXold, &xoldarr));
107: PetscCall(VecGetArray(localF, &farr));
109: for (i = 0; i < nv; i++) {
110: PetscCall(DMNetworkIsGhostVertex(networkdm, vtx[i], &ghostvtex));
111: if (ghostvtex) continue;
113: PetscCall(DMNetworkGetLocalVecOffset(networkdm, vtx[i], ALL_COMPONENTS, &offset));
114: PetscCall(DMNetworkGetComponent(networkdm, vtx[i], ALL_COMPONENTS, NULL, NULL, &nvar));
115: for (j = 0; j < nvar; j++) farr[offset + j] = xarr[offset + j] - xoldarr[offset + j];
116: }
118: PetscCall(VecRestoreArrayRead(localX, &xarr));
119: PetscCall(VecRestoreArrayRead(localXold, &xoldarr));
120: PetscCall(VecRestoreArray(localF, &farr));
121: PetscFunctionReturn(PETSC_SUCCESS);
122: }
124: PetscErrorCode FormFunction(SNES snes, Vec X, Vec F, void *appctx)
125: {
126: DM networkdm;
127: Vec localX, localF;
128: PetscInt nv, ne, v;
129: const PetscInt *vtx, *edges;
130: PetscMPIInt rank;
131: MPI_Comm comm;
132: UserCtx *user = (UserCtx *)appctx;
133: UserCtx_Power appctx_power = (*user).appctx_power;
134: AppCtx_Water appctx_water = (*user).appctx_water;
136: PetscFunctionBegin;
137: PetscCall(SNESGetDM(snes, &networkdm));
138: PetscCall(PetscObjectGetComm((PetscObject)networkdm, &comm));
139: PetscCallMPI(MPI_Comm_rank(comm, &rank));
141: PetscCall(DMGetLocalVector(networkdm, &localX));
142: PetscCall(DMGetLocalVector(networkdm, &localF));
143: PetscCall(VecSet(F, 0.0));
144: PetscCall(VecSet(localF, 0.0));
146: PetscCall(DMGlobalToLocalBegin(networkdm, X, INSERT_VALUES, localX));
147: PetscCall(DMGlobalToLocalEnd(networkdm, X, INSERT_VALUES, localX));
149: /* Form Function for power subnetwork */
150: PetscCall(DMNetworkGetSubnetwork(networkdm, 0, &nv, &ne, &vtx, &edges));
151: if (user->subsnes_id == 1) { /* snes_water only */
152: PetscCall(FormFunction_Dummy(networkdm, localX, localF, nv, ne, vtx, edges, user));
153: } else {
154: PetscCall(FormFunction_Power(networkdm, localX, localF, nv, ne, vtx, edges, &appctx_power));
155: }
157: /* Form Function for water subnetwork */
158: PetscCall(DMNetworkGetSubnetwork(networkdm, 1, &nv, &ne, &vtx, &edges));
159: if (user->subsnes_id == 0) { /* snes_power only */
160: PetscCall(FormFunction_Dummy(networkdm, localX, localF, nv, ne, vtx, edges, user));
161: } else {
162: PetscCall(FormFunction_Water(networkdm, localX, localF, nv, ne, vtx, edges, NULL));
163: }
165: /* Illustrate how to access the coupling vertex of the subnetworks without doing anything to F yet */
166: PetscCall(DMNetworkGetSharedVertices(networkdm, &nv, &vtx));
167: for (v = 0; v < nv; v++) {
168: PetscInt key, ncomp, nvar, nconnedges, k, e, keye, goffset[3];
169: void *component;
170: const PetscInt *connedges;
172: PetscCall(DMNetworkGetComponent(networkdm, vtx[v], ALL_COMPONENTS, NULL, NULL, &nvar));
173: PetscCall(DMNetworkGetNumComponents(networkdm, vtx[v], &ncomp));
174: /* printf(" [%d] coupling vertex[%" PetscInt_FMT "]: v %" PetscInt_FMT ", ncomp %" PetscInt_FMT "; nvar %" PetscInt_FMT "\n",rank,v,vtx[v], ncomp,nvar); */
176: for (k = 0; k < ncomp; k++) {
177: PetscCall(DMNetworkGetComponent(networkdm, vtx[v], k, &key, &component, &nvar));
178: PetscCall(DMNetworkGetGlobalVecOffset(networkdm, vtx[v], k, &goffset[k]));
180: /* Verify the coupling vertex is a powernet load vertex or a water vertex */
181: switch (k) {
182: case 0:
183: PetscCheck(key == appctx_power.compkey_bus && nvar == 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "key %" PetscInt_FMT " not a power bus vertex or nvar %" PetscInt_FMT " != 2", key, nvar);
184: break;
185: case 1:
186: PetscCheck(key == appctx_power.compkey_load && nvar == 0 && goffset[1] == goffset[0] + 2, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not a power load vertex");
187: break;
188: case 2:
189: PetscCheck(key == appctx_water.compkey_vtx && nvar == 1 && goffset[2] == goffset[1], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not a water vertex");
190: break;
191: default:
192: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "k %" PetscInt_FMT " is wrong", k);
193: }
194: /* printf(" [%d] coupling vertex[%" PetscInt_FMT "]: key %" PetscInt_FMT "; nvar %" PetscInt_FMT ", goffset %" PetscInt_FMT "\n",rank,v,key,nvar,goffset[k]); */
195: }
197: /* Get its supporting edges */
198: PetscCall(DMNetworkGetSupportingEdges(networkdm, vtx[v], &nconnedges, &connedges));
199: /* printf("\n[%d] coupling vertex: nconnedges %" PetscInt_FMT "\n",rank,nconnedges); */
200: for (k = 0; k < nconnedges; k++) {
201: e = connedges[k];
202: PetscCall(DMNetworkGetNumComponents(networkdm, e, &ncomp));
203: /* printf("\n [%d] connected edge[%" PetscInt_FMT "]=%" PetscInt_FMT " has ncomp %" PetscInt_FMT "\n",rank,k,e,ncomp); */
204: PetscCall(DMNetworkGetComponent(networkdm, e, 0, &keye, &component, NULL));
205: if (keye != appctx_water.compkey_edge) PetscCheck(keye == appctx_power.compkey_branch, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Not a power branch");
206: }
207: }
209: PetscCall(DMRestoreLocalVector(networkdm, &localX));
211: PetscCall(DMLocalToGlobalBegin(networkdm, localF, ADD_VALUES, F));
212: PetscCall(DMLocalToGlobalEnd(networkdm, localF, ADD_VALUES, F));
213: PetscCall(DMRestoreLocalVector(networkdm, &localF));
214: #if 0
215: if (rank == 0) printf("F:\n");
216: PetscCall(VecView(F,PETSC_VIEWER_STDOUT_WORLD));
217: #endif
218: PetscFunctionReturn(PETSC_SUCCESS);
219: }
221: PetscErrorCode SetInitialGuess(DM networkdm, Vec X, void *appctx)
222: {
223: PetscInt nv, ne, i, j, ncomp, offset, key;
224: const PetscInt *vtx, *edges;
225: UserCtx *user = (UserCtx *)appctx;
226: Vec localX = user->localXold;
227: UserCtx_Power appctx_power = (*user).appctx_power;
228: AppCtx_Water appctx_water = (*user).appctx_water;
229: PetscBool ghost;
230: PetscScalar *xarr;
231: VERTEX_Power bus;
232: VERTEX_Water vertex;
233: void *component;
234: GEN gen;
236: PetscFunctionBegin;
237: PetscCall(VecSet(X, 0.0));
238: PetscCall(VecSet(localX, 0.0));
240: /* Set initial guess for power subnetwork */
241: PetscCall(DMNetworkGetSubnetwork(networkdm, 0, &nv, &ne, &vtx, &edges));
242: PetscCall(SetInitialGuess_Power(networkdm, localX, nv, ne, vtx, edges, &appctx_power));
244: /* Set initial guess for water subnetwork */
245: PetscCall(DMNetworkGetSubnetwork(networkdm, 1, &nv, &ne, &vtx, &edges));
246: PetscCall(SetInitialGuess_Water(networkdm, localX, nv, ne, vtx, edges, NULL));
248: /* Set initial guess at the coupling vertex */
249: PetscCall(VecGetArray(localX, &xarr));
250: PetscCall(DMNetworkGetSharedVertices(networkdm, &nv, &vtx));
251: for (i = 0; i < nv; i++) {
252: PetscCall(DMNetworkIsGhostVertex(networkdm, vtx[i], &ghost));
253: if (ghost) continue;
255: PetscCall(DMNetworkGetNumComponents(networkdm, vtx[i], &ncomp));
256: for (j = 0; j < ncomp; j++) {
257: PetscCall(DMNetworkGetLocalVecOffset(networkdm, vtx[i], j, &offset));
258: PetscCall(DMNetworkGetComponent(networkdm, vtx[i], j, &key, (void **)&component, NULL));
259: if (key == appctx_power.compkey_bus) {
260: bus = (VERTEX_Power)(component);
261: xarr[offset] = bus->va * PETSC_PI / 180.0;
262: xarr[offset + 1] = bus->vm;
263: } else if (key == appctx_power.compkey_gen) {
264: gen = (GEN)(component);
265: if (!gen->status) continue;
266: xarr[offset + 1] = gen->vs;
267: } else if (key == appctx_water.compkey_vtx) {
268: vertex = (VERTEX_Water)(component);
269: if (vertex->type == VERTEX_TYPE_JUNCTION) {
270: xarr[offset] = 100;
271: } else if (vertex->type == VERTEX_TYPE_RESERVOIR) {
272: xarr[offset] = vertex->res.head;
273: } else {
274: xarr[offset] = vertex->tank.initlvl + vertex->tank.elev;
275: }
276: }
277: }
278: }
279: PetscCall(VecRestoreArray(localX, &xarr));
281: PetscCall(DMLocalToGlobalBegin(networkdm, localX, ADD_VALUES, X));
282: PetscCall(DMLocalToGlobalEnd(networkdm, localX, ADD_VALUES, X));
283: PetscFunctionReturn(PETSC_SUCCESS);
284: }
286: /* Set coordinates */
287: static PetscErrorCode CoordinateVecSetUp(DM dm, Vec coords)
288: {
289: DM dmclone;
290: PetscInt i, gidx, offset, v, nv, Nsubnet;
291: const PetscInt *vtx;
292: PetscScalar *carray;
294: PetscFunctionBeginUser;
295: PetscCall(DMGetCoordinateDM(dm, &dmclone));
296: PetscCall(VecGetArrayWrite(coords, &carray));
297: PetscCall(DMNetworkGetNumSubNetworks(dm, NULL, &Nsubnet));
298: for (i = 0; i < Nsubnet; i++) {
299: PetscCall(DMNetworkGetSubnetwork(dm, i, &nv, NULL, &vtx, NULL));
300: for (v = 0; v < nv; v++) {
301: PetscCall(DMNetworkGetGlobalVertexIndex(dm, vtx[v], &gidx));
302: PetscCall(DMNetworkGetLocalVecOffset(dmclone, vtx[v], 0, &offset));
303: switch (gidx) {
304: case 0:
305: carray[offset] = -1.0;
306: carray[offset + 1] = -1.0;
307: break;
308: case 1:
309: carray[offset] = -2.0;
310: carray[offset + 1] = 2.0;
311: break;
312: case 2:
313: carray[offset] = 0.0;
314: carray[offset + 1] = 2.0;
315: break;
316: case 3:
317: carray[offset] = -1.0;
318: carray[offset + 1] = 0.0;
319: break;
320: case 4:
321: carray[offset] = 0.0;
322: carray[offset + 1] = 0.0;
323: break;
324: case 5:
325: carray[offset] = 0.0;
326: carray[offset + 1] = 1.0;
327: break;
328: case 6:
329: carray[offset] = -1.0;
330: carray[offset + 1] = 1.0;
331: break;
332: case 7:
333: carray[offset] = -2.0;
334: carray[offset + 1] = 1.0;
335: break;
336: case 8:
337: carray[offset] = -2.0;
338: carray[offset + 1] = 0.0;
339: break;
340: case 9:
341: carray[offset] = 1.0;
342: carray[offset + 1] = 0.0;
343: break;
344: case 10:
345: carray[offset] = 1.0;
346: carray[offset + 1] = -1.0;
347: break;
348: case 11:
349: carray[offset] = 2.0;
350: carray[offset + 1] = -1.0;
351: break;
352: case 12:
353: carray[offset] = 2.0;
354: carray[offset + 1] = 0.0;
355: break;
356: case 13:
357: carray[offset] = 0.0;
358: carray[offset + 1] = -1.0;
359: break;
360: case 14:
361: carray[offset] = 2.0;
362: carray[offset + 1] = 1.0;
363: break;
364: default:
365: PetscCheck(gidx < 15 && gidx > -1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "gidx %" PetscInt_FMT "must between 0 and 14", gidx);
366: }
367: }
368: }
369: PetscCall(VecRestoreArrayWrite(coords, &carray));
370: PetscFunctionReturn(PETSC_SUCCESS);
371: }
373: static PetscErrorCode CoordinatePrint(DM dm)
374: {
375: DM dmclone;
376: PetscInt cdim, v, off, vglobal, vStart, vEnd;
377: const PetscScalar *carray;
378: Vec coords;
379: MPI_Comm comm;
380: PetscMPIInt rank;
382: PetscFunctionBegin;
383: PetscCall(PetscObjectGetComm((PetscObject)dm, &comm));
384: PetscCallMPI(MPI_Comm_rank(comm, &rank));
386: PetscCall(DMGetCoordinateDM(dm, &dmclone));
387: PetscCall(DMNetworkGetVertexRange(dm, &vStart, &vEnd));
388: PetscCall(DMGetCoordinatesLocal(dm, &coords));
390: PetscCall(DMGetCoordinateDim(dm, &cdim));
391: PetscCall(VecGetArrayRead(coords, &carray));
393: PetscCall(PetscPrintf(MPI_COMM_WORLD, "\nCoordinatePrint, cdim %" PetscInt_FMT ":\n", cdim));
394: PetscCall(PetscSynchronizedPrintf(MPI_COMM_WORLD, "[%i]\n", rank));
395: for (v = vStart; v < vEnd; v++) {
396: PetscCall(DMNetworkGetLocalVecOffset(dmclone, v, 0, &off));
397: PetscCall(DMNetworkGetGlobalVertexIndex(dmclone, v, &vglobal));
398: switch (cdim) {
399: case 2:
400: PetscCall(PetscSynchronizedPrintf(MPI_COMM_WORLD, "Vertex: %" PetscInt_FMT ", x = %f y = %f \n", vglobal, (double)PetscRealPart(carray[off]), (double)PetscRealPart(carray[off + 1])));
401: break;
402: default:
403: PetscCheck(cdim == 2, MPI_COMM_WORLD, PETSC_ERR_SUP, "Only supports Network embedding dimension of 2, not supplied %" PetscInt_FMT, cdim);
404: break;
405: }
406: }
407: PetscCall(PetscSynchronizedFlush(MPI_COMM_WORLD, NULL));
408: PetscCall(VecRestoreArrayRead(coords, &carray));
409: PetscFunctionReturn(PETSC_SUCCESS);
410: }
412: int main(int argc, char **argv)
413: {
414: DM networkdm, dmclone;
415: PetscLogStage stage[4];
416: PetscMPIInt rank, size;
417: PetscInt Nsubnet = 2, numVertices[2], numEdges[2], i, j, nv, ne, it_max = 10;
418: PetscInt vStart, vEnd, compkey;
419: const PetscInt *vtx, *edges;
420: Vec X, F, coords;
421: SNES snes, snes_power, snes_water;
422: Mat Jac;
423: PetscBool ghost, viewJ = PETSC_FALSE, viewX = PETSC_FALSE, test = PETSC_FALSE, distribute = PETSC_TRUE, flg, printCoord = PETSC_FALSE, viewCSV = PETSC_FALSE;
424: UserCtx user;
425: SNESConvergedReason reason;
427: /* Power subnetwork */
428: UserCtx_Power *appctx_power = &user.appctx_power;
429: char pfdata_file[PETSC_MAX_PATH_LEN] = "power/case9.m";
430: PFDATA *pfdata = NULL;
431: PetscInt genj, loadj, *edgelist_power = NULL, power_netnum;
432: PetscScalar Sbase = 0.0;
434: /* Water subnetwork */
435: AppCtx_Water *appctx_water = &user.appctx_water;
436: WATERDATA *waterdata = NULL;
437: char waterdata_file[PETSC_MAX_PATH_LEN] = "water/sample1.inp";
438: PetscInt *edgelist_water = NULL, water_netnum;
440: /* Shared vertices between subnetworks */
441: PetscInt power_svtx, water_svtx;
443: PetscFunctionBeginUser;
444: PetscCall(PetscInitialize(&argc, &argv, "ex1options", help));
445: PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
446: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
448: /* (1) Read Data - Only rank 0 reads the data */
449: PetscCall(PetscLogStageRegister("Read Data", &stage[0]));
450: PetscCall(PetscLogStagePush(stage[0]));
452: for (i = 0; i < Nsubnet; i++) {
453: numVertices[i] = 0;
454: numEdges[i] = 0;
455: }
457: /* All processes READ THE DATA FOR THE FIRST SUBNETWORK: Electric Power Grid */
458: /* Used for shared vertex, because currently the coupling info must be available in all processes!!! */
459: PetscCall(PetscOptionsGetString(NULL, NULL, "-pfdata", pfdata_file, PETSC_MAX_PATH_LEN - 1, NULL));
460: PetscCall(PetscNew(&pfdata));
461: PetscCall(PFReadMatPowerData(pfdata, pfdata_file));
462: if (rank == 0) PetscCall(PetscPrintf(PETSC_COMM_SELF, "Power network: nb = %" PetscInt_FMT ", ngen = %" PetscInt_FMT ", nload = %" PetscInt_FMT ", nbranch = %" PetscInt_FMT "\n", pfdata->nbus, pfdata->ngen, pfdata->nload, pfdata->nbranch));
463: Sbase = pfdata->sbase;
464: if (rank == 0) { /* proc[0] will create Electric Power Grid */
465: numEdges[0] = pfdata->nbranch;
466: numVertices[0] = pfdata->nbus;
468: PetscCall(PetscMalloc1(2 * numEdges[0], &edgelist_power));
469: PetscCall(GetListofEdges_Power(pfdata, edgelist_power));
470: }
471: /* Broadcast power Sbase to all processors */
472: PetscCallMPI(MPI_Bcast(&Sbase, 1, MPIU_SCALAR, 0, PETSC_COMM_WORLD));
473: appctx_power->Sbase = Sbase;
474: appctx_power->jac_error = PETSC_FALSE;
475: /* If external option activated. Introduce error in jacobian */
476: PetscCall(PetscOptionsHasName(NULL, NULL, "-jac_error", &appctx_power->jac_error));
478: /* All processes READ THE DATA FOR THE SECOND SUBNETWORK: Water */
479: /* Used for shared vertex, because currently the coupling info must be available in all processes!!! */
480: PetscCall(PetscNew(&waterdata));
481: PetscCall(PetscOptionsGetString(NULL, NULL, "-waterdata", waterdata_file, PETSC_MAX_PATH_LEN - 1, NULL));
482: PetscCall(WaterReadData(waterdata, waterdata_file));
483: if (size == 1 || (size > 1 && rank == 1)) {
484: PetscCall(PetscCalloc1(2 * waterdata->nedge, &edgelist_water));
485: PetscCall(GetListofEdges_Water(waterdata, edgelist_water));
486: numEdges[1] = waterdata->nedge;
487: numVertices[1] = waterdata->nvertex;
488: }
489: PetscCall(PetscLogStagePop());
491: /* (2) Create a network consist of two subnetworks */
492: PetscCall(PetscLogStageRegister("Net Setup", &stage[1]));
493: PetscCall(PetscLogStagePush(stage[1]));
495: PetscCall(PetscOptionsGetBool(NULL, NULL, "-viewCSV", &viewCSV, NULL));
496: PetscCall(PetscOptionsGetBool(NULL, NULL, "-printCoord", &printCoord, NULL));
497: PetscCall(PetscOptionsGetBool(NULL, NULL, "-test", &test, NULL));
498: PetscCall(PetscOptionsGetBool(NULL, NULL, "-distribute", &distribute, NULL));
500: /* Create an empty network object */
501: PetscCall(DMNetworkCreate(PETSC_COMM_WORLD, &networkdm));
503: /* Register the components in the network */
504: PetscCall(DMNetworkRegisterComponent(networkdm, "branchstruct", sizeof(struct _p_EDGE_Power), &appctx_power->compkey_branch));
505: PetscCall(DMNetworkRegisterComponent(networkdm, "busstruct", sizeof(struct _p_VERTEX_Power), &appctx_power->compkey_bus));
506: PetscCall(DMNetworkRegisterComponent(networkdm, "genstruct", sizeof(struct _p_GEN), &appctx_power->compkey_gen));
507: PetscCall(DMNetworkRegisterComponent(networkdm, "loadstruct", sizeof(struct _p_LOAD), &appctx_power->compkey_load));
509: PetscCall(DMNetworkRegisterComponent(networkdm, "edge_water", sizeof(struct _p_EDGE_Water), &appctx_water->compkey_edge));
510: PetscCall(DMNetworkRegisterComponent(networkdm, "vertex_water", sizeof(struct _p_VERTEX_Water), &appctx_water->compkey_vtx));
512: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "[%d] Total local nvertices %" PetscInt_FMT " + %" PetscInt_FMT " = %" PetscInt_FMT ", nedges %" PetscInt_FMT " + %" PetscInt_FMT " = %" PetscInt_FMT "\n", rank, numVertices[0], numVertices[1], numVertices[0] + numVertices[1], numEdges[0], numEdges[1], numEdges[0] + numEdges[1]));
513: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
515: PetscCall(DMNetworkSetNumSubNetworks(networkdm, PETSC_DECIDE, Nsubnet));
516: PetscCall(DMNetworkAddSubnetwork(networkdm, "power", numEdges[0], edgelist_power, &power_netnum));
517: PetscCall(DMNetworkAddSubnetwork(networkdm, "water", numEdges[1], edgelist_water, &water_netnum));
519: /* vertex subnet[0].4 shares with vertex subnet[1].0 */
520: power_svtx = 4;
521: water_svtx = 0;
522: PetscCall(DMNetworkAddSharedVertices(networkdm, power_netnum, water_netnum, 1, &power_svtx, &water_svtx));
524: /* Set up the network layout */
525: PetscCall(DMNetworkLayoutSetUp(networkdm));
527: /* ADD VARIABLES AND COMPONENTS FOR THE POWER SUBNETWORK */
528: genj = 0;
529: loadj = 0;
530: PetscCall(DMNetworkGetSubnetwork(networkdm, power_netnum, &nv, &ne, &vtx, &edges));
532: for (i = 0; i < ne; i++) PetscCall(DMNetworkAddComponent(networkdm, edges[i], appctx_power->compkey_branch, &pfdata->branch[i], 0));
534: for (i = 0; i < nv; i++) {
535: PetscCall(DMNetworkIsSharedVertex(networkdm, vtx[i], &flg));
536: if (flg) continue;
538: PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_bus, &pfdata->bus[i], 2));
539: if (pfdata->bus[i].ngen) {
540: for (j = 0; j < pfdata->bus[i].ngen; j++) PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_gen, &pfdata->gen[genj++], 0));
541: }
542: if (pfdata->bus[i].nload) {
543: for (j = 0; j < pfdata->bus[i].nload; j++) PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_load, &pfdata->load[loadj++], 0));
544: }
545: }
547: /* ADD VARIABLES AND COMPONENTS FOR THE WATER SUBNETWORK */
548: PetscCall(DMNetworkGetSubnetwork(networkdm, water_netnum, &nv, &ne, &vtx, &edges));
549: for (i = 0; i < ne; i++) PetscCall(DMNetworkAddComponent(networkdm, edges[i], appctx_water->compkey_edge, &waterdata->edge[i], 0));
551: for (i = 0; i < nv; i++) {
552: PetscCall(DMNetworkIsSharedVertex(networkdm, vtx[i], &flg));
553: if (flg) continue;
555: PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_water->compkey_vtx, &waterdata->vertex[i], 1));
556: }
558: /* ADD VARIABLES AND COMPONENTS AT THE SHARED VERTEX: net[0].4 coupls with net[1].0 -- owning and all ghost ranks of the vertex do this */
559: PetscCall(DMNetworkGetSharedVertices(networkdm, &nv, &vtx));
560: for (i = 0; i < nv; i++) {
561: /* power */
562: PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_bus, &pfdata->bus[4], 2));
563: /* bus[4] is a load, add its component */
564: PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_power->compkey_load, &pfdata->load[0], 0));
566: /* water */
567: PetscCall(DMNetworkAddComponent(networkdm, vtx[i], appctx_water->compkey_vtx, &waterdata->vertex[0], 1));
568: }
570: /* Set coordinates for visualization */
571: PetscCall(DMSetCoordinateDim(networkdm, 2));
572: PetscCall(DMGetCoordinateDM(networkdm, &dmclone));
573: PetscCall(DMNetworkGetVertexRange(dmclone, &vStart, &vEnd));
574: PetscCall(DMNetworkRegisterComponent(dmclone, "coordinates", 0, &compkey));
575: for (i = vStart; i < vEnd; i++) PetscCall(DMNetworkAddComponent(dmclone, i, compkey, NULL, 2));
576: PetscCall(DMNetworkFinalizeComponents(dmclone));
578: PetscCall(DMCreateLocalVector(dmclone, &coords));
579: PetscCall(DMSetCoordinatesLocal(networkdm, coords)); /* set/get coords to/from networkdm */
580: PetscCall(CoordinateVecSetUp(networkdm, coords));
581: if (printCoord) PetscCall(CoordinatePrint(networkdm));
583: /* Set up DM for use */
584: PetscCall(DMSetUp(networkdm));
586: /* Free user objects */
587: PetscCall(PetscFree(edgelist_power));
588: PetscCall(PetscFree(pfdata->bus));
589: PetscCall(PetscFree(pfdata->gen));
590: PetscCall(PetscFree(pfdata->branch));
591: PetscCall(PetscFree(pfdata->load));
592: PetscCall(PetscFree(pfdata));
594: PetscCall(PetscFree(edgelist_water));
595: PetscCall(PetscFree(waterdata->vertex));
596: PetscCall(PetscFree(waterdata->edge));
597: PetscCall(PetscFree(waterdata));
599: /* Re-distribute networkdm to multiple processes for better job balance */
600: if (distribute) {
601: PetscCall(DMNetworkDistribute(&networkdm, 0));
603: if (printCoord) PetscCall(CoordinatePrint(networkdm));
604: if (viewCSV) { /* CSV View of network with coordinates */
605: PetscCall(PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_CSV));
606: PetscCall(DMView(networkdm, PETSC_VIEWER_STDOUT_WORLD));
607: PetscCall(PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD));
608: }
609: }
610: PetscCall(VecDestroy(&coords));
612: /* Test DMNetworkGetSubnetwork() and DMNetworkGetSubnetworkSharedVertices() */
613: if (test) {
614: PetscInt v, gidx;
615: PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
616: for (i = 0; i < Nsubnet; i++) {
617: PetscCall(DMNetworkGetSubnetwork(networkdm, i, &nv, &ne, &vtx, &edges));
618: PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%d] After distribute, subnet[%" PetscInt_FMT "] ne %" PetscInt_FMT ", nv %" PetscInt_FMT "\n", rank, i, ne, nv));
619: PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
621: for (v = 0; v < nv; v++) {
622: PetscCall(DMNetworkIsGhostVertex(networkdm, vtx[v], &ghost));
623: PetscCall(DMNetworkGetGlobalVertexIndex(networkdm, vtx[v], &gidx));
624: PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%d] subnet[%" PetscInt_FMT "] v %" PetscInt_FMT " %" PetscInt_FMT "; ghost %d\n", rank, i, vtx[v], gidx, ghost));
625: }
626: PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
627: }
628: PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
630: PetscCall(DMNetworkGetSharedVertices(networkdm, &nv, &vtx));
631: PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%d] After distribute, num of shared vertices nsv = %" PetscInt_FMT "\n", rank, nv));
632: for (v = 0; v < nv; v++) {
633: PetscCall(DMNetworkGetGlobalVertexIndex(networkdm, vtx[v], &gidx));
634: PetscCall(PetscPrintf(PETSC_COMM_SELF, "[%d] sv %" PetscInt_FMT ", gidx=%" PetscInt_FMT "\n", rank, vtx[v], gidx));
635: }
636: PetscCallMPI(MPI_Barrier(PETSC_COMM_WORLD));
637: }
639: /* Create solution vector X */
640: PetscCall(DMCreateGlobalVector(networkdm, &X));
641: PetscCall(VecDuplicate(X, &F));
642: PetscCall(DMGetLocalVector(networkdm, &user.localXold));
643: PetscCall(PetscLogStagePop());
645: /* (3) Setup Solvers */
646: PetscCall(PetscOptionsGetBool(NULL, NULL, "-viewJ", &viewJ, NULL));
647: PetscCall(PetscOptionsGetBool(NULL, NULL, "-viewX", &viewX, NULL));
649: PetscCall(PetscLogStageRegister("SNES Setup", &stage[2]));
650: PetscCall(PetscLogStagePush(stage[2]));
652: PetscCall(SetInitialGuess(networkdm, X, &user));
654: /* Create coupled snes */
655: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "SNES_coupled setup ......\n"));
656: user.subsnes_id = Nsubnet;
657: PetscCall(SNESCreate(PETSC_COMM_WORLD, &snes));
658: PetscCall(SNESSetDM(snes, networkdm));
659: PetscCall(SNESSetOptionsPrefix(snes, "coupled_"));
660: PetscCall(SNESSetFunction(snes, F, FormFunction, &user));
661: PetscCall(SNESMonitorSet(snes, UserMonitor, &user, NULL));
662: PetscCall(SNESSetFromOptions(snes));
664: if (viewJ) {
665: /* View Jac structure */
666: PetscCall(SNESGetJacobian(snes, &Jac, NULL, NULL, NULL));
667: PetscCall(MatView(Jac, PETSC_VIEWER_DRAW_WORLD));
668: }
670: if (viewX) {
671: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Solution:\n"));
672: PetscCall(VecView(X, PETSC_VIEWER_STDOUT_WORLD));
673: }
675: if (viewJ) {
676: /* View assembled Jac */
677: PetscCall(MatView(Jac, PETSC_VIEWER_DRAW_WORLD));
678: }
680: /* Create snes_power */
681: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "SNES_power setup ......\n"));
683: user.subsnes_id = 0;
684: PetscCall(SNESCreate(PETSC_COMM_WORLD, &snes_power));
685: PetscCall(SNESSetDM(snes_power, networkdm));
686: PetscCall(SNESSetOptionsPrefix(snes_power, "power_"));
687: PetscCall(SNESSetFunction(snes_power, F, FormFunction, &user));
688: PetscCall(SNESMonitorSet(snes_power, UserMonitor, &user, NULL));
690: /* Use user-provide Jacobian */
691: PetscCall(DMCreateMatrix(networkdm, &Jac));
692: PetscCall(SNESSetJacobian(snes_power, Jac, Jac, FormJacobian_subPower, &user));
693: PetscCall(SNESSetFromOptions(snes_power));
695: if (viewX) {
696: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Power Solution:\n"));
697: PetscCall(VecView(X, PETSC_VIEWER_STDOUT_WORLD));
698: }
699: if (viewJ) {
700: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Power Jac:\n"));
701: PetscCall(SNESGetJacobian(snes_power, &Jac, NULL, NULL, NULL));
702: PetscCall(MatView(Jac, PETSC_VIEWER_DRAW_WORLD));
703: /* PetscCall(MatView(Jac,PETSC_VIEWER_STDOUT_WORLD)); */
704: }
706: /* Create snes_water */
707: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "SNES_water setup......\n"));
709: user.subsnes_id = 1;
710: PetscCall(SNESCreate(PETSC_COMM_WORLD, &snes_water));
711: PetscCall(SNESSetDM(snes_water, networkdm));
712: PetscCall(SNESSetOptionsPrefix(snes_water, "water_"));
713: PetscCall(SNESSetFunction(snes_water, F, FormFunction, &user));
714: PetscCall(SNESMonitorSet(snes_water, UserMonitor, &user, NULL));
715: PetscCall(SNESSetFromOptions(snes_water));
717: if (viewX) {
718: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Water Solution:\n"));
719: PetscCall(VecView(X, PETSC_VIEWER_STDOUT_WORLD));
720: }
721: PetscCall(PetscLogStagePop());
723: /* (4) Solve */
724: PetscCall(PetscLogStageRegister("SNES Solve", &stage[3]));
725: PetscCall(PetscLogStagePush(stage[3]));
726: user.it = 0;
727: reason = SNES_DIVERGED_DTOL;
728: while (user.it < it_max && (PetscInt)reason < 0) {
729: #if 0
730: user.subsnes_id = 0;
731: PetscCall(SNESSolve(snes_power,NULL,X));
733: user.subsnes_id = 1;
734: PetscCall(SNESSolve(snes_water,NULL,X));
735: #endif
736: user.subsnes_id = Nsubnet;
737: PetscCall(SNESSolve(snes, NULL, X));
739: PetscCall(SNESGetConvergedReason(snes, &reason));
740: user.it++;
741: }
742: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Coupled_SNES converged in %" PetscInt_FMT " iterations\n", user.it));
743: if (viewX) {
744: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Final Solution:\n"));
745: PetscCall(VecView(X, PETSC_VIEWER_STDOUT_WORLD));
746: }
747: PetscCall(PetscLogStagePop());
749: /* Free objects */
750: /* -------------*/
751: PetscCall(VecDestroy(&X));
752: PetscCall(VecDestroy(&F));
753: PetscCall(DMRestoreLocalVector(networkdm, &user.localXold));
755: PetscCall(SNESDestroy(&snes));
756: PetscCall(MatDestroy(&Jac));
757: PetscCall(SNESDestroy(&snes_power));
758: PetscCall(SNESDestroy(&snes_water));
760: PetscCall(DMDestroy(&networkdm));
761: PetscCall(PetscFinalize());
762: return 0;
763: }
765: /*TEST
767: build:
768: requires: !complex double defined(PETSC_HAVE_ATTRIBUTEALIGNED)
769: depends: power/PFReadData.c power/pffunctions.c water/waterreaddata.c water/waterfunctions.c
771: test:
772: args: -coupled_snes_converged_reason -options_left no -dmnetwork_view
773: localrunfiles: ex1options power/case9.m water/sample1.inp
774: output_file: output/ex1.out
776: test:
777: suffix: 2
778: nsize: 3
779: args: -coupled_snes_converged_reason -options_left no -petscpartitioner_type parmetis
780: localrunfiles: ex1options power/case9.m water/sample1.inp
781: output_file: output/ex1_2.out
782: requires: parmetis
784: test:
785: suffix: 3
786: nsize: 3
787: args: -coupled_snes_converged_reason -options_left no -distribute false
788: localrunfiles: ex1options power/case9.m water/sample1.inp
789: output_file: output/ex1_2.out
791: test:
792: suffix: 4
793: nsize: 4
794: args: -coupled_snes_converged_reason -options_left no -petscpartitioner_type simple -dmnetwork_view -dmnetwork_view_distributed
795: localrunfiles: ex1options power/case9.m water/sample1.inp
796: output_file: output/ex1_4.out
798: test:
799: suffix: 5
800: args: -coupled_snes_converged_reason -options_left no -viewCSV
801: localrunfiles: ex1options power/case9.m water/sample1.inp
802: output_file: output/ex1_5.out
804: test:
805: suffix: 6
806: nsize: 3
807: args: -coupled_snes_converged_reason -options_left no -petscpartitioner_type parmetis -dmnetwork_view_distributed draw:null
808: localrunfiles: ex1options power/case9.m water/sample1.inp
809: output_file: output/ex1_2.out
810: requires: parmetis
812: TEST*/