Actual source code: sliced.c
1: #include <petscdmsliced.h>
2: #include <petscmat.h>
3: #include <petsc/private/dmimpl.h>
5: /* CSR storage of the nonzero structure of a bs*bs matrix */
6: typedef struct {
7: PetscInt bs, nz, *i, *j;
8: } DMSlicedBlockFills;
10: typedef struct {
11: PetscInt bs, n, N, Nghosts, *ghosts;
12: PetscInt d_nz, o_nz, *d_nnz, *o_nnz;
13: DMSlicedBlockFills *dfill, *ofill;
14: } DM_Sliced;
16: PetscErrorCode DMCreateMatrix_Sliced(DM dm, Mat *J)
17: {
18: PetscInt *globals, *sd_nnz, *so_nnz, rstart, bs, i;
19: ISLocalToGlobalMapping lmap;
20: void (*aij)(void) = NULL;
21: DM_Sliced *slice = (DM_Sliced *)dm->data;
23: PetscFunctionBegin;
24: bs = slice->bs;
25: PetscCall(MatCreate(PetscObjectComm((PetscObject)dm), J));
26: PetscCall(MatSetSizes(*J, slice->n * bs, slice->n * bs, PETSC_DETERMINE, PETSC_DETERMINE));
27: PetscCall(MatSetBlockSize(*J, bs));
28: PetscCall(MatSetType(*J, dm->mattype));
29: PetscCall(MatSeqBAIJSetPreallocation(*J, bs, slice->d_nz, slice->d_nnz));
30: PetscCall(MatMPIBAIJSetPreallocation(*J, bs, slice->d_nz, slice->d_nnz, slice->o_nz, slice->o_nnz));
31: /* In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
32: * good before going on with it. */
33: PetscCall(PetscObjectQueryFunction((PetscObject)*J, "MatMPIAIJSetPreallocation_C", &aij));
34: if (!aij) PetscCall(PetscObjectQueryFunction((PetscObject)*J, "MatSeqAIJSetPreallocation_C", &aij));
35: if (aij) {
36: if (bs == 1) {
37: PetscCall(MatSeqAIJSetPreallocation(*J, slice->d_nz, slice->d_nnz));
38: PetscCall(MatMPIAIJSetPreallocation(*J, slice->d_nz, slice->d_nnz, slice->o_nz, slice->o_nnz));
39: } else if (!slice->d_nnz) {
40: PetscCall(MatSeqAIJSetPreallocation(*J, slice->d_nz * bs, NULL));
41: PetscCall(MatMPIAIJSetPreallocation(*J, slice->d_nz * bs, NULL, slice->o_nz * bs, NULL));
42: } else {
43: /* The user has provided preallocation per block-row, convert it to per scalar-row respecting DMSlicedSetBlockFills() if applicable */
44: PetscCall(PetscMalloc2(slice->n * bs, &sd_nnz, (!!slice->o_nnz) * slice->n * bs, &so_nnz));
45: for (i = 0; i < slice->n * bs; i++) {
46: sd_nnz[i] = (slice->d_nnz[i / bs] - 1) * (slice->ofill ? slice->ofill->i[i % bs + 1] - slice->ofill->i[i % bs] : bs) + (slice->dfill ? slice->dfill->i[i % bs + 1] - slice->dfill->i[i % bs] : bs);
47: if (so_nnz) so_nnz[i] = slice->o_nnz[i / bs] * (slice->ofill ? slice->ofill->i[i % bs + 1] - slice->ofill->i[i % bs] : bs);
48: }
49: PetscCall(MatSeqAIJSetPreallocation(*J, slice->d_nz * bs, sd_nnz));
50: PetscCall(MatMPIAIJSetPreallocation(*J, slice->d_nz * bs, sd_nnz, slice->o_nz * bs, so_nnz));
51: PetscCall(PetscFree2(sd_nnz, so_nnz));
52: }
53: }
55: /* Set up the local to global map. For the scalar map, we have to translate to entry-wise indexing instead of block-wise. */
56: PetscCall(PetscMalloc1(slice->n + slice->Nghosts, &globals));
57: PetscCall(MatGetOwnershipRange(*J, &rstart, NULL));
58: for (i = 0; i < slice->n; i++) globals[i] = rstart / bs + i;
60: for (i = 0; i < slice->Nghosts; i++) globals[slice->n + i] = slice->ghosts[i];
61: PetscCall(ISLocalToGlobalMappingCreate(PETSC_COMM_SELF, bs, slice->n + slice->Nghosts, globals, PETSC_OWN_POINTER, &lmap));
62: PetscCall(MatSetLocalToGlobalMapping(*J, lmap, lmap));
63: PetscCall(ISLocalToGlobalMappingDestroy(&lmap));
64: PetscCall(MatSetDM(*J, dm));
65: PetscFunctionReturn(PETSC_SUCCESS);
66: }
68: /*@C
69: DMSlicedSetGhosts - Sets the global indices of other processes elements that will
70: be ghosts on this process
72: Not Collective
74: Input Parameters:
75: + slice - the DM object
76: . bs - block size
77: . nlocal - number of local (owned, non-ghost) blocks
78: . Nghosts - number of ghost blocks on this process
79: - ghosts - global indices of each ghost block
81: Level: advanced
83: .seealso `DMDestroy()`, `DMCreateGlobalVector()`
85: @*/
86: PetscErrorCode DMSlicedSetGhosts(DM dm, PetscInt bs, PetscInt nlocal, PetscInt Nghosts, const PetscInt ghosts[])
87: {
88: DM_Sliced *slice = (DM_Sliced *)dm->data;
90: PetscFunctionBegin;
92: PetscCall(PetscFree(slice->ghosts));
93: PetscCall(PetscMalloc1(Nghosts, &slice->ghosts));
94: PetscCall(PetscArraycpy(slice->ghosts, ghosts, Nghosts));
95: slice->bs = bs;
96: slice->n = nlocal;
97: slice->Nghosts = Nghosts;
98: PetscFunctionReturn(PETSC_SUCCESS);
99: }
101: /*@C
102: DMSlicedSetPreallocation - sets the matrix memory preallocation for matrices computed by DMSliced
104: Not Collective
106: Input Parameters:
107: + slice - the DM object
108: . d_nz - number of block nonzeros per block row in diagonal portion of local
109: submatrix (same for all local rows)
110: . d_nnz - array containing the number of block nonzeros in the various block rows
111: of the in diagonal portion of the local (possibly different for each block
112: row) or NULL.
113: . o_nz - number of block nonzeros per block row in the off-diagonal portion of local
114: submatrix (same for all local rows).
115: - o_nnz - array containing the number of nonzeros in the various block rows of the
116: off-diagonal portion of the local submatrix (possibly different for
117: each block row) or NULL.
119: Notes:
120: See MatMPIBAIJSetPreallocation() for more details on preallocation. If a scalar matrix (AIJ) is
121: obtained with DMSlicedGetMatrix(), the correct preallocation will be set, respecting DMSlicedSetBlockFills().
123: Level: advanced
125: .seealso `DMDestroy()`, `DMCreateGlobalVector()`, `MatMPIAIJSetPreallocation()`,
126: `MatMPIBAIJSetPreallocation()`, `DMSlicedGetMatrix()`, `DMSlicedSetBlockFills()`
128: @*/
129: PetscErrorCode DMSlicedSetPreallocation(DM dm, PetscInt d_nz, const PetscInt d_nnz[], PetscInt o_nz, const PetscInt o_nnz[])
130: {
131: DM_Sliced *slice = (DM_Sliced *)dm->data;
133: PetscFunctionBegin;
135: slice->d_nz = d_nz;
136: slice->d_nnz = (PetscInt *)d_nnz;
137: slice->o_nz = o_nz;
138: slice->o_nnz = (PetscInt *)o_nnz;
139: PetscFunctionReturn(PETSC_SUCCESS);
140: }
142: static PetscErrorCode DMSlicedSetBlockFills_Private(PetscInt bs, const PetscInt *fill, DMSlicedBlockFills **inf)
143: {
144: PetscInt i, j, nz, *fi, *fj;
145: DMSlicedBlockFills *f;
147: PetscFunctionBegin;
149: if (*inf) PetscCall(PetscFree3(*inf, (*inf)->i, (*inf)->j));
150: if (!fill) PetscFunctionReturn(PETSC_SUCCESS);
151: for (i = 0, nz = 0; i < bs * bs; i++)
152: if (fill[i]) nz++;
153: PetscCall(PetscMalloc3(1, &f, bs + 1, &fi, nz, &fj));
154: f->bs = bs;
155: f->nz = nz;
156: f->i = fi;
157: f->j = fj;
158: for (i = 0, nz = 0; i < bs; i++) {
159: fi[i] = nz;
160: for (j = 0; j < bs; j++)
161: if (fill[i * bs + j]) fj[nz++] = j;
162: }
163: fi[i] = nz;
164: *inf = f;
165: PetscFunctionReturn(PETSC_SUCCESS);
166: }
168: /*@C
169: DMSlicedSetBlockFills - Sets the fill pattern in each block for a multi-component problem
170: of the matrix returned by DMSlicedGetMatrix().
172: Logically Collective on dm
174: Input Parameters:
175: + sliced - the DM object
176: . dfill - the fill pattern in the diagonal block (may be NULL, means use dense block)
177: - ofill - the fill pattern in the off-diagonal blocks
179: Notes:
180: This only makes sense for multicomponent problems using scalar matrix formats (AIJ).
181: See DMDASetBlockFills() for example usage.
183: Level: advanced
185: .seealso `DMSlicedGetMatrix()`, `DMDASetBlockFills()`
186: @*/
187: PetscErrorCode DMSlicedSetBlockFills(DM dm, const PetscInt *dfill, const PetscInt *ofill)
188: {
189: DM_Sliced *slice = (DM_Sliced *)dm->data;
191: PetscFunctionBegin;
193: PetscCall(DMSlicedSetBlockFills_Private(slice->bs, dfill, &slice->dfill));
194: PetscCall(DMSlicedSetBlockFills_Private(slice->bs, ofill, &slice->ofill));
195: PetscFunctionReturn(PETSC_SUCCESS);
196: }
198: static PetscErrorCode DMDestroy_Sliced(DM dm)
199: {
200: DM_Sliced *slice = (DM_Sliced *)dm->data;
202: PetscFunctionBegin;
203: PetscCall(PetscFree(slice->ghosts));
204: if (slice->dfill) PetscCall(PetscFree3(slice->dfill, slice->dfill->i, slice->dfill->j));
205: if (slice->ofill) PetscCall(PetscFree3(slice->ofill, slice->ofill->i, slice->ofill->j));
206: /* This was originally freed in DMDestroy(), but that prevents reference counting of backend objects */
207: PetscCall(PetscFree(slice));
208: PetscFunctionReturn(PETSC_SUCCESS);
209: }
211: static PetscErrorCode DMCreateGlobalVector_Sliced(DM dm, Vec *gvec)
212: {
213: DM_Sliced *slice = (DM_Sliced *)dm->data;
215: PetscFunctionBegin;
218: *gvec = NULL;
219: PetscCall(VecCreateGhostBlock(PetscObjectComm((PetscObject)dm), slice->bs, slice->n * slice->bs, PETSC_DETERMINE, slice->Nghosts, slice->ghosts, gvec));
220: PetscCall(VecSetDM(*gvec, dm));
221: PetscFunctionReturn(PETSC_SUCCESS);
222: }
224: static PetscErrorCode DMGlobalToLocalBegin_Sliced(DM da, Vec g, InsertMode mode, Vec l)
225: {
226: PetscBool flg;
228: PetscFunctionBegin;
229: PetscCall(VecGhostIsLocalForm(g, l, &flg));
230: PetscCheck(flg, PetscObjectComm((PetscObject)da), PETSC_ERR_ARG_WRONG, "Local vector is not local form of global vector");
231: PetscCall(VecGhostUpdateEnd(g, mode, SCATTER_FORWARD));
232: PetscCall(VecGhostUpdateBegin(g, mode, SCATTER_FORWARD));
233: PetscFunctionReturn(PETSC_SUCCESS);
234: }
236: static PetscErrorCode DMGlobalToLocalEnd_Sliced(DM da, Vec g, InsertMode mode, Vec l)
237: {
238: PetscBool flg;
240: PetscFunctionBegin;
241: PetscCall(VecGhostIsLocalForm(g, l, &flg));
242: PetscCheck(flg, PetscObjectComm((PetscObject)da), PETSC_ERR_ARG_WRONG, "Local vector is not local form of global vector");
243: PetscCall(VecGhostUpdateEnd(g, mode, SCATTER_FORWARD));
244: PetscFunctionReturn(PETSC_SUCCESS);
245: }
247: /*MC
248: DMSLICED = "sliced" - A DM object that is used to manage data for a general graph. Uses VecCreateGhost() ghosted vectors for storing the fields
250: See DMCreateSliced() for details.
252: Level: intermediate
254: .seealso: `DMType`, `DMCOMPOSITE`, `DMCreateSliced()`, `DMCreate()`
255: M*/
257: PETSC_EXTERN PetscErrorCode DMCreate_Sliced(DM p)
258: {
259: DM_Sliced *slice;
261: PetscFunctionBegin;
262: PetscCall(PetscNew(&slice));
263: p->data = slice;
265: p->ops->createglobalvector = DMCreateGlobalVector_Sliced;
266: p->ops->creatematrix = DMCreateMatrix_Sliced;
267: p->ops->globaltolocalbegin = DMGlobalToLocalBegin_Sliced;
268: p->ops->globaltolocalend = DMGlobalToLocalEnd_Sliced;
269: p->ops->destroy = DMDestroy_Sliced;
270: PetscFunctionReturn(PETSC_SUCCESS);
271: }
273: /*@C
274: DMSlicedCreate - Creates a DM object, used to manage data for a unstructured problem
276: Collective
278: Input Parameters:
279: + comm - the processors that will share the global vector
280: . bs - the block size
281: . nlocal - number of vector entries on this process
282: . Nghosts - number of ghost points needed on this process
283: . ghosts - global indices of all ghost points for this process
284: . d_nnz - matrix preallocation information representing coupling within this process
285: - o_nnz - matrix preallocation information representing coupling between this process and other processes
287: Output Parameters:
288: . slice - the slice object
290: Notes:
291: This DM does not support DMCreateLocalVector(), DMGlobalToLocalBegin(), and DMGlobalToLocalEnd() instead one directly uses
292: VecGhostGetLocalForm() and VecGhostRestoreLocalForm() to access the local representation and VecGhostUpdateBegin() and VecGhostUpdateEnd() to update
293: the ghost points.
295: One can use DMGlobalToLocalBegin(), and DMGlobalToLocalEnd() instead of VecGhostUpdateBegin() and VecGhostUpdateEnd().
297: Level: advanced
299: .seealso `DMDestroy()`, `DMCreateGlobalVector()`, `DMSetType()`, `DMSLICED`, `DMSlicedSetGhosts()`, `DMSlicedSetPreallocation()`, `VecGhostUpdateBegin()`, `VecGhostUpdateEnd()`,
300: `VecGhostGetLocalForm()`, `VecGhostRestoreLocalForm()`
302: @*/
303: PetscErrorCode DMSlicedCreate(MPI_Comm comm, PetscInt bs, PetscInt nlocal, PetscInt Nghosts, const PetscInt ghosts[], const PetscInt d_nnz[], const PetscInt o_nnz[], DM *dm)
304: {
305: PetscFunctionBegin;
307: PetscCall(DMCreate(comm, dm));
308: PetscCall(DMSetType(*dm, DMSLICED));
309: PetscCall(DMSlicedSetGhosts(*dm, bs, nlocal, Nghosts, ghosts));
310: if (d_nnz) PetscCall(DMSlicedSetPreallocation(*dm, 0, d_nnz, 0, o_nnz));
311: PetscFunctionReturn(PETSC_SUCCESS);
312: }