Actual source code: baijov.c


  2: /*
  3:    Routines to compute overlapping regions of a parallel MPI matrix
  4:   and to find submatrices that were shared across processors.
  5: */
  6: #include <../src/mat/impls/baij/mpi/mpibaij.h>
  7: #include <petscbt.h>

  9: static PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Local(Mat, PetscInt, char **, PetscInt *, PetscInt **);
 10: static PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Receive(Mat, PetscInt, PetscInt **, PetscInt **, PetscInt *);
 11: extern PetscErrorCode MatGetRow_MPIBAIJ(Mat, PetscInt, PetscInt *, PetscInt **, PetscScalar **);
 12: extern PetscErrorCode MatRestoreRow_MPIBAIJ(Mat, PetscInt, PetscInt *, PetscInt **, PetscScalar **);

 14: PetscErrorCode MatIncreaseOverlap_MPIBAIJ(Mat C, PetscInt imax, IS is[], PetscInt ov)
 15: {
 16:   PetscInt i, N = C->cmap->N, bs = C->rmap->bs;
 17:   IS      *is_new;

 19:   PetscFunctionBegin;
 20:   PetscCall(PetscMalloc1(imax, &is_new));
 21:   /* Convert the indices into block format */
 22:   PetscCall(ISCompressIndicesGeneral(N, C->rmap->n, bs, imax, is, is_new));
 23:   PetscCheck(ov >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Negative overlap specified");
 24:   for (i = 0; i < ov; ++i) PetscCall(MatIncreaseOverlap_MPIBAIJ_Once(C, imax, is_new));
 25:   for (i = 0; i < imax; i++) PetscCall(ISDestroy(&is[i]));
 26:   PetscCall(ISExpandIndicesGeneral(N, N, bs, imax, is_new, is));
 27:   for (i = 0; i < imax; i++) PetscCall(ISDestroy(&is_new[i]));
 28:   PetscCall(PetscFree(is_new));
 29:   PetscFunctionReturn(PETSC_SUCCESS);
 30: }

 32: /*
 33:   Sample message format:
 34:   If a processor A wants processor B to process some elements corresponding
 35:   to index sets is[1], is[5]
 36:   mesg [0] = 2   (no of index sets in the mesg)
 37:   -----------
 38:   mesg [1] = 1 => is[1]
 39:   mesg [2] = sizeof(is[1]);
 40:   -----------
 41:   mesg [5] = 5  => is[5]
 42:   mesg [6] = sizeof(is[5]);
 43:   -----------
 44:   mesg [7]
 45:   mesg [n]  data(is[1])
 46:   -----------
 47:   mesg[n+1]
 48:   mesg[m]  data(is[5])
 49:   -----------

 51:   Notes:
 52:   nrqs - no of requests sent (or to be sent out)
 53:   nrqr - no of requests received (which have to be or which have been processed)
 54: */
 55: PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Once(Mat C, PetscInt imax, IS is[])
 56: {
 57:   Mat_MPIBAIJ     *c = (Mat_MPIBAIJ *)C->data;
 58:   const PetscInt **idx, *idx_i;
 59:   PetscInt        *n, *w3, *w4, **data, len;
 60:   PetscMPIInt      size, rank, tag1, tag2, *w2, *w1, nrqr;
 61:   PetscInt         Mbs, i, j, k, **rbuf, row, nrqs, msz, **outdat, **ptr;
 62:   PetscInt        *ctr, *pa, *tmp, *isz, *isz1, **xdata, **rbuf2, *d_p;
 63:   PetscMPIInt     *onodes1, *olengths1, *onodes2, *olengths2, proc = -1;
 64:   PetscBT         *table;
 65:   MPI_Comm         comm, *iscomms;
 66:   MPI_Request     *s_waits1, *r_waits1, *s_waits2, *r_waits2;
 67:   char            *t_p;

 69:   PetscFunctionBegin;
 70:   PetscCall(PetscObjectGetComm((PetscObject)C, &comm));
 71:   size = c->size;
 72:   rank = c->rank;
 73:   Mbs  = c->Mbs;

 75:   PetscCall(PetscObjectGetNewTag((PetscObject)C, &tag1));
 76:   PetscCall(PetscObjectGetNewTag((PetscObject)C, &tag2));

 78:   PetscCall(PetscMalloc2(imax + 1, (PetscInt ***)&idx, imax, &n));

 80:   for (i = 0; i < imax; i++) {
 81:     PetscCall(ISGetIndices(is[i], &idx[i]));
 82:     PetscCall(ISGetLocalSize(is[i], &n[i]));
 83:   }

 85:   /* evaluate communication - mesg to who,length of mesg, and buffer space
 86:      required. Based on this, buffers are allocated, and data copied into them*/
 87:   PetscCall(PetscCalloc4(size, &w1, size, &w2, size, &w3, size, &w4));
 88:   for (i = 0; i < imax; i++) {
 89:     PetscCall(PetscArrayzero(w4, size)); /* initialise work vector*/
 90:     idx_i = idx[i];
 91:     len   = n[i];
 92:     for (j = 0; j < len; j++) {
 93:       row = idx_i[j];
 94:       PetscCheck(row >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Index set cannot have negative entries");
 95:       PetscCall(PetscLayoutFindOwner(C->rmap, row * C->rmap->bs, &proc));
 96:       w4[proc]++;
 97:     }
 98:     for (j = 0; j < size; j++) {
 99:       if (w4[j]) {
100:         w1[j] += w4[j];
101:         w3[j]++;
102:       }
103:     }
104:   }

106:   nrqs     = 0; /* no of outgoing messages */
107:   msz      = 0; /* total mesg length (for all proc */
108:   w1[rank] = 0; /* no mesg sent to itself */
109:   w3[rank] = 0;
110:   for (i = 0; i < size; i++) {
111:     if (w1[i]) {
112:       w2[i] = 1;
113:       nrqs++;
114:     } /* there exists a message to proc i */
115:   }
116:   /* pa - is list of processors to communicate with */
117:   PetscCall(PetscMalloc1(nrqs, &pa));
118:   for (i = 0, j = 0; i < size; i++) {
119:     if (w1[i]) {
120:       pa[j] = i;
121:       j++;
122:     }
123:   }

125:   /* Each message would have a header = 1 + 2*(no of IS) + data */
126:   for (i = 0; i < nrqs; i++) {
127:     j = pa[i];
128:     w1[j] += w2[j] + 2 * w3[j];
129:     msz += w1[j];
130:   }

132:   /* Determine the number of messages to expect, their lengths, from from-ids */
133:   PetscCall(PetscGatherNumberOfMessages(comm, w2, w1, &nrqr));
134:   PetscCall(PetscGatherMessageLengths(comm, nrqs, nrqr, w1, &onodes1, &olengths1));

136:   /* Now post the Irecvs corresponding to these messages */
137:   PetscCall(PetscPostIrecvInt(comm, tag1, nrqr, onodes1, olengths1, &rbuf, &r_waits1));

139:   /* Allocate Memory for outgoing messages */
140:   PetscCall(PetscMalloc4(size, &outdat, size, &ptr, msz, &tmp, size, &ctr));
141:   PetscCall(PetscArrayzero(outdat, size));
142:   PetscCall(PetscArrayzero(ptr, size));
143:   {
144:     PetscInt *iptr = tmp, ict = 0;
145:     for (i = 0; i < nrqs; i++) {
146:       j = pa[i];
147:       iptr += ict;
148:       outdat[j] = iptr;
149:       ict       = w1[j];
150:     }
151:   }

153:   /* Form the outgoing messages */
154:   /*plug in the headers*/
155:   for (i = 0; i < nrqs; i++) {
156:     j            = pa[i];
157:     outdat[j][0] = 0;
158:     PetscCall(PetscArrayzero(outdat[j] + 1, 2 * w3[j]));
159:     ptr[j] = outdat[j] + 2 * w3[j] + 1;
160:   }

162:   /* Memory for doing local proc's work*/
163:   {
164:     PetscCall(PetscCalloc5(imax, &table, imax, &data, imax, &isz, Mbs * imax, &d_p, (Mbs / PETSC_BITS_PER_BYTE + 1) * imax, &t_p));

166:     for (i = 0; i < imax; i++) {
167:       table[i] = t_p + (Mbs / PETSC_BITS_PER_BYTE + 1) * i;
168:       data[i]  = d_p + (Mbs)*i;
169:     }
170:   }

172:   /* Parse the IS and update local tables and the outgoing buf with the data*/
173:   {
174:     PetscInt n_i, *data_i, isz_i, *outdat_j, ctr_j;
175:     PetscBT  table_i;

177:     for (i = 0; i < imax; i++) {
178:       PetscCall(PetscArrayzero(ctr, size));
179:       n_i     = n[i];
180:       table_i = table[i];
181:       idx_i   = idx[i];
182:       data_i  = data[i];
183:       isz_i   = isz[i];
184:       for (j = 0; j < n_i; j++) { /* parse the indices of each IS */
185:         row = idx_i[j];
186:         PetscCall(PetscLayoutFindOwner(C->rmap, row * C->rmap->bs, &proc));
187:         if (proc != rank) { /* copy to the outgoing buffer */
188:           ctr[proc]++;
189:           *ptr[proc] = row;
190:           ptr[proc]++;
191:         } else { /* Update the local table */
192:           if (!PetscBTLookupSet(table_i, row)) data_i[isz_i++] = row;
193:         }
194:       }
195:       /* Update the headers for the current IS */
196:       for (j = 0; j < size; j++) { /* Can Optimise this loop by using pa[] */
197:         if ((ctr_j = ctr[j])) {
198:           outdat_j            = outdat[j];
199:           k                   = ++outdat_j[0];
200:           outdat_j[2 * k]     = ctr_j;
201:           outdat_j[2 * k - 1] = i;
202:         }
203:       }
204:       isz[i] = isz_i;
205:     }
206:   }

208:   /*  Now  post the sends */
209:   PetscCall(PetscMalloc1(nrqs, &s_waits1));
210:   for (i = 0; i < nrqs; ++i) {
211:     j = pa[i];
212:     PetscCallMPI(MPI_Isend(outdat[j], w1[j], MPIU_INT, j, tag1, comm, s_waits1 + i));
213:   }

215:   /* No longer need the original indices*/
216:   for (i = 0; i < imax; ++i) PetscCall(ISRestoreIndices(is[i], idx + i));
217:   PetscCall(PetscFree2(*(PetscInt ***)&idx, n));

219:   PetscCall(PetscMalloc1(imax, &iscomms));
220:   for (i = 0; i < imax; ++i) {
221:     PetscCall(PetscCommDuplicate(PetscObjectComm((PetscObject)is[i]), &iscomms[i], NULL));
222:     PetscCall(ISDestroy(&is[i]));
223:   }

225:   /* Do Local work*/
226:   PetscCall(MatIncreaseOverlap_MPIBAIJ_Local(C, imax, table, isz, data));

228:   /* Receive messages*/
229:   PetscCallMPI(MPI_Waitall(nrqr, r_waits1, MPI_STATUSES_IGNORE));
230:   PetscCallMPI(MPI_Waitall(nrqs, s_waits1, MPI_STATUSES_IGNORE));

232:   /* Phase 1 sends are complete - deallocate buffers */
233:   PetscCall(PetscFree4(outdat, ptr, tmp, ctr));
234:   PetscCall(PetscFree4(w1, w2, w3, w4));

236:   PetscCall(PetscMalloc1(nrqr, &xdata));
237:   PetscCall(PetscMalloc1(nrqr, &isz1));
238:   PetscCall(MatIncreaseOverlap_MPIBAIJ_Receive(C, nrqr, rbuf, xdata, isz1));
239:   if (rbuf) {
240:     PetscCall(PetscFree(rbuf[0]));
241:     PetscCall(PetscFree(rbuf));
242:   }

244:   /* Send the data back*/
245:   /* Do a global reduction to know the buffer space req for incoming messages*/
246:   {
247:     PetscMPIInt *rw1;

249:     PetscCall(PetscCalloc1(size, &rw1));

251:     for (i = 0; i < nrqr; ++i) {
252:       proc      = onodes1[i];
253:       rw1[proc] = isz1[i];
254:     }

256:     /* Determine the number of messages to expect, their lengths, from from-ids */
257:     PetscCall(PetscGatherMessageLengths(comm, nrqr, nrqs, rw1, &onodes2, &olengths2));
258:     PetscCall(PetscFree(rw1));
259:   }
260:   /* Now post the Irecvs corresponding to these messages */
261:   PetscCall(PetscPostIrecvInt(comm, tag2, nrqs, onodes2, olengths2, &rbuf2, &r_waits2));

263:   /*  Now  post the sends */
264:   PetscCall(PetscMalloc1(nrqr, &s_waits2));
265:   for (i = 0; i < nrqr; ++i) {
266:     j = onodes1[i];
267:     PetscCallMPI(MPI_Isend(xdata[i], isz1[i], MPIU_INT, j, tag2, comm, s_waits2 + i));
268:   }

270:   PetscCall(PetscFree(onodes1));
271:   PetscCall(PetscFree(olengths1));

273:   /* receive work done on other processors*/
274:   {
275:     PetscMPIInt idex;
276:     PetscInt    is_no, ct1, max, *rbuf2_i, isz_i, *data_i, jmax;
277:     PetscBT     table_i;

279:     for (i = 0; i < nrqs; ++i) {
280:       PetscCallMPI(MPI_Waitany(nrqs, r_waits2, &idex, MPI_STATUS_IGNORE));
281:       /* Process the message*/
282:       rbuf2_i = rbuf2[idex];
283:       ct1     = 2 * rbuf2_i[0] + 1;
284:       jmax    = rbuf2[idex][0];
285:       for (j = 1; j <= jmax; j++) {
286:         max     = rbuf2_i[2 * j];
287:         is_no   = rbuf2_i[2 * j - 1];
288:         isz_i   = isz[is_no];
289:         data_i  = data[is_no];
290:         table_i = table[is_no];
291:         for (k = 0; k < max; k++, ct1++) {
292:           row = rbuf2_i[ct1];
293:           if (!PetscBTLookupSet(table_i, row)) data_i[isz_i++] = row;
294:         }
295:         isz[is_no] = isz_i;
296:       }
297:     }
298:     PetscCallMPI(MPI_Waitall(nrqr, s_waits2, MPI_STATUSES_IGNORE));
299:   }

301:   for (i = 0; i < imax; ++i) {
302:     PetscCall(ISCreateGeneral(iscomms[i], isz[i], data[i], PETSC_COPY_VALUES, is + i));
303:     PetscCall(PetscCommDestroy(&iscomms[i]));
304:   }

306:   PetscCall(PetscFree(iscomms));
307:   PetscCall(PetscFree(onodes2));
308:   PetscCall(PetscFree(olengths2));

310:   PetscCall(PetscFree(pa));
311:   if (rbuf2) {
312:     PetscCall(PetscFree(rbuf2[0]));
313:     PetscCall(PetscFree(rbuf2));
314:   }
315:   PetscCall(PetscFree(s_waits1));
316:   PetscCall(PetscFree(r_waits1));
317:   PetscCall(PetscFree(s_waits2));
318:   PetscCall(PetscFree(r_waits2));
319:   PetscCall(PetscFree5(table, data, isz, d_p, t_p));
320:   if (xdata) {
321:     PetscCall(PetscFree(xdata[0]));
322:     PetscCall(PetscFree(xdata));
323:   }
324:   PetscCall(PetscFree(isz1));
325:   PetscFunctionReturn(PETSC_SUCCESS);
326: }

328: /*
329:    MatIncreaseOverlap_MPIBAIJ_Local - Called by MatincreaseOverlap, to do
330:        the work on the local processor.

332:      Inputs:
333:       C      - MAT_MPIBAIJ;
334:       imax - total no of index sets processed at a time;
335:       table  - an array of char - size = Mbs bits.

337:      Output:
338:       isz    - array containing the count of the solution elements corresponding
339:                to each index set;
340:       data   - pointer to the solutions
341: */
342: static PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Local(Mat C, PetscInt imax, PetscBT *table, PetscInt *isz, PetscInt **data)
343: {
344:   Mat_MPIBAIJ *c = (Mat_MPIBAIJ *)C->data;
345:   Mat          A = c->A, B = c->B;
346:   Mat_SeqBAIJ *a = (Mat_SeqBAIJ *)A->data, *b = (Mat_SeqBAIJ *)B->data;
347:   PetscInt     start, end, val, max, rstart, cstart, *ai, *aj;
348:   PetscInt    *bi, *bj, *garray, i, j, k, row, *data_i, isz_i;
349:   PetscBT      table_i;

351:   PetscFunctionBegin;
352:   rstart = c->rstartbs;
353:   cstart = c->cstartbs;
354:   ai     = a->i;
355:   aj     = a->j;
356:   bi     = b->i;
357:   bj     = b->j;
358:   garray = c->garray;

360:   for (i = 0; i < imax; i++) {
361:     data_i  = data[i];
362:     table_i = table[i];
363:     isz_i   = isz[i];
364:     for (j = 0, max = isz[i]; j < max; j++) {
365:       row   = data_i[j] - rstart;
366:       start = ai[row];
367:       end   = ai[row + 1];
368:       for (k = start; k < end; k++) { /* Amat */
369:         val = aj[k] + cstart;
370:         if (!PetscBTLookupSet(table_i, val)) data_i[isz_i++] = val;
371:       }
372:       start = bi[row];
373:       end   = bi[row + 1];
374:       for (k = start; k < end; k++) { /* Bmat */
375:         val = garray[bj[k]];
376:         if (!PetscBTLookupSet(table_i, val)) data_i[isz_i++] = val;
377:       }
378:     }
379:     isz[i] = isz_i;
380:   }
381:   PetscFunctionReturn(PETSC_SUCCESS);
382: }
383: /*
384:       MatIncreaseOverlap_MPIBAIJ_Receive - Process the received messages,
385:          and return the output

387:          Input:
388:            C    - the matrix
389:            nrqr - no of messages being processed.
390:            rbuf - an array of pointers to the received requests

392:          Output:
393:            xdata - array of messages to be sent back
394:            isz1  - size of each message

396:   For better efficiency perhaps we should malloc separately each xdata[i],
397: then if a remalloc is required we need only copy the data for that one row
398: rather than all previous rows as it is now where a single large chunk of
399: memory is used.

401: */
402: static PetscErrorCode MatIncreaseOverlap_MPIBAIJ_Receive(Mat C, PetscInt nrqr, PetscInt **rbuf, PetscInt **xdata, PetscInt *isz1)
403: {
404:   Mat_MPIBAIJ *c = (Mat_MPIBAIJ *)C->data;
405:   Mat          A = c->A, B = c->B;
406:   Mat_SeqBAIJ *a = (Mat_SeqBAIJ *)A->data, *b = (Mat_SeqBAIJ *)B->data;
407:   PetscInt     rstart, cstart, *ai, *aj, *bi, *bj, *garray, i, j, k;
408:   PetscInt     row, total_sz, ct, ct1, ct2, ct3, mem_estimate, oct2, l, start, end;
409:   PetscInt     val, max1, max2, Mbs, no_malloc = 0, *tmp, new_estimate, ctr;
410:   PetscInt    *rbuf_i, kmax, rbuf_0;
411:   PetscBT      xtable;

413:   PetscFunctionBegin;
414:   Mbs    = c->Mbs;
415:   rstart = c->rstartbs;
416:   cstart = c->cstartbs;
417:   ai     = a->i;
418:   aj     = a->j;
419:   bi     = b->i;
420:   bj     = b->j;
421:   garray = c->garray;

423:   for (i = 0, ct = 0, total_sz = 0; i < nrqr; ++i) {
424:     rbuf_i = rbuf[i];
425:     rbuf_0 = rbuf_i[0];
426:     ct += rbuf_0;
427:     for (j = 1; j <= rbuf_0; j++) total_sz += rbuf_i[2 * j];
428:   }

430:   if (c->Mbs) max1 = ct * (a->nz + b->nz) / c->Mbs;
431:   else max1 = 1;
432:   mem_estimate = 3 * ((total_sz > max1 ? total_sz : max1) + 1);
433:   if (nrqr) {
434:     PetscCall(PetscMalloc1(mem_estimate, &xdata[0]));
435:     ++no_malloc;
436:   }
437:   PetscCall(PetscBTCreate(Mbs, &xtable));
438:   PetscCall(PetscArrayzero(isz1, nrqr));

440:   ct3 = 0;
441:   for (i = 0; i < nrqr; i++) { /* for easch mesg from proc i */
442:     rbuf_i = rbuf[i];
443:     rbuf_0 = rbuf_i[0];
444:     ct1    = 2 * rbuf_0 + 1;
445:     ct2    = ct1;
446:     ct3 += ct1;
447:     for (j = 1; j <= rbuf_0; j++) { /* for each IS from proc i*/
448:       PetscCall(PetscBTMemzero(Mbs, xtable));
449:       oct2 = ct2;
450:       kmax = rbuf_i[2 * j];
451:       for (k = 0; k < kmax; k++, ct1++) {
452:         row = rbuf_i[ct1];
453:         if (!PetscBTLookupSet(xtable, row)) {
454:           if (!(ct3 < mem_estimate)) {
455:             new_estimate = (PetscInt)(1.5 * mem_estimate) + 1;
456:             PetscCall(PetscMalloc1(new_estimate, &tmp));
457:             PetscCall(PetscArraycpy(tmp, xdata[0], mem_estimate));
458:             PetscCall(PetscFree(xdata[0]));
459:             xdata[0]     = tmp;
460:             mem_estimate = new_estimate;
461:             ++no_malloc;
462:             for (ctr = 1; ctr <= i; ctr++) xdata[ctr] = xdata[ctr - 1] + isz1[ctr - 1];
463:           }
464:           xdata[i][ct2++] = row;
465:           ct3++;
466:         }
467:       }
468:       for (k = oct2, max2 = ct2; k < max2; k++) {
469:         row   = xdata[i][k] - rstart;
470:         start = ai[row];
471:         end   = ai[row + 1];
472:         for (l = start; l < end; l++) {
473:           val = aj[l] + cstart;
474:           if (!PetscBTLookupSet(xtable, val)) {
475:             if (!(ct3 < mem_estimate)) {
476:               new_estimate = (PetscInt)(1.5 * mem_estimate) + 1;
477:               PetscCall(PetscMalloc1(new_estimate, &tmp));
478:               PetscCall(PetscArraycpy(tmp, xdata[0], mem_estimate));
479:               PetscCall(PetscFree(xdata[0]));
480:               xdata[0]     = tmp;
481:               mem_estimate = new_estimate;
482:               ++no_malloc;
483:               for (ctr = 1; ctr <= i; ctr++) xdata[ctr] = xdata[ctr - 1] + isz1[ctr - 1];
484:             }
485:             xdata[i][ct2++] = val;
486:             ct3++;
487:           }
488:         }
489:         start = bi[row];
490:         end   = bi[row + 1];
491:         for (l = start; l < end; l++) {
492:           val = garray[bj[l]];
493:           if (!PetscBTLookupSet(xtable, val)) {
494:             if (!(ct3 < mem_estimate)) {
495:               new_estimate = (PetscInt)(1.5 * mem_estimate) + 1;
496:               PetscCall(PetscMalloc1(new_estimate, &tmp));
497:               PetscCall(PetscArraycpy(tmp, xdata[0], mem_estimate));
498:               PetscCall(PetscFree(xdata[0]));
499:               xdata[0]     = tmp;
500:               mem_estimate = new_estimate;
501:               ++no_malloc;
502:               for (ctr = 1; ctr <= i; ctr++) xdata[ctr] = xdata[ctr - 1] + isz1[ctr - 1];
503:             }
504:             xdata[i][ct2++] = val;
505:             ct3++;
506:           }
507:         }
508:       }
509:       /* Update the header*/
510:       xdata[i][2 * j]     = ct2 - oct2; /* Undo the vector isz1 and use only a var*/
511:       xdata[i][2 * j - 1] = rbuf_i[2 * j - 1];
512:     }
513:     xdata[i][0] = rbuf_0;
514:     if (i + 1 < nrqr) xdata[i + 1] = xdata[i] + ct2;
515:     isz1[i] = ct2; /* size of each message */
516:   }
517:   PetscCall(PetscBTDestroy(&xtable));
518:   PetscCall(PetscInfo(C, "Allocated %" PetscInt_FMT " bytes, required %" PetscInt_FMT ", no of mallocs = %" PetscInt_FMT "\n", mem_estimate, ct3, no_malloc));
519:   PetscFunctionReturn(PETSC_SUCCESS);
520: }

522: PetscErrorCode MatCreateSubMatrices_MPIBAIJ(Mat C, PetscInt ismax, const IS isrow[], const IS iscol[], MatReuse scall, Mat *submat[])
523: {
524:   IS          *isrow_block, *iscol_block;
525:   Mat_MPIBAIJ *c = (Mat_MPIBAIJ *)C->data;
526:   PetscInt     nmax, nstages_local, nstages, i, pos, max_no, N = C->cmap->N, bs = C->rmap->bs;
527:   Mat_SeqBAIJ *subc;
528:   Mat_SubSppt *smat;

530:   PetscFunctionBegin;
531:   /* The compression and expansion should be avoided. Doesn't point
532:      out errors, might change the indices, hence buggey */
533:   PetscCall(PetscMalloc2(ismax + 1, &isrow_block, ismax + 1, &iscol_block));
534:   PetscCall(ISCompressIndicesGeneral(N, C->rmap->n, bs, ismax, isrow, isrow_block));
535:   PetscCall(ISCompressIndicesGeneral(N, C->cmap->n, bs, ismax, iscol, iscol_block));

537:   /* Determine the number of stages through which submatrices are done */
538:   if (!C->cmap->N) nmax = 20 * 1000000 / sizeof(PetscInt);
539:   else nmax = 20 * 1000000 / (c->Nbs * sizeof(PetscInt));
540:   if (!nmax) nmax = 1;

542:   if (scall == MAT_INITIAL_MATRIX) {
543:     nstages_local = ismax / nmax + ((ismax % nmax) ? 1 : 0); /* local nstages */

545:     /* Make sure every processor loops through the nstages */
546:     PetscCall(MPIU_Allreduce(&nstages_local, &nstages, 1, MPIU_INT, MPI_MAX, PetscObjectComm((PetscObject)C)));

548:     /* Allocate memory to hold all the submatrices and dummy submatrices */
549:     PetscCall(PetscCalloc1(ismax + nstages, submat));
550:   } else { /* MAT_REUSE_MATRIX */
551:     if (ismax) {
552:       subc = (Mat_SeqBAIJ *)((*submat)[0]->data);
553:       smat = subc->submatis1;
554:     } else { /* (*submat)[0] is a dummy matrix */
555:       smat = (Mat_SubSppt *)(*submat)[0]->data;
556:     }
557:     PetscCheck(smat, PETSC_COMM_SELF, PETSC_ERR_ARG_NULL, "MatCreateSubMatrices(...,MAT_REUSE_MATRIX,...) requires submat");
558:     nstages = smat->nstages;
559:   }

561:   for (i = 0, pos = 0; i < nstages; i++) {
562:     if (pos + nmax <= ismax) max_no = nmax;
563:     else if (pos >= ismax) max_no = 0;
564:     else max_no = ismax - pos;

566:     PetscCall(MatCreateSubMatrices_MPIBAIJ_local(C, max_no, isrow_block + pos, iscol_block + pos, scall, *submat + pos));
567:     if (!max_no) {
568:       if (scall == MAT_INITIAL_MATRIX) { /* submat[pos] is a dummy matrix */
569:         smat          = (Mat_SubSppt *)(*submat)[pos]->data;
570:         smat->nstages = nstages;
571:       }
572:       pos++; /* advance to next dummy matrix if any */
573:     } else pos += max_no;
574:   }

576:   if (scall == MAT_INITIAL_MATRIX && ismax) {
577:     /* save nstages for reuse */
578:     subc          = (Mat_SeqBAIJ *)((*submat)[0]->data);
579:     smat          = subc->submatis1;
580:     smat->nstages = nstages;
581:   }

583:   for (i = 0; i < ismax; i++) {
584:     PetscCall(ISDestroy(&isrow_block[i]));
585:     PetscCall(ISDestroy(&iscol_block[i]));
586:   }
587:   PetscCall(PetscFree2(isrow_block, iscol_block));
588:   PetscFunctionReturn(PETSC_SUCCESS);
589: }

591: #if defined(PETSC_USE_CTABLE)
592: PetscErrorCode PetscGetProc(const PetscInt row, const PetscMPIInt size, const PetscInt proc_gnode[], PetscMPIInt *rank)
593: {
594:   PetscInt    nGlobalNd = proc_gnode[size];
595:   PetscMPIInt fproc;

597:   PetscFunctionBegin;
598:   PetscCall(PetscMPIIntCast((PetscInt)(((float)row * (float)size / (float)nGlobalNd + 0.5)), &fproc));
599:   if (fproc > size) fproc = size;
600:   while (row < proc_gnode[fproc] || row >= proc_gnode[fproc + 1]) {
601:     if (row < proc_gnode[fproc]) fproc--;
602:     else fproc++;
603:   }
604:   *rank = fproc;
605:   PetscFunctionReturn(PETSC_SUCCESS);
606: }
607: #endif

609: /* This code is used for BAIJ and SBAIJ matrices (unfortunate dependency) */
610: PetscErrorCode MatCreateSubMatrices_MPIBAIJ_local(Mat C, PetscInt ismax, const IS isrow[], const IS iscol[], MatReuse scall, Mat *submats)
611: {
612:   Mat_MPIBAIJ     *c = (Mat_MPIBAIJ *)C->data;
613:   Mat              A = c->A;
614:   Mat_SeqBAIJ     *a = (Mat_SeqBAIJ *)A->data, *b = (Mat_SeqBAIJ *)c->B->data, *subc;
615:   const PetscInt **icol, **irow;
616:   PetscInt        *nrow, *ncol, start;
617:   PetscMPIInt      rank, size, tag0, tag2, tag3, tag4, *w1, *w2, *w3, *w4, nrqr;
618:   PetscInt       **sbuf1, **sbuf2, *sbuf2_i, i, j, k, l, ct1, ct2, **rbuf1, row, proc = -1;
619:   PetscInt         nrqs = 0, msz, **ptr = NULL, *req_size = NULL, *ctr = NULL, *pa, *tmp = NULL, tcol;
620:   PetscInt       **rbuf3 = NULL, *req_source1 = NULL, *req_source2, **sbuf_aj, **rbuf2 = NULL, max1, max2;
621:   PetscInt       **lens, is_no, ncols, *cols, mat_i, *mat_j, tmp2, jmax;
622: #if defined(PETSC_USE_CTABLE)
623:   PetscHMapI *cmap, cmap_i = NULL, *rmap, rmap_i;
624: #else
625:   PetscInt **cmap, *cmap_i = NULL, **rmap, *rmap_i;
626: #endif
627:   const PetscInt *irow_i, *icol_i;
628:   PetscInt        ctr_j, *sbuf1_j, *sbuf_aj_i, *rbuf1_i, kmax, *lens_i;
629:   MPI_Request    *s_waits1, *r_waits1, *s_waits2, *r_waits2, *r_waits3;
630:   MPI_Request    *r_waits4, *s_waits3, *s_waits4;
631:   MPI_Comm        comm;
632:   PetscScalar   **rbuf4, *rbuf4_i = NULL, **sbuf_aa, *vals, *mat_a = NULL, *imat_a = NULL, *sbuf_aa_i;
633:   PetscMPIInt    *onodes1, *olengths1, end;
634:   PetscInt      **row2proc, *row2proc_i, *imat_ilen, *imat_j, *imat_i;
635:   Mat_SubSppt    *smat_i;
636:   PetscBool      *issorted, colflag, iscsorted = PETSC_TRUE;
637:   PetscInt       *sbuf1_i, *rbuf2_i, *rbuf3_i, ilen;
638:   PetscInt        bs = C->rmap->bs, bs2 = c->bs2, rstart = c->rstartbs;
639:   PetscBool       ijonly = c->ijonly; /* private flag indicates only matrix data structures are requested */
640:   PetscInt        nzA, nzB, *a_i = a->i, *b_i = b->i, *a_j = a->j, *b_j = b->j, ctmp, imark, *cworkA, *cworkB;
641:   PetscScalar    *vworkA = NULL, *vworkB = NULL, *a_a = a->a, *b_a = b->a;
642:   PetscInt        cstart = c->cstartbs, *bmap = c->garray;
643:   PetscBool      *allrows, *allcolumns;

645:   PetscFunctionBegin;
646:   PetscCall(PetscObjectGetComm((PetscObject)C, &comm));
647:   size = c->size;
648:   rank = c->rank;

650:   PetscCall(PetscMalloc5(ismax, &row2proc, ismax, &cmap, ismax, &rmap, ismax + 1, &allcolumns, ismax, &allrows));
651:   PetscCall(PetscMalloc5(ismax, (PetscInt ***)&irow, ismax, (PetscInt ***)&icol, ismax, &nrow, ismax, &ncol, ismax, &issorted));

653:   for (i = 0; i < ismax; i++) {
654:     PetscCall(ISSorted(iscol[i], &issorted[i]));
655:     if (!issorted[i]) iscsorted = issorted[i]; /* columns are not sorted! */
656:     PetscCall(ISSorted(isrow[i], &issorted[i]));

658:     /* Check for special case: allcolumns */
659:     PetscCall(ISIdentity(iscol[i], &colflag));
660:     PetscCall(ISGetLocalSize(iscol[i], &ncol[i]));

662:     if (colflag && ncol[i] == c->Nbs) {
663:       allcolumns[i] = PETSC_TRUE;
664:       icol[i]       = NULL;
665:     } else {
666:       allcolumns[i] = PETSC_FALSE;
667:       PetscCall(ISGetIndices(iscol[i], &icol[i]));
668:     }

670:     /* Check for special case: allrows */
671:     PetscCall(ISIdentity(isrow[i], &colflag));
672:     PetscCall(ISGetLocalSize(isrow[i], &nrow[i]));
673:     if (colflag && nrow[i] == c->Mbs) {
674:       allrows[i] = PETSC_TRUE;
675:       irow[i]    = NULL;
676:     } else {
677:       allrows[i] = PETSC_FALSE;
678:       PetscCall(ISGetIndices(isrow[i], &irow[i]));
679:     }
680:   }

682:   if (scall == MAT_REUSE_MATRIX) {
683:     /* Assumes new rows are same length as the old rows */
684:     for (i = 0; i < ismax; i++) {
685:       subc = (Mat_SeqBAIJ *)(submats[i]->data);
686:       PetscCheck(subc->mbs == nrow[i] && subc->nbs == ncol[i], PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Cannot reuse matrix. wrong size");

688:       /* Initial matrix as if empty */
689:       PetscCall(PetscArrayzero(subc->ilen, subc->mbs));

691:       /* Initial matrix as if empty */
692:       submats[i]->factortype = C->factortype;

694:       smat_i = subc->submatis1;

696:       nrqs        = smat_i->nrqs;
697:       nrqr        = smat_i->nrqr;
698:       rbuf1       = smat_i->rbuf1;
699:       rbuf2       = smat_i->rbuf2;
700:       rbuf3       = smat_i->rbuf3;
701:       req_source2 = smat_i->req_source2;

703:       sbuf1 = smat_i->sbuf1;
704:       sbuf2 = smat_i->sbuf2;
705:       ptr   = smat_i->ptr;
706:       tmp   = smat_i->tmp;
707:       ctr   = smat_i->ctr;

709:       pa          = smat_i->pa;
710:       req_size    = smat_i->req_size;
711:       req_source1 = smat_i->req_source1;

713:       allcolumns[i] = smat_i->allcolumns;
714:       allrows[i]    = smat_i->allrows;
715:       row2proc[i]   = smat_i->row2proc;
716:       rmap[i]       = smat_i->rmap;
717:       cmap[i]       = smat_i->cmap;
718:     }

720:     if (!ismax) { /* Get dummy submatrices and retrieve struct submatis1 */
721:       PetscCheck(submats[0], PETSC_COMM_SELF, PETSC_ERR_ARG_NULL, "submats are null, cannot reuse");
722:       smat_i = (Mat_SubSppt *)submats[0]->data;

724:       nrqs        = smat_i->nrqs;
725:       nrqr        = smat_i->nrqr;
726:       rbuf1       = smat_i->rbuf1;
727:       rbuf2       = smat_i->rbuf2;
728:       rbuf3       = smat_i->rbuf3;
729:       req_source2 = smat_i->req_source2;

731:       sbuf1 = smat_i->sbuf1;
732:       sbuf2 = smat_i->sbuf2;
733:       ptr   = smat_i->ptr;
734:       tmp   = smat_i->tmp;
735:       ctr   = smat_i->ctr;

737:       pa          = smat_i->pa;
738:       req_size    = smat_i->req_size;
739:       req_source1 = smat_i->req_source1;

741:       allcolumns[0] = PETSC_FALSE;
742:     }
743:   } else { /* scall == MAT_INITIAL_MATRIX */
744:     /* Get some new tags to keep the communication clean */
745:     PetscCall(PetscObjectGetNewTag((PetscObject)C, &tag2));
746:     PetscCall(PetscObjectGetNewTag((PetscObject)C, &tag3));

748:     /* evaluate communication - mesg to who, length of mesg, and buffer space
749:      required. Based on this, buffers are allocated, and data copied into them*/
750:     PetscCall(PetscCalloc4(size, &w1, size, &w2, size, &w3, size, &w4)); /* mesg size, initialize work vectors */

752:     for (i = 0; i < ismax; i++) {
753:       jmax   = nrow[i];
754:       irow_i = irow[i];

756:       PetscCall(PetscMalloc1(jmax, &row2proc_i));
757:       row2proc[i] = row2proc_i;

759:       if (issorted[i]) proc = 0;
760:       for (j = 0; j < jmax; j++) {
761:         if (!issorted[i]) proc = 0;
762:         if (allrows[i]) row = j;
763:         else row = irow_i[j];

765:         while (row >= c->rangebs[proc + 1]) proc++;
766:         w4[proc]++;
767:         row2proc_i[j] = proc; /* map row index to proc */
768:       }
769:       for (j = 0; j < size; j++) {
770:         if (w4[j]) {
771:           w1[j] += w4[j];
772:           w3[j]++;
773:           w4[j] = 0;
774:         }
775:       }
776:     }

778:     nrqs     = 0; /* no of outgoing messages */
779:     msz      = 0; /* total mesg length (for all procs) */
780:     w1[rank] = 0; /* no mesg sent to self */
781:     w3[rank] = 0;
782:     for (i = 0; i < size; i++) {
783:       if (w1[i]) {
784:         w2[i] = 1;
785:         nrqs++;
786:       } /* there exists a message to proc i */
787:     }
788:     PetscCall(PetscMalloc1(nrqs, &pa)); /*(proc -array)*/
789:     for (i = 0, j = 0; i < size; i++) {
790:       if (w1[i]) {
791:         pa[j] = i;
792:         j++;
793:       }
794:     }

796:     /* Each message would have a header = 1 + 2*(no of IS) + data */
797:     for (i = 0; i < nrqs; i++) {
798:       j = pa[i];
799:       w1[j] += w2[j] + 2 * w3[j];
800:       msz += w1[j];
801:     }
802:     PetscCall(PetscInfo(0, "Number of outgoing messages %" PetscInt_FMT " Total message length %" PetscInt_FMT "\n", nrqs, msz));

804:     /* Determine the number of messages to expect, their lengths, from from-ids */
805:     PetscCall(PetscGatherNumberOfMessages(comm, w2, w1, &nrqr));
806:     PetscCall(PetscGatherMessageLengths(comm, nrqs, nrqr, w1, &onodes1, &olengths1));

808:     /* Now post the Irecvs corresponding to these messages */
809:     PetscCall(PetscObjectGetNewTag((PetscObject)C, &tag0));
810:     PetscCall(PetscPostIrecvInt(comm, tag0, nrqr, onodes1, olengths1, &rbuf1, &r_waits1));

812:     /* Allocate Memory for outgoing messages */
813:     PetscCall(PetscMalloc4(size, &sbuf1, size, &ptr, 2 * msz, &tmp, size, &ctr));
814:     PetscCall(PetscArrayzero(sbuf1, size));
815:     PetscCall(PetscArrayzero(ptr, size));

817:     {
818:       PetscInt *iptr = tmp;
819:       k              = 0;
820:       for (i = 0; i < nrqs; i++) {
821:         j = pa[i];
822:         iptr += k;
823:         sbuf1[j] = iptr;
824:         k        = w1[j];
825:       }
826:     }

828:     /* Form the outgoing messages. Initialize the header space */
829:     for (i = 0; i < nrqs; i++) {
830:       j           = pa[i];
831:       sbuf1[j][0] = 0;
832:       PetscCall(PetscArrayzero(sbuf1[j] + 1, 2 * w3[j]));
833:       ptr[j] = sbuf1[j] + 2 * w3[j] + 1;
834:     }

836:     /* Parse the isrow and copy data into outbuf */
837:     for (i = 0; i < ismax; i++) {
838:       row2proc_i = row2proc[i];
839:       PetscCall(PetscArrayzero(ctr, size));
840:       irow_i = irow[i];
841:       jmax   = nrow[i];
842:       for (j = 0; j < jmax; j++) { /* parse the indices of each IS */
843:         proc = row2proc_i[j];
844:         if (allrows[i]) row = j;
845:         else row = irow_i[j];

847:         if (proc != rank) { /* copy to the outgoing buf*/
848:           ctr[proc]++;
849:           *ptr[proc] = row;
850:           ptr[proc]++;
851:         }
852:       }
853:       /* Update the headers for the current IS */
854:       for (j = 0; j < size; j++) { /* Can Optimise this loop too */
855:         if ((ctr_j = ctr[j])) {
856:           sbuf1_j            = sbuf1[j];
857:           k                  = ++sbuf1_j[0];
858:           sbuf1_j[2 * k]     = ctr_j;
859:           sbuf1_j[2 * k - 1] = i;
860:         }
861:       }
862:     }

864:     /*  Now  post the sends */
865:     PetscCall(PetscMalloc1(nrqs, &s_waits1));
866:     for (i = 0; i < nrqs; ++i) {
867:       j = pa[i];
868:       PetscCallMPI(MPI_Isend(sbuf1[j], w1[j], MPIU_INT, j, tag0, comm, s_waits1 + i));
869:     }

871:     /* Post Receives to capture the buffer size */
872:     PetscCall(PetscMalloc1(nrqs, &r_waits2));
873:     PetscCall(PetscMalloc3(nrqs, &req_source2, nrqs, &rbuf2, nrqs, &rbuf3));
874:     if (nrqs) rbuf2[0] = tmp + msz;
875:     for (i = 1; i < nrqs; ++i) rbuf2[i] = rbuf2[i - 1] + w1[pa[i - 1]];
876:     for (i = 0; i < nrqs; ++i) {
877:       j = pa[i];
878:       PetscCallMPI(MPI_Irecv(rbuf2[i], w1[j], MPIU_INT, j, tag2, comm, r_waits2 + i));
879:     }

881:     /* Send to other procs the buf size they should allocate */
882:     /* Receive messages*/
883:     PetscCall(PetscMalloc1(nrqr, &s_waits2));
884:     PetscCall(PetscMalloc3(nrqr, &sbuf2, nrqr, &req_size, nrqr, &req_source1));

886:     PetscCallMPI(MPI_Waitall(nrqr, r_waits1, MPI_STATUSES_IGNORE));
887:     for (i = 0; i < nrqr; ++i) {
888:       req_size[i] = 0;
889:       rbuf1_i     = rbuf1[i];
890:       start       = 2 * rbuf1_i[0] + 1;
891:       end         = olengths1[i];
892:       PetscCall(PetscMalloc1(end, &sbuf2[i]));
893:       sbuf2_i = sbuf2[i];
894:       for (j = start; j < end; j++) {
895:         row        = rbuf1_i[j] - rstart;
896:         ncols      = a_i[row + 1] - a_i[row] + b_i[row + 1] - b_i[row];
897:         sbuf2_i[j] = ncols;
898:         req_size[i] += ncols;
899:       }
900:       req_source1[i] = onodes1[i];
901:       /* form the header */
902:       sbuf2_i[0] = req_size[i];
903:       for (j = 1; j < start; j++) sbuf2_i[j] = rbuf1_i[j];

905:       PetscCallMPI(MPI_Isend(sbuf2_i, end, MPIU_INT, req_source1[i], tag2, comm, s_waits2 + i));
906:     }

908:     PetscCall(PetscFree(onodes1));
909:     PetscCall(PetscFree(olengths1));

911:     PetscCall(PetscFree(r_waits1));
912:     PetscCall(PetscFree4(w1, w2, w3, w4));

914:     /* Receive messages*/
915:     PetscCall(PetscMalloc1(nrqs, &r_waits3));

917:     PetscCallMPI(MPI_Waitall(nrqs, r_waits2, MPI_STATUSES_IGNORE));
918:     for (i = 0; i < nrqs; ++i) {
919:       PetscCall(PetscMalloc1(rbuf2[i][0], &rbuf3[i]));
920:       req_source2[i] = pa[i];
921:       PetscCallMPI(MPI_Irecv(rbuf3[i], rbuf2[i][0], MPIU_INT, req_source2[i], tag3, comm, r_waits3 + i));
922:     }
923:     PetscCall(PetscFree(r_waits2));

925:     /* Wait on sends1 and sends2 */
926:     PetscCallMPI(MPI_Waitall(nrqs, s_waits1, MPI_STATUSES_IGNORE));
927:     PetscCallMPI(MPI_Waitall(nrqr, s_waits2, MPI_STATUSES_IGNORE));
928:     PetscCall(PetscFree(s_waits1));
929:     PetscCall(PetscFree(s_waits2));

931:     /* Now allocate sending buffers for a->j, and send them off */
932:     PetscCall(PetscMalloc1(nrqr, &sbuf_aj));
933:     for (i = 0, j = 0; i < nrqr; i++) j += req_size[i];
934:     if (nrqr) PetscCall(PetscMalloc1(j, &sbuf_aj[0]));
935:     for (i = 1; i < nrqr; i++) sbuf_aj[i] = sbuf_aj[i - 1] + req_size[i - 1];

937:     PetscCall(PetscMalloc1(nrqr, &s_waits3));
938:     {
939:       for (i = 0; i < nrqr; i++) {
940:         rbuf1_i   = rbuf1[i];
941:         sbuf_aj_i = sbuf_aj[i];
942:         ct1       = 2 * rbuf1_i[0] + 1;
943:         ct2       = 0;
944:         for (j = 1, max1 = rbuf1_i[0]; j <= max1; j++) {
945:           kmax = rbuf1[i][2 * j];
946:           for (k = 0; k < kmax; k++, ct1++) {
947:             row    = rbuf1_i[ct1] - rstart;
948:             nzA    = a_i[row + 1] - a_i[row];
949:             nzB    = b_i[row + 1] - b_i[row];
950:             ncols  = nzA + nzB;
951:             cworkA = a_j + a_i[row];
952:             cworkB = b_j + b_i[row];

954:             /* load the column indices for this row into cols */
955:             cols = sbuf_aj_i + ct2;
956:             for (l = 0; l < nzB; l++) {
957:               if ((ctmp = bmap[cworkB[l]]) < cstart) cols[l] = ctmp;
958:               else break;
959:             }
960:             imark = l;
961:             for (l = 0; l < nzA; l++) cols[imark + l] = cstart + cworkA[l];
962:             for (l = imark; l < nzB; l++) cols[nzA + l] = bmap[cworkB[l]];
963:             ct2 += ncols;
964:           }
965:         }
966:         PetscCallMPI(MPI_Isend(sbuf_aj_i, req_size[i], MPIU_INT, req_source1[i], tag3, comm, s_waits3 + i));
967:       }
968:     }

970:     /* create col map: global col of C -> local col of submatrices */
971: #if defined(PETSC_USE_CTABLE)
972:     for (i = 0; i < ismax; i++) {
973:       if (!allcolumns[i]) {
974:         PetscCall(PetscHMapICreateWithSize(ncol[i], cmap + i));

976:         jmax   = ncol[i];
977:         icol_i = icol[i];
978:         cmap_i = cmap[i];
979:         for (j = 0; j < jmax; j++) PetscCall(PetscHMapISet(cmap[i], icol_i[j] + 1, j + 1));
980:       } else cmap[i] = NULL;
981:     }
982: #else
983:     for (i = 0; i < ismax; i++) {
984:       if (!allcolumns[i]) {
985:         PetscCall(PetscCalloc1(c->Nbs, &cmap[i]));
986:         jmax   = ncol[i];
987:         icol_i = icol[i];
988:         cmap_i = cmap[i];
989:         for (j = 0; j < jmax; j++) cmap_i[icol_i[j]] = j + 1;
990:       } else cmap[i] = NULL;
991:     }
992: #endif

994:     /* Create lens which is required for MatCreate... */
995:     for (i = 0, j = 0; i < ismax; i++) j += nrow[i];
996:     PetscCall(PetscMalloc1(ismax, &lens));

998:     if (ismax) PetscCall(PetscCalloc1(j, &lens[0]));
999:     for (i = 1; i < ismax; i++) lens[i] = lens[i - 1] + nrow[i - 1];

1001:     /* Update lens from local data */
1002:     for (i = 0; i < ismax; i++) {
1003:       row2proc_i = row2proc[i];
1004:       jmax       = nrow[i];
1005:       if (!allcolumns[i]) cmap_i = cmap[i];
1006:       irow_i = irow[i];
1007:       lens_i = lens[i];
1008:       for (j = 0; j < jmax; j++) {
1009:         if (allrows[i]) row = j;
1010:         else row = irow_i[j]; /* global blocked row of C */

1012:         proc = row2proc_i[j];
1013:         if (proc == rank) {
1014:           /* Get indices from matA and then from matB */
1015: #if defined(PETSC_USE_CTABLE)
1016:           PetscInt tt;
1017: #endif
1018:           row    = row - rstart;
1019:           nzA    = a_i[row + 1] - a_i[row];
1020:           nzB    = b_i[row + 1] - b_i[row];
1021:           cworkA = a_j + a_i[row];
1022:           cworkB = b_j + b_i[row];

1024:           if (!allcolumns[i]) {
1025: #if defined(PETSC_USE_CTABLE)
1026:             for (k = 0; k < nzA; k++) {
1027:               PetscCall(PetscHMapIGetWithDefault(cmap_i, cstart + cworkA[k] + 1, 0, &tt));
1028:               if (tt) lens_i[j]++;
1029:             }
1030:             for (k = 0; k < nzB; k++) {
1031:               PetscCall(PetscHMapIGetWithDefault(cmap_i, bmap[cworkB[k]] + 1, 0, &tt));
1032:               if (tt) lens_i[j]++;
1033:             }

1035: #else
1036:             for (k = 0; k < nzA; k++) {
1037:               if (cmap_i[cstart + cworkA[k]]) lens_i[j]++;
1038:             }
1039:             for (k = 0; k < nzB; k++) {
1040:               if (cmap_i[bmap[cworkB[k]]]) lens_i[j]++;
1041:             }
1042: #endif
1043:           } else { /* allcolumns */
1044:             lens_i[j] = nzA + nzB;
1045:           }
1046:         }
1047:       }
1048:     }

1050:     /* Create row map: global row of C -> local row of submatrices */
1051:     for (i = 0; i < ismax; i++) {
1052:       if (!allrows[i]) {
1053: #if defined(PETSC_USE_CTABLE)
1054:         PetscCall(PetscHMapICreateWithSize(nrow[i], rmap + i));
1055:         irow_i = irow[i];
1056:         jmax   = nrow[i];
1057:         for (j = 0; j < jmax; j++) {
1058:           if (allrows[i]) {
1059:             PetscCall(PetscHMapISet(rmap[i], j + 1, j + 1));
1060:           } else {
1061:             PetscCall(PetscHMapISet(rmap[i], irow_i[j] + 1, j + 1));
1062:           }
1063:         }
1064: #else
1065:         PetscCall(PetscCalloc1(c->Mbs, &rmap[i]));
1066:         rmap_i = rmap[i];
1067:         irow_i = irow[i];
1068:         jmax   = nrow[i];
1069:         for (j = 0; j < jmax; j++) {
1070:           if (allrows[i]) rmap_i[j] = j;
1071:           else rmap_i[irow_i[j]] = j;
1072:         }
1073: #endif
1074:       } else rmap[i] = NULL;
1075:     }

1077:     /* Update lens from offproc data */
1078:     {
1079:       PetscInt *rbuf2_i, *rbuf3_i, *sbuf1_i;

1081:       PetscCallMPI(MPI_Waitall(nrqs, r_waits3, MPI_STATUSES_IGNORE));
1082:       for (tmp2 = 0; tmp2 < nrqs; tmp2++) {
1083:         sbuf1_i = sbuf1[pa[tmp2]];
1084:         jmax    = sbuf1_i[0];
1085:         ct1     = 2 * jmax + 1;
1086:         ct2     = 0;
1087:         rbuf2_i = rbuf2[tmp2];
1088:         rbuf3_i = rbuf3[tmp2];
1089:         for (j = 1; j <= jmax; j++) {
1090:           is_no  = sbuf1_i[2 * j - 1];
1091:           max1   = sbuf1_i[2 * j];
1092:           lens_i = lens[is_no];
1093:           if (!allcolumns[is_no]) cmap_i = cmap[is_no];
1094:           rmap_i = rmap[is_no];
1095:           for (k = 0; k < max1; k++, ct1++) {
1096:             if (allrows[is_no]) {
1097:               row = sbuf1_i[ct1];
1098:             } else {
1099: #if defined(PETSC_USE_CTABLE)
1100:               PetscCall(PetscHMapIGetWithDefault(rmap_i, sbuf1_i[ct1] + 1, 0, &row));
1101:               row--;
1102:               PetscCheck(row >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "row not found in table");
1103: #else
1104:               row = rmap_i[sbuf1_i[ct1]]; /* the val in the new matrix to be */
1105: #endif
1106:             }
1107:             max2 = rbuf2_i[ct1];
1108:             for (l = 0; l < max2; l++, ct2++) {
1109:               if (!allcolumns[is_no]) {
1110: #if defined(PETSC_USE_CTABLE)
1111:                 PetscCall(PetscHMapIGetWithDefault(cmap_i, rbuf3_i[ct2] + 1, 0, &tcol));
1112: #else
1113:                 tcol = cmap_i[rbuf3_i[ct2]];
1114: #endif
1115:                 if (tcol) lens_i[row]++;
1116:               } else {         /* allcolumns */
1117:                 lens_i[row]++; /* lens_i[row] += max2 ? */
1118:               }
1119:             }
1120:           }
1121:         }
1122:       }
1123:     }
1124:     PetscCall(PetscFree(r_waits3));
1125:     PetscCallMPI(MPI_Waitall(nrqr, s_waits3, MPI_STATUSES_IGNORE));
1126:     PetscCall(PetscFree(s_waits3));

1128:     /* Create the submatrices */
1129:     for (i = 0; i < ismax; i++) {
1130:       PetscInt bs_tmp;
1131:       if (ijonly) bs_tmp = 1;
1132:       else bs_tmp = bs;

1134:       PetscCall(MatCreate(PETSC_COMM_SELF, submats + i));
1135:       PetscCall(MatSetSizes(submats[i], nrow[i] * bs_tmp, ncol[i] * bs_tmp, PETSC_DETERMINE, PETSC_DETERMINE));

1137:       PetscCall(MatSetType(submats[i], ((PetscObject)A)->type_name));
1138:       PetscCall(MatSeqBAIJSetPreallocation(submats[i], bs_tmp, 0, lens[i]));
1139:       PetscCall(MatSeqSBAIJSetPreallocation(submats[i], bs_tmp, 0, lens[i])); /* this subroutine is used by SBAIJ routines */

1141:       /* create struct Mat_SubSppt and attached it to submat */
1142:       PetscCall(PetscNew(&smat_i));
1143:       subc            = (Mat_SeqBAIJ *)submats[i]->data;
1144:       subc->submatis1 = smat_i;

1146:       smat_i->destroy          = submats[i]->ops->destroy;
1147:       submats[i]->ops->destroy = MatDestroySubMatrix_SeqBAIJ;
1148:       submats[i]->factortype   = C->factortype;

1150:       smat_i->id          = i;
1151:       smat_i->nrqs        = nrqs;
1152:       smat_i->nrqr        = nrqr;
1153:       smat_i->rbuf1       = rbuf1;
1154:       smat_i->rbuf2       = rbuf2;
1155:       smat_i->rbuf3       = rbuf3;
1156:       smat_i->sbuf2       = sbuf2;
1157:       smat_i->req_source2 = req_source2;

1159:       smat_i->sbuf1 = sbuf1;
1160:       smat_i->ptr   = ptr;
1161:       smat_i->tmp   = tmp;
1162:       smat_i->ctr   = ctr;

1164:       smat_i->pa          = pa;
1165:       smat_i->req_size    = req_size;
1166:       smat_i->req_source1 = req_source1;

1168:       smat_i->allcolumns = allcolumns[i];
1169:       smat_i->allrows    = allrows[i];
1170:       smat_i->singleis   = PETSC_FALSE;
1171:       smat_i->row2proc   = row2proc[i];
1172:       smat_i->rmap       = rmap[i];
1173:       smat_i->cmap       = cmap[i];
1174:     }

1176:     if (!ismax) { /* Create dummy submats[0] for reuse struct subc */
1177:       PetscCall(MatCreate(PETSC_COMM_SELF, &submats[0]));
1178:       PetscCall(MatSetSizes(submats[0], 0, 0, PETSC_DETERMINE, PETSC_DETERMINE));
1179:       PetscCall(MatSetType(submats[0], MATDUMMY));

1181:       /* create struct Mat_SubSppt and attached it to submat */
1182:       PetscCall(PetscNew(&smat_i));
1183:       submats[0]->data = (void *)smat_i;

1185:       smat_i->destroy          = submats[0]->ops->destroy;
1186:       submats[0]->ops->destroy = MatDestroySubMatrix_Dummy;
1187:       submats[0]->factortype   = C->factortype;

1189:       smat_i->id          = 0;
1190:       smat_i->nrqs        = nrqs;
1191:       smat_i->nrqr        = nrqr;
1192:       smat_i->rbuf1       = rbuf1;
1193:       smat_i->rbuf2       = rbuf2;
1194:       smat_i->rbuf3       = rbuf3;
1195:       smat_i->sbuf2       = sbuf2;
1196:       smat_i->req_source2 = req_source2;

1198:       smat_i->sbuf1 = sbuf1;
1199:       smat_i->ptr   = ptr;
1200:       smat_i->tmp   = tmp;
1201:       smat_i->ctr   = ctr;

1203:       smat_i->pa          = pa;
1204:       smat_i->req_size    = req_size;
1205:       smat_i->req_source1 = req_source1;

1207:       smat_i->allcolumns = PETSC_FALSE;
1208:       smat_i->singleis   = PETSC_FALSE;
1209:       smat_i->row2proc   = NULL;
1210:       smat_i->rmap       = NULL;
1211:       smat_i->cmap       = NULL;
1212:     }

1214:     if (ismax) PetscCall(PetscFree(lens[0]));
1215:     PetscCall(PetscFree(lens));
1216:     if (sbuf_aj) {
1217:       PetscCall(PetscFree(sbuf_aj[0]));
1218:       PetscCall(PetscFree(sbuf_aj));
1219:     }

1221:   } /* endof scall == MAT_INITIAL_MATRIX */

1223:   /* Post recv matrix values */
1224:   if (!ijonly) {
1225:     PetscCall(PetscObjectGetNewTag((PetscObject)C, &tag4));
1226:     PetscCall(PetscMalloc1(nrqs, &rbuf4));
1227:     PetscCall(PetscMalloc1(nrqs, &r_waits4));
1228:     for (i = 0; i < nrqs; ++i) {
1229:       PetscCall(PetscMalloc1(rbuf2[i][0] * bs2, &rbuf4[i]));
1230:       PetscCallMPI(MPI_Irecv(rbuf4[i], rbuf2[i][0] * bs2, MPIU_SCALAR, req_source2[i], tag4, comm, r_waits4 + i));
1231:     }

1233:     /* Allocate sending buffers for a->a, and send them off */
1234:     PetscCall(PetscMalloc1(nrqr, &sbuf_aa));
1235:     for (i = 0, j = 0; i < nrqr; i++) j += req_size[i];

1237:     if (nrqr) PetscCall(PetscMalloc1(j * bs2, &sbuf_aa[0]));
1238:     for (i = 1; i < nrqr; i++) sbuf_aa[i] = sbuf_aa[i - 1] + req_size[i - 1] * bs2;

1240:     PetscCall(PetscMalloc1(nrqr, &s_waits4));

1242:     for (i = 0; i < nrqr; i++) {
1243:       rbuf1_i   = rbuf1[i];
1244:       sbuf_aa_i = sbuf_aa[i];
1245:       ct1       = 2 * rbuf1_i[0] + 1;
1246:       ct2       = 0;
1247:       for (j = 1, max1 = rbuf1_i[0]; j <= max1; j++) {
1248:         kmax = rbuf1_i[2 * j];
1249:         for (k = 0; k < kmax; k++, ct1++) {
1250:           row    = rbuf1_i[ct1] - rstart;
1251:           nzA    = a_i[row + 1] - a_i[row];
1252:           nzB    = b_i[row + 1] - b_i[row];
1253:           ncols  = nzA + nzB;
1254:           cworkB = b_j + b_i[row];
1255:           vworkA = a_a + a_i[row] * bs2;
1256:           vworkB = b_a + b_i[row] * bs2;

1258:           /* load the column values for this row into vals*/
1259:           vals = sbuf_aa_i + ct2 * bs2;
1260:           for (l = 0; l < nzB; l++) {
1261:             if ((bmap[cworkB[l]]) < cstart) {
1262:               PetscCall(PetscArraycpy(vals + l * bs2, vworkB + l * bs2, bs2));
1263:             } else break;
1264:           }
1265:           imark = l;
1266:           for (l = 0; l < nzA; l++) PetscCall(PetscArraycpy(vals + (imark + l) * bs2, vworkA + l * bs2, bs2));
1267:           for (l = imark; l < nzB; l++) PetscCall(PetscArraycpy(vals + (nzA + l) * bs2, vworkB + l * bs2, bs2));

1269:           ct2 += ncols;
1270:         }
1271:       }
1272:       PetscCallMPI(MPI_Isend(sbuf_aa_i, req_size[i] * bs2, MPIU_SCALAR, req_source1[i], tag4, comm, s_waits4 + i));
1273:     }
1274:   }

1276:   /* Assemble the matrices */
1277:   /* First assemble the local rows */
1278:   for (i = 0; i < ismax; i++) {
1279:     row2proc_i = row2proc[i];
1280:     subc       = (Mat_SeqBAIJ *)submats[i]->data;
1281:     imat_ilen  = subc->ilen;
1282:     imat_j     = subc->j;
1283:     imat_i     = subc->i;
1284:     imat_a     = subc->a;

1286:     if (!allcolumns[i]) cmap_i = cmap[i];
1287:     rmap_i = rmap[i];
1288:     irow_i = irow[i];
1289:     jmax   = nrow[i];
1290:     for (j = 0; j < jmax; j++) {
1291:       if (allrows[i]) row = j;
1292:       else row = irow_i[j];
1293:       proc = row2proc_i[j];

1295:       if (proc == rank) {
1296:         row    = row - rstart;
1297:         nzA    = a_i[row + 1] - a_i[row];
1298:         nzB    = b_i[row + 1] - b_i[row];
1299:         cworkA = a_j + a_i[row];
1300:         cworkB = b_j + b_i[row];
1301:         if (!ijonly) {
1302:           vworkA = a_a + a_i[row] * bs2;
1303:           vworkB = b_a + b_i[row] * bs2;
1304:         }

1306:         if (allrows[i]) {
1307:           row = row + rstart;
1308:         } else {
1309: #if defined(PETSC_USE_CTABLE)
1310:           PetscCall(PetscHMapIGetWithDefault(rmap_i, row + rstart + 1, 0, &row));
1311:           row--;

1313:           PetscCheck(row >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "row not found in table");
1314: #else
1315:           row = rmap_i[row + rstart];
1316: #endif
1317:         }
1318:         mat_i = imat_i[row];
1319:         if (!ijonly) mat_a = imat_a + mat_i * bs2;
1320:         mat_j = imat_j + mat_i;
1321:         ilen  = imat_ilen[row];

1323:         /* load the column indices for this row into cols*/
1324:         if (!allcolumns[i]) {
1325:           for (l = 0; l < nzB; l++) {
1326:             if ((ctmp = bmap[cworkB[l]]) < cstart) {
1327: #if defined(PETSC_USE_CTABLE)
1328:               PetscCall(PetscHMapIGetWithDefault(cmap_i, ctmp + 1, 0, &tcol));
1329:               if (tcol) {
1330: #else
1331:               if ((tcol = cmap_i[ctmp])) {
1332: #endif
1333:                 *mat_j++ = tcol - 1;
1334:                 PetscCall(PetscArraycpy(mat_a, vworkB + l * bs2, bs2));
1335:                 mat_a += bs2;
1336:                 ilen++;
1337:               }
1338:             } else break;
1339:           }
1340:           imark = l;
1341:           for (l = 0; l < nzA; l++) {
1342: #if defined(PETSC_USE_CTABLE)
1343:             PetscCall(PetscHMapIGetWithDefault(cmap_i, cstart + cworkA[l] + 1, 0, &tcol));
1344:             if (tcol) {
1345: #else
1346:             if ((tcol = cmap_i[cstart + cworkA[l]])) {
1347: #endif
1348:               *mat_j++ = tcol - 1;
1349:               if (!ijonly) {
1350:                 PetscCall(PetscArraycpy(mat_a, vworkA + l * bs2, bs2));
1351:                 mat_a += bs2;
1352:               }
1353:               ilen++;
1354:             }
1355:           }
1356:           for (l = imark; l < nzB; l++) {
1357: #if defined(PETSC_USE_CTABLE)
1358:             PetscCall(PetscHMapIGetWithDefault(cmap_i, bmap[cworkB[l]] + 1, 0, &tcol));
1359:             if (tcol) {
1360: #else
1361:             if ((tcol = cmap_i[bmap[cworkB[l]]])) {
1362: #endif
1363:               *mat_j++ = tcol - 1;
1364:               if (!ijonly) {
1365:                 PetscCall(PetscArraycpy(mat_a, vworkB + l * bs2, bs2));
1366:                 mat_a += bs2;
1367:               }
1368:               ilen++;
1369:             }
1370:           }
1371:         } else { /* allcolumns */
1372:           for (l = 0; l < nzB; l++) {
1373:             if ((ctmp = bmap[cworkB[l]]) < cstart) {
1374:               *mat_j++ = ctmp;
1375:               PetscCall(PetscArraycpy(mat_a, vworkB + l * bs2, bs2));
1376:               mat_a += bs2;
1377:               ilen++;
1378:             } else break;
1379:           }
1380:           imark = l;
1381:           for (l = 0; l < nzA; l++) {
1382:             *mat_j++ = cstart + cworkA[l];
1383:             if (!ijonly) {
1384:               PetscCall(PetscArraycpy(mat_a, vworkA + l * bs2, bs2));
1385:               mat_a += bs2;
1386:             }
1387:             ilen++;
1388:           }
1389:           for (l = imark; l < nzB; l++) {
1390:             *mat_j++ = bmap[cworkB[l]];
1391:             if (!ijonly) {
1392:               PetscCall(PetscArraycpy(mat_a, vworkB + l * bs2, bs2));
1393:               mat_a += bs2;
1394:             }
1395:             ilen++;
1396:           }
1397:         }
1398:         imat_ilen[row] = ilen;
1399:       }
1400:     }
1401:   }

1403:   /* Now assemble the off proc rows */
1404:   if (!ijonly) PetscCallMPI(MPI_Waitall(nrqs, r_waits4, MPI_STATUSES_IGNORE));
1405:   for (tmp2 = 0; tmp2 < nrqs; tmp2++) {
1406:     sbuf1_i = sbuf1[pa[tmp2]];
1407:     jmax    = sbuf1_i[0];
1408:     ct1     = 2 * jmax + 1;
1409:     ct2     = 0;
1410:     rbuf2_i = rbuf2[tmp2];
1411:     rbuf3_i = rbuf3[tmp2];
1412:     if (!ijonly) rbuf4_i = rbuf4[tmp2];
1413:     for (j = 1; j <= jmax; j++) {
1414:       is_no  = sbuf1_i[2 * j - 1];
1415:       rmap_i = rmap[is_no];
1416:       if (!allcolumns[is_no]) cmap_i = cmap[is_no];
1417:       subc      = (Mat_SeqBAIJ *)submats[is_no]->data;
1418:       imat_ilen = subc->ilen;
1419:       imat_j    = subc->j;
1420:       imat_i    = subc->i;
1421:       if (!ijonly) imat_a = subc->a;
1422:       max1 = sbuf1_i[2 * j];
1423:       for (k = 0; k < max1; k++, ct1++) { /* for each recved block row */
1424:         row = sbuf1_i[ct1];

1426:         if (allrows[is_no]) {
1427:           row = sbuf1_i[ct1];
1428:         } else {
1429: #if defined(PETSC_USE_CTABLE)
1430:           PetscCall(PetscHMapIGetWithDefault(rmap_i, row + 1, 0, &row));
1431:           row--;
1432:           PetscCheck(row >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "row not found in table");
1433: #else
1434:           row = rmap_i[row];
1435: #endif
1436:         }
1437:         ilen  = imat_ilen[row];
1438:         mat_i = imat_i[row];
1439:         if (!ijonly) mat_a = imat_a + mat_i * bs2;
1440:         mat_j = imat_j + mat_i;
1441:         max2  = rbuf2_i[ct1];
1442:         if (!allcolumns[is_no]) {
1443:           for (l = 0; l < max2; l++, ct2++) {
1444: #if defined(PETSC_USE_CTABLE)
1445:             PetscCall(PetscHMapIGetWithDefault(cmap_i, rbuf3_i[ct2] + 1, 0, &tcol));
1446: #else
1447:             tcol = cmap_i[rbuf3_i[ct2]];
1448: #endif
1449:             if (tcol) {
1450:               *mat_j++ = tcol - 1;
1451:               if (!ijonly) {
1452:                 PetscCall(PetscArraycpy(mat_a, rbuf4_i + ct2 * bs2, bs2));
1453:                 mat_a += bs2;
1454:               }
1455:               ilen++;
1456:             }
1457:           }
1458:         } else { /* allcolumns */
1459:           for (l = 0; l < max2; l++, ct2++) {
1460:             *mat_j++ = rbuf3_i[ct2]; /* same global column index of C */
1461:             if (!ijonly) {
1462:               PetscCall(PetscArraycpy(mat_a, rbuf4_i + ct2 * bs2, bs2));
1463:               mat_a += bs2;
1464:             }
1465:             ilen++;
1466:           }
1467:         }
1468:         imat_ilen[row] = ilen;
1469:       }
1470:     }
1471:   }

1473:   if (!iscsorted) { /* sort column indices of the rows */
1474:     MatScalar *work;

1476:     PetscCall(PetscMalloc1(bs2, &work));
1477:     for (i = 0; i < ismax; i++) {
1478:       subc      = (Mat_SeqBAIJ *)submats[i]->data;
1479:       imat_ilen = subc->ilen;
1480:       imat_j    = subc->j;
1481:       imat_i    = subc->i;
1482:       if (!ijonly) imat_a = subc->a;
1483:       if (allcolumns[i]) continue;

1485:       jmax = nrow[i];
1486:       for (j = 0; j < jmax; j++) {
1487:         mat_i = imat_i[j];
1488:         mat_j = imat_j + mat_i;
1489:         ilen  = imat_ilen[j];
1490:         if (ijonly) {
1491:           PetscCall(PetscSortInt(ilen, mat_j));
1492:         } else {
1493:           mat_a = imat_a + mat_i * bs2;
1494:           PetscCall(PetscSortIntWithDataArray(ilen, mat_j, mat_a, bs2 * sizeof(MatScalar), work));
1495:         }
1496:       }
1497:     }
1498:     PetscCall(PetscFree(work));
1499:   }

1501:   if (!ijonly) {
1502:     PetscCall(PetscFree(r_waits4));
1503:     PetscCallMPI(MPI_Waitall(nrqr, s_waits4, MPI_STATUSES_IGNORE));
1504:     PetscCall(PetscFree(s_waits4));
1505:   }

1507:   /* Restore the indices */
1508:   for (i = 0; i < ismax; i++) {
1509:     if (!allrows[i]) PetscCall(ISRestoreIndices(isrow[i], irow + i));
1510:     if (!allcolumns[i]) PetscCall(ISRestoreIndices(iscol[i], icol + i));
1511:   }

1513:   for (i = 0; i < ismax; i++) {
1514:     PetscCall(MatAssemblyBegin(submats[i], MAT_FINAL_ASSEMBLY));
1515:     PetscCall(MatAssemblyEnd(submats[i], MAT_FINAL_ASSEMBLY));
1516:   }

1518:   PetscCall(PetscFree5(*(PetscInt ***)&irow, *(PetscInt ***)&icol, nrow, ncol, issorted));
1519:   PetscCall(PetscFree5(row2proc, cmap, rmap, allcolumns, allrows));

1521:   if (!ijonly) {
1522:     if (sbuf_aa) {
1523:       PetscCall(PetscFree(sbuf_aa[0]));
1524:       PetscCall(PetscFree(sbuf_aa));
1525:     }

1527:     for (i = 0; i < nrqs; ++i) PetscCall(PetscFree(rbuf4[i]));
1528:     PetscCall(PetscFree(rbuf4));
1529:   }
1530:   c->ijonly = PETSC_FALSE; /* set back to the default */
1531:   PetscFunctionReturn(PETSC_SUCCESS);
1532: }