Actual source code: vscat.c

  1: #define PETSCVEC_DLL
  2: /*
  3:      Code for creating scatters between vectors. This file 
  4:   includes the code for scattering between sequential vectors and
  5:   some special cases for parallel scatters.
  6: */

 8:  #include private/isimpl.h
 9:  #include private/vecimpl.h

 11: /* Logging support */
 12: PetscCookie  VEC_SCATTER_COOKIE;

 14: #if defined(PETSC_USE_DEBUG)
 15: /*
 16:      Checks if any indices are less than zero and generates an error
 17: */
 20: static PetscErrorCode VecScatterCheckIndices_Private(PetscInt nmax,PetscInt n,const PetscInt *idx)
 21: {
 22:   PetscInt i;

 25:   for (i=0; i<n; i++) {
 26:     if (idx[i] < 0)     SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Negative index %D at %D location",idx[i],i);
 27:     if (idx[i] >= nmax) SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Index %D at %D location greater than max %D",idx[i],i,nmax);
 28:   }
 29:   return(0);
 30: }
 31: #endif

 33: /*
 34:       This is special scatter code for when the entire parallel vector is copied to each processor.

 36:    This code was written by Cameron Cooper, Occidental College, Fall 1995,
 37:    will working at ANL as a SERS student.
 38: */
 41: PetscErrorCode VecScatterBegin_MPI_ToAll(VecScatter ctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
 42: {
 44:   PetscInt       yy_n,xx_n;
 45:   PetscScalar    *xv,*yv;

 48:   VecGetLocalSize(y,&yy_n);
 49:   VecGetLocalSize(x,&xx_n);
 50:   VecGetArray(y,&yv);
 51:   if (x != y) {VecGetArray(x,&xv);}

 53:   if (mode & SCATTER_REVERSE) {
 54:     PetscScalar          *xvt,*xvt2;
 55:     VecScatter_MPI_ToAll *scat = (VecScatter_MPI_ToAll*)ctx->todata;
 56:     PetscInt             i;
 57:     PetscMPIInt          *disply = scat->displx;

 59:     if (addv == INSERT_VALUES) {
 60:       PetscInt rstart,rend;
 61:       /* 
 62:          copy the correct part of the local vector into the local storage of 
 63:          the MPI one.  Note: this operation only makes sense if all the local 
 64:          vectors have the same values
 65:       */
 66:       VecGetOwnershipRange(y,&rstart,&rend);
 67:       PetscMemcpy(yv,xv+rstart,yy_n*sizeof(PetscScalar));
 68:     } else {
 69:       MPI_Comm    comm;
 70:       PetscMPIInt rank;
 71:       PetscObjectGetComm((PetscObject)y,&comm);
 72:       MPI_Comm_rank(comm,&rank);
 73:       if (scat->work1) xvt = scat->work1;
 74:       else {
 75:         PetscMalloc(xx_n*sizeof(PetscScalar),&xvt);
 76:         scat->work1 = xvt;
 77:       }
 78:       if (!rank) { /* I am the zeroth processor, values are accumulated here */
 79:         if   (scat->work2) xvt2 = scat->work2;
 80:         else {
 81:           PetscMalloc(xx_n*sizeof(PetscScalar),& xvt2);
 82:           scat->work2 = xvt2;
 83:         }
 84:         MPI_Gatherv(yv,yy_n,MPIU_SCALAR,xvt2,scat->count,disply,MPIU_SCALAR,0,((PetscObject)ctx)->comm);
 85: #if defined(PETSC_USE_COMPLEX)
 86:         MPI_Reduce(xv,xvt,2*xx_n,MPIU_REAL,MPI_SUM,0,((PetscObject)ctx)->comm);
 87: #else
 88:         MPI_Reduce(xv,xvt,xx_n,MPIU_SCALAR,MPI_SUM,0,((PetscObject)ctx)->comm);
 89: #endif
 90:         if (addv == ADD_VALUES) {
 91:           for (i=0; i<xx_n; i++) {
 92:             xvt[i] += xvt2[i];
 93:           }
 94: #if !defined(PETSC_USE_COMPLEX)
 95:         } else if (addv == MAX_VALUES) {
 96:           for (i=0; i<xx_n; i++) {
 97:             xvt[i] = PetscMax(xvt[i],xvt2[i]);
 98:           }
 99: #endif
100:         } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
101:         MPI_Scatterv(xvt,scat->count,disply,MPIU_SCALAR,yv,yy_n,MPIU_SCALAR,0,((PetscObject)ctx)->comm);
102:       } else {
103:         MPI_Gatherv(yv,yy_n,MPIU_SCALAR,0, 0,0,MPIU_SCALAR,0,((PetscObject)ctx)->comm);
104: #if defined(PETSC_USE_COMPLEX)
105:         MPI_Reduce(xv,xvt,2*xx_n,MPIU_REAL,MPI_SUM,0,((PetscObject)ctx)->comm);
106: #else
107:         MPI_Reduce(xv,xvt,xx_n,MPIU_SCALAR,MPI_SUM,0,((PetscObject)ctx)->comm);
108: #endif
109:         MPI_Scatterv(0,scat->count,disply,MPIU_SCALAR,yv,yy_n,MPIU_SCALAR,0,((PetscObject)ctx)->comm);
110:       }
111:     }
112:   } else {
113:     PetscScalar          *yvt;
114:     VecScatter_MPI_ToAll *scat = (VecScatter_MPI_ToAll*)ctx->todata;
115:     PetscInt             i;
116:     PetscMPIInt          *displx = scat->displx;

118:     if (addv == INSERT_VALUES) {
119:       MPI_Allgatherv(xv,xx_n,MPIU_SCALAR,yv,scat->count,displx,MPIU_SCALAR,((PetscObject)ctx)->comm);
120:     } else {
121:       if (scat->work1) yvt = scat->work1;
122:       else {
123:         PetscMalloc(yy_n*sizeof(PetscScalar),&yvt);
124:         scat->work1 = yvt;
125:       }
126:       MPI_Allgatherv(xv,xx_n,MPIU_SCALAR,yvt,scat->count,displx,MPIU_SCALAR,((PetscObject)ctx)->comm);
127:       if (addv == ADD_VALUES){
128:         for (i=0; i<yy_n; i++) {
129:           yv[i] += yvt[i];
130:         }
131: #if !defined(PETSC_USE_COMPLEX)
132:       } else if (addv == MAX_VALUES) {
133:         for (i=0; i<yy_n; i++) {
134:           yv[i] = PetscMax(yv[i],yvt[i]);
135:         }
136: #endif
137:       } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
138:     }
139:   }
140:   VecRestoreArray(y,&yv);
141:   if (x != y) {VecRestoreArray(x,&xv);}
142:   return(0);
143: }

145: /*
146:       This is special scatter code for when the entire parallel vector is  copied to processor 0.

148: */
151: PetscErrorCode VecScatterBegin_MPI_ToOne(VecScatter ctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
152: {
154:   PetscMPIInt    rank;
155:   PetscInt       yy_n,xx_n;
156:   PetscScalar    *xv,*yv;
157:   MPI_Comm       comm;

160:   VecGetLocalSize(y,&yy_n);
161:   VecGetLocalSize(x,&xx_n);
162:   VecGetArray(x,&xv);
163:   VecGetArray(y,&yv);

165:   PetscObjectGetComm((PetscObject)x,&comm);
166:   MPI_Comm_rank(comm,&rank);

168:   /* --------  Reverse scatter; spread from processor 0 to other processors */
169:   if (mode & SCATTER_REVERSE) {
170:     PetscScalar          *yvt;
171:     VecScatter_MPI_ToAll *scat = (VecScatter_MPI_ToAll*)ctx->todata;
172:     PetscInt             i;
173:     PetscMPIInt          *disply = scat->displx;

175:     if (addv == INSERT_VALUES) {
176:       MPI_Scatterv(xv,scat->count,disply,MPIU_SCALAR,yv,yy_n,MPIU_SCALAR,0,((PetscObject)ctx)->comm);
177:     } else {
178:       if (scat->work2) yvt = scat->work2;
179:       else {
180:         PetscMalloc(xx_n*sizeof(PetscScalar),&yvt);
181:         scat->work2 = yvt;
182:       }
183:       MPI_Scatterv(xv,scat->count,disply,MPIU_SCALAR,yvt,yy_n,MPIU_SCALAR,0,((PetscObject)ctx)->comm);
184:       if (addv == ADD_VALUES) {
185:         for (i=0; i<yy_n; i++) {
186:           yv[i] += yvt[i];
187:         }
188: #if !defined(PETSC_USE_COMPLEX)
189:       } else if (addv == MAX_VALUES) {
190:         for (i=0; i<yy_n; i++) {
191:           yv[i] = PetscMax(yv[i],yvt[i]);
192:         }
193: #endif
194:       } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
195:     }
196:   /* ---------  Forward scatter; gather all values onto processor 0 */
197:   } else {
198:     PetscScalar          *yvt  = 0;
199:     VecScatter_MPI_ToAll *scat = (VecScatter_MPI_ToAll*)ctx->todata;
200:     PetscInt             i;
201:     PetscMPIInt          *displx = scat->displx;

203:     if (addv == INSERT_VALUES) {
204:       MPI_Gatherv(xv,xx_n,MPIU_SCALAR,yv,scat->count,displx,MPIU_SCALAR,0,((PetscObject)ctx)->comm);
205:     } else {
206:       if (!rank) {
207:         if (scat->work1) yvt = scat->work1;
208:         else {
209:           PetscMalloc(yy_n*sizeof(PetscScalar),&yvt);
210:           scat->work1 = yvt;
211:         }
212:       }
213:       MPI_Gatherv(xv,xx_n,MPIU_SCALAR,yvt,scat->count,displx,MPIU_SCALAR,0,((PetscObject)ctx)->comm);
214:       if (!rank) {
215:         if (addv == ADD_VALUES) {
216:           for (i=0; i<yy_n; i++) {
217:             yv[i] += yvt[i];
218:           }
219: #if !defined(PETSC_USE_COMPLEX)
220:         } else if (addv == MAX_VALUES) {
221:           for (i=0; i<yy_n; i++) {
222:             yv[i] = PetscMax(yv[i],yvt[i]);
223:           }
224: #endif
225:         }  else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
226:       }
227:     }
228:   }
229:   VecRestoreArray(x,&xv);
230:   VecRestoreArray(y,&yv);
231:   return(0);
232: }

234: /*
235:        The follow to are used for both VecScatterBegin_MPI_ToAll() and VecScatterBegin_MPI_ToOne()
236: */
239: PetscErrorCode VecScatterDestroy_MPI_ToAll(VecScatter ctx)
240: {
241:   VecScatter_MPI_ToAll *scat = (VecScatter_MPI_ToAll*)ctx->todata;
242:   PetscErrorCode       ierr;

245:   PetscFree(scat->work1);
246:   PetscFree(scat->work2);
247:   PetscFree(scat->displx);
248:   PetscFree2(ctx->todata,scat->count);
249:   PetscHeaderDestroy(ctx);
250:   return(0);
251: }

255: PetscErrorCode VecScatterDestroy_SGtoSG(VecScatter ctx)
256: {
257:   PetscErrorCode         ierr;

260:   PetscFree2(((VecScatter_Seq_General*)ctx->todata)->vslots,((VecScatter_Seq_General*)ctx->fromdata)->vslots);
261:   PetscFree2(ctx->todata,ctx->fromdata);
262:   PetscHeaderDestroy(ctx);
263:   return(0);
264: }

268: PetscErrorCode VecScatterDestroy_SGtoSS(VecScatter ctx)
269: {
270:   PetscErrorCode         ierr;

273:   PetscFree(((VecScatter_Seq_General*)ctx->fromdata)->vslots);
274:   PetscFree2(ctx->todata,ctx->fromdata);
275:   PetscHeaderDestroy(ctx);
276:   return(0);
277: }

281: PetscErrorCode VecScatterDestroy_SStoSG(VecScatter ctx)
282: {
283:   PetscErrorCode         ierr;

286:   PetscFree(((VecScatter_Seq_General*)ctx->todata)->vslots);
287:   PetscFree2(ctx->todata,ctx->fromdata);
288:   PetscHeaderDestroy(ctx);
289:   return(0);
290: }

294: PetscErrorCode VecScatterDestroy_SStoSS(VecScatter ctx)
295: {

299:   PetscFree2(ctx->todata,ctx->fromdata);
300:   PetscHeaderDestroy(ctx);
301:   return(0);
302: }

304: /* -------------------------------------------------------------------------*/
307: PetscErrorCode VecScatterCopy_MPI_ToAll(VecScatter in,VecScatter out)
308: {
309:   VecScatter_MPI_ToAll *in_to = (VecScatter_MPI_ToAll*)in->todata,*sto;
310:   PetscErrorCode       ierr;
311:   PetscMPIInt          size;
312:   PetscInt             i;

315:   out->begin          = in->begin;
316:   out->end            = in->end;
317:   out->copy           = in->copy;
318:   out->destroy        = in->destroy;
319:   out->view           = in->view;

321:   MPI_Comm_size(((PetscObject)out)->comm,&size);
322:   PetscMalloc2(1,VecScatter_MPI_ToAll,&sto,size,PetscMPIInt,&sto->count);
323:   PetscMalloc(size*sizeof(PetscMPIInt),&sto->displx);
324:   sto->type           = in_to->type;
325:   for (i=0; i<size; i++) {
326:     sto->count[i]  = in_to->count[i];
327:     sto->displx[i] = in_to->displx[i];
328:   }
329:   sto->work1         = 0;
330:   sto->work2         = 0;
331:   out->todata        = (void*)sto;
332:   out->fromdata      = (void*)0;
333:   return(0);
334: }

336: /* --------------------------------------------------------------------------------------*/
337: /* 
338:         Scatter: sequential general to sequential general 
339: */
342: PetscErrorCode VecScatterBegin_SGtoSG(VecScatter ctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
343: {
344:   VecScatter_Seq_General *gen_to = (VecScatter_Seq_General*)ctx->todata;
345:   VecScatter_Seq_General *gen_from = (VecScatter_Seq_General*)ctx->fromdata;
346:   PetscErrorCode         ierr;
347:   PetscInt               i,n = gen_from->n,*fslots,*tslots;
348:   PetscScalar            *xv,*yv;
349: 
351:   VecGetArray(x,&xv);
352:   if (x != y) {VecGetArray(y,&yv);} else {yv = xv;}
353:   if (mode & SCATTER_REVERSE){
354:     gen_to   = (VecScatter_Seq_General*)ctx->fromdata;
355:     gen_from = (VecScatter_Seq_General*)ctx->todata;
356:   }
357:   fslots = gen_from->vslots;
358:   tslots = gen_to->vslots;

360:   if (addv == INSERT_VALUES) {
361:     for (i=0; i<n; i++) {yv[tslots[i]] = xv[fslots[i]];}
362:   } else if (addv == ADD_VALUES) {
363:     for (i=0; i<n; i++) {yv[tslots[i]] += xv[fslots[i]];}
364: #if !defined(PETSC_USE_COMPLEX)
365:   } else  if (addv == MAX_VALUES) {
366:     for (i=0; i<n; i++) {yv[tslots[i]] = PetscMax(yv[tslots[i]],xv[fslots[i]]);}
367: #endif
368:   } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
369:   VecRestoreArray(x,&xv);
370:   if (x != y) {VecRestoreArray(y,&yv);}
371:   return(0);
372: }

374: /* 
375:     Scatter: sequential general to sequential stride 1 
376: */
379: PetscErrorCode VecScatterBegin_SGtoSS_Stride1(VecScatter ctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
380: {
381:   VecScatter_Seq_Stride  *gen_to   = (VecScatter_Seq_Stride*)ctx->todata;
382:   VecScatter_Seq_General *gen_from = (VecScatter_Seq_General*)ctx->fromdata;
383:   PetscInt               i,n = gen_from->n,*fslots = gen_from->vslots;
384:   PetscErrorCode         ierr;
385:   PetscInt               first = gen_to->first;
386:   PetscScalar            *xv,*yv;
387: 
389:   VecGetArray(x,&xv);
390:   if (x != y) {VecGetArray(y,&yv);} else {yv = xv;}
391:   if (mode & SCATTER_REVERSE){
392:     xv += first;
393:     if (addv == INSERT_VALUES) {
394:       for (i=0; i<n; i++) {yv[fslots[i]] = xv[i];}
395:     } else  if (addv == ADD_VALUES) {
396:       for (i=0; i<n; i++) {yv[fslots[i]] += xv[i];}
397: #if !defined(PETSC_USE_COMPLEX)
398:     } else  if (addv == MAX_VALUES) {
399:       for (i=0; i<n; i++) {yv[fslots[i]] = PetscMax(yv[fslots[i]],xv[i]);}
400: #endif
401:     } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
402:   } else {
403:     yv += first;
404:     if (addv == INSERT_VALUES) {
405:       for (i=0; i<n; i++) {yv[i] = xv[fslots[i]];}
406:     } else  if (addv == ADD_VALUES) {
407:       for (i=0; i<n; i++) {yv[i] += xv[fslots[i]];}
408: #if !defined(PETSC_USE_COMPLEX)
409:     } else if (addv == MAX_VALUES) {
410:       for (i=0; i<n; i++) {yv[i] = PetscMax(yv[i],xv[fslots[i]]);}
411: #endif
412:     } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
413:   }
414:   VecRestoreArray(x,&xv);
415:   if (x != y) {VecRestoreArray(y,&yv);}
416:   return(0);
417: }

419: /* 
420:    Scatter: sequential general to sequential stride 
421: */
424: PetscErrorCode VecScatterBegin_SGtoSS(VecScatter ctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
425: {
426:   VecScatter_Seq_Stride  *gen_to   = (VecScatter_Seq_Stride*)ctx->todata;
427:   VecScatter_Seq_General *gen_from = (VecScatter_Seq_General*)ctx->fromdata;
428:   PetscInt               i,n = gen_from->n,*fslots = gen_from->vslots;
429:   PetscErrorCode         ierr;
430:   PetscInt               first = gen_to->first,step = gen_to->step;
431:   PetscScalar            *xv,*yv;
432: 
434:   VecGetArray(x,&xv);
435:   if (x != y) {VecGetArray(y,&yv);} else {yv = xv;}

437:   if (mode & SCATTER_REVERSE){
438:     if (addv == INSERT_VALUES) {
439:       for (i=0; i<n; i++) {yv[fslots[i]] = xv[first + i*step];}
440:     } else if (addv == ADD_VALUES) {
441:       for (i=0; i<n; i++) {yv[fslots[i]] += xv[first + i*step];}
442: #if !defined(PETSC_USE_COMPLEX)
443:     } else if (addv == MAX_VALUES) {
444:       for (i=0; i<n; i++) {yv[fslots[i]] = PetscMax(yv[fslots[i]],xv[first + i*step]);}
445: #endif
446:     } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
447:   } else {
448:     if (addv == INSERT_VALUES) {
449:       for (i=0; i<n; i++) {yv[first + i*step] = xv[fslots[i]];}
450:     } else if (addv == ADD_VALUES) {
451:       for (i=0; i<n; i++) {yv[first + i*step] += xv[fslots[i]];}
452: #if !defined(PETSC_USE_COMPLEX)
453:     } else if (addv == MAX_VALUES) {
454:       for (i=0; i<n; i++) {yv[first + i*step] = PetscMax(yv[first + i*step],xv[fslots[i]]);}
455: #endif
456:     } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
457:   }
458:   VecRestoreArray(x,&xv);
459:   if (x != y) {VecRestoreArray(y,&yv);}
460:   return(0);
461: }

463: /* 
464:     Scatter: sequential stride 1 to sequential general 
465: */
468: PetscErrorCode VecScatterBegin_SStoSG_Stride1(VecScatter ctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
469: {
470:   VecScatter_Seq_Stride  *gen_from = (VecScatter_Seq_Stride*)ctx->fromdata;
471:   VecScatter_Seq_General *gen_to   = (VecScatter_Seq_General*)ctx->todata;
472:   PetscInt               i,n = gen_from->n,*fslots = gen_to->vslots;
473:   PetscErrorCode         ierr;
474:   PetscInt               first = gen_from->first;
475:   PetscScalar            *xv,*yv;
476: 
478:   VecGetArray(x,&xv);
479:   if (x != y) {VecGetArray(y,&yv);} else {yv = xv;}

481:   if (mode & SCATTER_REVERSE){
482:     yv += first;
483:     if (addv == INSERT_VALUES) {
484:       for (i=0; i<n; i++) {yv[i] = xv[fslots[i]];}
485:     } else  if (addv == ADD_VALUES) {
486:       for (i=0; i<n; i++) {yv[i] += xv[fslots[i]];}
487: #if !defined(PETSC_USE_COMPLEX)
488:     } else  if (addv == MAX_VALUES) {
489:       for (i=0; i<n; i++) {yv[i] = PetscMax(yv[i],xv[fslots[i]]);}
490: #endif
491:     } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
492:   } else {
493:     xv += first;
494:     if (addv == INSERT_VALUES) {
495:       for (i=0; i<n; i++) {yv[fslots[i]] = xv[i];}
496:     } else  if (addv == ADD_VALUES) {
497:       for (i=0; i<n; i++) {yv[fslots[i]] += xv[i];}
498: #if !defined(PETSC_USE_COMPLEX)
499:     } else  if (addv == MAX_VALUES) {
500:       for (i=0; i<n; i++) {yv[fslots[i]] = PetscMax(yv[fslots[i]],xv[i]);}
501: #endif
502:     } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
503:   }
504:   VecRestoreArray(x,&xv);
505:   if (x != y) {VecRestoreArray(y,&yv);}
506:   return(0);
507: }

511: /* 
512:    Scatter: sequential stride to sequential general 
513: */
514: PetscErrorCode VecScatterBegin_SStoSG(VecScatter ctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
515: {
516:   VecScatter_Seq_Stride  *gen_from = (VecScatter_Seq_Stride*)ctx->fromdata;
517:   VecScatter_Seq_General *gen_to   = (VecScatter_Seq_General*)ctx->todata;
518:   PetscInt               i,n = gen_from->n,*fslots = gen_to->vslots;
519:   PetscErrorCode         ierr;
520:   PetscInt               first = gen_from->first,step = gen_from->step;
521:   PetscScalar            *xv,*yv;
522: 
524:   VecGetArray(x,&xv);
525:   if (x != y) {VecGetArray(y,&yv);} else {yv = xv;}

527:   if (mode & SCATTER_REVERSE){
528:     if (addv == INSERT_VALUES) {
529:       for (i=0; i<n; i++) {yv[first + i*step] = xv[fslots[i]];}
530:     } else  if (addv == ADD_VALUES) {
531:       for (i=0; i<n; i++) {yv[first + i*step] += xv[fslots[i]];}
532: #if !defined(PETSC_USE_COMPLEX)
533:     } else  if (addv == MAX_VALUES) {
534:       for (i=0; i<n; i++) {yv[first + i*step] = PetscMax(yv[first + i*step],xv[fslots[i]]);}
535: #endif
536:     } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
537:   } else {
538:     if (addv == INSERT_VALUES) {
539:       for (i=0; i<n; i++) {yv[fslots[i]] = xv[first + i*step];}
540:     } else  if (addv == ADD_VALUES) {
541:       for (i=0; i<n; i++) {yv[fslots[i]] += xv[first + i*step];}
542: #if !defined(PETSC_USE_COMPLEX)
543:     } else  if (addv == MAX_VALUES) {
544:       for (i=0; i<n; i++) {yv[fslots[i]] = PetscMax(yv[fslots[i]],xv[first + i*step]);}
545: #endif
546:     } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
547:   }
548:   VecRestoreArray(x,&xv);
549:   if (x != y) {VecRestoreArray(y,&yv);}
550:   return(0);
551: }

553: /* 
554:      Scatter: sequential stride to sequential stride 
555: */
558: PetscErrorCode VecScatterBegin_SStoSS(VecScatter ctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
559: {
560:   VecScatter_Seq_Stride *gen_to   = (VecScatter_Seq_Stride*)ctx->todata;
561:   VecScatter_Seq_Stride *gen_from = (VecScatter_Seq_Stride*)ctx->fromdata;
562:   PetscInt              i,n = gen_from->n,to_first = gen_to->first,to_step = gen_to->step;
563:   PetscErrorCode        ierr;
564:   PetscInt              from_first = gen_from->first,from_step = gen_from->step;
565:   PetscScalar           *xv,*yv;
566: 
568:   VecGetArray(x,&xv);
569:   if (x != y) {VecGetArray(y,&yv);} else {yv = xv;}

571:   if (mode & SCATTER_REVERSE){
572:     from_first = gen_to->first;
573:     to_first   = gen_from->first;
574:     from_step  = gen_to->step;
575:     to_step    = gen_from->step;
576:   }

578:   if (addv == INSERT_VALUES) {
579:     if (to_step == 1 && from_step == 1) {
580:       PetscMemcpy(yv+to_first,xv+from_first,n*sizeof(PetscScalar));
581:     } else  {
582:       for (i=0; i<n; i++) {
583:         yv[to_first + i*to_step] = xv[from_first+i*from_step];
584:       }
585:     }
586:   } else if (addv == ADD_VALUES) {
587:     if (to_step == 1 && from_step == 1) {
588:       yv += to_first; xv += from_first;
589:       for (i=0; i<n; i++) {
590:         yv[i] += xv[i];
591:       }
592:     } else {
593:       for (i=0; i<n; i++) {
594:         yv[to_first + i*to_step] += xv[from_first+i*from_step];
595:       }
596:     }
597: #if !defined(PETSC_USE_COMPLEX)
598:   } else if (addv == MAX_VALUES) {
599:     if (to_step == 1 && from_step == 1) {
600:       yv += to_first; xv += from_first;
601:       for (i=0; i<n; i++) {
602:         yv[i] = PetscMax(yv[i],xv[i]);
603:       }
604:     } else {
605:       for (i=0; i<n; i++) {
606:         yv[to_first + i*to_step] = PetscMax(yv[to_first + i*to_step],xv[from_first+i*from_step]);
607:       }
608:     }
609: #endif
610:   } else {SETERRQ(PETSC_ERR_ARG_UNKNOWN_TYPE,"Wrong insert option");}
611:   VecRestoreArray(x,&xv);
612:   if (x != y) {VecRestoreArray(y,&yv);}
613:   return(0);
614: }

616: /* --------------------------------------------------------------------------*/


621: PetscErrorCode VecScatterCopy_SGToSG(VecScatter in,VecScatter out)
622: {
623:   PetscErrorCode         ierr;
624:   VecScatter_Seq_General *in_to   = (VecScatter_Seq_General*)in->todata,*out_to = PETSC_NULL;
625:   VecScatter_Seq_General *in_from = (VecScatter_Seq_General*)in->fromdata,*out_from = PETSC_NULL;
626: 
628:   out->begin         = in->begin;
629:   out->end           = in->end;
630:   out->copy          = in->copy;
631:   out->destroy       = in->destroy;
632:   out->view          = in->view;

634:   PetscMalloc2(1,VecScatter_Seq_General,&out_to,1,VecScatter_Seq_General,&out_from);
635:   PetscMalloc2(in_to->n,PetscInt,&out_to->vslots,in_from->n,PetscInt,&out_from->vslots);
636:   out_to->n                      = in_to->n;
637:   out_to->type                   = in_to->type;
638:   out_to->nonmatching_computed   = PETSC_FALSE;
639:   out_to->slots_nonmatching      = 0;
640:   out_to->is_copy                = PETSC_FALSE;
641:   PetscMemcpy(out_to->vslots,in_to->vslots,(out_to->n)*sizeof(PetscInt));

643:   out_from->n                    = in_from->n;
644:   out_from->type                 = in_from->type;
645:   out_from->nonmatching_computed = PETSC_FALSE;
646:   out_from->slots_nonmatching    = 0;
647:   out_from->is_copy              = PETSC_FALSE;
648:   PetscMemcpy(out_from->vslots,in_from->vslots,(out_from->n)*sizeof(PetscInt));

650:   out->todata     = (void*)out_to;
651:   out->fromdata   = (void*)out_from;
652:   return(0);
653: }


658: PetscErrorCode VecScatterCopy_SGToSS(VecScatter in,VecScatter out)
659: {
660:   PetscErrorCode         ierr;
661:   VecScatter_Seq_Stride  *in_to   = (VecScatter_Seq_Stride*)in->todata,*out_to = PETSC_NULL;
662:   VecScatter_Seq_General *in_from = (VecScatter_Seq_General*)in->fromdata,*out_from = PETSC_NULL;
663: 
665:   out->begin         = in->begin;
666:   out->end           = in->end;
667:   out->copy          = in->copy;
668:   out->destroy       = in->destroy;
669:   out->view          = in->view;

671:   PetscMalloc2(1,VecScatter_Seq_Stride,&out_to,1,VecScatter_Seq_General,&out_from);
672:   PetscMalloc(in_from->n*sizeof(PetscInt),&out_from->vslots);
673:   out_to->n       = in_to->n;
674:   out_to->type    = in_to->type;
675:   out_to->first   = in_to->first;
676:   out_to->step    = in_to->step;
677:   out_to->type    = in_to->type;

679:   out_from->n                    = in_from->n;
680:   out_from->type                 = in_from->type;
681:   out_from->nonmatching_computed = PETSC_FALSE;
682:   out_from->slots_nonmatching    = 0;
683:   out_from->is_copy              = PETSC_FALSE;
684:   PetscMemcpy(out_from->vslots,in_from->vslots,(out_from->n)*sizeof(PetscInt));

686:   out->todata     = (void*)out_to;
687:   out->fromdata   = (void*)out_from;
688:   return(0);
689: }

691: /* --------------------------------------------------------------------------*/
692: /* 
693:     Scatter: parallel to sequential vector, sequential strides for both. 
694: */
697: PetscErrorCode VecScatterCopy_SStoSS(VecScatter in,VecScatter out)
698: {
699:   VecScatter_Seq_Stride *in_to   = (VecScatter_Seq_Stride*)in->todata,*out_to = PETSC_NULL;
700:   VecScatter_Seq_Stride *in_from = (VecScatter_Seq_Stride*)in->fromdata,*out_from = PETSC_NULL;
701:   PetscErrorCode        ierr;

704:   out->begin      = in->begin;
705:   out->end        = in->end;
706:   out->copy       = in->copy;
707:   out->destroy    = in->destroy;
708:   out->view       = in->view;

710:   PetscMalloc2(1,VecScatter_Seq_Stride,&out_to,1,VecScatter_Seq_Stride,&out_from);
711:   out_to->n       = in_to->n;
712:   out_to->type    = in_to->type;
713:   out_to->first   = in_to->first;
714:   out_to->step    = in_to->step;
715:   out_to->type    = in_to->type;
716:   out_from->n     = in_from->n;
717:   out_from->type  = in_from->type;
718:   out_from->first = in_from->first;
719:   out_from->step  = in_from->step;
720:   out_from->type  = in_from->type;
721:   out->todata     = (void*)out_to;
722:   out->fromdata   = (void*)out_from;
723:   return(0);
724: }

726: EXTERN PetscErrorCode VecScatterCreate_PtoS(PetscInt,const PetscInt *,PetscInt,const PetscInt *,Vec,Vec,PetscInt,VecScatter);
727: EXTERN PetscErrorCode VecScatterCreate_PtoP(PetscInt,const PetscInt *,PetscInt,const PetscInt *,Vec,Vec,VecScatter);
728: EXTERN PetscErrorCode VecScatterCreate_StoP(PetscInt,const PetscInt *,PetscInt,const PetscInt *,Vec,Vec,PetscInt,VecScatter);

730: /* =======================================================================*/
731: #define VEC_SEQ_ID 0
732: #define VEC_MPI_ID 1

734: /*
735:    Blocksizes we have optimized scatters for 
736: */

738: #define VecScatterOptimizedBS(mbs) ((2 <= mbs && mbs <= 8) || mbs == 12)

740: PetscErrorCode  VecScatterCreateEmpty(MPI_Comm comm,VecScatter *newctx)
741: {
742:   VecScatter     ctx;

746:   PetscHeaderCreate(ctx,_p_VecScatter,int,VEC_SCATTER_COOKIE,0,"VecScatter",comm,VecScatterDestroy,VecScatterView);
747:   ctx->inuse               = PETSC_FALSE;
748:   ctx->beginandendtogether = PETSC_FALSE;
749:   PetscOptionsGetTruth(PETSC_NULL,"-vecscatter_merge",&ctx->beginandendtogether,PETSC_NULL);
750:   if (ctx->beginandendtogether) {
751:     PetscInfo(ctx,"Using combined (merged) vector scatter begin and end\n");
752:   }
753:   ctx->packtogether = PETSC_FALSE;
754:   PetscOptionsGetTruth(PETSC_NULL,"-vecscatter_packtogether",&ctx->packtogether,PETSC_NULL);
755:   if (ctx->packtogether) {
756:     PetscInfo(ctx,"Pack all messages before sending\n");
757:   }
758:   *newctx = ctx;
759:   return(0);
760: }

764: /*@C
765:    VecScatterCreate - Creates a vector scatter context.

767:    Collective on Vec

769:    Input Parameters:
770: +  xin - a vector that defines the shape (parallel data layout of the vector)
771:          of vectors from which we scatter
772: .  yin - a vector that defines the shape (parallel data layout of the vector)
773:          of vectors to which we scatter
774: .  ix - the indices of xin to scatter (if PETSC_NULL scatters all values)
775: -  iy - the indices of yin to hold results (if PETSC_NULL fills entire vector yin)

777:    Output Parameter:
778: .  newctx - location to store the new scatter context

780:    Options Database Keys: (uses regular MPI_Sends by default)
781: +  -vecscatter_view         - Prints detail of communications
782: .  -vecscatter_view_info    - Print less details about communication
783: .  -vecscatter_ssend        - Uses MPI_Ssend_init() instead of MPI_Send_init() 
784: .  -vecscatter_rsend           - use ready receiver mode for MPI sends 
785: .  -vecscatter_merge        - VecScatterBegin() handles all of the communication, VecScatterEnd() is a nop 
786:                               eliminates the chance for overlap of computation and communication 
787: .  -vecscatter_sendfirst    - Posts sends before receives 
788: .  -vecscatter_packtogether - Pack all messages before sending, receive all messages before unpacking
789: .  -vecscatter_alltoall     - Uses MPI all to all communication for scatter
790: .  -vecscatter_window       - Use MPI 2 window operations to move data
791: -  -vecscatter_nopack       - Avoid packing to work vector when possible (if used with -vecscatter_alltoall then will use MPI_Alltoallw()

793: $
794: $                                                                                    --When packing is used--
795: $                               MPI Datatypes (no packing)  sendfirst   merge        packtogether  persistent*    
796: $                                _nopack                   _sendfirst    _merge      _packtogether                -vecscatter_
797: $ ----------------------------------------------------------------------------------------------------------------------------
798: $    Message passing    Send       p                           X            X           X         always
799: $                      Ssend       p                           X            X           X         always          _ssend
800: $                      Rsend       p                        nonsense        X           X         always          _rsend
801: $    AlltoAll  v or w              X                        nonsense     always         X         nonsense        _alltoall
802: $    MPI_Win                       p                        nonsense        p           p         nonsense        _window
803: $                              
804: $   Since persistent sends and receives require a constant memory address they can only be used when data is packed into the work vector
805: $   because the in and out array may be different for each call to VecScatterBegin/End().
806: $
807: $    p indicates possible, but not implemented. X indicates implemented
808: $

810:     Level: intermediate

812:   Notes:
813:    In calls to VecScatter() you can use different vectors than the xin and 
814:    yin you used above; BUT they must have the same parallel data layout, for example,
815:    they could be obtained from VecDuplicate().
816:    A VecScatter context CANNOT be used in two or more simultaneous scatters;
817:    that is you cannot call a second VecScatterBegin() with the same scatter
818:    context until the VecScatterEnd() has been called on the first VecScatterBegin().
819:    In this case a separate VecScatter is needed for each concurrent scatter.

821:    Currently the MPI_Send(), MPI_Ssend() and MPI_Rsend() all use PERSISTENT versions.
822:    (this unfortunately requires that the same in and out arrays be used for each use, this
823:     is why when not using MPI_alltoallw() we always need to pack the input into the work array before sending
824:     and unpack upon receeving instead of using MPI datatypes to avoid the packing/unpacking).

826:    Concepts: scatter^between vectors
827:    Concepts: gather^between vectors

829: .seealso: VecScatterDestroy(), VecScatterCreateToAll(), VecScatterCreateToZero()
830: @*/
831: PetscErrorCode  VecScatterCreate(Vec xin,IS ix,Vec yin,IS iy,VecScatter *newctx)
832: {
833:   VecScatter     ctx;
835:   PetscMPIInt    size;
836:   PetscInt       totalv,xin_type = VEC_SEQ_ID,yin_type = VEC_SEQ_ID,*range;
837:   MPI_Comm       comm,ycomm;
838:   PetscTruth     ixblock,iyblock,iystride,islocal,cando,flag;
839:   IS             tix = 0,tiy = 0;


843:   /*
844:       Determine if the vectors are "parallel", ie. it shares a comm with other processors, or
845:       sequential (it does not share a comm). The difference is that parallel vectors treat the 
846:       index set as providing indices in the global parallel numbering of the vector, with 
847:       sequential vectors treat the index set as providing indices in the local sequential
848:       numbering
849:   */
850:   PetscObjectGetComm((PetscObject)xin,&comm);
851:   MPI_Comm_size(comm,&size);
852:   if (size > 1) {xin_type = VEC_MPI_ID;}

854:   PetscObjectGetComm((PetscObject)yin,&ycomm);
855:   MPI_Comm_size(ycomm,&size);
856:   if (size > 1) {comm = ycomm; yin_type = VEC_MPI_ID;}
857: 
858:   /* generate the Scatter context */
859:   PetscHeaderCreate(ctx,_p_VecScatter,int,VEC_SCATTER_COOKIE,0,"VecScatter",comm,VecScatterDestroy,VecScatterView);
860:   ctx->inuse               = PETSC_FALSE;

862:   ctx->beginandendtogether = PETSC_FALSE;
863:   PetscOptionsGetTruth(PETSC_NULL,"-vecscatter_merge",&ctx->beginandendtogether,PETSC_NULL);
864:   if (ctx->beginandendtogether) {
865:     PetscInfo(ctx,"Using combined (merged) vector scatter begin and end\n");
866:   }
867:   ctx->packtogether = PETSC_FALSE;
868:   PetscOptionsGetTruth(PETSC_NULL,"-vecscatter_packtogether",&ctx->packtogether,PETSC_NULL);
869:   if (ctx->packtogether) {
870:     PetscInfo(ctx,"Pack all messages before sending\n");
871:   }

873:   VecGetLocalSize(xin,&ctx->from_n);
874:   VecGetLocalSize(yin,&ctx->to_n);

876:   /*
877:       if ix or iy is not included; assume just grabbing entire vector
878:   */
879:   if (!ix && xin_type == VEC_SEQ_ID) {
880:     ISCreateStride(comm,ctx->from_n,0,1,&ix);
881:     tix  = ix;
882:   } else if (!ix && xin_type == VEC_MPI_ID) {
883:     if (yin_type == VEC_MPI_ID) {
884:       PetscInt ntmp, low;
885:       VecGetLocalSize(xin,&ntmp);
886:       VecGetOwnershipRange(xin,&low,PETSC_NULL);
887:       ISCreateStride(comm,ntmp,low,1,&ix);
888:     } else{
889:       PetscInt Ntmp;
890:       VecGetSize(xin,&Ntmp);
891:       ISCreateStride(comm,Ntmp,0,1,&ix);
892:     }
893:     tix  = ix;
894:   } else if (!ix) {
895:     SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"ix not given, but not Seq or MPI vector");
896:   }

898:   if (!iy && yin_type == VEC_SEQ_ID) {
899:     ISCreateStride(comm,ctx->to_n,0,1,&iy);
900:     tiy  = iy;
901:   } else if (!iy && yin_type == VEC_MPI_ID) {
902:     if (xin_type == VEC_MPI_ID) {
903:       PetscInt ntmp, low;
904:       VecGetLocalSize(yin,&ntmp);
905:       VecGetOwnershipRange(yin,&low,PETSC_NULL);
906:       ISCreateStride(comm,ntmp,low,1,&iy);
907:     } else{
908:       PetscInt Ntmp;
909:       VecGetSize(yin,&Ntmp);
910:       ISCreateStride(comm,Ntmp,0,1,&iy);
911:     }
912:     tiy  = iy;
913:   } else if (!iy) {
914:     SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"iy not given, but not Seq or MPI vector");
915:   }

917:   /* ===========================================================================================================
918:         Check for special cases
919:      ==========================================================================================================*/
920:   /* ---------------------------------------------------------------------------*/
921:   if (xin_type == VEC_SEQ_ID && yin_type == VEC_SEQ_ID) {
922:     if (((PetscObject)ix)->type == IS_GENERAL && ((PetscObject)iy)->type == IS_GENERAL){
923:       PetscInt               nx,ny;
924:       const PetscInt         *idx,*idy;
925:       VecScatter_Seq_General *to = PETSC_NULL,*from = PETSC_NULL;

927:       ISGetLocalSize(ix,&nx);
928:       ISGetLocalSize(iy,&ny);
929:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
930:       ISGetIndices(ix,&idx);
931:       ISGetIndices(iy,&idy);
932:       PetscMalloc2(1,VecScatter_Seq_General,&to,1,VecScatter_Seq_General,&from);
933:       PetscMalloc2(nx,PetscInt,&to->vslots,nx,PetscInt,&from->vslots);
934:       to->n             = nx;
935: #if defined(PETSC_USE_DEBUG)
936:       VecScatterCheckIndices_Private(ctx->to_n,ny,idy);
937: #endif
938:       PetscMemcpy(to->vslots,idy,nx*sizeof(PetscInt));
939:       from->n           = nx;
940: #if defined(PETSC_USE_DEBUG)
941:       VecScatterCheckIndices_Private(ctx->from_n,nx,idx);
942: #endif
943:        PetscMemcpy(from->vslots,idx,nx*sizeof(PetscInt));
944:       to->type          = VEC_SCATTER_SEQ_GENERAL;
945:       from->type        = VEC_SCATTER_SEQ_GENERAL;
946:       ctx->todata       = (void*)to;
947:       ctx->fromdata     = (void*)from;
948:       ctx->begin        = VecScatterBegin_SGtoSG;
949:       ctx->end          = 0;
950:       ctx->destroy      = VecScatterDestroy_SGtoSG;
951:       ctx->copy         = VecScatterCopy_SGToSG;
952:       PetscInfo(xin,"Special case: sequential vector general scatter\n");
953:       goto functionend;
954:     } else if (((PetscObject)ix)->type == IS_STRIDE &&  ((PetscObject)iy)->type == IS_STRIDE){
955:       PetscInt               nx,ny,to_first,to_step,from_first,from_step;
956:       VecScatter_Seq_Stride  *from8 = PETSC_NULL,*to8 = PETSC_NULL;

958:       ISGetLocalSize(ix,&nx);
959:       ISGetLocalSize(iy,&ny);
960:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
961:       ISStrideGetInfo(iy,&to_first,&to_step);
962:       ISStrideGetInfo(ix,&from_first,&from_step);
963:       PetscMalloc2(1,VecScatter_Seq_Stride,&to8,1,VecScatter_Seq_Stride,&from8);
964:       to8->n             = nx;
965:       to8->first         = to_first;
966:       to8->step          = to_step;
967:       from8->n           = nx;
968:       from8->first       = from_first;
969:       from8->step        = from_step;
970:       to8->type          = VEC_SCATTER_SEQ_STRIDE;
971:       from8->type        = VEC_SCATTER_SEQ_STRIDE;
972:       ctx->todata       = (void*)to8;
973:       ctx->fromdata     = (void*)from8;
974:       ctx->begin        = VecScatterBegin_SStoSS;
975:       ctx->end          = 0;
976:       ctx->destroy      = VecScatterDestroy_SStoSS;
977:       ctx->copy         = VecScatterCopy_SStoSS;
978:       PetscInfo(xin,"Special case: sequential vector stride to stride\n");
979:       goto functionend;
980:     } else if (((PetscObject)ix)->type == IS_GENERAL && ((PetscObject)iy)->type == IS_STRIDE){
981:       PetscInt               nx,ny,first,step;
982:       const PetscInt         *idx;
983:       VecScatter_Seq_General *from9 = PETSC_NULL;
984:       VecScatter_Seq_Stride  *to9 = PETSC_NULL;

986:       ISGetLocalSize(ix,&nx);
987:       ISGetIndices(ix,&idx);
988:       ISGetLocalSize(iy,&ny);
989:       ISStrideGetInfo(iy,&first,&step);
990:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
991:       PetscMalloc2(1,VecScatter_Seq_Stride,&to9,1,VecScatter_Seq_General,&from9);
992:       PetscMalloc(nx*sizeof(PetscInt),&from9->vslots);
993:       to9->n         = nx;
994:       to9->first     = first;
995:       to9->step      = step;
996:       from9->n       = nx;
997: #if defined(PETSC_USE_DEBUG)
998:       VecScatterCheckIndices_Private(ctx->from_n,nx,idx);
999: #endif
1000:       PetscMemcpy(from9->vslots,idx,nx*sizeof(PetscInt));
1001:       ctx->todata    = (void*)to9; ctx->fromdata = (void*)from9;
1002:       if (step == 1)  ctx->begin = VecScatterBegin_SGtoSS_Stride1;
1003:       else            ctx->begin = VecScatterBegin_SGtoSS;
1004:       ctx->destroy   = VecScatterDestroy_SGtoSS;
1005:       ctx->end       = 0;
1006:       ctx->copy      = VecScatterCopy_SGToSS;
1007:       to9->type      = VEC_SCATTER_SEQ_STRIDE;
1008:       from9->type    = VEC_SCATTER_SEQ_GENERAL;
1009:       PetscInfo(xin,"Special case: sequential vector general to stride\n");
1010:       goto functionend;
1011:     } else if (((PetscObject)ix)->type == IS_STRIDE && ((PetscObject)iy)->type == IS_GENERAL){
1012:       PetscInt               nx,ny,first,step;
1013:       const PetscInt         *idy;
1014:       VecScatter_Seq_General *to10 = PETSC_NULL;
1015:       VecScatter_Seq_Stride  *from10 = PETSC_NULL;

1017:       ISGetLocalSize(ix,&nx);
1018:       ISGetIndices(iy,&idy);
1019:       ISGetLocalSize(iy,&ny);
1020:       ISStrideGetInfo(ix,&first,&step);
1021:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1022:       PetscMalloc2(1,VecScatter_Seq_General,&to10,1,VecScatter_Seq_Stride,&from10);
1023:       PetscMalloc(nx*sizeof(PetscInt),&to10->vslots);
1024:       from10->n         = nx;
1025:       from10->first     = first;
1026:       from10->step      = step;
1027:       to10->n           = nx;
1028: #if defined(PETSC_USE_DEBUG)
1029:       VecScatterCheckIndices_Private(ctx->to_n,ny,idy);
1030: #endif
1031:       PetscMemcpy(to10->vslots,idy,nx*sizeof(PetscInt));
1032:       ctx->todata     = (void*)to10;
1033:       ctx->fromdata   = (void*)from10;
1034:       if (step == 1) ctx->begin = VecScatterBegin_SStoSG_Stride1;
1035:       else           ctx->begin = VecScatterBegin_SStoSG;
1036:       ctx->destroy    = VecScatterDestroy_SStoSG;
1037:       ctx->end        = 0;
1038:       ctx->copy       = 0;
1039:       to10->type      = VEC_SCATTER_SEQ_GENERAL;
1040:       from10->type    = VEC_SCATTER_SEQ_STRIDE;
1041:       PetscInfo(xin,"Special case: sequential vector stride to general\n");
1042:       goto functionend;
1043:     } else {
1044:       PetscInt               nx,ny;
1045:       const PetscInt         *idx,*idy;
1046:       VecScatter_Seq_General *to11 = PETSC_NULL,*from11 = PETSC_NULL;
1047:       PetscTruth             idnx,idny;

1049:       ISGetLocalSize(ix,&nx);
1050:       ISGetLocalSize(iy,&ny);
1051:       if (nx != ny) SETERRQ2(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match, in %D out %D",nx,ny);

1053:       ISIdentity(ix,&idnx);
1054:       ISIdentity(iy,&idny);
1055:       if (idnx && idny) {
1056:         VecScatter_Seq_Stride *to13 = PETSC_NULL,*from13 = PETSC_NULL;
1057:         PetscMalloc2(1,VecScatter_Seq_Stride,&to13,1,VecScatter_Seq_Stride,&from13);
1058:         to13->n           = nx;
1059:         to13->first       = 0;
1060:         to13->step        = 1;
1061:         from13->n         = nx;
1062:         from13->first     = 0;
1063:         from13->step      = 1;
1064:         to13->type        = VEC_SCATTER_SEQ_STRIDE;
1065:         from13->type      = VEC_SCATTER_SEQ_STRIDE;
1066:         ctx->todata       = (void*)to13;
1067:         ctx->fromdata     = (void*)from13;
1068:         ctx->begin        = VecScatterBegin_SStoSS;
1069:         ctx->end          = 0;
1070:         ctx->destroy      = VecScatterDestroy_SStoSS;
1071:         ctx->copy         = VecScatterCopy_SStoSS;
1072:         PetscInfo(xin,"Special case: sequential copy\n");
1073:         goto functionend;
1074:       }

1076:       ISGetIndices(iy,&idy);
1077:       ISGetIndices(ix,&idx);
1078:       PetscMalloc2(1,VecScatter_Seq_General,&to11,1,VecScatter_Seq_General,&from11);
1079:       PetscMalloc2(nx,PetscInt,&to11->vslots,nx,PetscInt,&from11->vslots);
1080:       to11->n           = nx;
1081: #if defined(PETSC_USE_DEBUG)
1082:       VecScatterCheckIndices_Private(ctx->to_n,ny,idy);
1083: #endif
1084:       PetscMemcpy(to11->vslots,idy,nx*sizeof(PetscInt));
1085:       from11->n         = nx;
1086: #if defined(PETSC_USE_DEBUG)
1087:       VecScatterCheckIndices_Private(ctx->from_n,nx,idx);
1088: #endif
1089:       PetscMemcpy(from11->vslots,idx,nx*sizeof(PetscInt));
1090:       to11->type        = VEC_SCATTER_SEQ_GENERAL;
1091:       from11->type      = VEC_SCATTER_SEQ_GENERAL;
1092:       ctx->todata       = (void*)to11;
1093:       ctx->fromdata     = (void*)from11;
1094:       ctx->begin        = VecScatterBegin_SGtoSG;
1095:       ctx->end          = 0;
1096:       ctx->destroy      = VecScatterDestroy_SGtoSG;
1097:       ctx->copy         = VecScatterCopy_SGToSG;
1098:       ISRestoreIndices(ix,&idx);
1099:       ISRestoreIndices(iy,&idy);
1100:       PetscInfo(xin,"Sequential vector scatter with block indices\n");
1101:       goto functionend;
1102:     }
1103:   }
1104:   /* ---------------------------------------------------------------------------*/
1105:   if (xin_type == VEC_MPI_ID && yin_type == VEC_SEQ_ID) {

1107:   /* ===========================================================================================================
1108:         Check for special cases
1109:      ==========================================================================================================*/
1110:     islocal = PETSC_FALSE;
1111:     /* special case extracting (subset of) local portion */
1112:     if (((PetscObject)ix)->type == IS_STRIDE && ((PetscObject)iy)->type == IS_STRIDE){
1113:       PetscInt              nx,ny,to_first,to_step,from_first,from_step;
1114:       PetscInt              start,end;
1115:       VecScatter_Seq_Stride *from12 = PETSC_NULL,*to12 = PETSC_NULL;

1117:       VecGetOwnershipRange(xin,&start,&end);
1118:       ISGetLocalSize(ix,&nx);
1119:       ISStrideGetInfo(ix,&from_first,&from_step);
1120:       ISGetLocalSize(iy,&ny);
1121:       ISStrideGetInfo(iy,&to_first,&to_step);
1122:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1123:       if (ix->min >= start && ix->max < end) islocal = PETSC_TRUE; else islocal = PETSC_FALSE;
1124:       MPI_Allreduce(&islocal,&cando,1,MPI_INT,MPI_LAND,((PetscObject)xin)->comm);
1125:       if (cando) {
1126:         PetscMalloc2(1,VecScatter_Seq_Stride,&to12,1,VecScatter_Seq_Stride,&from12);
1127:         to12->n             = nx;
1128:         to12->first         = to_first;
1129:         to12->step          = to_step;
1130:         from12->n           = nx;
1131:         from12->first       = from_first-start;
1132:         from12->step        = from_step;
1133:         to12->type          = VEC_SCATTER_SEQ_STRIDE;
1134:         from12->type        = VEC_SCATTER_SEQ_STRIDE;
1135:         ctx->todata         = (void*)to12;
1136:         ctx->fromdata       = (void*)from12;
1137:         ctx->begin          = VecScatterBegin_SStoSS;
1138:         ctx->end            = 0;
1139:         ctx->destroy        = VecScatterDestroy_SStoSS;
1140:         ctx->copy           = VecScatterCopy_SStoSS;
1141:         PetscInfo(xin,"Special case: processors only getting local values\n");
1142:         goto functionend;
1143:       }
1144:     } else {
1145:       MPI_Allreduce(&islocal,&cando,1,MPI_INT,MPI_LAND,((PetscObject)xin)->comm);
1146:     }

1148:     /* test for special case of all processors getting entire vector */
1149:     /* contains check that PetscMPIInt can handle the sizes needed */
1150:     totalv = 0;
1151:     if (((PetscObject)ix)->type == IS_STRIDE && ((PetscObject)iy)->type == IS_STRIDE){
1152:       PetscInt             i,nx,ny,to_first,to_step,from_first,from_step,N;
1153:       PetscMPIInt          *count = PETSC_NULL,*displx;
1154:       VecScatter_MPI_ToAll *sto = PETSC_NULL;

1156:       ISGetLocalSize(ix,&nx);
1157:       ISStrideGetInfo(ix,&from_first,&from_step);
1158:       ISGetLocalSize(iy,&ny);
1159:       ISStrideGetInfo(iy,&to_first,&to_step);
1160:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1161:       VecGetSize(xin,&N);
1162:       if (nx != N) {
1163:         totalv = 0;
1164:       } else if (from_first == 0 && from_step == 1 && from_first == to_first && from_step == to_step){
1165:         totalv = 1;
1166:       } else totalv = 0;
1167:       MPI_Allreduce(&totalv,&cando,1,MPI_INT,MPI_LAND,((PetscObject)xin)->comm);

1169: #if defined(PETSC_USE_64BIT_INDICES)
1170:       if (cando && (yin->map->N < PETSC_MPI_INT_MAX)) {
1171: #else
1172:       if (cando) {
1173: #endif
1174:         MPI_Comm_size(((PetscObject)ctx)->comm,&size);
1175:         PetscMalloc(size*sizeof(PetscMPIInt),&displx);
1176:         PetscMalloc2(1,VecScatter_MPI_ToAll,&sto,size,PetscMPIInt,&count);
1177:         range = xin->map->range;
1178:         for (i=0; i<size; i++) {
1179:           count[i]  = PetscMPIIntCast(range[i+1] - range[i]);
1180:           displx[i] = PetscMPIIntCast(range[i]);
1181:         }
1182:         sto->count        = count;
1183:         sto->displx       = displx;
1184:         sto->work1        = 0;
1185:         sto->work2        = 0;
1186:         sto->type         = VEC_SCATTER_MPI_TOALL;
1187:         ctx->todata       = (void*)sto;
1188:         ctx->fromdata     = 0;
1189:         ctx->begin        = VecScatterBegin_MPI_ToAll;
1190:         ctx->end          = 0;
1191:         ctx->destroy      = VecScatterDestroy_MPI_ToAll;
1192:         ctx->copy         = VecScatterCopy_MPI_ToAll;
1193:         PetscInfo(xin,"Special case: all processors get entire parallel vector\n");
1194:         goto functionend;
1195:       }
1196:     } else {
1197:       MPI_Allreduce(&totalv,&cando,1,MPI_INT,MPI_LAND,((PetscObject)xin)->comm);
1198:     }

1200:     /* test for special case of processor 0 getting entire vector */
1201:     /* contains check that PetscMPIInt can handle the sizes needed */
1202:     totalv = 0;
1203:     if (((PetscObject)ix)->type == IS_STRIDE && ((PetscObject)iy)->type == IS_STRIDE){
1204:       PetscInt             i,nx,ny,to_first,to_step,from_first,from_step,N;
1205:       PetscMPIInt          rank,*count = PETSC_NULL,*displx;
1206:       VecScatter_MPI_ToAll *sto = PETSC_NULL;

1208:       PetscObjectGetComm((PetscObject)xin,&comm);
1209:       MPI_Comm_rank(comm,&rank);
1210:       ISGetLocalSize(ix,&nx);
1211:       ISStrideGetInfo(ix,&from_first,&from_step);
1212:       ISGetLocalSize(iy,&ny);
1213:       ISStrideGetInfo(iy,&to_first,&to_step);
1214:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1215:       if (!rank) {
1216:         VecGetSize(xin,&N);
1217:         if (nx != N) {
1218:           totalv = 0;
1219:         } else if (from_first == 0        && from_step == 1 &&
1220:                    from_first == to_first && from_step == to_step){
1221:           totalv = 1;
1222:         } else totalv = 0;
1223:       } else {
1224:         if (!nx) totalv = 1;
1225:         else     totalv = 0;
1226:       }
1227:       MPI_Allreduce(&totalv,&cando,1,MPI_INT,MPI_LAND,((PetscObject)xin)->comm);

1229: #if defined(PETSC_USE_64BIT_INDICES)
1230:       if (cando && (yin->map->N < PETSC_MPI_INT_MAX)) {
1231: #else
1232:       if (cando) {
1233: #endif
1234:         MPI_Comm_size(((PetscObject)ctx)->comm,&size);
1235:         PetscMalloc(size*sizeof(PetscMPIInt),&displx);
1236:         PetscMalloc2(1,VecScatter_MPI_ToAll,&sto,size,PetscMPIInt,&count);
1237:         range = xin->map->range;
1238:         for (i=0; i<size; i++) {
1239:           count[i] = PetscMPIIntCast(range[i+1] - range[i]);
1240:           displx[i] = PetscMPIIntCast(range[i]);
1241:         }
1242:         sto->count        = count;
1243:         sto->displx       = displx;
1244:         sto->work1        = 0;
1245:         sto->work2        = 0;
1246:         sto->type         = VEC_SCATTER_MPI_TOONE;
1247:         ctx->todata       = (void*)sto;
1248:         ctx->fromdata     = 0;
1249:         ctx->begin        = VecScatterBegin_MPI_ToOne;
1250:         ctx->end          = 0;
1251:         ctx->destroy      = VecScatterDestroy_MPI_ToAll;
1252:         ctx->copy         = VecScatterCopy_MPI_ToAll;
1253:         PetscInfo(xin,"Special case: processor zero gets entire parallel vector, rest get none\n");
1254:         goto functionend;
1255:       }
1256:     } else {
1257:       MPI_Allreduce(&totalv,&cando,1,MPI_INT,MPI_LAND,((PetscObject)xin)->comm);
1258:     }

1260:     ISBlock(ix,&ixblock);
1261:     ISBlock(iy,&iyblock);
1262:     ISStride(iy,&iystride);
1263:     if (ixblock) {
1264:       /* special case block to block */
1265:       if (iyblock) {
1266:         PetscInt       nx,ny,bsx,bsy;
1267:         const PetscInt *idx,*idy;
1268:         ISBlockGetBlockSize(iy,&bsy);
1269:         ISBlockGetBlockSize(ix,&bsx);
1270:         if (bsx == bsy && VecScatterOptimizedBS(bsx)) {
1271:           ISBlockGetLocalSize(ix,&nx);
1272:           ISBlockGetIndices(ix,&idx);
1273:           ISBlockGetLocalSize(iy,&ny);
1274:           ISBlockGetIndices(iy,&idy);
1275:           if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1276:           VecScatterCreate_PtoS(nx,idx,ny,idy,xin,yin,bsx,ctx);
1277:           ISBlockRestoreIndices(ix,&idx);
1278:           ISBlockRestoreIndices(iy,&idy);
1279:           PetscInfo(xin,"Special case: blocked indices\n");
1280:           goto functionend;
1281:         }
1282:       /* special case block to stride */
1283:       } else if (iystride) {
1284:         PetscInt ystart,ystride,ysize,bsx;
1285:         ISStrideGetInfo(iy,&ystart,&ystride);
1286:         ISGetLocalSize(iy,&ysize);
1287:         ISBlockGetBlockSize(ix,&bsx);
1288:         /* see if stride index set is equivalent to block index set */
1289:         if (VecScatterOptimizedBS(bsx) && ((ystart % bsx) == 0) && (ystride == 1) && ((ysize % bsx) == 0)) {
1290:           PetscInt       nx,il,*idy;
1291:           const PetscInt *idx;
1292:           ISBlockGetLocalSize(ix,&nx); ISBlockGetIndices(ix,&idx);
1293:           if (ysize != bsx*nx) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1294:           PetscMalloc(nx*sizeof(PetscInt),&idy);
1295:           if (nx) {
1296:             idy[0] = ystart;
1297:             for (il=1; il<nx; il++) idy[il] = idy[il-1] + bsx;
1298:           }
1299:           VecScatterCreate_PtoS(nx,idx,nx,idy,xin,yin,bsx,ctx);
1300:           PetscFree(idy);
1301:           ISBlockRestoreIndices(ix,&idx);
1302:           PetscInfo(xin,"Special case: blocked indices to stride\n");
1303:           goto functionend;
1304:         }
1305:       }
1306:     }
1307:     /* left over general case */
1308:     {
1309:       PetscInt       nx,ny;
1310:       const PetscInt *idx,*idy;
1311:       ISGetLocalSize(ix,&nx);
1312:       ISGetIndices(ix,&idx);
1313:       ISGetLocalSize(iy,&ny);
1314:       ISGetIndices(iy,&idy);
1315:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1316:       VecScatterCreate_PtoS(nx,idx,ny,idy,xin,yin,1,ctx);
1317:       ISRestoreIndices(ix,&idx);
1318:       ISRestoreIndices(iy,&idy);
1319:       PetscInfo(xin,"General case: MPI to Seq\n");
1320:       goto functionend;
1321:     }
1322:   }
1323:   /* ---------------------------------------------------------------------------*/
1324:   if (xin_type == VEC_SEQ_ID && yin_type == VEC_MPI_ID) {
1325:   /* ===========================================================================================================
1326:         Check for special cases
1327:      ==========================================================================================================*/
1328:     /* special case local copy portion */
1329:     islocal = PETSC_FALSE;
1330:     if (((PetscObject)ix)->type == IS_STRIDE && ((PetscObject)iy)->type == IS_STRIDE){
1331:       PetscInt              nx,ny,to_first,to_step,from_step,start,end,from_first;
1332:       VecScatter_Seq_Stride *from = PETSC_NULL,*to = PETSC_NULL;

1334:       VecGetOwnershipRange(yin,&start,&end);
1335:       ISGetLocalSize(ix,&nx);
1336:       ISStrideGetInfo(ix,&from_first,&from_step);
1337:       ISGetLocalSize(iy,&ny);
1338:       ISStrideGetInfo(iy,&to_first,&to_step);
1339:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1340:       if (iy->min >= start && iy->max < end) islocal = PETSC_TRUE; else islocal = PETSC_FALSE;
1341:       MPI_Allreduce(&islocal,&cando,1,MPI_INT,MPI_LAND,((PetscObject)yin)->comm);
1342:       if (cando) {
1343:         PetscMalloc2(1,VecScatter_Seq_Stride,&to,1,VecScatter_Seq_Stride,&from);
1344:         to->n             = nx;
1345:         to->first         = to_first-start;
1346:         to->step          = to_step;
1347:         from->n           = nx;
1348:         from->first       = from_first;
1349:         from->step        = from_step;
1350:         to->type          = VEC_SCATTER_SEQ_STRIDE;
1351:         from->type        = VEC_SCATTER_SEQ_STRIDE;
1352:         ctx->todata       = (void*)to;
1353:         ctx->fromdata     = (void*)from;
1354:         ctx->begin        = VecScatterBegin_SStoSS;
1355:         ctx->end          = 0;
1356:         ctx->destroy      = VecScatterDestroy_SStoSS;
1357:         ctx->copy         = VecScatterCopy_SStoSS;
1358:         PetscInfo(xin,"Special case: sequential stride to MPI stride\n");
1359:         goto functionend;
1360:       }
1361:     } else {
1362:       MPI_Allreduce(&islocal,&cando,1,MPI_INT,MPI_LAND,((PetscObject)yin)->comm);
1363:     }
1364:       /* special case block to stride */
1365:     if (((PetscObject)ix)->type == IS_BLOCK && ((PetscObject)iy)->type == IS_STRIDE){
1366:       PetscInt ystart,ystride,ysize,bsx;
1367:       ISStrideGetInfo(iy,&ystart,&ystride);
1368:       ISGetLocalSize(iy,&ysize);
1369:       ISBlockGetBlockSize(ix,&bsx);
1370:       /* see if stride index set is equivalent to block index set */
1371:       if (VecScatterOptimizedBS(bsx) && ((ystart % bsx) == 0) && (ystride == 1) && ((ysize % bsx) == 0)) {
1372:         PetscInt       nx,il,*idy;
1373:         const PetscInt *idx;
1374:         ISBlockGetLocalSize(ix,&nx); ISBlockGetIndices(ix,&idx);
1375:         if (ysize != bsx*nx) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1376:         PetscMalloc(nx*sizeof(PetscInt),&idy);
1377:         if (nx) {
1378:           idy[0] = ystart;
1379:           for (il=1; il<nx; il++) idy[il] = idy[il-1] + bsx;
1380:         }
1381:         VecScatterCreate_StoP(nx,idx,nx,idy,xin,yin,bsx,ctx);
1382:         PetscFree(idy);
1383:         ISBlockRestoreIndices(ix,&idx);
1384:         PetscInfo(xin,"Special case: Blocked indices to stride\n");
1385:         goto functionend;
1386:       }
1387:     }

1389:     /* general case */
1390:     {
1391:       PetscInt       nx,ny;
1392:       const PetscInt *idx,*idy;
1393:       ISGetLocalSize(ix,&nx);
1394:       ISGetIndices(ix,&idx);
1395:       ISGetLocalSize(iy,&ny);
1396:       ISGetIndices(iy,&idy);
1397:       if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1398:       VecScatterCreate_StoP(nx,idx,ny,idy,xin,yin,1,ctx);
1399:       ISRestoreIndices(ix,&idx);
1400:       ISRestoreIndices(iy,&idy);
1401:       PetscInfo(xin,"General case: Seq to MPI\n");
1402:       goto functionend;
1403:     }
1404:   }
1405:   /* ---------------------------------------------------------------------------*/
1406:   if (xin_type == VEC_MPI_ID && yin_type == VEC_MPI_ID) {
1407:     /* no special cases for now */
1408:     PetscInt       nx,ny;
1409:     const PetscInt *idx,*idy;
1410:     ISGetLocalSize(ix,&nx);
1411:     ISGetIndices(ix,&idx);
1412:     ISGetLocalSize(iy,&ny);
1413:     ISGetIndices(iy,&idy);
1414:     if (nx != ny) SETERRQ(PETSC_ERR_ARG_SIZ,"Local scatter sizes don't match");
1415:     VecScatterCreate_PtoP(nx,idx,ny,idy,xin,yin,ctx);
1416:     ISRestoreIndices(ix,&idx);
1417:     ISRestoreIndices(iy,&idy);
1418:     PetscInfo(xin,"General case: MPI to MPI\n");
1419:     goto functionend;
1420:   }

1422:   functionend:
1423:   *newctx = ctx;
1424:   if (tix) {ISDestroy(tix);}
1425:   if (tiy) {ISDestroy(tiy);}
1426:   flag = PETSC_FALSE;
1427:   PetscOptionsGetTruth(PETSC_NULL,"-vecscatter_view_info",&flag,PETSC_NULL);
1428:   if (flag) {
1429:     PetscViewer viewer;
1430:     PetscViewerASCIIGetStdout(comm,&viewer);
1431:     PetscViewerPushFormat(viewer,PETSC_VIEWER_ASCII_INFO);
1432:     VecScatterView(ctx,viewer);
1433:     PetscViewerPopFormat(viewer);
1434:   }
1435:   flag = PETSC_FALSE;
1436:   PetscOptionsGetTruth(PETSC_NULL,"-vecscatter_view",&flag,PETSC_NULL);
1437:   if (flag) {
1438:     PetscViewer viewer;
1439:     PetscViewerASCIIGetStdout(comm,&viewer);
1440:     VecScatterView(ctx,viewer);
1441:   }
1442:   return(0);
1443: }

1445: /* ------------------------------------------------------------------*/
1448: /*@
1449:    VecScatterGetMerged - Returns true if the scatter is completed in the VecScatterBegin()
1450:       and the VecScatterEnd() does nothing

1452:    Not Collective

1454:    Input Parameter:
1455: .   ctx - scatter context created with VecScatterCreate()

1457:    Output Parameter:
1458: .   flg - PETSC_TRUE if the VecScatterBegin/End() are all done during the VecScatterBegin()

1460:    Level: developer

1462: .seealso: VecScatterCreate(), VecScatterEnd(), VecScatterBegin()
1463: @*/
1464: PetscErrorCode  VecScatterGetMerged(VecScatter ctx,PetscTruth *flg)
1465: {
1468:   *flg = ctx->beginandendtogether;
1469:   return(0);
1470: }

1474: /*@
1475:    VecScatterBegin - Begins a generalized scatter from one vector to
1476:    another. Complete the scattering phase with VecScatterEnd().

1478:    Collective on VecScatter and Vec

1480:    Input Parameters:
1481: +  inctx - scatter context generated by VecScatterCreate()
1482: .  x - the vector from which we scatter
1483: .  y - the vector to which we scatter
1484: .  addv - either ADD_VALUES or INSERT_VALUES, with INSERT_VALUES mode any location 
1485:           not scattered to retains its old value; i.e. the vector is NOT first zeroed.
1486: -  mode - the scattering mode, usually SCATTER_FORWARD.  The available modes are:
1487:     SCATTER_FORWARD or SCATTER_REVERSE


1490:    Level: intermediate

1492:    Options Database: See VecScatterCreate()

1494:    Notes:
1495:    The vectors x and y need not be the same vectors used in the call 
1496:    to VecScatterCreate(), but x must have the same parallel data layout
1497:    as that passed in as the x to VecScatterCreate(), similarly for the y.
1498:    Most likely they have been obtained from VecDuplicate().

1500:    You cannot change the values in the input vector between the calls to VecScatterBegin()
1501:    and VecScatterEnd().

1503:    If you use SCATTER_REVERSE the two arguments x and y should be reversed, from 
1504:    the SCATTER_FORWARD.
1505:    
1506:    y[iy[i]] = x[ix[i]], for i=0,...,ni-1

1508:    This scatter is far more general than the conventional
1509:    scatter, since it can be a gather or a scatter or a combination,
1510:    depending on the indices ix and iy.  If x is a parallel vector and y
1511:    is sequential, VecScatterBegin() can serve to gather values to a
1512:    single processor.  Similarly, if y is parallel and x sequential, the
1513:    routine can scatter from one processor to many processors.

1515:    Concepts: scatter^between vectors
1516:    Concepts: gather^between vectors

1518: .seealso: VecScatterCreate(), VecScatterEnd()
1519: @*/
1520: PetscErrorCode  VecScatterBegin(VecScatter inctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
1521: {
1523: #if defined(PETSC_USE_DEBUG)
1524:   PetscInt      to_n,from_n;
1525: #endif

1531:   if (inctx->inuse) SETERRQ(PETSC_ERR_ARG_WRONGSTATE," Scatter ctx already in use");
1532:   CHKMEMQ;

1534: #if defined(PETSC_USE_DEBUG)
1535:   /*
1536:      Error checking to make sure these vectors match the vectors used
1537:    to create the vector scatter context. -1 in the from_n and to_n indicate the
1538:    vector lengths are unknown (for example with mapped scatters) and thus 
1539:    no error checking is performed.
1540:   */
1541:   if (inctx->from_n >= 0 && inctx->to_n >= 0) {
1542:     VecGetLocalSize(x,&from_n);
1543:     VecGetLocalSize(y,&to_n);
1544:     if (mode & SCATTER_REVERSE) {
1545:       if (to_n != inctx->from_n) SETERRQ2(PETSC_ERR_ARG_SIZ,"Vector wrong size %D for scatter %D (scatter reverse and vector to != ctx from size)",to_n,inctx->from_n);
1546:       if (from_n != inctx->to_n) SETERRQ2(PETSC_ERR_ARG_SIZ,"Vector wrong size %D for scatter %D (scatter reverse and vector from != ctx to size)",from_n,inctx->to_n);
1547:     } else {
1548:       if (to_n != inctx->to_n)     SETERRQ2(PETSC_ERR_ARG_SIZ,"Vector wrong size %D for scatter %D (scatter forward and vector to != ctx to size)",to_n,inctx->to_n);
1549:       if (from_n != inctx->from_n) SETERRQ2(PETSC_ERR_ARG_SIZ,"Vector wrong size %D for scatter %D (scatter forward and vector from != ctx from size)",from_n,inctx->from_n);
1550:     }
1551:   }
1552: #endif

1554:   inctx->inuse = PETSC_TRUE;
1555:   PetscLogEventBarrierBegin(VEC_ScatterBarrier,0,0,0,0,((PetscObject)inctx)->comm);
1556:   (*inctx->begin)(inctx,x,y,addv,mode);
1557:   if (inctx->beginandendtogether && inctx->end) {
1558:     inctx->inuse = PETSC_FALSE;
1559:     (*inctx->end)(inctx,x,y,addv,mode);
1560:   }
1561:   PetscLogEventBarrierEnd(VEC_ScatterBarrier,0,0,0,0,((PetscObject)inctx)->comm);
1562:   CHKMEMQ;
1563:   return(0);
1564: }

1566: /* --------------------------------------------------------------------*/
1569: /*@
1570:    VecScatterEnd - Ends a generalized scatter from one vector to another.  Call
1571:    after first calling VecScatterBegin().

1573:    Collective on VecScatter and Vec

1575:    Input Parameters:
1576: +  ctx - scatter context generated by VecScatterCreate()
1577: .  x - the vector from which we scatter
1578: .  y - the vector to which we scatter
1579: .  addv - either ADD_VALUES or INSERT_VALUES.
1580: -  mode - the scattering mode, usually SCATTER_FORWARD.  The available modes are:
1581:      SCATTER_FORWARD, SCATTER_REVERSE

1583:    Level: intermediate

1585:    Notes:
1586:    If you use SCATTER_REVERSE the arguments x and y should be reversed, from the SCATTER_FORWARD.

1588:    y[iy[i]] = x[ix[i]], for i=0,...,ni-1

1590: .seealso: VecScatterBegin(), VecScatterCreate()
1591: @*/
1592: PetscErrorCode  VecScatterEnd(VecScatter ctx,Vec x,Vec y,InsertMode addv,ScatterMode mode)
1593: {

1600:   ctx->inuse = PETSC_FALSE;
1601:   if (!ctx->end) return(0);
1602:   CHKMEMQ;
1603:   if (!ctx->beginandendtogether) {
1604:     PetscLogEventBegin(VEC_ScatterEnd,ctx,x,y,0);
1605:     (*(ctx)->end)(ctx,x,y,addv,mode);
1606:     PetscLogEventEnd(VEC_ScatterEnd,ctx,x,y,0);
1607:   }
1608:   CHKMEMQ;
1609:   return(0);
1610: }

1614: /*@
1615:    VecScatterDestroy - Destroys a scatter context created by 
1616:    VecScatterCreate().

1618:    Collective on VecScatter

1620:    Input Parameter:
1621: .  ctx - the scatter context

1623:    Level: intermediate

1625: .seealso: VecScatterCreate(), VecScatterCopy()
1626: @*/
1627: PetscErrorCode  VecScatterDestroy(VecScatter ctx)
1628: {

1633:   if (--((PetscObject)ctx)->refct > 0) return(0);

1635:   /* if memory was published with AMS then destroy it */
1636:   PetscObjectDepublish(ctx);

1638:   (*ctx->destroy)(ctx);
1639:   return(0);
1640: }

1644: /*@
1645:    VecScatterCopy - Makes a copy of a scatter context.

1647:    Collective on VecScatter

1649:    Input Parameter:
1650: .  sctx - the scatter context

1652:    Output Parameter:
1653: .  ctx - the context copy

1655:    Level: advanced

1657: .seealso: VecScatterCreate(), VecScatterDestroy()
1658: @*/
1659: PetscErrorCode  VecScatterCopy(VecScatter sctx,VecScatter *ctx)
1660: {

1666:   if (!sctx->copy) SETERRQ(PETSC_ERR_SUP,"Cannot copy this type");
1667:   PetscHeaderCreate(*ctx,_p_VecScatter,int,VEC_SCATTER_COOKIE,0,"VecScatter",((PetscObject)sctx)->comm,VecScatterDestroy,VecScatterView);
1668:   (*ctx)->to_n   = sctx->to_n;
1669:   (*ctx)->from_n = sctx->from_n;
1670:   (*sctx->copy)(sctx,*ctx);
1671:   return(0);
1672: }


1675: /* ------------------------------------------------------------------*/
1678: /*@
1679:    VecScatterView - Views a vector scatter context.

1681:    Collective on VecScatter

1683:    Input Parameters:
1684: +  ctx - the scatter context
1685: -  viewer - the viewer for displaying the context

1687:    Level: intermediate

1689: @*/
1690: PetscErrorCode  VecScatterView(VecScatter ctx,PetscViewer viewer)
1691: {

1696:   if (!viewer) {
1697:     PetscViewerASCIIGetStdout(((PetscObject)ctx)->comm,&viewer);
1698:   }
1700:   if (ctx->view) {
1701:     (*ctx->view)(ctx,viewer);
1702:   }
1703:   return(0);
1704: }

1708: /*@C
1709:    VecScatterRemap - Remaps the "from" and "to" indices in a 
1710:    vector scatter context. FOR EXPERTS ONLY!

1712:    Collective on VecScatter

1714:    Input Parameters:
1715: +  scat - vector scatter context
1716: .  from - remapping for "from" indices (may be PETSC_NULL)
1717: -  to   - remapping for "to" indices (may be PETSC_NULL)

1719:    Level: developer

1721:    Notes: In the parallel case the todata is actually the indices
1722:           from which the data is TAKEN! The from stuff is where the 
1723:           data is finally put. This is VERY VERY confusing!

1725:           In the sequential case the todata is the indices where the 
1726:           data is put and the fromdata is where it is taken from.
1727:           This is backwards from the paralllel case! CRY! CRY! CRY!

1729: @*/
1730: PetscErrorCode  VecScatterRemap(VecScatter scat,PetscInt *rto,PetscInt *rfrom)
1731: {
1732:   VecScatter_Seq_General *to,*from;
1733:   VecScatter_MPI_General *mto;
1734:   PetscInt               i;


1741:   from = (VecScatter_Seq_General *)scat->fromdata;
1742:   mto  = (VecScatter_MPI_General *)scat->todata;

1744:   if (mto->type == VEC_SCATTER_MPI_TOALL) SETERRQ(PETSC_ERR_ARG_SIZ,"Not for to all scatter");

1746:   if (rto) {
1747:     if (mto->type == VEC_SCATTER_MPI_GENERAL) {
1748:       /* handle off processor parts */
1749:       for (i=0; i<mto->starts[mto->n]; i++) {
1750:         mto->indices[i] = rto[mto->indices[i]];
1751:       }
1752:       /* handle local part */
1753:       to = &mto->local;
1754:       for (i=0; i<to->n; i++) {
1755:         to->vslots[i] = rto[to->vslots[i]];
1756:       }
1757:     } else if (from->type == VEC_SCATTER_SEQ_GENERAL) {
1758:       for (i=0; i<from->n; i++) {
1759:         from->vslots[i] = rto[from->vslots[i]];
1760:       }
1761:     } else if (from->type == VEC_SCATTER_SEQ_STRIDE) {
1762:       VecScatter_Seq_Stride *sto = (VecScatter_Seq_Stride*)from;
1763: 
1764:       /* if the remapping is the identity and stride is identity then skip remap */
1765:       if (sto->step == 1 && sto->first == 0) {
1766:         for (i=0; i<sto->n; i++) {
1767:           if (rto[i] != i) {
1768:             SETERRQ(PETSC_ERR_ARG_SIZ,"Unable to remap such scatters");
1769:           }
1770:         }
1771:       } else SETERRQ(PETSC_ERR_ARG_SIZ,"Unable to remap such scatters");
1772:     } else SETERRQ(PETSC_ERR_ARG_SIZ,"Unable to remap such scatters");
1773:   }

1775:   if (rfrom) {
1776:     SETERRQ(PETSC_ERR_SUP,"Unable to remap the FROM in scatters yet");
1777:   }

1779:   /*
1780:      Mark then vector lengths as unknown because we do not know the 
1781:    lengths of the remapped vectors
1782:   */
1783:   scat->from_n = -1;
1784:   scat->to_n   = -1;

1786:   return(0);
1787: }