Actual source code: pdvec.c

  1: /* $Id: pdvec.c,v 1.154 2001/09/11 16:32:01 bsmith Exp $*/
  2: /*
  3:      Code for some of the parallel vector primatives.
  4: */
 5:  #include src/vec/impls/mpi/pvecimpl.h
  6: #if defined(PETSC_HAVE_PNETCDF)
  7: EXTERN_C_BEGIN
  8: #include "pnetcdf.h"
  9: EXTERN_C_END
 10: #endif

 14: int VecDestroy_MPI(Vec v)
 15: {
 16:   Vec_MPI *x = (Vec_MPI*)v->data;
 17:   int     ierr;


 21:   /* if memory was published with AMS then destroy it */
 22:   PetscObjectDepublish(v);

 24: #if defined(PETSC_USE_LOG)
 25:   PetscLogObjectState((PetscObject)v,"Length=%d",v->N);
 26: #endif  
 27:   if (x->array_allocated) {PetscFree(x->array_allocated);}

 29:   /* Destroy local representation of vector if it exists */
 30:   if (x->localrep) {
 31:     VecDestroy(x->localrep);
 32:     if (x->localupdate) {VecScatterDestroy(x->localupdate);}
 33:   }
 34:   /* Destroy the stashes: note the order - so that the tags are freed properly */
 35:   VecStashDestroy_Private(&v->bstash);
 36:   VecStashDestroy_Private(&v->stash);
 37:   PetscFree(x);
 38:   return(0);
 39: }

 43: int VecView_MPI_ASCII(Vec xin,PetscViewer viewer)
 44: {
 45:   int               i,rank,len,work = xin->n,n,j,size,ierr,cnt,tag = ((PetscObject)viewer)->tag;
 46:   MPI_Status        status;
 47:   PetscScalar       *values,*xarray;
 48:   char              *name;
 49:   PetscViewerFormat format;

 52:   VecGetArrayFast(xin,&xarray);
 53:   /* determine maximum message to arrive */
 54:   MPI_Comm_rank(xin->comm,&rank);
 55:   MPI_Reduce(&work,&len,1,MPI_INT,MPI_MAX,0,xin->comm);
 56:   MPI_Comm_size(xin->comm,&size);

 58:   if (!rank) {
 59:     PetscMalloc((len+1)*sizeof(PetscScalar),&values);
 60:     PetscViewerGetFormat(viewer,&format);
 61:     /*
 62:         Matlab format and ASCII format are very similar except 
 63:         Matlab uses %18.16e format while ASCII uses %g
 64:     */
 65:     if (format == PETSC_VIEWER_ASCII_MATLAB) {
 66:       PetscObjectGetName((PetscObject)xin,&name);
 67:       PetscViewerASCIIPrintf(viewer,"%s = [\n",name);
 68:       for (i=0; i<xin->n; i++) {
 69: #if defined(PETSC_USE_COMPLEX)
 70:         if (PetscImaginaryPart(xarray[i]) > 0.0) {
 71:           PetscViewerASCIIPrintf(viewer,"%18.16e + %18.16ei\n",PetscRealPart(xarray[i]),PetscImaginaryPart(xarray[i]));
 72:         } else if (PetscImaginaryPart(xarray[i]) < 0.0) {
 73:           PetscViewerASCIIPrintf(viewer,"%18.16e - %18.16ei\n",PetscRealPart(xarray[i]),-PetscImaginaryPart(xarray[i]));
 74:         } else {
 75:           PetscViewerASCIIPrintf(viewer,"%18.16e\n",PetscRealPart(xarray[i]));
 76:         }
 77: #else
 78:         PetscViewerASCIIPrintf(viewer,"%18.16e\n",xarray[i]);
 79: #endif
 80:       }
 81:       /* receive and print messages */
 82:       for (j=1; j<size; j++) {
 83:         MPI_Recv(values,len,MPIU_SCALAR,j,tag,xin->comm,&status);
 84:         MPI_Get_count(&status,MPIU_SCALAR,&n);
 85:         for (i=0; i<n; i++) {
 86: #if defined(PETSC_USE_COMPLEX)
 87:           if (PetscImaginaryPart(values[i]) > 0.0) {
 88:             PetscViewerASCIIPrintf(viewer,"%18.16e + %18.16e i\n",PetscRealPart(values[i]),PetscImaginaryPart(values[i]));
 89:           } else if (PetscImaginaryPart(values[i]) < 0.0) {
 90:             PetscViewerASCIIPrintf(viewer,"%18.16e - %18.16e i\n",PetscRealPart(values[i]),-PetscImaginaryPart(values[i]));
 91:           } else {
 92:             PetscViewerASCIIPrintf(viewer,"%18.16e\n",PetscRealPart(values[i]));
 93:           }
 94: #else
 95:           PetscViewerASCIIPrintf(viewer,"%18.16e\n",values[i]);
 96: #endif
 97:         }
 98:       }
 99:       PetscViewerASCIIPrintf(viewer,"];\n");

101:     } else if (format == PETSC_VIEWER_ASCII_SYMMODU) {
102:       for (i=0; i<xin->n; i++) {
103: #if defined(PETSC_USE_COMPLEX)
104:         PetscViewerASCIIPrintf(viewer,"%18.16e %18.16e\n",PetscRealPart(xarray[i]),PetscImaginaryPart(xarray[i]));
105: #else
106:         PetscViewerASCIIPrintf(viewer,"%18.16e\n",xarray[i]);
107: #endif
108:       }
109:       /* receive and print messages */
110:       for (j=1; j<size; j++) {
111:         MPI_Recv(values,len,MPIU_SCALAR,j,tag,xin->comm,&status);
112:         MPI_Get_count(&status,MPIU_SCALAR,&n);
113:         for (i=0; i<n; i++) {
114: #if defined(PETSC_USE_COMPLEX)
115:           PetscViewerASCIIPrintf(viewer,"%18.16e %18.16e\n",PetscRealPart(values[i]),PetscImaginaryPart(values[i]));
116: #else
117:           PetscViewerASCIIPrintf(viewer,"%18.16e\n",values[i]);
118: #endif
119:         }
120:       }

122:     } else {
123:       if (format != PETSC_VIEWER_ASCII_COMMON) {PetscViewerASCIIPrintf(viewer,"Process [%d]\n",rank);}
124:       cnt = 0;
125:       for (i=0; i<xin->n; i++) {
126:         if (format == PETSC_VIEWER_ASCII_INDEX) {
127:           PetscViewerASCIIPrintf(viewer,"%d: ",cnt++);
128:         }
129: #if defined(PETSC_USE_COMPLEX)
130:         if (PetscImaginaryPart(xarray[i]) > 0.0) {
131:           PetscViewerASCIIPrintf(viewer,"%g + %g i\n",PetscRealPart(xarray[i]),PetscImaginaryPart(xarray[i]));
132:         } else if (PetscImaginaryPart(xarray[i]) < 0.0) {
133:           PetscViewerASCIIPrintf(viewer,"%g - %g i\n",PetscRealPart(xarray[i]),-PetscImaginaryPart(xarray[i]));
134:         } else {
135:           PetscViewerASCIIPrintf(viewer,"%g\n",PetscRealPart(xarray[i]));
136:         }
137: #else
138:         PetscViewerASCIIPrintf(viewer,"%g\n",xarray[i]);
139: #endif
140:       }
141:       /* receive and print messages */
142:       for (j=1; j<size; j++) {
143:         MPI_Recv(values,len,MPIU_SCALAR,j,tag,xin->comm,&status);
144:         MPI_Get_count(&status,MPIU_SCALAR,&n);
145:         if (format != PETSC_VIEWER_ASCII_COMMON) {
146:           PetscViewerASCIIPrintf(viewer,"Process [%d]\n",j);
147:         }
148:         for (i=0; i<n; i++) {
149:           if (format == PETSC_VIEWER_ASCII_INDEX) {
150:             PetscViewerASCIIPrintf(viewer,"%d: ",cnt++);
151:           }
152: #if defined(PETSC_USE_COMPLEX)
153:           if (PetscImaginaryPart(values[i]) > 0.0) {
154:             PetscViewerASCIIPrintf(viewer,"%g + %g i\n",PetscRealPart(values[i]),PetscImaginaryPart(values[i]));
155:           } else if (PetscImaginaryPart(values[i]) < 0.0) {
156:             PetscViewerASCIIPrintf(viewer,"%g - %g i\n",PetscRealPart(values[i]),-PetscImaginaryPart(values[i]));
157:           } else {
158:             PetscViewerASCIIPrintf(viewer,"%g\n",PetscRealPart(values[i]));
159:           }
160: #else
161:           PetscViewerASCIIPrintf(viewer,"%g\n",values[i]);
162: #endif
163:         }
164:       }
165:     }
166:     PetscFree(values);
167:   } else {
168:     /* send values */
169:     MPI_Send(xarray,xin->n,MPIU_SCALAR,0,tag,xin->comm);
170:   }
171:   PetscViewerFlush(viewer);
172:   VecRestoreArrayFast(xin,&xarray);
173:   return(0);
174: }

178: int VecView_MPI_Binary(Vec xin,PetscViewer viewer)
179: {
180:   int         rank,ierr,len,work = xin->n,n,j,size,fdes,tag = ((PetscObject)viewer)->tag;
181:   MPI_Status  status;
182:   PetscScalar *values,*xarray;
183:   FILE        *file;

186:   VecGetArrayFast(xin,&xarray);
187:   PetscViewerBinaryGetDescriptor(viewer,&fdes);

189:   /* determine maximum message to arrive */
190:   MPI_Comm_rank(xin->comm,&rank);
191:   MPI_Reduce(&work,&len,1,MPI_INT,MPI_MAX,0,xin->comm);
192:   MPI_Comm_size(xin->comm,&size);

194:   if (!rank) {
195:     int cookie = VEC_FILE_COOKIE;
196:     PetscBinaryWrite(fdes,&cookie,1,PETSC_INT,0);
197:     PetscBinaryWrite(fdes,&xin->N,1,PETSC_INT,0);
198:     PetscBinaryWrite(fdes,xarray,xin->n,PETSC_SCALAR,0);

200:     PetscMalloc((len+1)*sizeof(PetscScalar),&values);
201:     /* receive and print messages */
202:     for (j=1; j<size; j++) {
203:       MPI_Recv(values,len,MPIU_SCALAR,j,tag,xin->comm,&status);
204:       MPI_Get_count(&status,MPIU_SCALAR,&n);
205:       PetscBinaryWrite(fdes,values,n,PETSC_SCALAR,0);
206:     }
207:     PetscFree(values);
208:     PetscViewerBinaryGetInfoPointer(viewer,&file);
209:     if (file && xin->bs > 1) {
210:       if (xin->prefix) {
211:         fprintf(file,"-%svecload_block_size %d\n",xin->prefix,xin->bs);
212:       } else {
213:         fprintf(file,"-vecload_block_size %d\n",xin->bs);
214:       }
215:     }
216:   } else {
217:     /* send values */
218:     MPI_Send(xarray,xin->n,MPIU_SCALAR,0,tag,xin->comm);
219:   }
220:   VecRestoreArrayFast(xin,&xarray);
221:   return(0);
222: }

226: int VecView_MPI_Draw_LG(Vec xin,PetscViewer viewer)
227: {
228:   int         i,rank,size,N = xin->N,*lens,ierr;
229:   PetscDraw   draw;
230:   PetscReal   *xx,*yy;
231:   PetscDrawLG lg;
232:   PetscTruth  isnull;
233:   PetscScalar *xarray;

236:   PetscViewerDrawGetDraw(viewer,0,&draw);
237:   PetscDrawIsNull(draw,&isnull);
238:   if (isnull) return(0);

240:   VecGetArrayFast(xin,&xarray);
241:   PetscViewerDrawGetDrawLG(viewer,0,&lg);
242:   PetscDrawCheckResizedWindow(draw);
243:   MPI_Comm_rank(xin->comm,&rank);
244:   MPI_Comm_size(xin->comm,&size);
245:   if (!rank) {
246:     PetscDrawLGReset(lg);
247:     PetscMalloc(2*(N+1)*sizeof(PetscReal),&xx);
248:     for (i=0; i<N; i++) {xx[i] = (PetscReal) i;}
249:     yy   = xx + N;
250:     PetscMalloc(size*sizeof(int),&lens);
251:     for (i=0; i<size; i++) {
252:       lens[i] = xin->map->range[i+1] - xin->map->range[i];
253:     }
254: #if !defined(PETSC_USE_COMPLEX)
255:     MPI_Gatherv(xarray,xin->n,MPIU_REAL,yy,lens,xin->map->range,MPIU_REAL,0,xin->comm);
256: #else
257:     {
258:       PetscReal *xr;
259:       PetscMalloc((xin->n+1)*sizeof(PetscReal),&xr);
260:       for (i=0; i<xin->n; i++) {
261:         xr[i] = PetscRealPart(xarray[i]);
262:       }
263:       MPI_Gatherv(xr,xin->n,MPIU_REAL,yy,lens,xin->map->range,MPIU_REAL,0,xin->comm);
264:       PetscFree(xr);
265:     }
266: #endif
267:     PetscFree(lens);
268:     PetscDrawLGAddPoints(lg,N,&xx,&yy);
269:     PetscFree(xx);
270:   } else {
271: #if !defined(PETSC_USE_COMPLEX)
272:     MPI_Gatherv(xarray,xin->n,MPIU_REAL,0,0,0,MPIU_REAL,0,xin->comm);
273: #else
274:     {
275:       PetscReal *xr;
276:       PetscMalloc((xin->n+1)*sizeof(PetscReal),&xr);
277:       for (i=0; i<xin->n; i++) {
278:         xr[i] = PetscRealPart(xarray[i]);
279:       }
280:       MPI_Gatherv(xr,xin->n,MPIU_REAL,0,0,0,MPIU_REAL,0,xin->comm);
281:       PetscFree(xr);
282:     }
283: #endif
284:   }
285:   PetscDrawLGDraw(lg);
286:   PetscDrawSynchronizedFlush(draw);
287:   VecRestoreArrayFast(xin,&xarray);
288:   return(0);
289: }

291: EXTERN_C_BEGIN
294: int VecView_MPI_Draw(Vec xin,PetscViewer viewer)
295: {
296:   int           i,rank,size,ierr,start,end,tag = ((PetscObject)viewer)->tag;
297:   MPI_Status    status;
298:   PetscReal     coors[4],ymin,ymax,xmin,xmax,tmp;
299:   PetscDraw     draw;
300:   PetscTruth    isnull;
301:   PetscDrawAxis axis;
302:   PetscScalar   *xarray;

305:   PetscViewerDrawGetDraw(viewer,0,&draw);
306:   PetscDrawIsNull(draw,&isnull); if (isnull) return(0);

308:   VecGetArrayFast(xin,&xarray);
309:   PetscDrawCheckResizedWindow(draw);
310:   xmin = 1.e20; xmax = -1.e20;
311:   for (i=0; i<xin->n; i++) {
312: #if defined(PETSC_USE_COMPLEX)
313:     if (PetscRealPart(xarray[i]) < xmin) xmin = PetscRealPart(xarray[i]);
314:     if (PetscRealPart(xarray[i]) > xmax) xmax = PetscRealPart(xarray[i]);
315: #else
316:     if (xarray[i] < xmin) xmin = xarray[i];
317:     if (xarray[i] > xmax) xmax = xarray[i];
318: #endif
319:   }
320:   if (xmin + 1.e-10 > xmax) {
321:     xmin -= 1.e-5;
322:     xmax += 1.e-5;
323:   }
324:   MPI_Reduce(&xmin,&ymin,1,MPIU_REAL,MPI_MIN,0,xin->comm);
325:   MPI_Reduce(&xmax,&ymax,1,MPIU_REAL,MPI_MAX,0,xin->comm);
326:   MPI_Comm_size(xin->comm,&size);
327:   MPI_Comm_rank(xin->comm,&rank);
328:   PetscDrawAxisCreate(draw,&axis);
329:   PetscLogObjectParent(draw,axis);
330:   if (!rank) {
331:     PetscDrawClear(draw);
332:     PetscDrawFlush(draw);
333:     PetscDrawAxisSetLimits(axis,0.0,(double)xin->N,ymin,ymax);
334:     PetscDrawAxisDraw(axis);
335:     PetscDrawGetCoordinates(draw,coors,coors+1,coors+2,coors+3);
336:   }
337:   PetscDrawAxisDestroy(axis);
338:   MPI_Bcast(coors,4,MPIU_REAL,0,xin->comm);
339:   if (rank) {PetscDrawSetCoordinates(draw,coors[0],coors[1],coors[2],coors[3]);}
340:   /* draw local part of vector */
341:   VecGetOwnershipRange(xin,&start,&end);
342:   if (rank < size-1) { /*send value to right */
343:     MPI_Send(&xarray[xin->n-1],1,MPIU_REAL,rank+1,tag,xin->comm);
344:   }
345:   for (i=1; i<xin->n; i++) {
346: #if !defined(PETSC_USE_COMPLEX)
347:     PetscDrawLine(draw,(PetscReal)(i-1+start),xarray[i-1],(PetscReal)(i+start),
348:                    xarray[i],PETSC_DRAW_RED);
349: #else
350:     PetscDrawLine(draw,(PetscReal)(i-1+start),PetscRealPart(xarray[i-1]),(PetscReal)(i+start),
351:                    PetscRealPart(xarray[i]),PETSC_DRAW_RED);
352: #endif
353:   }
354:   if (rank) { /* receive value from right */
355:     MPI_Recv(&tmp,1,MPIU_REAL,rank-1,tag,xin->comm,&status);
356: #if !defined(PETSC_USE_COMPLEX)
357:     PetscDrawLine(draw,(PetscReal)start-1,tmp,(PetscReal)start,xarray[0],PETSC_DRAW_RED);
358: #else
359:     PetscDrawLine(draw,(PetscReal)start-1,tmp,(PetscReal)start,PetscRealPart(xarray[0]),PETSC_DRAW_RED);
360: #endif
361:   }
362:   PetscDrawSynchronizedFlush(draw);
363:   PetscDrawPause(draw);
364:   VecRestoreArrayFast(xin,&xarray);
365:   return(0);
366: }
367: EXTERN_C_END

371: int VecView_MPI_Socket(Vec xin,PetscViewer viewer)
372: {
373:   int         i,rank,size,N = xin->N,*lens,ierr;
374:   PetscScalar *xx,*xarray;

377:   VecGetArrayFast(xin,&xarray);
378:   MPI_Comm_rank(xin->comm,&rank);
379:   MPI_Comm_size(xin->comm,&size);
380:   if (!rank) {
381:     PetscMalloc((N+1)*sizeof(PetscScalar),&xx);
382:     PetscMalloc(size*sizeof(int),&lens);
383:     for (i=0; i<size; i++) {
384:       lens[i] = xin->map->range[i+1] - xin->map->range[i];
385:     }
386:     MPI_Gatherv(xarray,xin->n,MPIU_SCALAR,xx,lens,xin->map->range,MPIU_SCALAR,0,xin->comm);
387:     PetscFree(lens);
388:     PetscViewerSocketPutScalar(viewer,N,1,xx);
389:     PetscFree(xx);
390:   } else {
391:     MPI_Gatherv(xarray,xin->n,MPIU_REAL,0,0,0,MPIU_REAL,0,xin->comm);
392:   }
393:   VecRestoreArrayFast(xin,&xarray);
394:   return(0);
395: }

399: int VecView_MPI_Netcdf(Vec xin,PetscViewer v)
400: {
401: #if defined(PETSC_HAVE_PNETCDF)
402:   int         n = xin->n,ierr,ncid,xdim,xdim_num=1,xin_id,xstart;
403:   MPI_Comm    comm = xin->comm;
404:   PetscScalar *xarray;

407: #if !defined(PETSC_USE_COMPLEX)
408:   VecGetArrayFast(xin,&xarray);
409:   PetscViewerNetcdfGetID(v,&ncid);
410:   if (ncid < 0) SETERRQ(1,"First call PetscViewerNetcdfOpen to create NetCDF dataset");
411:   /* define dimensions */
412:   ncmpi_def_dim(ncid,"PETSc_Vector_Global_Size",xin->N,&xdim);
413:   /* define variables */
414:   ncmpi_def_var(ncid,"PETSc_Vector_MPI",NC_DOUBLE,xdim_num,&xdim,&xin_id);
415:   /* leave define mode */
416:   ncmpi_enddef(ncid);
417:   /* store the vector */
418:   VecGetOwnershipRange(xin,&xstart,PETSC_NULL);
419:   ncmpi_put_vara_double_all(ncid,xin_id,(const size_t*)&xstart,(const size_t*)&n,xarray);
420:   VecRestoreArrayFast(xin,&xarray);
421: #else 
422:     PetscPrintf(PETSC_COMM_WORLD,"NetCDF viewer not supported for complex numbers\n");
423: #endif
424:   return(0);
425: #else /* !defined(PETSC_HAVE_PNETCDF) */
427:   SETERRQ(1,"Build PETSc with NetCDF to use this viewer");
428: 
429: #endif
430: }

434: int VecView_MPI_HDF4_Ex(Vec X, PetscViewer viewer, int d, int *dims)
435: {
436: #if defined(PETSC_HAVE_HDF4)
437:   int         rank, ierr, len, i, j, k, size, cur, bs, n, N;
438:   int         tag = ((PetscObject)viewer)->tag;
439:   MPI_Status  status;
440:   PetscScalar *x;
441:   float       *xlf, *xf;


445:   bs = X->bs > 0 ? X->bs : 1;
446:   N  = X->N / bs;
447:   n  = X->n / bs;

449:   // For now, always convert to float
450:   PetscMalloc(N * sizeof(float), &xf);
451:   PetscMalloc(n * sizeof(float), &xlf);

453:   MPI_Comm_rank(X->comm, &rank);
454:   MPI_Comm_size(X->comm, &size);

456:   VecGetArrayFast(X, &x);

458:   for (k = 0; k < bs; k++) {
459:     for (i = 0; i < n; i++) {
460:       xlf[i] = (float) x[i*bs + k];
461:     }
462:     if (rank == 0) {
463:       cur = 0;
464:       PetscMemcpy(xf + cur, xlf, n * sizeof(float));
465:       cur += n;
466:       for (j = 1; j < size; j++) {
467:  MPI_Recv(xf + cur, N - cur, MPI_FLOAT, j, tag, X->comm,
468:  &status);
469:  MPI_Get_count(&status, MPI_FLOAT, &len);
470:  cur += len;
471:       }
472:       if (cur != N) {
473:  SETERRQ2(1, "? %d %d", cur, N);
474:       }
475:       PetscViewerHDF4WriteSDS(viewer, xf, 2, dims, bs);
476:     } else {
477:       MPI_Send(xlf, n, MPI_FLOAT, 0, tag, X->comm);
478:     }
479:   }
480:   VecRestoreArrayFast(X, &x);
481:   PetscFree(xlf);
482:   PetscFree(xf);
483:   return(0);
484: #else /* !defined(PETSC_HAVE_HDF4) */
486:   SETERRQ(1,"Build PETSc with HDF4 to use this viewer");
487: 
488: #endif
489: }

493: int VecView_MPI_HDF4(Vec xin,PetscViewer viewer)
494: {
495: #if defined(PETSC_HAVE_HDF4)  
496:   int ierr, bs, dims[1];

498:   bs = xin->bs > 0 ? xin->bs : 1;
499:   dims[0] = xin->N / bs;
500:   VecView_MPI_HDF4_Ex(xin, viewer, 1, dims);
501:   return(0);
502: #else /* !defined(PETSC_HAVE_HDF4) */
504:   SETERRQ(1,"Build PETSc with HDF4 to use this viewer");
505: #endif
506: }

510: int VecView_MPI(Vec xin,PetscViewer viewer)
511: {
512:   int        ierr;
513:   PetscTruth isascii,issocket,isbinary,isdraw,ismathematica,isnetcdf,ishdf4;

516:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&isascii);
517:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_SOCKET,&issocket);
518:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_BINARY,&isbinary);
519:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_DRAW,&isdraw);
520:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_MATHEMATICA,&ismathematica);
521:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_NETCDF,&isnetcdf);
522:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_HDF4,&ishdf4);
523:   if (isascii){
524:     VecView_MPI_ASCII(xin,viewer);
525:   } else if (issocket) {
526:     VecView_MPI_Socket(xin,viewer);
527:   } else if (isbinary) {
528:     VecView_MPI_Binary(xin,viewer);
529:   } else if (isdraw) {
530:     PetscViewerFormat format;

532:     PetscViewerGetFormat(viewer,&format);
533:     if (format == PETSC_VIEWER_DRAW_LG) {
534:       VecView_MPI_Draw_LG(xin,viewer);
535:     } else {
536:       VecView_MPI_Draw(xin,viewer);
537:     }
538:   } else if (ismathematica) {
539:     PetscViewerMathematicaPutVector(viewer,xin);
540:   } else if (isnetcdf) {
541:     VecView_MPI_Netcdf(xin,viewer);
542:   } else if (ishdf4) {
543:     VecView_MPI_HDF4(xin,viewer);
544:   } else {
545:     SETERRQ1(1,"Viewer type %s not supported for this object",((PetscObject)viewer)->type_name);
546:   }
547:   return(0);
548: }

552: int VecGetSize_MPI(Vec xin,int *N)
553: {
555:   *N = xin->N;
556:   return(0);
557: }

561: int VecSetValues_MPI(Vec xin,int ni,const int ix[],const PetscScalar y[],InsertMode addv)
562: {
563:   int           rank = xin->stash.rank, *owners = xin->map->range,start = owners[rank];
564:   int           end = owners[rank+1],i,row,ierr;
565:   PetscScalar   *xx;

568: #if defined(PETSC_USE_BOPT_g)
569:   if (xin->stash.insertmode == INSERT_VALUES && addv == ADD_VALUES) {
570:    SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"You have already inserted values; you cannot now add");
571:   } else if (xin->stash.insertmode == ADD_VALUES && addv == INSERT_VALUES) {
572:    SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"You have already added values; you cannot now insert");
573:   }
574: #endif
575:   VecGetArrayFast(xin,&xx);
576:   xin->stash.insertmode = addv;

578:   if (addv == INSERT_VALUES) {
579:     for (i=0; i<ni; i++) {
580:       if ((row = ix[i]) >= start && row < end) {
581:         xx[row-start] = y[i];
582:       } else if (!xin->stash.donotstash) {
583:         if (ix[i] < 0) continue;
584: #if defined(PETSC_USE_BOPT_g)
585:         if (ix[i] >= xin->N) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Out of range index value %d maximum %d",ix[i],xin->N);
586: #endif
587:         VecStashValue_Private(&xin->stash,row,y[i]);
588:       }
589:     }
590:   } else {
591:     for (i=0; i<ni; i++) {
592:       if ((row = ix[i]) >= start && row < end) {
593:         xx[row-start] += y[i];
594:       } else if (!xin->stash.donotstash) {
595:         if (ix[i] < 0) continue;
596: #if defined(PETSC_USE_BOPT_g)
597:         if (ix[i] > xin->N) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Out of range index value %d maximum %d",ix[i],xin->N);
598: #endif        
599:         VecStashValue_Private(&xin->stash,row,y[i]);
600:       }
601:     }
602:   }
603:   VecRestoreArrayFast(xin,&xx);
604:   return(0);
605: }

609: int VecSetValuesBlocked_MPI(Vec xin,int ni,const int ix[],const PetscScalar yin[],InsertMode addv)
610: {
611:   int           rank = xin->stash.rank,*owners = xin->map->range,start = owners[rank];
612:   int           end = owners[rank+1],i,row,bs = xin->bs,j,ierr;
613:   PetscScalar   *xx,*y = (PetscScalar*)yin;

616:   VecGetArrayFast(xin,&xx);
617: #if defined(PETSC_USE_BOPT_g)
618:   if (xin->stash.insertmode == INSERT_VALUES && addv == ADD_VALUES) {
619:    SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"You have already inserted values; you cannot now add");
620:   }
621:   else if (xin->stash.insertmode == ADD_VALUES && addv == INSERT_VALUES) {
622:    SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"You have already added values; you cannot now insert");
623:   }
624: #endif
625:   xin->stash.insertmode = addv;

627:   if (addv == INSERT_VALUES) {
628:     for (i=0; i<ni; i++) {
629:       if ((row = bs*ix[i]) >= start && row < end) {
630:         for (j=0; j<bs; j++) {
631:           xx[row-start+j] = y[j];
632:         }
633:       } else if (!xin->stash.donotstash) {
634:         if (ix[i] < 0) continue;
635: #if defined(PETSC_USE_BOPT_g)
636:         if (ix[i] >= xin->N) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Out of range index value %d max %d",ix[i],xin->N);
637: #endif
638:         VecStashValuesBlocked_Private(&xin->bstash,ix[i],y);
639:       }
640:       y += bs;
641:     }
642:   } else {
643:     for (i=0; i<ni; i++) {
644:       if ((row = bs*ix[i]) >= start && row < end) {
645:         for (j=0; j<bs; j++) {
646:           xx[row-start+j] += y[j];
647:         }
648:       } else if (!xin->stash.donotstash) {
649:         if (ix[i] < 0) continue;
650: #if defined(PETSC_USE_BOPT_g)
651:         if (ix[i] > xin->N) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Out of range index value %d max %d",ix[i],xin->N);
652: #endif
653:         VecStashValuesBlocked_Private(&xin->bstash,ix[i],y);
654:       }
655:       y += bs;
656:     }
657:   }
658:   VecRestoreArrayFast(xin,&xx);
659:   return(0);
660: }

662: /*
663:    Since nsends or nreceives may be zero we add 1 in certain mallocs
664: to make sure we never malloc an empty one.      
665: */
668: int VecAssemblyBegin_MPI(Vec xin)
669: {
670:   int         *owners = xin->map->range,*bowners,ierr,size,i,bs,nstash,reallocs;
671:   InsertMode  addv;
672:   MPI_Comm    comm = xin->comm;

675:   if (xin->stash.donotstash) {
676:     return(0);
677:   }

679:   MPI_Allreduce(&xin->stash.insertmode,&addv,1,MPI_INT,MPI_BOR,comm);
680:   if (addv == (ADD_VALUES|INSERT_VALUES)) {
681:     SETERRQ(PETSC_ERR_ARG_NOTSAMETYPE,"Some processors inserted values while others added");
682:   }
683:   xin->stash.insertmode = addv; /* in case this processor had no cache */
684: 
685:   bs = xin->bs;
686:   MPI_Comm_size(xin->comm,&size);
687:   if (!xin->bstash.bowners && xin->bs != -1) {
688:     PetscMalloc((size+1)*sizeof(int),&bowners);
689:     for (i=0; i<size+1; i++){ bowners[i] = owners[i]/bs;}
690:     xin->bstash.bowners = bowners;
691:   } else {
692:     bowners = xin->bstash.bowners;
693:   }
694:   VecStashScatterBegin_Private(&xin->stash,owners);
695:   VecStashScatterBegin_Private(&xin->bstash,bowners);
696:   VecStashGetInfo_Private(&xin->stash,&nstash,&reallocs);
697:   PetscLogInfo(0,"VecAssemblyBegin_MPI:Stash has %d entries, uses %d mallocs.\n",nstash,reallocs);
698:   VecStashGetInfo_Private(&xin->bstash,&nstash,&reallocs);
699:   PetscLogInfo(0,"VecAssemblyBegin_MPI:Block-Stash has %d entries, uses %d mallocs.\n",nstash,reallocs);

701:   return(0);
702: }

706: int VecAssemblyEnd_MPI(Vec vec)
707: {
708:   int         ierr,base,i,j,n,*row,flg,bs;
709:   PetscScalar *val,*vv,*array,*xarray;

712:   if (!vec->stash.donotstash) {
713:     VecGetArrayFast(vec,&xarray);
714:     base = vec->map->range[vec->stash.rank];
715:     bs   = vec->bs;

717:     /* Process the stash */
718:     while (1) {
719:       VecStashScatterGetMesg_Private(&vec->stash,&n,&row,&val,&flg);
720:       if (!flg) break;
721:       if (vec->stash.insertmode == ADD_VALUES) {
722:         for (i=0; i<n; i++) { xarray[row[i] - base] += val[i]; }
723:       } else if (vec->stash.insertmode == INSERT_VALUES) {
724:         for (i=0; i<n; i++) { xarray[row[i] - base] = val[i]; }
725:       } else {
726:         SETERRQ(PETSC_ERR_ARG_CORRUPT,"Insert mode is not set correctly; corrupted vector");
727:       }
728:     }
729:     VecStashScatterEnd_Private(&vec->stash);

731:     /* now process the block-stash */
732:     while (1) {
733:       VecStashScatterGetMesg_Private(&vec->bstash,&n,&row,&val,&flg);
734:       if (!flg) break;
735:       for (i=0; i<n; i++) {
736:         array = xarray+row[i]*bs-base;
737:         vv    = val+i*bs;
738:         if (vec->stash.insertmode == ADD_VALUES) {
739:           for (j=0; j<bs; j++) { array[j] += vv[j];}
740:         } else if (vec->stash.insertmode == INSERT_VALUES) {
741:           for (j=0; j<bs; j++) { array[j] = vv[j]; }
742:         } else {
743:           SETERRQ(PETSC_ERR_ARG_CORRUPT,"Insert mode is not set correctly; corrupted vector");
744:         }
745:       }
746:     }
747:     VecStashScatterEnd_Private(&vec->bstash);
748:     VecRestoreArrayFast(vec,&xarray);
749:   }
750:   vec->stash.insertmode = NOT_SET_VALUES;
751:   return(0);
752: }