Actual source code: dmmbvec.cxx
 
   petsc-3.7.7 2017-09-25
   
  1: #include <petsc/private/dmmbimpl.h> /*I  "petscdmmoab.h"   I*/
  2: #include <petsc/private/vecimpl.h>
  4: #include <petscdmmoab.h>
  5: #include <MBTagConventions.hpp>
  7: /* declare some private DMMoab specific overrides */
  8: static PetscErrorCode DMCreateVector_Moab_Private(DM dm,moab::Tag tag,const moab::Range* userrange,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec);
  9: static PetscErrorCode DMVecUserDestroy_Moab(void *user);
 10: static PetscErrorCode DMVecDuplicate_Moab(Vec x,Vec *y);
 11: static PetscErrorCode DMVecCreateTagName_Moab_Private(moab::ParallelComm *pcomm,char** tag_name);
 15: /*@C
 16:   DMMoabCreateVector - Create a Vec from either an existing tag, or a specified tag size, and a range of entities
 18:   Collective on MPI_Comm
 20:   Input Parameter:
 21: + dm              - The DMMoab object being set
 22: . tag             - If non-zero, block size will be taken from the tag size
 23: . range           - If non-empty, Vec corresponds to these entities, otherwise to the entities set on the DMMoab
 24: . is_global_vec   - If true, this is a local representation of the Vec (including ghosts in parallel), otherwise a truly parallel one
 25: . destroy_tag     - If true, MOAB tag is destroyed with Vec, otherwise it is left on MOAB
 27:   Output Parameter:
 28: . vec             - The created vector
 30:   Level: beginner
 32: .keywords: Vec, create
 34: .seealso: VecCreate()
 35: @*/
 36: PetscErrorCode DMMoabCreateVector(DM dm,moab::Tag tag,const moab::Range* range,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec)
 37: {
 38:   PetscErrorCode     ierr;
 41:   if(!tag && (!range || range->empty())) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Both tag and range cannot be null.");
 43:   DMCreateVector_Moab_Private(dm,tag,range,is_global_vec,destroy_tag,vec);
 44:   return(0);
 45: }
 50: /*@C
 51:   DMMoabGetVecTag - Get the MOAB tag associated with this Vec
 53:   Input Parameter:
 54: . vec - Vec being queried
 56:   Output Parameter:
 57: . tag - Tag associated with this Vec. NULL if vec is a native PETSc Vec.
 59:   Level: beginner
 61: .keywords: DMMoab, MOAB tag
 63: .seealso: DMMoabCreateVector(), DMMoabGetVecRange()
 64: @*/
 65: PetscErrorCode DMMoabGetVecTag(Vec vec,moab::Tag *tag)
 66: {
 67:   PetscContainer  moabdata;
 68:   Vec_MOAB        *vmoab;
 69:   PetscErrorCode  ierr;
 74:   /* Get the MOAB private data */
 75:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
 76:   PetscContainerGetPointer(moabdata, (void**) &vmoab);
 78:   *tag = vmoab->tag;
 79:   return(0);
 80: }
 85: /*@C
 86:   DMMoabGetVecRange - Get the MOAB entities associated with this Vec
 88:   Input Parameter:
 89: . vec   - Vec being queried
 91:   Output Parameter:
 92: . range - Entities associated with this Vec. NULL if vec is a native PETSc Vec.
 94:   Level: beginner
 96: .keywords: DMMoab, MOAB range
 98: .seealso: DMMoabCreateVector(), DMMoabGetVecTag()
 99: @*/
100: PetscErrorCode DMMoabGetVecRange(Vec vec,moab::Range *range)
101: {
102:   PetscContainer  moabdata;
103:   Vec_MOAB        *vmoab;
104:   PetscErrorCode  ierr;
109:   /* Get the MOAB private data handle */
110:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
111:   PetscContainerGetPointer(moabdata, (void**) &vmoab);
113:   *range = *vmoab->tag_range;
114:   return(0);
115: }
120: /*@C
121:   DMMoabVecGetArray - Returns the writable direct access array to the local representation of MOAB tag data for the underlying vector using locally owned+ghosted range of entities
123:   Collective on MPI_Comm
125:   Input Parameter:
126: . dm              - The DMMoab object being set
127: . vec             - The Vector whose underlying data is requested
129:   Output Parameter:
130: . array           - The local data array
132:   Level: intermediate
134: .keywords: MOAB, distributed array
136: .seealso: DMMoabVecRestoreArray(), DMMoabVecGetArrayRead(), DMMoabVecRestoreArrayRead()
137: @*/
138: PetscErrorCode  DMMoabVecGetArray(DM dm,Vec vec,void* array)
139: {
140:   DM_Moab        *dmmoab;
141:   moab::ErrorCode merr;
142:   PetscErrorCode  ierr;
143:   PetscInt        count,i,f;
144:   moab::Tag       vtag;
145:   PetscScalar     **varray;
146:   PetscScalar     *marray;
147:   PetscContainer  moabdata;
148:   Vec_MOAB        *vmoab,*xmoab;
154:   dmmoab=(DM_Moab*)dm->data;
156:   /* Get the Vec_MOAB struct for the original vector */
157:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
158:   PetscContainerGetPointer(moabdata, (void**)&vmoab);
160:   /* Get the real scalar array handle */
161:   varray = reinterpret_cast<PetscScalar**>(array);
163:   if (vmoab->is_native_vec) {
165:     /* get the local representation of the arrays from Vectors */
166:     DMGetLocalVector(dm,&vmoab->local);
167:     DMGlobalToLocalBegin(dm,vec,INSERT_VALUES,vmoab->local);
168:     DMGlobalToLocalEnd(dm,vec,INSERT_VALUES,vmoab->local);
170:     /* Get the Vec_MOAB struct for the original vector */
171:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
172:     PetscContainerGetPointer(moabdata, (void**)&xmoab);
174:     /* get the local representation of the arrays from Vectors */
175:     VecGhostGetLocalForm(vmoab->local,&xmoab->local);
176:     VecGhostUpdateBegin(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
177:     VecGhostUpdateEnd(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
179:     VecGetArray(xmoab->local, varray);
180:   }
181:   else {
183:     /* Get the MOAB private data */
184:     DMMoabGetVecTag(vec,&vtag);
186:     /* exchange the data into ghost cells first */
187:     merr = dmmoab->pcomm->exchange_tags(vtag,*dmmoab->vlocal);MBERRNM(merr);
189:     PetscMalloc1((dmmoab->nloc+dmmoab->nghost)*dmmoab->numFields,varray);
191:     /* Get the array data for local entities */
192:     merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
193:     if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);
195:     i=0;
196:     for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
197:       for (f=0;f<dmmoab->numFields;f++,i++)
198:         (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f]=marray[i];
199:     }
200:   }
201:   return(0);
202: }
207: /*@C
208:   DMMoabVecRestoreArray - Restores the writable direct access array obtained via DMMoabVecGetArray
210:   Collective on MPI_Comm
212:   Input Parameter:
213: + dm              - The DMMoab object being set
214: . vec             - The Vector whose underlying data is requested
215: . array           - The local data array
217:   Level: intermediate
219: .keywords: MOAB, distributed array
221: .seealso: DMMoabVecGetArray(), DMMoabVecGetArrayRead(), DMMoabVecRestoreArrayRead()
222: @*/
223: PetscErrorCode  DMMoabVecRestoreArray(DM dm,Vec vec,void* array)
224: {
225:   DM_Moab        *dmmoab;
226:   moab::ErrorCode merr;
227:   PetscErrorCode  ierr;
228:   moab::Tag       vtag;
229:   PetscInt        count,i,f;
230:   PetscScalar     **varray;
231:   PetscScalar     *marray;
232:   PetscContainer  moabdata;
233:   Vec_MOAB        *vmoab,*xmoab;
239:   dmmoab=(DM_Moab*)dm->data;
241:   /* Get the Vec_MOAB struct for the original vector */
242:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
243:   PetscContainerGetPointer(moabdata, (void**)&vmoab);
245:   /* Get the real scalar array handle */
246:   varray = reinterpret_cast<PetscScalar**>(array);
248:   if (vmoab->is_native_vec) {
250:     /* Get the Vec_MOAB struct for the original vector */
251:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
252:     PetscContainerGetPointer(moabdata, (void**)&xmoab);
254:     /* get the local representation of the arrays from Vectors */
255:     VecRestoreArray(xmoab->local, varray);
256:     VecGhostUpdateBegin(vmoab->local,ADD_VALUES,SCATTER_REVERSE);
257:     VecGhostUpdateEnd(vmoab->local,ADD_VALUES,SCATTER_REVERSE);
258:     VecGhostRestoreLocalForm(vmoab->local,&xmoab->local);
260:     /* restore local pieces */
261:     DMLocalToGlobalBegin(dm,vmoab->local,INSERT_VALUES,vec);
262:     DMLocalToGlobalEnd(dm,vmoab->local,INSERT_VALUES,vec);
263:     DMRestoreLocalVector(dm, &vmoab->local);
264:   }
265:   else {
267:     /* Get the MOAB private data */
268:     DMMoabGetVecTag(vec,&vtag);
270:     /* Get the array data for local entities */
271:     merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
272:     if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);
274:     i=0;
275:     for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
276:       for (f=0;f<dmmoab->numFields;f++,i++)
277:       marray[i] = (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f];
278:     }
280:     /* reduce the tags correctly -> should probably let the user choose how to reduce in the future
281:       For all FEM residual based assembly calculations, MPI_SUM should serve well */
282:     merr = dmmoab->pcomm->reduce_tags(vtag,MPI_SUM,*dmmoab->vlocal);MBERRV(dmmoab->mbiface,merr);
284:     PetscFree(*varray);
285:   }
286:   return(0);
287: }
291: /*@C
292:   DMMoabVecGetArrayRead - Returns the read-only direct access array to the local representation of MOAB tag data for the underlying vector using locally owned+ghosted range of entities
294:   Collective on MPI_Comm
296:   Input Parameter:
297: + dm              - The DMMoab object being set
298: . vec             - The Vector whose underlying data is requested
300:   Output Parameter:
301: . array           - The local data array
303:   Level: intermediate
305: .keywords: MOAB, distributed array
307: .seealso: DMMoabVecRestoreArrayRead(), DMMoabVecGetArray(), DMMoabVecRestoreArray()
308: @*/
309: PetscErrorCode  DMMoabVecGetArrayRead(DM dm,Vec vec,void* array)
310: {
311:   DM_Moab        *dmmoab;
312:   moab::ErrorCode merr;
313:   PetscErrorCode  ierr;
314:   PetscInt        count,i,f;
315:   moab::Tag       vtag;
316:   PetscScalar     **varray;
317:   PetscScalar     *marray;
318:   PetscContainer  moabdata;
319:   Vec_MOAB        *vmoab,*xmoab;
325:   dmmoab=(DM_Moab*)dm->data;
327:   /* Get the Vec_MOAB struct for the original vector */
328:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
329:   PetscContainerGetPointer(moabdata, (void**)&vmoab);
331:   /* Get the real scalar array handle */
332:   varray = reinterpret_cast<PetscScalar**>(array);
334:   if (vmoab->is_native_vec) {
335:     /* get the local representation of the arrays from Vectors */
336:     DMGetLocalVector(dm,&vmoab->local);
337:     DMGlobalToLocalBegin(dm,vec,INSERT_VALUES,vmoab->local);
338:     DMGlobalToLocalEnd(dm,vec,INSERT_VALUES,vmoab->local);
340:     /* Get the Vec_MOAB struct for the original vector */
341:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
342:     PetscContainerGetPointer(moabdata, (void**)&xmoab);
344:     /* get the local representation of the arrays from Vectors */
345:     VecGhostGetLocalForm(vmoab->local,&xmoab->local);
346:     VecGhostUpdateBegin(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
347:     VecGhostUpdateEnd(vmoab->local,INSERT_VALUES,SCATTER_FORWARD);
348:     VecGetArray(xmoab->local, varray);
349:   }
350:   else {
351:     /* Get the MOAB private data */
352:     DMMoabGetVecTag(vec,&vtag);
354:     /* exchange the data into ghost cells first */
355:     merr = dmmoab->pcomm->exchange_tags(vtag,*dmmoab->vlocal);MBERRNM(merr);
357:     PetscMalloc1((dmmoab->nloc+dmmoab->nghost)*dmmoab->numFields,varray);
359:     /* Get the array data for local entities */
360:     merr = dmmoab->mbiface->tag_iterate(vtag,dmmoab->vlocal->begin(),dmmoab->vlocal->end(),count,reinterpret_cast<void*&>(marray),false);MBERRNM(merr);
361:     if (count!=dmmoab->nloc+dmmoab->nghost) SETERRQ2(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Mismatch between local vertices and tag partition for Vec. %D != %D.",count,dmmoab->nloc+dmmoab->nghost);
363:     i=0;
364:     for(moab::Range::iterator iter = dmmoab->vlocal->begin(); iter != dmmoab->vlocal->end(); iter++) {
365:       for (f=0;f<dmmoab->numFields;f++,i++)
366:         (*varray)[dmmoab->lidmap[(PetscInt)*iter]*dmmoab->numFields+f]=marray[i];
367:     }
368:   }
369:   return(0);
370: }
375: /*@C
376:   DMMoabVecRestoreArray - Restores the read-only direct access array obtained via DMMoabVecGetArray
378:   Collective on MPI_Comm
380:   Input Parameter:
381: + dm              - The DMMoab object being set
382: . vec             - The Vector whose underlying data is requested
383: . array           - The local data array
385:   Level: intermediate
387: .keywords: MOAB, distributed array
389: .seealso: DMMoabVecGetArrayRead(), DMMoabVecGetArray(), DMMoabVecRestoreArray()
390: @*/
391: PetscErrorCode  DMMoabVecRestoreArrayRead(DM dm,Vec vec,void* array)
392: {
393:   PetscErrorCode  ierr;
394:   PetscScalar     **varray;
395:   PetscContainer  moabdata;
396:   Vec_MOAB        *vmoab,*xmoab;
403:   /* Get the Vec_MOAB struct for the original vector */
404:   PetscObjectQuery((PetscObject)vec,"MOABData", (PetscObject*) &moabdata);
405:   PetscContainerGetPointer(moabdata, (void**)&vmoab);
407:   /* Get the real scalar array handle */
408:   varray = reinterpret_cast<PetscScalar**>(array);
410:   if (vmoab->is_native_vec) {
411:     /* Get the Vec_MOAB struct for the original vector */
412:     PetscObjectQuery((PetscObject)vmoab->local,"MOABData", (PetscObject*) &moabdata);
413:     PetscContainerGetPointer(moabdata, (void**)&xmoab);
415:     /* restore the local representation of the arrays from Vectors */
416:     VecRestoreArray(xmoab->local, varray);
417:     VecGhostRestoreLocalForm(vmoab->local,&xmoab->local);
419:     /* restore local pieces */
420:     DMRestoreLocalVector(dm, &vmoab->local);
421:   }
422:   else {
423:     /* Nothing to do but just free the memory allocated before */
424:     PetscFree(*varray);
426:   }
427:   return(0);
428: }
433: PetscErrorCode DMCreateVector_Moab_Private(DM dm,moab::Tag tag,const moab::Range* userrange,PetscBool is_global_vec,PetscBool destroy_tag,Vec *vec)
434: {
435:   PetscErrorCode    ierr;
436:   moab::ErrorCode   merr;
437:   PetscBool         is_newtag;
438:   const moab::Range *range;
439:   PetscInt          count,lnative_vec,gnative_vec;
440:   std::string       ttname;
441:   PetscScalar       *data_ptr,*defaultvals;
443:   Vec_MOAB *vmoab;
444:   DM_Moab *dmmoab = (DM_Moab*)dm->data;
445:   moab::ParallelComm *pcomm = dmmoab->pcomm;
446:   moab::Interface *mbiface = dmmoab->mbiface;
449:   if(sizeof(PetscReal) != sizeof(PetscScalar)) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "MOAB tags only support Real types (Complex-type unsupported)");
450:   if(!userrange) range = dmmoab->vowned;
451:   else range = userrange;
452:   if(!range) SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_ARG_WRONG, "Input range cannot be empty or call DMSetUp first.");
454:   /* If the tag data is in a single sequence, use PETSc native vector since tag_iterate isn't useful anymore */
455:   lnative_vec=(range->psize()-1);
457: #if 0
458: //  lnative_vec=1; /* NOTE: Testing PETSc vector: will force to create native vector all the time */
459: //  lnative_vec=0; /* NOTE: Testing MOAB vector: will force to create MOAB tag_iterate based vector all the time */
460: #endif
461:   MPIU_Allreduce(&lnative_vec, &gnative_vec, 1, MPI_INT, MPI_MAX, pcomm->comm());
463:   /* Create the MOAB internal data object */
464:   PetscNew(&vmoab);
465:   vmoab->is_native_vec=(gnative_vec>0?PETSC_TRUE:PETSC_FALSE);
467:   if (!vmoab->is_native_vec) {
468:     merr = mbiface->tag_get_name(tag, ttname);
469:     if (!ttname.length() && merr !=moab::MB_SUCCESS) {
470:       /* get the new name for the anonymous MOABVec -> the tag_name will be destroyed along with Tag */
471:       char *tag_name = PETSC_NULL;
472:       DMVecCreateTagName_Moab_Private(pcomm,&tag_name);
473:       is_newtag = PETSC_TRUE;
475:       /* Create the default value for the tag (all zeros) */
476:       PetscCalloc1(dmmoab->numFields,&defaultvals);
478:       /* Create the tag */
479:       merr = mbiface->tag_get_handle(tag_name,dmmoab->numFields,moab::MB_TYPE_DOUBLE,tag,
480:                                     moab::MB_TAG_DENSE|moab::MB_TAG_CREAT,defaultvals);MBERRNM(merr);
481:       PetscFree(tag_name);
482:       PetscFree(defaultvals);
483:     }
484:     else {
485:       /* Make sure the tag data is of type "double" */
486:       moab::DataType tag_type;
487:       merr = mbiface->tag_get_data_type(tag, tag_type);MBERRNM(merr);
488:       if(tag_type != moab::MB_TYPE_DOUBLE) SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Tag data type must be MB_TYPE_DOUBLE");
489:       is_newtag = destroy_tag;
490:     }
492:     vmoab->tag = tag;
493:     vmoab->new_tag = is_newtag;
494:   }
495:   vmoab->mbiface = mbiface;
496:   vmoab->pcomm = pcomm;
497:   vmoab->is_global_vec = is_global_vec;
498:   vmoab->tag_size=dmmoab->bs;
499: 
500:   if (vmoab->is_native_vec) {
502:     /* Create the PETSc Vector directly and attach our functions accordingly */
503:     if (!is_global_vec) {
504:       /* This is an MPI Vector with ghosted padding */
505:       VecCreateGhostBlock(pcomm->comm(),dmmoab->bs,dmmoab->numFields*dmmoab->nloc,
506:                                  dmmoab->numFields*dmmoab->n,dmmoab->nghost,&dmmoab->gsindices[dmmoab->nloc],vec);
507:     }
508:     else {
509:       /* This is an MPI/SEQ Vector */
510:       VecCreate(pcomm->comm(),vec);
511:       VecSetSizes(*vec,dmmoab->numFields*dmmoab->nloc,PETSC_DECIDE);
512:       VecSetBlockSize(*vec,dmmoab->bs);
513:       VecSetType(*vec, VECMPI);
514:     }
515:   }
516:   else {
517:     /* Call tag_iterate. This will cause MOAB to allocate memory for the
518:        tag data if it hasn't already happened */
519:     merr = mbiface->tag_iterate(tag,range->begin(),range->end(),count,(void*&)data_ptr);MBERRNM(merr);
521:     /* set the reference for vector range */
522:     vmoab->tag_range = new moab::Range(*range);
523:     merr = mbiface->tag_get_length(tag,dmmoab->numFields);MBERRNM(merr);
525:     /* Create the PETSc Vector
526:       Query MOAB mesh to check if there are any ghosted entities
527:         -> if we do, create a ghosted vector to map correctly to the same layout
528:         -> else, create a non-ghosted parallel vector */
529:     if (!is_global_vec) {
530:       /* This is an MPI Vector with ghosted padding */
531:       VecCreateGhostBlockWithArray(pcomm->comm(),dmmoab->bs,dmmoab->numFields*dmmoab->nloc,
532:                                 dmmoab->numFields*dmmoab->n,dmmoab->nghost,&dmmoab->gsindices[dmmoab->nloc],data_ptr,vec);
533:     }
534:     else {
535:       /* This is an MPI Vector without ghosted padding */
536:       VecCreateMPIWithArray(pcomm->comm(),dmmoab->bs,dmmoab->numFields*range->size(),
537:                                 PETSC_DECIDE,data_ptr,vec);
538:     }
539:   }
540:   VecSetFromOptions(*vec);
542:   /* create a container and store the internal MOAB data for faster access based on Entities etc */
543:   PetscContainer moabdata;
544:   PetscContainerCreate(PETSC_COMM_WORLD,&moabdata);
545:   PetscContainerSetPointer(moabdata,vmoab);
546:   PetscContainerSetUserDestroy(moabdata,DMVecUserDestroy_Moab);
547:   PetscObjectCompose((PetscObject)*vec,"MOABData",(PetscObject)moabdata);
548:   (*vec)->ops->duplicate = DMVecDuplicate_Moab;
549:   PetscContainerDestroy(&moabdata);
551:   /* Vector created, manually set local to global mapping */
552:   if (dmmoab->ltog_map) {
553:     VecSetLocalToGlobalMapping(*vec,dmmoab->ltog_map);
554:   }
556:   /* set the DM reference to the vector */
557:   VecSetDM(*vec, dm);
558:   return(0);
559: }
564: /*  DMVecCreateTagName_Moab_Private
565:  *
566:  *  Creates a unique tag name that will be shared across processes. If
567:  *  pcomm is NULL, then this is a serial vector. A unique tag name
568:  *  will be returned in tag_name in either case.
569:  *
570:  *  The tag names have the format _PETSC_VEC_N where N is some integer.
571:  *
572:  *  NOTE: The tag_name is allocated in this routine; The user needs to free 
573:  *        the character array.
574:  */
575: PetscErrorCode DMVecCreateTagName_Moab_Private(moab::ParallelComm *pcomm,char** tag_name)
576: {
577:   moab::ErrorCode mberr;
578:   PetscErrorCode  ierr;
579:   PetscInt        n,global_n;
580:   moab::Tag indexTag;
583:   const char*       PVEC_PREFIX      = "__PETSC_VEC_";
584:   PetscMalloc(PETSC_MAX_PATH_LEN, tag_name);
586:   /* Check to see if there are any PETSc vectors defined */
587:   moab::Interface  *mbiface = pcomm->get_moab();
588:   moab::EntityHandle rootset = mbiface->get_root_set();
589: 
590:   /* Create a tag in MOAB mesh to index and keep track of number of Petsc vec tags */
591:   mberr = mbiface->tag_get_handle("__PETSC_VECS__",1,moab::MB_TYPE_INTEGER,indexTag,
592:                                   moab::MB_TAG_SPARSE | moab::MB_TAG_CREAT,0);MBERRNM(mberr);
593:   mberr = mbiface->tag_get_data(indexTag, &rootset, 1, &n);
594:   if (mberr == moab::MB_TAG_NOT_FOUND) n=0;  /* this is the first temporary vector */
595:   else MBERRNM(mberr);
597:   /* increment the new value of n */
598:   ++n;
600:   /* Make sure that n is consistent across all processes */
601:   MPIU_Allreduce(&n,&global_n,1,MPI_INT,MPI_MAX,pcomm->comm());
603:   /* Set the new name accordingly and return */
604:   PetscSNPrintf(*tag_name, PETSC_MAX_PATH_LEN-1, "%s_%D", PVEC_PREFIX, global_n);
605:   mberr = mbiface->tag_set_data(indexTag, &rootset, 1, (const void*)&global_n);MBERRNM(mberr);
606:   return(0);
607: }
612: PetscErrorCode DMCreateGlobalVector_Moab(DM dm,Vec *gvec)
613: {
614:   PetscErrorCode  ierr;
615:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;
620:   DMCreateVector_Moab_Private(dm,PETSC_NULL,dmmoab->vowned,PETSC_TRUE,PETSC_TRUE,gvec);
621:   return(0);
622: }
627: PetscErrorCode DMCreateLocalVector_Moab(DM dm,Vec *lvec)
628: {
629:   PetscErrorCode  ierr;
630:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;
635:   DMCreateVector_Moab_Private(dm,PETSC_NULL,dmmoab->vlocal,PETSC_FALSE,PETSC_TRUE,lvec);
636:   return(0);
637: }
642: PetscErrorCode DMVecDuplicate_Moab(Vec x,Vec *y)
643: {
645:   DM             dm;
646:   PetscContainer  moabdata;
647:   Vec_MOAB        *vmoab;
653:   /* Get the Vec_MOAB struct for the original vector */
654:   PetscObjectQuery((PetscObject)x,"MOABData", (PetscObject*) &moabdata);
655:   PetscContainerGetPointer(moabdata, (void**)&vmoab);
657:   VecGetDM(x, &dm);
660:   DMCreateVector_Moab_Private(dm,PETSC_NULL,vmoab->tag_range,vmoab->is_global_vec,PETSC_TRUE,y);
661:   VecSetDM(*y, dm);
662:   return(0);
663: }
668: PetscErrorCode DMVecUserDestroy_Moab(void *user)
669: {
670:   Vec_MOAB        *vmoab=(Vec_MOAB*)user;
671:   PetscErrorCode  ierr;
672:   moab::ErrorCode merr;
675:   if(vmoab->new_tag && vmoab->tag) {
676:     /* Tag was created via a call to VecDuplicate, delete the underlying tag in MOAB */
677:     merr = vmoab->mbiface->tag_delete(vmoab->tag);MBERRNM(merr);
678:   }
679:   delete vmoab->tag_range;
680:   vmoab->tag = PETSC_NULL;
681:   vmoab->mbiface = PETSC_NULL;
682:   vmoab->pcomm = PETSC_NULL;
683:   PetscFree(vmoab);
684:   return(0);
685: }
690: PetscErrorCode  DMGlobalToLocalBegin_Moab(DM dm,Vec g,InsertMode mode,Vec l)
691: {
692:   PetscErrorCode    ierr;
693:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;
696:   VecScatterBegin(dmmoab->ltog_sendrecv,g,l,mode,SCATTER_REVERSE);
697:   return(0);
698: }
703: PetscErrorCode  DMGlobalToLocalEnd_Moab(DM dm,Vec g,InsertMode mode,Vec l)
704: {
705:   PetscErrorCode    ierr;
706:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;
709:   VecScatterEnd(dmmoab->ltog_sendrecv,g,l,mode,SCATTER_REVERSE);
710:   return(0);
711: }
716: PetscErrorCode  DMLocalToGlobalBegin_Moab(DM dm,Vec l,InsertMode mode,Vec g)
717: {
718:   PetscErrorCode    ierr;
719:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;
722:   VecScatterBegin(dmmoab->ltog_sendrecv,l,g,mode,SCATTER_FORWARD);
723:   return(0);
724: }
729: PetscErrorCode  DMLocalToGlobalEnd_Moab(DM dm,Vec l,InsertMode mode,Vec g)
730: {
731:   PetscErrorCode    ierr;
732:   DM_Moab         *dmmoab = (DM_Moab*)dm->data;
735:   VecScatterEnd(dmmoab->ltog_sendrecv,l,g,mode,SCATTER_FORWARD);
736:   return(0);
737: }