Hello,
I am trying to use Generic Hyperslabbing to change an initial array from the normal Cactus MPI distribution to one where all the GF points can exist in one loop on one processor. For example if phi_temp is an N^3 array and phi_in is an (N-2)^3 array,
sizex = cctk_gsh[0];
sizey = cctk_gsh[1];
sizez = cctk_gsh[2];
fsizex = cctk_gsh[0]-2;
fsizey = cctk_gsh[1]-2;
fsizez = cctk_gsh[2]-2;
struct xferinfo info1[3];
for (d=0; d<3; ++d) {
/* Source array: Cactus layout */
info1[d].src.gsh = cctk_gsh[d];
info1[d].src.lbnd = cctk_lbnd[d];
info1[d].src.lsh = cctk_lsh[d];
info1[d].src.lbbox = cctk_bbox[2*d];
info1[d].src.ubbox = cctk_bbox[2*d+1];
info1[d].src.nghostzones = cctk_nghostzones[d];
/* Destination array: New layout */
info1[d].dst.gsh = cctk_gsh[d];
info1[d].dst.lbnd = 0;
info1[d].dst.lsh = cctk_gsh[d];
info1[d].dst.lbbox = cctk_bbox[2*d];
info1[d].dst.ubbox = cctk_bbox[2*d+1];
info1[d].dst.nghostzones = cctk_nghostzones[d];
/* Source slab: whole array */
info1[d].src.off = 0;
info1[d].src.len = cctk_gsh[d];
info1[d].src.str = 1;
/* Destination slab: whole array */
info1[d].dst.off = 0;
info1[d].dst.len = cctk_gsh[d];
info1[d].dst.str = 1;
/* No transformation */
info1[d].xpose = d;
info1[d].flip = 0;
}
ierr = Slab_Transfer
(cctkGH, cctk_dim, info1, -1,
CCTK_VARIABLE_REAL, phi_temp,
CCTK_VARIABLE_REAL, phi_temp);
assert (! ierr);
phi_in = malloc(sizeof(CCTK_REAL)*fsizex*fsizey*fsizez);
for(k = 1; k <= fsizez; k++)
{
for(j = 1; j <= fsizey; j++)
{
for(i = 1; i <= fsizex; i++)
{
phi_in[(i-1) + (j-1)*fsizex + (k-1)*fsizex*fsizey] = phi_temp[i + j*sizex + k*sizex*sizey];
}
}
}
Running the code as is causes segmentation fault and some of the output GFs are distorted. If I comment out the Slab_Transfer I don't get a Segmentation fault. If I run the code with a single processor, there is no segmentation fault and no distortion
in the GFs. Can you see what I'm doing wrong with the hyperslabbing? There must be a better way to get the transfer to work while avoiding the errors.
-DG