> cat test_mpi_0-dim_topology.c #include #include /* for sleep() */ #include #define Uflag -99235 void printival(char *s, int ival) { printf(s); if (ival==Uflag) { printf("U"); } else if (ival==MPI_PROC_NULL) { printf("MPI_PROC_NULL"); } else if (ival==MPI_UNDEFINED) { printf("MPI_UNDEFINED"); } else { printf("%1d",ival); } } void report_topo(int myrank, char *name, MPI_Comm cart) { int status; MPI_Topo_test(cart, &status); printf("[%02d] %s: %s", myrank, name, (status==MPI_CART ? "CART" : "NOT-CART") ); if (status==MPI_CART) { int ndims; MPI_Cartdim_get(cart,&ndims); printf(" ndims=%1d", ndims); if (ndims >= 0) { int dims, periods, coords; dims=Uflag; periods=Uflag; coords=Uflag; MPI_Cart_get(cart,1,&dims,&periods,&coords); printival(" dims=",dims); printival(" periods=",periods); printival(" coords=",coords); } } printf("\n"); fflush(stdout); } int main(int argc, char *argv[]) { int myrank, size, periods, reorder, remain_dims, status, coords, rank, src, dest, dims; MPI_Comm cart1dim, sub0dim, subsub0dim, cart0dim; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &size); MPI_Comm_rank(MPI_COMM_WORLD, &myrank); for (periods=0; periods<=1; periods++) { MPI_Barrier(MPI_COMM_WORLD); sleep(1); if (myrank==0) printf("\nndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=%1d\n\n",periods); fflush(stdout); sleep(1); MPI_Barrier(MPI_COMM_WORLD); reorder=0; MPI_Cart_create(MPI_COMM_WORLD,1,&size,&periods,reorder,&cart1dim); report_topo(myrank, "cart1dim", cart1dim); remain_dims=0; MPI_Cart_sub(cart1dim,&remain_dims,&sub0dim); report_topo(myrank, "sub0dim", sub0dim); MPI_Topo_test(sub0dim, &status); if (status==MPI_CART) { coords=0; rank=Uflag; MPI_Cart_rank(sub0dim, &coords, &rank); printf("[%02d] MPI_Cart_rank(sub0dim,coords=0,OUT rank) -->",myrank); printival(" rank=",rank); printf("\n"); fflush(stdout); rank=0; coords=Uflag; MPI_Cart_coords(sub0dim, rank, 1, &coords); printf("[%02d] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) -->",myrank); printival(" coords=",coords); printf("\n"); fflush(stdout); src=Uflag; dest=Uflag; MPI_Cart_shift(sub0dim, 0, 1, &src, &dest); printf("[%02d] MPI_Cart_shift(sub0dim,dir=0,disp=1,OUT src, OUT dest) -->",myrank); printival(" src=",src); printival(" dest=",dest); printf("\n"); fflush(stdout); MPI_Cart_sub(sub0dim,&remain_dims,&subsub0dim); report_topo(myrank, "subsub0dim", subsub0dim); } } for (periods=0; periods<=1; periods++) { MPI_Barrier(MPI_COMM_WORLD); sleep(1); if (myrank==0) printf("\nndim=0 generated directly with MPI_Cart_create(ndims=0) -- periods=%1d\n\n",periods); fflush(stdout); sleep(1); MPI_Barrier(MPI_COMM_WORLD); reorder=0; dims=1; MPI_Cart_create(MPI_COMM_WORLD,0,&dims,&periods,reorder,&cart0dim); report_topo(myrank, "cart0dim", cart0dim); MPI_Topo_test(cart0dim, &status); if (status==MPI_CART) { coords=0; rank=Uflag; MPI_Cart_rank(cart0dim, &coords, &rank); printf("[%02d] MPI_Cart_rank(cart0dim,coords=0,OUT rank) -->",myrank); printival(" rank=",rank); printf("\n"); fflush(stdout); rank=0; coords=Uflag; MPI_Cart_coords(cart0dim, rank, 1, &coords); printf("[%02d] MPI_Cart_coords(cart0dim,rank=0,maxdims=1,OUT coords) -->",myrank); printival(" coords=",coords); printf("\n"); fflush(stdout); src=Uflag; dest=Uflag; MPI_Cart_shift(cart0dim, 0, 1, &src, &dest); printf("[%02d] MPI_Cart_shift(cart0dim,dir=0,disp=1,OUT src, OUT dest) -->",myrank); printival(" src=",src); printival(" dest=",dest); printf("\n"); fflush(stdout); MPI_Cart_sub(cart0dim,&remain_dims,&subsub0dim); report_topo(myrank, "subsub0dim", subsub0dim); } } MPI_Finalize(); } ________________________________________________________________________________________________________ IBM: ---- With MPI_Cart_shift: -------------------- kurs02@j36.!: mpcc -o test_mpi_0-dim_topology test_mpi_0-dim_topology.c kurs02@j36.!: llrun -p 2 ./test_mpi_0-dim_topology FZJ-INF: @data limit=3.000GB, @stack limit=0.500GB inserted llsubmit: Processed command file through Submit Filter: "/admin/loadl/filter". ATTENTION: 0031-408 2 tasks allocated by LoadLeveler, continuing... periods=0 [00] cart1dim: CART ndims=1 dims=2 periods=0 coords=0 [01] cart1dim: CART ndims=1 dims=2 periods=0 coords=1 [00] sub0dim: CART ndims=0 dims=U periods=U coords=U [00] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=-767705708 [01] sub0dim: CART ndims=0 dims=U periods=U coords=U [01] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=-767705708 [01] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [00] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U ERROR: 0032-170 Invalid direction (0) in MPI_Cart_shift, task 1 ERROR: 0032-170 Invalid direction (0) in MPI_Cart_shift, task 0 ERROR: 0031-250 task 0: Terminated ERROR: 0031-250 task 1: Terminated Without MPI_Cart_shift: ----------------------- kurs02@j36.!: mpcc -o test_mpi_0-dim_topology test_mpi_0-dim_topology.c kurs02@j36.!: llrun -p 2 ./test_mpi_0-dim_topology FZJ-INF: @data limit=3.000GB, @stack limit=0.500GB inserted llsubmit: Processed command file through Submit Filter: "/admin/loadl/filter". ATTENTION: 0031-408 2 tasks allocated by LoadLeveler, continuing... ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=0 [00] cart1dim: CART ndims=1 dims=2 periods=0 coords=0 [01] cart1dim: CART ndims=1 dims=2 periods=0 coords=1 [00] sub0dim: CART ndims=0 dims=U periods=U coords=U [00] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=-767705708 [01] sub0dim: CART ndims=0 dims=U periods=U coords=U [01] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=-767705708 [01] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [00] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [00] subsub0dim: CART ndims=0 dims=U periods=U coords=U [01] subsub0dim: CART ndims=0 dims=U periods=U coords=U ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=1 [00] cart1dim: CART ndims=1 dims=2 periods=1 coords=0 [01] cart1dim: CART ndims=1 dims=2 periods=1 coords=1 [00] sub0dim: CART ndims=0 dims=U periods=U coords=U [00] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=-767705708 [00] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [00] subsub0dim: CART ndims=0 dims=U periods=U coords=U [01] sub0dim: CART ndims=0 dims=U periods=U coords=U [01] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=-767705708 [01] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [01] subsub0dim: CART ndims=0 dims=U periods=U coords=U ndim=0 generated directly with MPI_Cart_create(ndims=0) -- periods=0 ERROR: 0032-143 Invalid dimension count (0) in MPI_Cart_create, task 0 ERROR: 0032-143 Invalid dimension count (0) in MPI_Cart_create, task 1 ERROR: 0031-250 task 0: Terminated ERROR: 0031-250 task 1: Terminated kurs02@j36.!: ________________________________________________________________________________________________________ OpenMPI 1.2.4 ------------- hpc43598 noco206.nec 203$module switch mpi/voltaire_icc mpi/openmpi-1.2.4 hpc43598 noco206.nec 204$mpicc -o test_mpi_0-dim_topology test_mpi_0-dim_topology.c hpc43598 noco206.nec 205$mpirun -np 2 ./test_mpi_0-dim_topology ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=0 [00] cart1dim: CART ndims=1 dims=2 periods=0 coords=0 [00] sub0dim: CART ndims=1 dims=1 periods=0 coords=0 [00] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [00] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=0 [00] MPI_Cart_shift(sub0dim,dir=0,disp=1,OUT src, OUT dest) --> src=MPI_PROC_NULL dest=MPI_PROC_NULL [00] subsub0dim: CART ndims=1 dims=1 periods=0 coords=0 [01] cart1dim: CART ndims=1 dims=2 periods=0 coords=1 [01] sub0dim: CART ndims=1 dims=1 periods=0 coords=0 [01] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [01] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=0 [01] MPI_Cart_shift(sub0dim,dir=0,disp=1,OUT src, OUT dest) --> src=MPI_PROC_NULL dest=MPI_PROC_NULL [01] subsub0dim: CART ndims=1 dims=1 periods=0 coords=0 ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=1 [00] cart1dim: CART ndims=1 dims=2 periods=1 coords=0 [00] sub0dim: CART ndims=1 dims=1 periods=0 coords=0 [00] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [00] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=0 [00] MPI_Cart_shift(sub0dim,dir=0,disp=1,OUT src, OUT dest) --> src=MPI_PROC_NULL dest=MPI_PROC_NULL [00] subsub0dim: CART ndims=1 dims=1 periods=0 coords=0 [01] cart1dim: CART ndims=1 dims=2 periods=1 coords=1 [01] sub0dim: CART ndims=1 dims=1 periods=0 coords=0 [01] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [01] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=0 [01] MPI_Cart_shift(sub0dim,dir=0,disp=1,OUT src, OUT dest) --> src=MPI_PROC_NULL dest=MPI_PROC_NULL [01] subsub0dim: CART ndims=1 dims=1 periods=0 coords=0 ndim=0 generated directly with MPI_Cart_create(ndims=0) -- periods=0 [noco206.nec:32298] *** An error occurred in MPI_Cart_create [noco206.nec:32298] *** on communicator MPI_COMM_WORLD [noco206.nec:32298] *** MPI_ERR_ARG: invalid argument of some other kind [noco206.nec:32298] *** MPI_ERRORS_ARE_FATAL (goodbye) [noco205.nec:15179] *** An error occurred in MPI_Cart_create [noco205.nec:15179] *** on communicator MPI_COMM_WORLD [noco205.nec:15179] *** MPI_ERR_ARG: invalid argument of some other kind [noco205.nec:15179] *** MPI_ERRORS_ARE_FATAL (goodbye) ________________________________________________________________________________________________________ NEC MPI/EX (4.0-3) on TX7 (a1.hww.de) ------------------------- hpc43598 a1 204$mpicc -o test_mpi_0-dim_topology test_mpi_0-dim_topology.c hpc43598 a1 205$mpirun -np 2 ./test_mpi_0-dim_topology ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=0 [00] cart1dim: CART ndims=1 dims=2 periods=0 coords=0 [00] sub0dim: NOT-CART [01] cart1dim: CART ndims=1 dims=2 periods=0 coords=1 [01] sub0dim: NOT-CART ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=1 [00] cart1dim: CART ndims=1 dims=2 periods=1 coords=0 [00] sub0dim: NOT-CART [01] cart1dim: CART ndims=1 dims=2 periods=1 coords=1 [01] sub0dim: NOT-CART ndim=0 generated directly with MPI_Cart_create(ndims=0) -- periods=0 [0,0] - MPI_CART_CREATE : Illegal dimension argument %d [0,0] Aborting program ! [0,0] Aborting program! MPI process (universe = 0, rank = 0) terminated by exit(11) a1: mpid: Program terminated unexpectedly ________________________________________________________________________________________________________ Voltaire MPI ------------ hpc43598 noco206.nec 201$mpicc -o test_mpi_0-dim_topology test_mpi_0-dim_topology.c hpc43598 noco206.nec 202$mpirun_ssh -np 2 -hostfile $PBS_NODEFILE ./test_mpi_0-dim_topology ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=0 [00] cart1dim: CART ndims=1 dims=2 periods=0 coords=0 [00] sub0dim: NOT-CART [01] cart1dim: CART ndims=1 dims=2 periods=0 coords=1 [01] sub0dim: NOT-CART ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=1 [00] cart1dim: CART ndims=1 dims=2 periods=1 coords=0 [00] sub0dim: NOT-CART [01] cart1dim: CART ndims=1 dims=2 periods=1 coords=1 [01] sub0dim: NOT-CART ndim=0 generated directly with MPI_Cart_create(ndims=0) -- periods=0 Rank 0 call MPI_Abort. Cleaning up all processes ... 1 - MPI_CART_CREATE : Invalid dimension argument 0 - MPI_CART_CREATE : Invalid dimension argument Rank 1 exited without finalize. done. ________________________________________________________________________________________________________ mpich2 1.0.3 ------------ With MPI_Cart_shift: -------------------- hpcklima@lapklimach:~/rolf-mpi$ mpicc test_mpi_0-dim_topology.c hpcklima@lapklimach:~/rolf-mpi$ mpd & [1] 29150 hpcklima@lapklimach:~/rolf-mpi$ mpirun -np 2 ./a.out periods=0 [00] cart1dim: CART ndims=1 dims=2 periods=0 coords=0 [01] cart1dim: CART ndims=1 dims=2 periods=0 coords=1 [00] sub0dim: CART ndims=0 dims=U periods=U coords=U [00] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [00] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [cli_0]: aborting job: Fatal error in MPI_Cart_shift: Invalid argument, error stack: MPI_Cart_shift(172): MPI_Cart_shift(comm=0x84000001, direction=0, displ=1, source=0xbf831664, dest=0xbf831660) failed MPI_Cart_shift(120): Number of dimensions 0 is too large (maximum is 0) [01] sub0dim: CART ndims=0 dims=U periods=U coords=U [01] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [01] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [cli_1]: aborting job: Fatal error in MPI_Cart_shift: Invalid argument, error stack: MPI_Cart_shift(172): MPI_Cart_shift(comm=0x84000001, direction=0, displ=1, source=0xbfa60954, dest=0xbfa60950) failed MPI_Cart_shift(120): Number of dimensions 0 is too large (maximum is 0) rank 1 in job 1 lapklimach_41642 caused collective abort of all ranks exit status of rank 1: return code 13 rank 0 in job 1 lapklimach_41642 caused collective abort of all ranks exit status of rank 0: return code 13 Without MPI_Cart_shift: ----------------------- hpcklima@lapklimach:~/rolf-mpi/v2$ mpicc test_mpi_0-dim_topology.c hpcklima@lapklimach:~/rolf-mpi/v2$ mpirun -np 2 ./a.out ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=0 [00] cart1dim: CART ndims=1 dims=2 periods=0 coords=0 [00] sub0dim: CART ndims=0 dims=U periods=U coords=U [00] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [00] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [00] subsub0dim: CART ndims=0 dims=U periods=U coords=U [01] cart1dim: CART ndims=1 dims=2 periods=0 coords=1 [01] sub0dim: CART ndims=0 dims=U periods=U coords=U [01] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [01] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [01] subsub0dim: CART ndims=0 dims=U periods=U coords=U ndim=0 generated via MPI_Cart_sub(remain_dims=0) -- periods=1 [00] cart1dim: CART ndims=1 dims=2 periods=1 coords=0 [00] sub0dim: CART ndims=0 dims=U periods=U coords=U [00] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [00] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [00] subsub0dim: CART ndims=0 dims=U periods=U coords=U [01] cart1dim: CART ndims=1 dims=2 periods=1 coords=1 [01] sub0dim: CART ndims=0 dims=U periods=U coords=U [01] MPI_Cart_rank(sub0dim,coords=0,OUT rank) --> rank=0 [01] MPI_Cart_coords(sub0dim,rank=0,maxdims=1,OUT coords) --> coords=U [01] subsub0dim: CART ndims=0 dims=U periods=U coords=U ndim=0 generated directly with MPI_Cart_create(ndims=0) -- periods=0 [cli_0]: aborting job: Fatal error in MPI_Cart_create: Invalid dimension argument, error stack: MPI_Cart_create(252): MPI_Cart_create(MPI_COMM_WORLD, ndims=0, dims=0xbffe0a7c, periods=0xbffe0a90, reorder=0, comm_cart=0xbffe0a6c) failed MPI_Cart_create(212): Invalid dimension argument (value is 0) rank 1 in job 2 lapklimach_41642 caused collective abort of all ranks exit status of rank 1: return code 13 [cli_1]: aborting job: Fatal error in MPI_Cart_create: Invalid dimension argument, error stack: MPI_Cart_create(252): MPI_Cart_create(MPI_COMM_WORLD, ndims=0, dims=0xbffe0a7c, periods=0xbffe0a90, reorder=0, comm_cart=0xbffe0a6c) failed MPI_Cart_create(212): Invalid dimension argument (value is 0) rank 0 in job 2 lapklimach_41642 caused collective abort of all ranks exit status of rank 0: return code 13 ________________________________________________________________________________________________________