/* ag_cube_nblk.c -- hypercube allgather using nonblocking sends and receives
 *
 * Input: series of blocksizes for allgather, 0 to stop.
 * Output: Contents of gathered array on each process -- list of
 *     process ranks, each rank appearing in a block of size blocksize.
 *
 * Note:  array sizes are hardwired in MAX and LOCAL_MAX.
 *
 * See Chap 13, pp. 299 & ff, in PPMPI.
 */

#include <stdio.h>
#include <string.h>
#include "mpi.h"
#include "cio.h"

#define MAX 128
#define LOCAL_MAX 128

void  Allgather_cube(
         float    x[]        /* in  */, 
         int      blocksize  /* in  */, 
         float    y[]        /* out */, 
         MPI_Comm comm       /* in  */);
void Print_arrays(MPI_Comm io_comm, char* title, 
         float y[], int blocksize);


/********************************************************************/
main(int argc, char* argv[]) {
    int       p;
    int       my_rank;
    float     x[LOCAL_MAX];
    float     y[MAX];
    int       blocksize;
    MPI_Comm  io_comm;
    int       i;

    MPI_Init(&argc, &argv);
    MPI_Comm_size(MPI_COMM_WORLD, &p);
    MPI_Comm_rank(MPI_COMM_WORLD, &my_rank);
    MPI_Comm_dup(MPI_COMM_WORLD, &io_comm);
    Cache_io_rank(MPI_COMM_WORLD, io_comm);
/*
    Cscanf(io_comm,"Enter the local array size","%d", &blocksize);
*/
    blocksize = 2;
    while(blocksize > 0) {
        for (i = 0; i < blocksize; i++)
            x[i] = (float) my_rank;
        Allgather_cube(x, blocksize, y, MPI_COMM_WORLD);
        Print_arrays(io_comm, "Gathered_arrays", y, blocksize);
        /* Enter 0 to stop. */
/*
        Cscanf(io_comm,"Enter the local array size",
            "%d", &blocksize);
*/
        blocksize = 0;
    }

    MPI_Finalize();
}  /* main */


/********************************************************************/
void Print_arrays(
         MPI_Comm  io_comm    /* in */,
         char*     title      /* in */, 
         float     y[]        /* in */, 
         int       blocksize  /* in */) {

    char item[16];
    char list[4*MAX];
    int  i;
    int  p;

    MPI_Comm_size(io_comm, &p);

    list[0] = '\0';
    for (i = 0; i < blocksize*p; i++) {
        sprintf(item, "%3.1f ", y[i]);
        strcat(list, item);
    }
    Cprintf(io_comm, title, "%s", list);
}  /* Print_arrays */


/********************************************************************/
int log_base2(int p) {
/* Just counts number of bits to right of most significant
 * bit.  So for p not a power of 2, it returns the floor
 * of log_2(p).
 */
    int return_val = 0;
    unsigned q;

    q = (unsigned) p;
    while(q != 1) {
        q = q >> 1;
        return_val++;
    }
    return return_val;
}  /* log_base2 */


/********************************************************************/
void  Allgather_cube(
         float    x[]        /* in  */, 
         int      blocksize  /* in  */, 
         float    y[]        /* out */, 
         MPI_Comm comm       /* in  */) {

    int           i, d, p, my_rank;
    unsigned      eor_bit;
    unsigned      and_bits;
    int           stage, partner;
    MPI_Datatype  hole_type;
    int           send_offset, recv_offset;
    MPI_Status    status;
    MPI_Request   send_request;
    MPI_Request   recv_request;

    int log_base2(int p);

    MPI_Comm_size(comm, &p);
    MPI_Comm_rank(comm, &my_rank);

    /* Copy x into correct location in y */
    for (i = 0; i < blocksize; i++)
        y[i + my_rank*blocksize] = x[i];

    /* Set up */
    d = log_base2(p);
    eor_bit = 1 << (d-1);
    and_bits = (1 << d) - 1;

    partner = my_rank ^ eor_bit;
    send_offset = (my_rank & and_bits)*blocksize;
    recv_offset = (partner & and_bits)*blocksize;
    MPI_Type_contiguous(blocksize, MPI_FLOAT, &hole_type);
    MPI_Type_commit(&hole_type);

    for (stage = 0; stage < d; stage++) {
        MPI_Isend(y + send_offset, 1, hole_type,
            partner, 0, comm, &send_request);
        MPI_Irecv(y + recv_offset, 1, hole_type,
            partner, 0, comm, &recv_request);

        if (stage < d-1) {
            eor_bit >>= 1;
            and_bits >>= 1;
            partner = my_rank ^ eor_bit;
            send_offset = (my_rank & and_bits)*blocksize;
            recv_offset = (partner & and_bits)*blocksize;
            MPI_Type_free(&hole_type);
            MPI_Type_vector(1 << (stage+1), blocksize,
                (1 << (d-stage-1))*blocksize, MPI_FLOAT,
                &hole_type);
            MPI_Type_commit(&hole_type);
        }

        MPI_Wait(&send_request, &status);
        MPI_Wait(&recv_request, &status);
    }  /* for */
} /* Allgather_cube */