/*
    Copyright 2003-2019 Intel Corporation.
    
    This software and the related documents are Intel copyrighted materials, and
    your use of them is governed by the express license under which they were
    provided to you (License). Unless the License provides otherwise, you may
    not use, modify, copy, publish, distribute, disclose or transmit this
    software or the related documents without Intel's prior written permission.
    
    This software and the related documents are provided as is, with no express
    or implied warranties, other than those that are expressly stated in the
    License.
*/
#include <mpi.h>
#include <stdio.h>

#define PROGRESS_THREAD_COUNT (16)
#define CHUNK_SIZE            (1024)

int main(int argc, char *argv[])
{
    MPI_Comm comms[PROGRESS_THREAD_COUNT];
    MPI_Request requests[PROGRESS_THREAD_COUNT];
    MPI_Status statuses[PROGRESS_THREAD_COUNT];

    int send_buffer[CHUNK_SIZE * PROGRESS_THREAD_COUNT];
    int recv_buffer[CHUNK_SIZE * PROGRESS_THREAD_COUNT];

    MPI_Info info;
    char thread_id_str[16];
    int idx, size, has_error = 0;

    MPI_Init(&argc, &argv);
    MPI_Comm_size(MPI_COMM_WORLD, &size);
    MPI_Info_create(&info);

    /* create "per-thread" communicators and assign thread_id for each communicator */
    for (idx = 0; idx < PROGRESS_THREAD_COUNT; idx++) {
        MPI_Comm_dup(MPI_COMM_WORLD, &comms[idx]);
        sprintf(thread_id_str, "%d", idx);
        MPI_Info_set(info, "thread_id", thread_id_str);
        MPI_Comm_set_info(comms[idx], info);
    }
    MPI_Info_free(&info);

    for (idx = 0; idx < CHUNK_SIZE * PROGRESS_THREAD_COUNT; idx++) {
        send_buffer[idx] = 1;
        recv_buffer[idx] = 0;
    }

    /* distribute MPI operations between communicators – i.e. between progress threads */
    for (idx = 0; idx < PROGRESS_THREAD_COUNT; idx++) {
        MPI_Iallreduce(send_buffer + idx * CHUNK_SIZE,
                       recv_buffer + idx * CHUNK_SIZE,
                       CHUNK_SIZE, MPI_INT, MPI_SUM, comms[idx], &requests[idx]);
    }
    MPI_Waitall(PROGRESS_THREAD_COUNT, requests, statuses);

    for (idx = 0; idx < CHUNK_SIZE * PROGRESS_THREAD_COUNT; idx++) {
        if (send_buffer[idx] != 1) {
            has_error = 1;
            break;
        }

        if (recv_buffer[idx] != size) {
            has_error = 1;
            break;
        }
    }

    printf((has_error) ? "FAIL\n" : "OK\n");

    MPI_Finalize();
    return 0;
}