[Mpi-forum] Missing user data in MPI_User_function?

[Phil Ruffwind]

On Mon, Apr 10, 2017, at 10:58, Jeff Hammond wrote:
> I do not understand why creating a wrapper function dynamically at run
> time creates a portability problem.  Can you elaborate?

It needs to generate machine code at run time, like the example below. 
(A more sane way to do this would be through libffi or something.)

    #include <stdio.h>
    #include <stdlib.h>
    #include <string.h>
    #include <sys/mman.h>
    #include <unistd.h>
    #include <mpi.h>

    typedef void (*MpiUserFunctionPtr)(void *, void *, int *,
    MPI_Datatype *);

    // operations provided by user
    typedef void (*Operation)(double *, double *);

    _Thread_local Operation current_operation;

    // converts Operation to MPI_User_function
    void converter(void *invec, void *inoutvec, int *len, MPI_Datatype
    *datatype)
    {
        current_operation(invec, inoutvec);
    }

    MpiUserFunctionPtr create_dynamic_wrapper(Operation operation)
    {
        MpiUserFunctionPtr converter_ptr = converter;
        Operation *current_operation_ptr = &current_operation;
        char *m = aligned_alloc(sysconf(_SC_PAGESIZE), 0x23);
        // unportable: assumes x86-64
        memcpy(m + 0x00, "\x49\xba", 2);     // movabs operation,%r10
        memcpy(m + 0x02, &operation, 8);
        memcpy(m + 0x0a, "\x48\xb8", 2);     // movabs
        &current_operation,%rax
        memcpy(m + 0x0c, &current_operation_ptr, 8);
        memcpy(m + 0x14, "\x4c\x89\x10", 3); // movabs %r10,(%rax)
        memcpy(m + 0x17, "\x48\xb8", 2);     // movabs &converter,%rax
        memcpy(m + 0x19, &converter_ptr, 8);
        memcpy(m + 0x21, "\xff\xe0", 2);     // jmpq   *%rax
        mprotect(m, 0x23, PROT_READ | PROT_WRITE | PROT_EXEC);
        return (MpiUserFunctionPtr)m;
    }

    // === demonstration ===

    // some test operations provided by the user (unaware of MPI)
    void add(double *invec, double *inoutvec) { *inoutvec += *invec; }
    void multiply(double *invec, double *inoutvec) { *inoutvec *=
    *invec; }

    int main(void)
    {
        int rank;
        MPI_Init(NULL, NULL);
        MPI_Comm_rank(MPI_COMM_WORLD, &rank);

        MPI_Op op_add;
        MPI_Op_create(create_dynamic_wrapper(&add), 1, &op_add);

        MPI_Op op_multiply;
        MPI_Op_create(create_dynamic_wrapper(&multiply), 1,
        &op_multiply);

        double d = rank + 1;
        double r = 0.0;
        MPI_Reduce(&d, &r, 1, MPI_DOUBLE, op_add, 0, MPI_COMM_WORLD);
        if (rank == 0) {
            printf("∑(rank+1) = %.0f\n", r);
        }

        r = 1.0;
        MPI_Reduce(&d, &r, 1, MPI_DOUBLE, op_multiply, 0,
        MPI_COMM_WORLD);
        if (rank == 0) {
            printf("∏(rank+1) = %.0f\n", r);
        }

        MPI_Finalize();
    }