prov/util/src/util_atomic.c (879 lines of code) (raw):
/*
* Copyright (c) 2013-2017 Intel Corporation. All rights reserved.
* Copyright (c) 2018 Cray Inc. All rights reserved.
* Copyright (c) 2018 System Fabric Works, Inc. All rights reserved.
*
* This software is available to you under a choice of one of two
* licenses. You may choose to be licensed under the terms of the GNU
* General Public License (GPL) Version 2, available from the file
* COPYING in the main directory of this source tree, or the
* BSD license below:
*
* Redistribution and use in source and binary forms, with or
* without modification, are permitted provided that the following
* conditions are met:
*
* - Redistributions of source code must retain the above
* copyright notice, this list of conditions and the following
* disclaimer.
*
* - Redistributions in binary form must reproduce the above
* copyright notice, this list of conditions and the following
* disclaimer in the documentation and/or other materials
* provided with the distribution.
*
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
* SOFTWARE.
*/
#include "ofi_atomic.h"
static const size_t ofi_datatype_size_table[] = {
[FI_INT8] = sizeof(int8_t),
[FI_UINT8] = sizeof(uint8_t),
[FI_INT16] = sizeof(int16_t),
[FI_UINT16] = sizeof(uint16_t),
[FI_INT32] = sizeof(int32_t),
[FI_UINT32] = sizeof(uint32_t),
[FI_INT64] = sizeof(int64_t),
[FI_UINT64] = sizeof(uint64_t),
[FI_FLOAT] = sizeof(float),
[FI_DOUBLE] = sizeof(double),
[FI_FLOAT_COMPLEX] = sizeof(ofi_complex_float),
[FI_DOUBLE_COMPLEX] = sizeof(ofi_complex_double),
[FI_LONG_DOUBLE] = sizeof(long double),
[FI_LONG_DOUBLE_COMPLEX] = sizeof(ofi_complex_long_double),
};
size_t ofi_datatype_size(enum fi_datatype datatype)
{
if (datatype >= FI_DATATYPE_LAST) {
errno = FI_EINVAL;
return 0;
}
return ofi_datatype_size_table[datatype];
}
/*
* Basic atomic operations
*/
#ifdef HAVE_BUILTIN_MM_ATOMICS
#define OFI_OP_MIN(type,dst,src) (dst) > (src)
#define OFI_OP_MAX(type,dst,src) (dst) < (src)
#define OFI_OP_SUM(type,dst,src) (dst) + (src)
#define OFI_OP_PROD(type,dst,src) (dst) * (src)
#define OFI_OP_LOR(type,dst,src) (dst) || (src)
#define OFI_OP_LAND(type,dst,src) (dst) && (src)
#define OFI_OP_BOR(type,dst,src) \
__atomic_fetch_or(&(dst), (src), __ATOMIC_SEQ_CST)
#define OFI_OP_BAND(type,dst,src) \
__atomic_fetch_and(&(dst), (src), __ATOMIC_SEQ_CST)
#define OFI_OP_LXOR(type,dst,src) \
((dst) && !(src)) || (!(dst) && (src))
#define OFI_OP_BXOR(type,dst,src) \
__atomic_fetch_xor(&(dst), (src), __ATOMIC_SEQ_CST)
#define OFI_OP_WRITE(type,dst,src) \
__atomic_store(&(dst), &(src), __ATOMIC_SEQ_CST)
#define OFI_OP_READ(type,dst,res) \
__atomic_load(&(dst), &(res), __ATOMIC_SEQ_CST)
#define OFI_OP_READWRITE(type,dst,src,res) \
__atomic_exchange(&(dst), &(src), &(res), __ATOMIC_SEQ_CST)
#define OFI_OP_CSWAP_EQ(type,dst,src,cmp) \
__atomic_compare_exchange(&(dst),&(cmp),&(src),0, \
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
#define OFI_OP_CSWAP_NE(type,dst,src,cmp) ((cmp) != (dst))
#define OFI_OP_CSWAP_LE(type,dst,src,cmp) ((cmp) <= (dst))
#define OFI_OP_CSWAP_LT(type,dst,src,cmp) ((cmp) < (dst))
#define OFI_OP_CSWAP_GE(type,dst,src,cmp) ((cmp) >= (dst))
#define OFI_OP_CSWAP_GT(type,dst,src,cmp) ((cmp) > (dst))
#define OFI_OP_MSWAP(type,dst,src,cmp) \
(((src) & (cmp)) | ((dst) & ~(cmp)))
/* Need special handlers for OFI complex datatypes for portability */
#define OFI_OP_SUM_COMPLEX(type,dst,src) ofi_complex_sum_##type(dst,src)
#define OFI_OP_PROD_COMPLEX(type,dst,src) ofi_complex_prod_##type(dst,src)
#define OFI_OP_LOR_COMPLEX(type,dst,src) ofi_complex_lor_##type(dst,src)
#define OFI_OP_LAND_COMPLEX(type,dst,src) ofi_complex_land_##type(dst,src)
#define OFI_OP_LXOR_COMPLEX(type,dst,src) ofi_complex_lxor_##type(dst,src)
#define OFI_OP_CSWAP_EQ_COMPLEX(type,dst,src,cmp) \
__atomic_compare_exchange(&(dst),&(cmp),&(src),0, \
__ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST)
#define OFI_OP_CSWAP_NE_COMPLEX(type,dst,src,cmp) \
(!ofi_complex_eq_##type(dst,cmp))
#define OFI_OP_READWRITE_COMPLEX OFI_OP_READWRITE
#else /* HAVE_BUILTIN_MM_ATOMICS */
#define OFI_OP_MIN(type,dst,src) if ((dst) > (src)) (dst) = (src)
#define OFI_OP_MAX(type,dst,src) if ((dst) < (src)) (dst) = (src)
#define OFI_OP_SUM(type,dst,src) (dst) += (src)
#define OFI_OP_PROD(type,dst,src) (dst) *= (src)
#define OFI_OP_LOR(type,dst,src) (dst) = (dst) || (src)
#define OFI_OP_LAND(type,dst,src) (dst) = (dst) && (src)
#define OFI_OP_BOR(type,dst,src) (dst) |= (src)
#define OFI_OP_BAND(type,dst,src) (dst) &= (src)
#define OFI_OP_LXOR(type,dst,src) (dst) = ((dst) && !(src)) || (!(dst) && (src))
#define OFI_OP_BXOR(type,dst,src) (dst) ^= (src)
#define OFI_OP_READ(type,dst,src) (dst)
#define OFI_OP_WRITE(type,dst,src) (dst) = (src)
#define OFI_OP_CSWAP_EQ(type,dst,src,cmp) if ((cmp) == (dst)) (dst) = (src)
#define OFI_OP_CSWAP_NE(type,dst,src,cmp) if ((cmp) != (dst)) (dst) = (src)
#define OFI_OP_CSWAP_LE(type,dst,src,cmp) if ((cmp) <= (dst)) (dst) = (src)
#define OFI_OP_CSWAP_LT(type,dst,src,cmp) if ((cmp) < (dst)) (dst) = (src)
#define OFI_OP_CSWAP_GE(type,dst,src,cmp) if ((cmp) >= (dst)) (dst) = (src)
#define OFI_OP_CSWAP_GT(type,dst,src,cmp) if ((cmp) > (dst)) (dst) = (src)
#define OFI_OP_MSWAP(type,dst,src,cmp) (dst) = (((src) & (cmp)) | \
((dst) & ~(cmp)))
/* Need special handlers for complex datatypes for portability */
#define OFI_OP_SUM_COMPLEX(type,dst,src) (dst) = ofi_complex_sum_##type(dst,src)
#define OFI_OP_PROD_COMPLEX(type,dst,src) (dst) = ofi_complex_prod_##type(dst,src)
#define OFI_OP_LOR_COMPLEX(type,dst,src) (dst) = ofi_complex_lor_##type(dst,src)
#define OFI_OP_LAND_COMPLEX(type,dst,src) (dst) = ofi_complex_land_##type(dst,src)
#define OFI_OP_LXOR_COMPLEX(type,dst,src) (dst) = ofi_complex_lxor_##type(dst,src)
#define OFI_OP_CSWAP_EQ_COMPLEX(type,dst,src,cmp) \
if (ofi_complex_eq_##type(dst,cmp)) (dst) = (src)
#define OFI_OP_CSWAP_NE_COMPLEX(type,dst,src,cmp) \
if (!ofi_complex_eq_##type(dst,cmp)) (dst) = (src)
#endif /* HAVE_BUILTIN_MM_ATOMICS */
#define OFI_OP_READ_COMPLEX OFI_OP_READ
#define OFI_OP_WRITE_COMPLEX OFI_OP_WRITE
/********************************
* ATOMIC TYPE function templates
********************************/
#define OFI_DEF_NOOP_NAME NULL,
#define OFI_DEF_NOOP_FUNC
/*
* WRITE
*/
#define OFI_DEF_WRITE_NAME(op, type) ofi_write_## op ##_## type,
#define OFI_DEF_WRITE_FUNC(op, type) \
static void ofi_write_## op ##_## type \
(void *dst, const void *src, size_t cnt) \
{ \
size_t i; \
type *d = (dst); \
const type *s = (src); \
type temp_s; \
for (i = 0; i < cnt; i++) { \
temp_s = s[i]; \
op(type, d[i], temp_s); \
} \
}
#define OFI_DEF_WRITE_COMPLEX_NAME(op, type) ofi_write_## op ##_## type,
#define OFI_DEF_WRITE_COMPLEX_FUNC(op, type) \
static void ofi_write_## op ##_## type \
(void *dst, const void *src, size_t cnt) \
{ \
size_t i; \
ofi_complex_##type *d = (dst); \
const ofi_complex_##type *s = (src); \
ofi_complex_##type temp_s; \
for (i = 0; i < cnt; i++) { \
temp_s = s[i]; \
op(type, d[i], temp_s); \
} \
}
#ifdef HAVE_BUILTIN_MM_ATOMICS
#define OFI_DEF_WRITEEXT_NAME(op, type) ofi_write_## op ##_## type,
#define OFI_DEF_WRITEEXT_FUNC(op, type) \
static void ofi_write_## op ##_## type \
(void *dst, const void *src, size_t cnt) \
{ \
type target; \
type val; \
size_t i; \
type *d = (dst); \
const type *s = (src); \
int success; \
\
for (i = 0; i < cnt; i++) { \
success = 0; \
do { \
target = d[i]; \
val = op(type, d[i], s[i]); \
success = __atomic_compare_exchange( \
&d[i],&target,&val,0, \
__ATOMIC_SEQ_CST, \
__ATOMIC_SEQ_CST); \
} while (!success); \
} \
}
#define OFI_DEF_WRITEEXT_CMP_NAME(op, type) ofi_write_## op ##_## type,
#define OFI_DEF_WRITEEXT_CMP_FUNC(op, type) \
static void ofi_write_## op ##_## type \
(void *dst, const void *src, size_t cnt) \
{ \
type target; \
size_t i; \
type *d = (dst); \
const type *s = (src); \
type temp_s; \
int success; \
\
for (i = 0; i < cnt; i++) { \
do { \
success = 1; \
target = d[i]; \
if (op(type, d[i], s[i])) { \
temp_s = s[i]; \
success = __atomic_compare_exchange( \
&d[i],&target,&temp_s, 0, \
__ATOMIC_SEQ_CST, \
__ATOMIC_SEQ_CST); \
} \
} while (!success); \
} \
}
#define OFI_DEF_WRITEEXT_COMPLEX_NAME(op, type) ofi_write_## op ##_## type,
#define OFI_DEF_WRITEEXT_COMPLEX_FUNC(op, type) \
static void ofi_write_## op ##_## type \
(void *dst, const void *src, size_t cnt) \
{ \
ofi_complex_##type target; \
ofi_complex_##type val; \
ofi_complex_##type *d = (dst); \
const ofi_complex_##type *s = (src); \
size_t i; \
int success; \
\
for (i = 0; i < cnt; i++) { \
success = 0; \
do { \
target = d[i]; \
val = op(type, d[i], s[i]); \
success = __atomic_compare_exchange( \
&d[i],&target,&val,0, \
__ATOMIC_SEQ_CST, \
__ATOMIC_SEQ_CST); \
} while (!success); \
} \
}
#endif /* HAVE BUILTIN_MM_ATOMICS */
/*
* READ (fetch)
*/
#define OFI_DEF_READ_NAME(op, type) ofi_read_## op ##_## type,
#define OFI_DEF_READ_COMPLEX_NAME(op, type) ofi_read_## op ##_## type,
#ifdef HAVE_BUILTIN_MM_ATOMICS
#define OFI_DEF_READ_FUNC(op, type) \
static void ofi_read_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
size_t i; \
type *d = (dst); \
type *r = (res); \
OFI_UNUSED(src); \
for (i = 0; i < cnt; i++) \
op(type, d[i], r[i]); \
}
#define OFI_DEF_READ_COMPLEX_FUNC(op, type) \
static void ofi_read_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
size_t i; \
ofi_complex_##type *d = (dst); \
ofi_complex_##type *r = (res); \
OFI_UNUSED(src); \
for (i = 0; i < cnt; i++) \
op(type, d[i], r[i]); \
}
#else /* HAVE_BUILTIN_MM_ATOMICS */
#define OFI_DEF_READ_FUNC(op, type) \
static void ofi_read_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
size_t i; \
type *d = (dst); \
type *r = (res); \
OFI_UNUSED(src); \
for (i = 0; i < cnt; i++) \
r[i] = op(type, d[i], r[i]); \
}
#define OFI_DEF_READ_COMPLEX_FUNC(op, type) \
static void ofi_read_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
size_t i; \
ofi_complex_##type *d = (dst); \
ofi_complex_##type *r = (res); \
OFI_UNUSED(src); \
for (i = 0; i < cnt; i++) \
r[i] = op(type, d[i], s[i]); \
}
#endif /* HAVE_BUILTIN_MM_ATOMICS */
/*
* READWRITE (fetch-write)
*/
#ifdef HAVE_BUILTIN_MM_ATOMICS
#define OFI_DEF_READWRITE_NAME(op, type) ofi_readwrite_## op ##_## type,
#define OFI_DEF_READWRITE_FUNC(op, type) \
static void ofi_readwrite_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
size_t i; \
type *d = (dst); \
const type *s = (src); \
type *r = (res); \
type temp_s; \
for (i = 0; i < cnt; i++) { \
temp_s = s[i]; \
r[i] = op(type, d[i], temp_s); \
} \
}
#define OFI_DEF_READWRITEEXT_NAME(op, type) ofi_readwrite_## op ##_## type,
#define OFI_DEF_READWRITEEXT_FUNC(op, type) \
static void ofi_readwrite_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
type target; \
type val; \
size_t i; \
type *d = (dst); \
type *r = (res); \
const type *s = (src); \
int success; \
\
for (i = 0; i < cnt; i++) { \
success = 0; \
do { \
target = d[i]; \
val = op(type, d[i], s[i]); \
success = __atomic_compare_exchange( \
&d[i],&target,&val,0, \
__ATOMIC_SEQ_CST, \
__ATOMIC_SEQ_CST); \
} while (!success); \
r[i] = target; \
} \
}
#define OFI_DEF_READWRITEEXT_CMP_NAME(op, type) ofi_readwrite_## op ##_## type,
#define OFI_DEF_READWRITEEXT_CMP_FUNC(op, type) \
static void ofi_readwrite_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
type target; \
size_t i; \
type *d = (dst); \
type *r = (res); \
const type *s = (src); \
type temp_s; \
int success; \
\
for (i = 0; i < cnt; i++) { \
do { \
target = d[i]; \
success = 1; \
if (op(type, d[i], s[i])) { \
temp_s = s[i]; \
success = __atomic_compare_exchange( \
&d[i],&target,&temp_s, 0, \
__ATOMIC_SEQ_CST, \
__ATOMIC_SEQ_CST); \
} \
} while (!success); \
r[i] = target; \
} \
}
#define OFI_DEF_EXCHANGE_NAME(op, type) ofi_readwrite_## op ##_## type,
#define OFI_DEF_EXCHANGE_FUNC(op, type) \
static void ofi_readwrite_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
size_t i; \
type *d = dst; \
const type *s = src; \
type *r = res; \
type temp_s; \
for (i = 0; i < cnt; i++) { \
temp_s = s[i]; \
op(type, d[i], temp_s, r[i]); \
} \
}
#define OFI_DEF_READWRITE_COMPLEX_NAME(op, type) ofi_readwrite_## op ##_## type,
#define OFI_DEF_READWRITE_COMPLEX_FUNC(op, type) \
static void ofi_readwrite_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
ofi_complex_##type *d = dst; \
const ofi_complex_##type *s = src; \
ofi_complex_##type *r = res; \
ofi_complex_##type temp_s; \
size_t i; \
for (i = 0; i < cnt; i++) { \
temp_s = s[i]; \
r[i] = op(type, d[i], temp_s); \
} \
}
#define OFI_DEF_READWRITEEXT_COMPLEX_NAME(op, type) ofi_readwrite_## op ##_## type,
#define OFI_DEF_READWRITEEXT_COMPLEX_FUNC(op, type) \
static void ofi_readwrite_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
ofi_complex_##type target; \
ofi_complex_##type val; \
ofi_complex_##type *d = dst; \
ofi_complex_##type *r = res; \
const ofi_complex_##type *s = src; \
size_t i; \
int success; \
\
for (i = 0; i < cnt; i++) { \
success = 0; \
do { \
target = d[i]; \
val = op(type, d[i], s[i]); \
success = __atomic_compare_exchange( \
&d[i],&target,&val,0, \
__ATOMIC_SEQ_CST, \
__ATOMIC_SEQ_CST); \
} while (!success); \
r[i] = target; \
} \
}
#define OFI_DEF_EXCHANGE_COMPLEX_NAME(op, type) ofi_readwrite_## op ##_## type,
#define OFI_DEF_EXCHANGE_COMPLEX_FUNC(op, type) \
static void ofi_readwrite_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
ofi_complex_##type *d = dst; \
const ofi_complex_##type *s = src; \
ofi_complex_##type *r = res; \
ofi_complex_##type temp_s; \
size_t i; \
for (i = 0; i < cnt; i++) { \
temp_s = s[i]; \
op(type, d[i], temp_s, r[i]); \
} \
}
#else /* HAVE_BUILTIN_MM_ATOMICS */
#define OFI_DEF_READWRITE_NAME(op, type) ofi_readwrite_## op ##_## type,
#define OFI_DEF_READWRITE_FUNC(op, type) \
static void ofi_readwrite_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
size_t i; \
type *d = dst; \
const type *s = src; \
type *r = res; \
for (i = 0; i < cnt; i++) { \
r[i] = d[i]; \
op(type, d[i], s[i]); \
} \
}
#define OFI_DEF_READWRITE_COMPLEX_NAME(op, type) ofi_readwrite_## op ##_## type,
#define OFI_DEF_READWRITE_COMPLEX_FUNC(op, type) \
static void ofi_readwrite_## op ##_## type \
(void *dst, const void *src, void *res, size_t cnt) \
{ \
size_t i; \
ofi_complex_##type *d = dst; \
const ofi_complex_##type *s = src; \
ofi_complex_##type *r = res; \
for (i = 0; i < cnt; i++) { \
r[i] = d[i]; \
op(type, d[i], s[i]); \
} \
}
#endif /* HAVE_BUILTIN_MM_ATOMICS */
/*
* CSWAP
*/
#ifdef HAVE_BUILTIN_MM_ATOMICS
#define OFI_DEF_CSWAP_NAME(op, type) ofi_cswap_## op ##_## type,
#define OFI_DEF_CSWAP_FUNC(op, type) \
static void ofi_cswap_## op ##_## type \
(void *dst, const void *src, const void *cmp, \
void *res, size_t cnt) \
{ \
size_t i; \
type *d = dst; \
const type *s = src; \
const type *c = cmp; \
type *r = res; \
type temp_c; \
type temp_s; \
\
for (i = 0; i < cnt; i++) { \
temp_c = c[i]; \
temp_s = s[i]; \
/* We never use weak operations */ \
(void) op(type, d[i], temp_s, temp_c); \
/* If d[i] != temp_c then d[i] -> temp_c */ \
r[i] = temp_c; \
} \
}
#define OFI_DEF_CSWAPEXT_NAME(op, type) ofi_cswap_## op ##_## type,
#define OFI_DEF_CSWAPEXT_FUNC(op, type) \
static void ofi_cswap_## op ##_## type \
(void *dst, const void *src, const void *cmp, \
void *res, size_t cnt) \
{ \
type target; \
size_t i; \
type *d = dst; \
type *r = res; \
const type *c = cmp; \
const type *s = src; \
type val; \
int success; \
\
for (i = 0; i < cnt; i++) { \
success = 0; \
do { \
target = d[i]; \
val = op(type, d[i], s[i], c[i]); \
success = __atomic_compare_exchange( \
&d[i],&target,&val, 0, \
__ATOMIC_SEQ_CST, \
__ATOMIC_SEQ_CST); \
} while (!success); \
r[i] = target; \
} \
}
#define OFI_DEF_CSWAPEXT_CMP_NAME(op, type) ofi_cswap_## op ##_## type,
#define OFI_DEF_CSWAPEXT_CMP_FUNC(op, type) \
static void ofi_cswap_## op ##_## type \
(void *dst, const void *src, const void *cmp, \
void *res, size_t cnt) \
{ \
type target; \
size_t i; \
type *d = dst; \
type *r = res; \
const type *c = cmp; \
const type *s = src; \
type temp_s; \
int success; \
\
for (i = 0; i < cnt; i++) { \
do { \
success = 1; \
target = d[i]; \
if (op(type, d[i], s[i], c[i])) { \
temp_s = s[i]; \
success = __atomic_compare_exchange( \
&d[i],&target,&temp_s, 0, \
__ATOMIC_SEQ_CST, \
__ATOMIC_SEQ_CST); \
} \
} while (!success); \
r[i] = target; \
} \
}
#define OFI_DEF_CSWAP_COMPLEX_NAME(op, type) ofi_cswap_## op ##_## type,
#define OFI_DEF_CSWAP_COMPLEX_FUNC(op, type) \
static void ofi_cswap_## op ##_## type \
(void *dst, const void *src, const void *cmp, \
void *res, size_t cnt) \
{ \
ofi_complex_##type *d = dst; \
const ofi_complex_##type *s = src; \
const ofi_complex_##type *c = cmp; \
ofi_complex_##type *r = res; \
ofi_complex_##type temp_c; \
ofi_complex_##type temp_s; \
size_t i; \
\
for (i = 0; i < cnt; i++) { \
temp_c = c[i]; \
temp_s = s[i]; \
(void) op(type, d[i], temp_s, temp_c); \
/* If d[i] != temp_c then d[i] -> temp_c */ \
r[i] = temp_c; \
} \
}
#define OFI_DEF_CSWAPEXT_CMP_COMPLEX_NAME(op, type) ofi_cswap_## op ##_## type,
#define OFI_DEF_CSWAPEXT_CMP_COMPLEX_FUNC(op, type) \
static void ofi_cswap_## op ##_## type \
(void *dst, const void *src, const void *cmp, \
void *res, size_t cnt) \
{ \
ofi_complex_##type target; \
ofi_complex_##type *d = dst; \
ofi_complex_##type *r = res; \
const ofi_complex_##type *c = cmp; \
const ofi_complex_##type *s = src; \
ofi_complex_##type temp_s; \
size_t i; \
int success; \
\
for (i = 0; i < cnt; i++) { \
do { \
success = 1; \
target = d[i]; \
if (op(type, d[i], s[i], c[i])) { \
temp_s = s[i]; \
success = __atomic_compare_exchange( \
&d[i],&target,&temp_s, 0, \
__ATOMIC_SEQ_CST, \
__ATOMIC_SEQ_CST); \
} \
} while (!success); \
r[i] = target; \
} \
}
#else /* HAVE_BUILTIN_MM_ATOMICS */
#define OFI_DEF_CSWAP_NAME(op, type) ofi_cswap_## op ##_## type,
#define OFI_DEF_CSWAP_FUNC(op, type) \
static void ofi_cswap_## op ##_## type \
(void *dst, const void *src, const void *cmp, \
void *res, size_t cnt) \
{ \
size_t i; \
type *d = dst; \
const type *s = src; \
const type *c = cmp; \
type *r = (res); \
for (i = 0; i < cnt; i++) { \
r[i] = d[i]; \
op(type, d[i], s[i], c[i]); \
} \
}
#define OFI_DEF_CSWAP_COMPLEX_NAME(op, type) ofi_cswap_## op ##_## type,
#define OFI_DEF_CSWAP_COMPLEX_FUNC(op, type) \
static void ofi_cswap_## op ##_## type \
(void *dst, const void *src, const void *cmp, \
void *res, size_t cnt) \
{ \
size_t i; \
ofi_complex_##type *d = dst; \
const ofi_complex_##type *s = src; \
const ofi_complex_##type *c = cmp; \
ofi_complex_##type *r = res; \
for (i = 0; i < cnt; i++) { \
r[i] = d[i]; \
op(type, d[i], s[i], c[i]); \
} \
}
#endif /* HAVE_BUILTIN_MM_ATOMICS */
/*********************************************************************
* Macros create atomic functions for each operation for each datatype
*********************************************************************/
/*
* Define all handlers in order to populate the dispatch table correctly.
*
* ATOMICTYPE - WRITE, READ, READWRITE, CSWAP, MSWAP
* FUNCNAME - Define function or simply generate function name
* The latter is needed to populate the dispatch table
* op - OFI_OP_XXX function should perform (e.g. OFI_OP_MIN)
*/
#define OFI_DEFINE_INT_HANDLERS(ATOMICTYPE, FUNCNAME, op) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int64_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint64_t) \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME
#ifdef HAVE_BUILTIN_MM_ATOMICS
/* Only support 8 byte and under datatypes */
#define OFI_DEFINE_ALL_HANDLERS(ATOMICTYPE, FUNCNAME, op) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int64_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint64_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, float) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, double) \
OFI_DEF_##ATOMICTYPE##_COMPLEX_##FUNCNAME(op ##_COMPLEX, float) \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME
#define OFI_DEFINE_REALNO_HANDLERS(ATOMICTYPE, FUNCNAME, op) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int64_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint64_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, float) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, double) \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME
#else /* HAVE_BUILTIN_MM_ATOMICS */
#define OFI_DEFINE_ALL_HANDLERS(ATOMICTYPE, FUNCNAME, op) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int64_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint64_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, float) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, double) \
OFI_DEF_##ATOMICTYPE##_COMPLEX_##FUNCNAME(op ##_COMPLEX, float) \
OFI_DEF_##ATOMICTYPE##_COMPLEX_##FUNCNAME(op ##_COMPLEX, double)\
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, long_double) \
OFI_DEF_##ATOMICTYPE##_COMPLEX_##FUNCNAME(op ##_COMPLEX, long_double)
#define OFI_DEFINE_REALNO_HANDLERS(ATOMICTYPE, FUNCNAME, op) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint8_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint16_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint32_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, int64_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, uint64_t) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, float) \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, double) \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_NOOP_##FUNCNAME \
OFI_DEF_##ATOMICTYPE##_##FUNCNAME(op, long_double) \
OFI_DEF_NOOP_##FUNCNAME
#endif /* HAVE_BUILTIN_MM_ATOMICS */
#define OFI_OP_NOT_SUPPORTED(op) NULL, NULL, NULL, NULL, NULL, \
NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
#ifdef HAVE_BUILTIN_MM_ATOMICS
/**********************
* Compiler built-in atomics write dispatch table
**********************/
OFI_DEFINE_REALNO_HANDLERS(WRITEEXT_CMP, FUNC, OFI_OP_MIN)
OFI_DEFINE_REALNO_HANDLERS(WRITEEXT_CMP, FUNC, OFI_OP_MAX)
OFI_DEFINE_ALL_HANDLERS(WRITEEXT, FUNC, OFI_OP_SUM)
OFI_DEFINE_ALL_HANDLERS(WRITEEXT, FUNC, OFI_OP_PROD)
OFI_DEFINE_ALL_HANDLERS(WRITEEXT, FUNC, OFI_OP_LOR)
OFI_DEFINE_ALL_HANDLERS(WRITEEXT, FUNC, OFI_OP_LAND)
OFI_DEFINE_INT_HANDLERS(WRITE, FUNC, OFI_OP_BOR)
OFI_DEFINE_INT_HANDLERS(WRITE, FUNC, OFI_OP_BAND)
OFI_DEFINE_ALL_HANDLERS(WRITEEXT, FUNC, OFI_OP_LXOR)
OFI_DEFINE_INT_HANDLERS(WRITE, FUNC, OFI_OP_BXOR)
OFI_DEFINE_ALL_HANDLERS(WRITE, FUNC, OFI_OP_WRITE)
void (*ofi_atomic_write_handlers[OFI_WRITE_OP_LAST][FI_DATATYPE_LAST])
(void *dst, const void *src, size_t cnt) =
{
{ OFI_DEFINE_REALNO_HANDLERS(WRITEEXT_CMP, NAME, OFI_OP_MIN) },
{ OFI_DEFINE_REALNO_HANDLERS(WRITEEXT_CMP, NAME, OFI_OP_MAX) },
{ OFI_DEFINE_ALL_HANDLERS(WRITEEXT, NAME, OFI_OP_SUM) },
{ OFI_DEFINE_ALL_HANDLERS(WRITEEXT, NAME, OFI_OP_PROD) },
{ OFI_DEFINE_ALL_HANDLERS(WRITEEXT, NAME, OFI_OP_LOR) },
{ OFI_DEFINE_ALL_HANDLERS(WRITEEXT, NAME, OFI_OP_LAND) },
{ OFI_DEFINE_INT_HANDLERS(WRITE, NAME, OFI_OP_BOR) },
{ OFI_DEFINE_INT_HANDLERS(WRITE, NAME, OFI_OP_BAND) },
{ OFI_DEFINE_ALL_HANDLERS(WRITEEXT, NAME, OFI_OP_LXOR) },
{ OFI_DEFINE_INT_HANDLERS(WRITE, NAME, OFI_OP_BXOR) },
{ OFI_OP_NOT_SUPPORTED(FI_ATOMIC_READ) },
{ OFI_DEFINE_ALL_HANDLERS(WRITE, NAME, OFI_OP_WRITE) },
};
/***************************
* Compiler built-in atomics read-write dispatch table
***************************/
OFI_DEFINE_REALNO_HANDLERS(READWRITEEXT_CMP, FUNC, OFI_OP_MIN)
OFI_DEFINE_REALNO_HANDLERS(READWRITEEXT_CMP, FUNC, OFI_OP_MAX)
OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, FUNC, OFI_OP_SUM)
OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, FUNC, OFI_OP_PROD)
OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, FUNC, OFI_OP_LOR)
OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, FUNC, OFI_OP_LAND)
OFI_DEFINE_INT_HANDLERS(READWRITE, FUNC, OFI_OP_BOR)
OFI_DEFINE_INT_HANDLERS(READWRITE, FUNC, OFI_OP_BAND)
OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, FUNC, OFI_OP_LXOR)
OFI_DEFINE_INT_HANDLERS(READWRITE, FUNC, OFI_OP_BXOR)
OFI_DEFINE_ALL_HANDLERS(READ, FUNC, OFI_OP_READ)
OFI_DEFINE_ALL_HANDLERS(EXCHANGE, FUNC, OFI_OP_READWRITE)
void (*ofi_atomic_readwrite_handlers[OFI_READWRITE_OP_LAST][FI_DATATYPE_LAST])
(void *dst, const void *src, void *res, size_t cnt) =
{
{ OFI_DEFINE_REALNO_HANDLERS(READWRITEEXT_CMP, NAME, OFI_OP_MIN) },
{ OFI_DEFINE_REALNO_HANDLERS(READWRITEEXT_CMP, NAME, OFI_OP_MAX) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, NAME, OFI_OP_SUM) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, NAME, OFI_OP_PROD) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, NAME, OFI_OP_LOR) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, NAME, OFI_OP_LAND) },
{ OFI_DEFINE_INT_HANDLERS(READWRITE, NAME, OFI_OP_BOR) },
{ OFI_DEFINE_INT_HANDLERS(READWRITE, NAME, OFI_OP_BAND) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITEEXT, NAME, OFI_OP_LXOR) },
{ OFI_DEFINE_INT_HANDLERS(READWRITE, NAME, OFI_OP_BXOR) },
{ OFI_DEFINE_ALL_HANDLERS(READ, NAME, OFI_OP_READ) },
{ OFI_DEFINE_ALL_HANDLERS(EXCHANGE, NAME, OFI_OP_READWRITE) },
};
/*****************************
* Compiler built-in atomics compare-swap dispatch table
*****************************/
OFI_DEFINE_ALL_HANDLERS(CSWAP, FUNC, OFI_OP_CSWAP_EQ)
OFI_DEFINE_ALL_HANDLERS(CSWAPEXT_CMP, FUNC, OFI_OP_CSWAP_NE)
OFI_DEFINE_REALNO_HANDLERS(CSWAPEXT_CMP, FUNC, OFI_OP_CSWAP_LE)
OFI_DEFINE_REALNO_HANDLERS(CSWAPEXT_CMP, FUNC, OFI_OP_CSWAP_LT)
OFI_DEFINE_REALNO_HANDLERS(CSWAPEXT_CMP, FUNC, OFI_OP_CSWAP_GE)
OFI_DEFINE_REALNO_HANDLERS(CSWAPEXT_CMP, FUNC, OFI_OP_CSWAP_GT)
OFI_DEFINE_INT_HANDLERS(CSWAPEXT, FUNC, OFI_OP_MSWAP)
void (*ofi_atomic_swap_handlers[OFI_SWAP_OP_LAST][FI_DATATYPE_LAST])
(void *dst, const void *src, const void *cmp, void *res, size_t cnt) =
{
{ OFI_DEFINE_ALL_HANDLERS(CSWAP, NAME, OFI_OP_CSWAP_EQ) },
{ OFI_DEFINE_ALL_HANDLERS(CSWAPEXT_CMP, NAME, OFI_OP_CSWAP_NE) },
{ OFI_DEFINE_REALNO_HANDLERS(CSWAPEXT_CMP, NAME, OFI_OP_CSWAP_LE) },
{ OFI_DEFINE_REALNO_HANDLERS(CSWAPEXT_CMP, NAME, OFI_OP_CSWAP_LT) },
{ OFI_DEFINE_REALNO_HANDLERS(CSWAPEXT_CMP, NAME, OFI_OP_CSWAP_GE) },
{ OFI_DEFINE_REALNO_HANDLERS(CSWAPEXT_CMP, NAME, OFI_OP_CSWAP_GT) },
{ OFI_DEFINE_INT_HANDLERS(CSWAPEXT, NAME, OFI_OP_MSWAP) },
};
static void ofi_log_atomic_info(const struct fi_provider *prov)
{
FI_INFO(prov, FI_LOG_DOMAIN, "Using built-in memory model atomics.\n");
}
#else /* HAVE_BUILTIN_MM_ATOMICS */
/**********************
* Write dispatch table
**********************/
OFI_DEFINE_REALNO_HANDLERS(WRITE, FUNC, OFI_OP_MIN)
OFI_DEFINE_REALNO_HANDLERS(WRITE, FUNC, OFI_OP_MAX)
OFI_DEFINE_ALL_HANDLERS(WRITE, FUNC, OFI_OP_SUM)
OFI_DEFINE_ALL_HANDLERS(WRITE, FUNC, OFI_OP_PROD)
OFI_DEFINE_ALL_HANDLERS(WRITE, FUNC, OFI_OP_LOR)
OFI_DEFINE_ALL_HANDLERS(WRITE, FUNC, OFI_OP_LAND)
OFI_DEFINE_INT_HANDLERS(WRITE, FUNC, OFI_OP_BOR)
OFI_DEFINE_INT_HANDLERS(WRITE, FUNC, OFI_OP_BAND)
OFI_DEFINE_ALL_HANDLERS(WRITE, FUNC, OFI_OP_LXOR)
OFI_DEFINE_INT_HANDLERS(WRITE, FUNC, OFI_OP_BXOR)
OFI_DEFINE_ALL_HANDLERS(WRITE, FUNC, OFI_OP_WRITE)
void (*ofi_atomic_write_handlers[OFI_WRITE_OP_LAST][FI_DATATYPE_LAST])
(void *dst, const void *src, size_t cnt) =
{
{ OFI_DEFINE_REALNO_HANDLERS(WRITE, NAME, OFI_OP_MIN) },
{ OFI_DEFINE_REALNO_HANDLERS(WRITE, NAME, OFI_OP_MAX) },
{ OFI_DEFINE_ALL_HANDLERS(WRITE, NAME, OFI_OP_SUM) },
{ OFI_DEFINE_ALL_HANDLERS(WRITE, NAME, OFI_OP_PROD) },
{ OFI_DEFINE_ALL_HANDLERS(WRITE, NAME, OFI_OP_LOR) },
{ OFI_DEFINE_ALL_HANDLERS(WRITE, NAME, OFI_OP_LAND) },
{ OFI_DEFINE_INT_HANDLERS(WRITE, NAME, OFI_OP_BOR) },
{ OFI_DEFINE_INT_HANDLERS(WRITE, NAME, OFI_OP_BAND) },
{ OFI_DEFINE_ALL_HANDLERS(WRITE, NAME, OFI_OP_LXOR) },
{ OFI_DEFINE_INT_HANDLERS(WRITE, NAME, OFI_OP_BXOR) },
{ OFI_OP_NOT_SUPPORTED(FI_ATOMIC_READ) },
{ OFI_DEFINE_ALL_HANDLERS(WRITE, NAME, OFI_OP_WRITE) },
};
/***************************
* Read-write dispatch table
***************************/
OFI_DEFINE_REALNO_HANDLERS(READWRITE, FUNC, OFI_OP_MIN)
OFI_DEFINE_REALNO_HANDLERS(READWRITE, FUNC, OFI_OP_MAX)
OFI_DEFINE_ALL_HANDLERS(READWRITE, FUNC, OFI_OP_SUM)
OFI_DEFINE_ALL_HANDLERS(READWRITE, FUNC, OFI_OP_PROD)
OFI_DEFINE_ALL_HANDLERS(READWRITE, FUNC, OFI_OP_LOR)
OFI_DEFINE_ALL_HANDLERS(READWRITE, FUNC, OFI_OP_LAND)
OFI_DEFINE_INT_HANDLERS(READWRITE, FUNC, OFI_OP_BOR)
OFI_DEFINE_INT_HANDLERS(READWRITE, FUNC, OFI_OP_BAND)
OFI_DEFINE_ALL_HANDLERS(READWRITE, FUNC, OFI_OP_LXOR)
OFI_DEFINE_INT_HANDLERS(READWRITE, FUNC, OFI_OP_BXOR)
OFI_DEFINE_ALL_HANDLERS(READ, FUNC, OFI_OP_READ)
OFI_DEFINE_ALL_HANDLERS(READWRITE, FUNC, OFI_OP_WRITE)
void (*ofi_atomic_readwrite_handlers[OFI_READWRITE_OP_LAST][FI_DATATYPE_LAST])
(void *dst, const void *src, void *res, size_t cnt) =
{
{ OFI_DEFINE_REALNO_HANDLERS(READWRITE, NAME, OFI_OP_MIN) },
{ OFI_DEFINE_REALNO_HANDLERS(READWRITE, NAME, OFI_OP_MAX) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITE, NAME, OFI_OP_SUM) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITE, NAME, OFI_OP_PROD) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITE, NAME, OFI_OP_LOR) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITE, NAME, OFI_OP_LAND) },
{ OFI_DEFINE_INT_HANDLERS(READWRITE, NAME, OFI_OP_BOR) },
{ OFI_DEFINE_INT_HANDLERS(READWRITE, NAME, OFI_OP_BAND) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITE, NAME, OFI_OP_LXOR) },
{ OFI_DEFINE_INT_HANDLERS(READWRITE, NAME, OFI_OP_BXOR) },
{ OFI_DEFINE_ALL_HANDLERS(READ, NAME, OFI_OP_READ) },
{ OFI_DEFINE_ALL_HANDLERS(READWRITE, NAME, OFI_OP_WRITE) },
};
/*****************************
* Compare-swap dispatch table
*****************************/
OFI_DEFINE_ALL_HANDLERS(CSWAP, FUNC, OFI_OP_CSWAP_EQ)
OFI_DEFINE_ALL_HANDLERS(CSWAP, FUNC, OFI_OP_CSWAP_NE)
OFI_DEFINE_REALNO_HANDLERS(CSWAP, FUNC, OFI_OP_CSWAP_LE)
OFI_DEFINE_REALNO_HANDLERS(CSWAP, FUNC, OFI_OP_CSWAP_LT)
OFI_DEFINE_REALNO_HANDLERS(CSWAP, FUNC, OFI_OP_CSWAP_GE)
OFI_DEFINE_REALNO_HANDLERS(CSWAP, FUNC, OFI_OP_CSWAP_GT)
OFI_DEFINE_INT_HANDLERS(CSWAP, FUNC, OFI_OP_MSWAP)
void (*ofi_atomic_swap_handlers[OFI_SWAP_OP_LAST][FI_DATATYPE_LAST])
(void *dst, const void *src, const void *cmp, void *res, size_t cnt) =
{
{ OFI_DEFINE_ALL_HANDLERS(CSWAP, NAME, OFI_OP_CSWAP_EQ) },
{ OFI_DEFINE_ALL_HANDLERS(CSWAP, NAME, OFI_OP_CSWAP_NE) },
{ OFI_DEFINE_REALNO_HANDLERS(CSWAP, NAME, OFI_OP_CSWAP_LE) },
{ OFI_DEFINE_REALNO_HANDLERS(CSWAP, NAME, OFI_OP_CSWAP_LT) },
{ OFI_DEFINE_REALNO_HANDLERS(CSWAP, NAME, OFI_OP_CSWAP_GE) },
{ OFI_DEFINE_REALNO_HANDLERS(CSWAP, NAME, OFI_OP_CSWAP_GT) },
{ OFI_DEFINE_INT_HANDLERS(CSWAP, NAME, OFI_OP_MSWAP) },
};
static void ofi_log_atomic_info(const struct fi_provider *prov)
{
FI_INFO(prov, FI_LOG_DOMAIN, "Using open-coded atomics. "
"Use requires single-threaded access by provider.\n");
}
#endif /* HAVE_BUILTIN_MM_ATOMICS */
int ofi_atomic_valid(const struct fi_provider *prov,
enum fi_datatype datatype, enum fi_op op, uint64_t flags)
{
int have_func;
ofi_log_atomic_info(prov);
if (flags & FI_TAGGED) {
/* Only tagged atomic write operations currently supported */
if (flags & (FI_FETCH_ATOMIC | FI_COMPARE_ATOMIC)) {
FI_INFO(prov, FI_LOG_DOMAIN,
"Only tagged atomic writes supported\n");
return -FI_ENOSYS;
}
} else if (flags & ~(FI_FETCH_ATOMIC | FI_COMPARE_ATOMIC)) {
FI_INFO(prov, FI_LOG_DOMAIN, "Unknown flag specified\n");
return -FI_EBADFLAGS;
} else if ((flags & FI_FETCH_ATOMIC) && (flags & FI_COMPARE_ATOMIC)) {
FI_INFO(prov, FI_LOG_DOMAIN, "Invalid flag combination\n");
return -FI_EBADFLAGS;
}
if (datatype >= FI_DATATYPE_LAST) {
FI_INFO(prov, FI_LOG_DOMAIN, "Invalid datatype\n");
return -FI_EOPNOTSUPP;
}
if (flags & FI_FETCH_ATOMIC) {
if (op >= OFI_READWRITE_OP_LAST) {
FI_INFO(prov, FI_LOG_DOMAIN, "Invalid fetch operation\n");
return -FI_EOPNOTSUPP;
}
have_func = ofi_atomic_readwrite_handlers[op][datatype] != NULL;
} else if (flags & FI_COMPARE_ATOMIC) {
if (op < FI_CSWAP || op > FI_MSWAP) {
FI_INFO(prov, FI_LOG_DOMAIN, "Invalid swap operation\n");
return -FI_EOPNOTSUPP;
}
have_func = ofi_atomic_swap_handlers[op - FI_CSWAP][datatype] != NULL;
} else {
if (op >= OFI_WRITE_OP_LAST) {
FI_INFO(prov, FI_LOG_DOMAIN, "Invalid write operation\n");
return -FI_EOPNOTSUPP;
}
have_func = ofi_atomic_write_handlers[op][datatype] != NULL;
}
if (!have_func) {
FI_INFO(prov, FI_LOG_DOMAIN, "Datatype/op combo not supported\n");
return -FI_EOPNOTSUPP;
}
return 0;
}