36#ifndef EIGEN_HALF_CUDA_H
37#define EIGEN_HALF_CUDA_H
39#if __cplusplus > 199711L
40#define EIGEN_EXPLICIT_CAST(tgt_type) explicit operator tgt_type()
42#define EIGEN_EXPLICIT_CAST(tgt_type) operator tgt_type()
52#if !defined(EIGEN_HAS_CUDA_FP16)
56 EIGEN_DEVICE_FUNC __half() {}
57 explicit EIGEN_DEVICE_FUNC __half(
unsigned short raw) : x(raw) {}
63EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half raw_uint16_to_half(
unsigned short x);
64EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half float_to_half_rtne(
float ff);
65EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
float half_to_float(__half h);
67struct half_base :
public __half {
68 EIGEN_DEVICE_FUNC half_base() {}
69 EIGEN_DEVICE_FUNC half_base(
const half_base& h) : __half(h) {}
70 EIGEN_DEVICE_FUNC half_base(
const __half& h) : __half(h) {}
76struct half :
public half_impl::half_base {
77 #if !defined(EIGEN_HAS_CUDA_FP16)
78 typedef half_impl::__half __half;
81 EIGEN_DEVICE_FUNC half() {}
83 EIGEN_DEVICE_FUNC half(
const __half& h) : half_impl::half_base(h) {}
84 EIGEN_DEVICE_FUNC half(
const half& h) : half_impl::half_base(h) {}
86 explicit EIGEN_DEVICE_FUNC half(
bool b)
87 : half_impl::half_base(half_impl::raw_uint16_to_half(b ? 0x3c00 : 0)) {}
89 explicit EIGEN_DEVICE_FUNC half(
const T& val)
90 : half_impl::half_base(half_impl::float_to_half_rtne(static_cast<float>(val))) {}
91 explicit EIGEN_DEVICE_FUNC half(
float f)
92 : half_impl::half_base(half_impl::float_to_half_rtne(f)) {}
94 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
bool)
const {
96 return (x & 0x7fff) != 0;
98 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
signed char)
const {
99 return static_cast<signed char>(half_impl::half_to_float(*
this));
101 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
unsigned char)
const {
102 return static_cast<unsigned char>(half_impl::half_to_float(*
this));
104 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
short)
const {
105 return static_cast<short>(half_impl::half_to_float(*
this));
107 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
unsigned short)
const {
108 return static_cast<unsigned short>(half_impl::half_to_float(*
this));
110 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
int)
const {
111 return static_cast<int>(half_impl::half_to_float(*
this));
113 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
unsigned int)
const {
114 return static_cast<unsigned int>(half_impl::half_to_float(*
this));
116 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
long)
const {
117 return static_cast<long>(half_impl::half_to_float(*
this));
119 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
unsigned long)
const {
120 return static_cast<unsigned long>(half_impl::half_to_float(*
this));
122 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
long long)
const {
123 return static_cast<long long>(half_impl::half_to_float(*
this));
125 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
unsigned long long)
const {
126 return static_cast<unsigned long long>(half_to_float(*
this));
128 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
float)
const {
129 return half_impl::half_to_float(*
this);
131 EIGEN_DEVICE_FUNC EIGEN_EXPLICIT_CAST(
double)
const {
132 return static_cast<double>(half_impl::half_to_float(*
this));
135 EIGEN_DEVICE_FUNC half& operator=(
const half& other) {
143#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
150__device__ half operator + (
const half& a,
const half& b) {
153__device__ half operator * (
const half& a,
const half& b) {
156__device__ half operator - (
const half& a,
const half& b) {
159__device__ half operator / (
const half& a,
const half& b) {
160 float num = __half2float(a);
161 float denom = __half2float(b);
162 return __float2half(num / denom);
164__device__ half operator - (
const half& a) {
167__device__ half& operator += (half& a,
const half& b) {
171__device__ half& operator *= (half& a,
const half& b) {
175__device__ half& operator -= (half& a,
const half& b) {
179__device__ half& operator /= (half& a,
const half& b) {
183__device__
bool operator == (
const half& a,
const half& b) {
186__device__
bool operator != (
const half& a,
const half& b) {
189__device__
bool operator < (
const half& a,
const half& b) {
192__device__
bool operator <= (
const half& a,
const half& b) {
195__device__
bool operator > (
const half& a,
const half& b) {
198__device__
bool operator >= (
const half& a,
const half& b) {
207EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator + (
const half& a,
const half& b) {
208 return half(
float(a) +
float(b));
210EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator * (
const half& a,
const half& b) {
211 return half(
float(a) *
float(b));
213EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (
const half& a,
const half& b) {
214 return half(
float(a) -
float(b));
216EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator / (
const half& a,
const half& b) {
217 return half(
float(a) /
float(b));
219EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator - (
const half& a) {
221 result.x = a.x ^ 0x8000;
224EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator += (half& a,
const half& b) {
225 a = half(
float(a) +
float(b));
228EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator *= (half& a,
const half& b) {
229 a = half(
float(a) *
float(b));
232EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator -= (half& a,
const half& b) {
233 a = half(
float(a) -
float(b));
236EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half& operator /= (half& a,
const half& b) {
237 a = half(
float(a) /
float(b));
240EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator == (
const half& a,
const half& b) {
241 return float(a) == float(b);
243EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator != (
const half& a,
const half& b) {
244 return float(a) != float(b);
246EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator < (
const half& a,
const half& b) {
247 return float(a) < float(b);
249EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator <= (
const half& a,
const half& b) {
250 return float(a) <= float(b);
252EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator > (
const half& a,
const half& b) {
253 return float(a) > float(b);
255EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
bool operator >= (
const half& a,
const half& b) {
256 return float(a) >= float(b);
263EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half operator / (
const half& a,
Index b) {
264 return half(
static_cast<float>(a) /
static_cast<float>(b));
272EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half raw_uint16_to_half(
unsigned short x) {
283EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC __half float_to_half_rtne(
float ff) {
284#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300
285 return __float2half(ff);
287#elif defined(EIGEN_HAS_FP16_C)
289 h.x = _cvtss_sh(ff, 0);
295 const FP32 f32infty = { 255 << 23 };
296 const FP32 f16max = { (127 + 16) << 23 };
297 const FP32 denorm_magic = { ((127 - 15) + (23 - 10) + 1) << 23 };
298 unsigned int sign_mask = 0x80000000u;
300 o.x =
static_cast<unsigned short>(0x0u);
302 unsigned int sign = f.u & sign_mask;
310 if (f.u >= f16max.u) {
311 o.x = (f.u > f32infty.u) ? 0x7e00 : 0x7c00;
313 if (f.u < (113 << 23)) {
317 f.f += denorm_magic.f;
320 o.x =
static_cast<unsigned short>(f.u - denorm_magic.u);
322 unsigned int mant_odd = (f.u >> 13) & 1;
325 f.u += ((
unsigned int)(15 - 127) << 23) + 0xfff;
329 o.x =
static_cast<unsigned short>(f.u >> 13);
333 o.x |=
static_cast<unsigned short>(
sign >> 16);
338EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
float half_to_float(__half h) {
339#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300
340 return __half2float(h);
342#elif defined(EIGEN_HAS_FP16_C)
343 return _cvtsh_ss(h.x);
346 const FP32 magic = { 113 << 23 };
347 const unsigned int shifted_exp = 0x7c00 << 13;
350 o.u = (h.x & 0x7fff) << 13;
351 unsigned int exp = shifted_exp & o.u;
352 o.u += (127 - 15) << 23;
355 if (exp == shifted_exp) {
356 o.u += (128 - 16) << 23;
357 }
else if (exp == 0) {
362 o.u |= (h.x & 0x8000) << 16;
369EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isinf)(
const half& a) {
370 return (a.x & 0x7fff) == 0x7c00;
372EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isnan)(
const half& a) {
373#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
376 return (a.x & 0x7fff) > 0x7c00;
379EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC bool (isfinite)(
const half& a) {
380 return !(isinf EIGEN_NOT_A_MACRO (a)) && !(isnan EIGEN_NOT_A_MACRO (a));
383EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half abs(
const half& a) {
385 result.x = a.x & 0x7FFF;
388EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half exp(
const half& a) {
389 return half(::expf(
float(a)));
391EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log(
const half& a) {
392#if defined(EIGEN_HAS_CUDA_FP16) && defined __CUDACC_VER__ && __CUDACC_VER__ >= 80000 && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
393 return Eigen::half(::hlog(a));
395 return half(::logf(
float(a)));
398EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log1p(
const half& a) {
399 return half(numext::log1p(
float(a)));
401EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half log10(
const half& a) {
402 return half(::log10f(
float(a)));
404EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sqrt(
const half& a) {
405 return half(::sqrtf(
float(a)));
407EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half pow(
const half& a,
const half& b) {
408 return half(::powf(
float(a),
float(b)));
410EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half sin(
const half& a) {
411 return half(::sinf(
float(a)));
413EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half cos(
const half& a) {
414 return half(::cosf(
float(a)));
416EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half tan(
const half& a) {
417 return half(::tanf(
float(a)));
419EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half tanh(
const half& a) {
420 return half(::tanhf(
float(a)));
422EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half floor(
const half& a) {
423 return half(::floorf(
float(a)));
425EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half ceil(
const half& a) {
426 return half(::ceilf(
float(a)));
429EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half (min)(
const half& a,
const half& b) {
430#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
431 return __hlt(b, a) ? b : a;
433 const float f1 =
static_cast<float>(a);
434 const float f2 =
static_cast<float>(b);
435 return f2 < f1 ? b : a;
438EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC half (max)(
const half& a,
const half& b) {
439#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
440 return __hlt(a, b) ? b : a;
442 const float f1 =
static_cast<float>(a);
443 const float f2 =
static_cast<float>(b);
444 return f1 < f2 ? b : a;
448EIGEN_ALWAYS_INLINE std::ostream& operator << (std::ostream& os,
const half& v) {
449 os << static_cast<float>(v);
461struct random_default_impl<half, false, false>
463 static inline half run(
const half& x,
const half& y)
465 return x + (y-x) * half(
float(std::rand()) / float(RAND_MAX));
467 static inline half run()
469 return run(half(-1.f), half(1.f));
473template<>
struct is_arithmetic<half> {
enum { value =
true }; };
477template<>
struct NumTraits<
Eigen::half>
478 : GenericNumTraits<Eigen::half>
480 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Eigen::half epsilon() {
481 return half_impl::raw_uint16_to_half(0x0800);
483 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Eigen::half dummy_precision() {
return Eigen::half(1e-2f); }
484 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Eigen::half highest() {
485 return half_impl::raw_uint16_to_half(0x7bff);
487 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Eigen::half lowest() {
488 return half_impl::raw_uint16_to_half(0xfbff);
490 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Eigen::half infinity() {
491 return half_impl::raw_uint16_to_half(0x7c00);
493 EIGEN_DEVICE_FUNC
static EIGEN_STRONG_INLINE Eigen::half quiet_NaN() {
494 return half_impl::raw_uint16_to_half(0x7c01);
501EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half fabsh(
const Eigen::half& a) {
503 result.x = a.x & 0x7FFF;
506EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half exph(
const Eigen::half& a) {
507 return Eigen::half(::expf(
float(a)));
509EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half logh(
const Eigen::half& a) {
510#if defined __CUDACC_VER__ && __CUDACC_VER__ >= 80000 && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 530
511 return Eigen::half(::hlog(a));
513 return Eigen::half(::logf(
float(a)));
516EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half sqrth(
const Eigen::half& a) {
517 return Eigen::half(::sqrtf(
float(a)));
519EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half powh(
const Eigen::half& a,
const Eigen::half& b) {
520 return Eigen::half(::powf(
float(a),
float(b)));
522EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half floorh(
const Eigen::half& a) {
523 return Eigen::half(::floorf(
float(a)));
525EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half ceilh(
const Eigen::half& a) {
526 return Eigen::half(::ceilf(
float(a)));
531#if __cplusplus > 199711L
533struct hash<
Eigen::half> {
534 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::size_t operator()(
const Eigen::half& a)
const {
535 return static_cast<std::size_t
>(a.x);
544#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300
545__device__ EIGEN_STRONG_INLINE Eigen::half __shfl_xor(Eigen::half var,
int laneMask,
int width=warpSize) {
546 return static_cast<Eigen::half
>(__shfl_xor(
static_cast<float>(var), laneMask, width));
551#if defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 350
552EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC Eigen::half __ldg(
const Eigen::half* ptr) {
553 return Eigen::half_impl::raw_uint16_to_half(
554 __ldg(
reinterpret_cast<const unsigned short*
>(ptr)));
559#if defined(__CUDA_ARCH__)
564EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
565bool (
isnan)(
const Eigen::half& h) {
566 return (half_impl::isnan)(h);
570EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
571bool (
isinf)(
const Eigen::half& h) {
572 return (half_impl::isinf)(h);
576EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE
577bool (
isfinite)(
const Eigen::half& h) {
578 return (half_impl::isfinite)(h);
Namespace containing all symbols from the Eigen library.
Definition: Core:287
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_sign_op< typename Derived::Scalar >, const Derived > sign(const Eigen::ArrayBase< Derived > &x)
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:33
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_isinf_op< typename Derived::Scalar >, const Derived > isinf(const Eigen::ArrayBase< Derived > &x)
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_isfinite_op< typename Derived::Scalar >, const Derived > isfinite(const Eigen::ArrayBase< Derived > &x)
const Eigen::CwiseUnaryOp< Eigen::internal::scalar_isnan_op< typename Derived::Scalar >, const Derived > isnan(const Eigen::ArrayBase< Derived > &x)