Eigen  3.3.0
 
Loading...
Searching...
No Matches
PacketMathHalf.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2016 Benoit Steiner <benoit.steiner.goog@gmail.com>
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9
10#ifndef EIGEN_PACKET_MATH_HALF_CUDA_H
11#define EIGEN_PACKET_MATH_HALF_CUDA_H
12
13
14namespace Eigen {
15namespace internal {
16
17// Most of the following operations require arch >= 3.0
18#if defined(EIGEN_HAS_CUDA_FP16) && defined(__CUDACC__) && defined(__CUDA_ARCH__) && __CUDA_ARCH__ >= 300
19
20template<> struct is_arithmetic<half2> { enum { value = true }; };
21
22template<> struct packet_traits<Eigen::half> : default_packet_traits
23{
24 typedef half2 type;
25 typedef half2 half;
26 enum {
27 Vectorizable = 1,
28 AlignedOnScalar = 1,
29 size=2,
30 HasHalfPacket = 0,
31 HasAdd = 1,
32 HasMul = 1,
33 HasDiv = 1,
34 HasSqrt = 1,
35 HasRsqrt = 1,
36 HasExp = 1,
37 HasLog = 1,
38 HasLog1p = 1
39 };
40};
41
42template<> struct unpacket_traits<half2> { typedef Eigen::half type; enum {size=2, alignment=Aligned16}; typedef half2 half; };
43
44template<> __device__ EIGEN_STRONG_INLINE half2 pset1<half2>(const Eigen::half& from) {
45 return __half2half2(from);
46}
47
48template<> __device__ EIGEN_STRONG_INLINE half2 pload<half2>(const Eigen::half* from) {
49 return *reinterpret_cast<const half2*>(from);
50}
51
52template<> __device__ EIGEN_STRONG_INLINE half2 ploadu<half2>(const Eigen::half* from) {
53 return __halves2half2(from[0], from[1]);
54}
55
56template<> EIGEN_STRONG_INLINE half2 ploaddup<half2>(const Eigen::half* from) {
57 return __halves2half2(from[0], from[0]);
58}
59
60template<> __device__ EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const half2& from) {
61 *reinterpret_cast<half2*>(to) = from;
62}
63
64template<> __device__ EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const half2& from) {
65 to[0] = __low2half(from);
66 to[1] = __high2half(from);
67}
68
69template<>
70 __device__ EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Aligned>(const Eigen::half* from) {
71#if __CUDA_ARCH__ >= 350
72 return __ldg((const half2*)from);
73#else
74 return __halves2half2(*(from+0), *(from+1));
75#endif
76}
77
78template<>
79__device__ EIGEN_ALWAYS_INLINE half2 ploadt_ro<half2, Unaligned>(const Eigen::half* from) {
80#if __CUDA_ARCH__ >= 350
81 return __halves2half2(__ldg(from+0), __ldg(from+1));
82#else
83 return __halves2half2(*(from+0), *(from+1));
84#endif
85}
86
87template<> __device__ EIGEN_STRONG_INLINE half2 pgather<Eigen::half, half2>(const Eigen::half* from, Index stride) {
88 return __halves2half2(from[0*stride], from[1*stride]);
89}
90
91template<> __device__ EIGEN_STRONG_INLINE void pscatter<Eigen::half, half2>(Eigen::half* to, const half2& from, Index stride) {
92 to[stride*0] = __low2half(from);
93 to[stride*1] = __high2half(from);
94}
95
96template<> __device__ EIGEN_STRONG_INLINE Eigen::half pfirst<half2>(const half2& a) {
97 return __low2half(a);
98}
99
100template<> __device__ EIGEN_STRONG_INLINE half2 pabs<half2>(const half2& a) {
101 half2 result;
102 result.x = a.x & 0x7FFF7FFF;
103 return result;
104}
105
106
107__device__ EIGEN_STRONG_INLINE void
108ptranspose(PacketBlock<half2,2>& kernel) {
109 __half a1 = __low2half(kernel.packet[0]);
110 __half a2 = __high2half(kernel.packet[0]);
111 __half b1 = __low2half(kernel.packet[1]);
112 __half b2 = __high2half(kernel.packet[1]);
113 kernel.packet[0] = __halves2half2(a1, b1);
114 kernel.packet[1] = __halves2half2(a2, b2);
115}
116
117template<> __device__ EIGEN_STRONG_INLINE half2 plset<half2>(const Eigen::half& a) {
118#if __CUDA_ARCH__ >= 530
119 return __halves2half2(a, __hadd(a, __float2half(1.0f)));
120#else
121 float f = __half2float(a) + 1.0f;
122 return __halves2half2(a, __float2half(f));
123#endif
124}
125
126template<> __device__ EIGEN_STRONG_INLINE half2 padd<half2>(const half2& a, const half2& b) {
127#if __CUDA_ARCH__ >= 530
128 return __hadd2(a, b);
129#else
130 float a1 = __low2float(a);
131 float a2 = __high2float(a);
132 float b1 = __low2float(b);
133 float b2 = __high2float(b);
134 float r1 = a1 + b1;
135 float r2 = a2 + b2;
136 return __floats2half2_rn(r1, r2);
137#endif
138}
139
140template<> __device__ EIGEN_STRONG_INLINE half2 psub<half2>(const half2& a, const half2& b) {
141#if __CUDA_ARCH__ >= 530
142 return __hsub2(a, b);
143#else
144 float a1 = __low2float(a);
145 float a2 = __high2float(a);
146 float b1 = __low2float(b);
147 float b2 = __high2float(b);
148 float r1 = a1 - b1;
149 float r2 = a2 - b2;
150 return __floats2half2_rn(r1, r2);
151#endif
152}
153
154template<> __device__ EIGEN_STRONG_INLINE half2 pnegate(const half2& a) {
155#if __CUDA_ARCH__ >= 530
156 return __hneg2(a);
157#else
158 float a1 = __low2float(a);
159 float a2 = __high2float(a);
160 return __floats2half2_rn(-a1, -a2);
161#endif
162}
163
164template<> __device__ EIGEN_STRONG_INLINE half2 pconj(const half2& a) { return a; }
165
166template<> __device__ EIGEN_STRONG_INLINE half2 pmul<half2>(const half2& a, const half2& b) {
167#if __CUDA_ARCH__ >= 530
168 return __hmul2(a, b);
169#else
170 float a1 = __low2float(a);
171 float a2 = __high2float(a);
172 float b1 = __low2float(b);
173 float b2 = __high2float(b);
174 float r1 = a1 * b1;
175 float r2 = a2 * b2;
176 return __floats2half2_rn(r1, r2);
177#endif
178}
179
180template<> __device__ EIGEN_STRONG_INLINE half2 pmadd<half2>(const half2& a, const half2& b, const half2& c) {
181#if __CUDA_ARCH__ >= 530
182 return __hfma2(a, b, c);
183#else
184 float a1 = __low2float(a);
185 float a2 = __high2float(a);
186 float b1 = __low2float(b);
187 float b2 = __high2float(b);
188 float c1 = __low2float(c);
189 float c2 = __high2float(c);
190 float r1 = a1 * b1 + c1;
191 float r2 = a2 * b2 + c2;
192 return __floats2half2_rn(r1, r2);
193#endif
194}
195
196template<> __device__ EIGEN_STRONG_INLINE half2 pdiv<half2>(const half2& a, const half2& b) {
197 float a1 = __low2float(a);
198 float a2 = __high2float(a);
199 float b1 = __low2float(b);
200 float b2 = __high2float(b);
201 float r1 = a1 / b1;
202 float r2 = a2 / b2;
203 return __floats2half2_rn(r1, r2);
204}
205
206template<> __device__ EIGEN_STRONG_INLINE half2 pmin<half2>(const half2& a, const half2& b) {
207 float a1 = __low2float(a);
208 float a2 = __high2float(a);
209 float b1 = __low2float(b);
210 float b2 = __high2float(b);
211 __half r1 = a1 < b1 ? __low2half(a) : __low2half(b);
212 __half r2 = a2 < b2 ? __high2half(a) : __high2half(b);
213 return __halves2half2(r1, r2);
214}
215
216template<> __device__ EIGEN_STRONG_INLINE half2 pmax<half2>(const half2& a, const half2& b) {
217 float a1 = __low2float(a);
218 float a2 = __high2float(a);
219 float b1 = __low2float(b);
220 float b2 = __high2float(b);
221 __half r1 = a1 > b1 ? __low2half(a) : __low2half(b);
222 __half r2 = a2 > b2 ? __high2half(a) : __high2half(b);
223 return __halves2half2(r1, r2);
224}
225
226template<> __device__ EIGEN_STRONG_INLINE Eigen::half predux<half2>(const half2& a) {
227#if __CUDA_ARCH__ >= 530
228 return __hadd(__low2half(a), __high2half(a));
229#else
230 float a1 = __low2float(a);
231 float a2 = __high2float(a);
232 return Eigen::half(half_impl::raw_uint16_to_half(__float2half_rn(a1 + a2)));
233#endif
234}
235
236template<> __device__ EIGEN_STRONG_INLINE Eigen::half predux_max<half2>(const half2& a) {
237#if __CUDA_ARCH__ >= 530
238 __half first = __low2half(a);
239 __half second = __high2half(a);
240 return __hgt(first, second) ? first : second;
241#else
242 float a1 = __low2float(a);
243 float a2 = __high2float(a);
244 return a1 > a2 ? __low2half(a) : __high2half(a);
245#endif
246}
247
248template<> __device__ EIGEN_STRONG_INLINE Eigen::half predux_min<half2>(const half2& a) {
249#if __CUDA_ARCH__ >= 530
250 __half first = __low2half(a);
251 __half second = __high2half(a);
252 return __hlt(first, second) ? first : second;
253#else
254 float a1 = __low2float(a);
255 float a2 = __high2float(a);
256 return a1 < a2 ? __low2half(a) : __high2half(a);
257#endif
258}
259
260template<> __device__ EIGEN_STRONG_INLINE Eigen::half predux_mul<half2>(const half2& a) {
261#if __CUDA_ARCH__ >= 530
262 return __hmul(__low2half(a), __high2half(a));
263#else
264 float a1 = __low2float(a);
265 float a2 = __high2float(a);
266 return Eigen::half(half_impl::raw_uint16_to_half(__float2half_rn(a1 * a2)));
267#endif
268}
269
270template<> __device__ EIGEN_STRONG_INLINE half2 plog1p<half2>(const half2& a) {
271 float a1 = __low2float(a);
272 float a2 = __high2float(a);
273 float r1 = log1pf(a1);
274 float r2 = log1pf(a2);
275 return __floats2half2_rn(r1, r2);
276}
277
278#if defined __CUDACC_VER__ && __CUDACC_VER__ >= 80000 && defined __CUDA_ARCH__ && __CUDA_ARCH__ >= 530
279
280template<> __device__ EIGEN_STRONG_INLINE
281half2 plog<half2>(const half2& a) {
282 return h2log(a);
283}
284
285template<> __device__ EIGEN_STRONG_INLINE
286half2 pexp<half2>(const half2& a) {
287 return h2exp(a);
288}
289
290template<> __device__ EIGEN_STRONG_INLINE
291half2 psqrt<half2>(const half2& a) {
292 return h2sqrt(a);
293}
294
295template<> __device__ EIGEN_STRONG_INLINE
296half2 prsqrt<half2>(const half2& a) {
297 return h2rsqrt(a);
298}
299
300#else
301
302template<> __device__ EIGEN_STRONG_INLINE half2 plog<half2>(const half2& a) {
303 float a1 = __low2float(a);
304 float a2 = __high2float(a);
305 float r1 = logf(a1);
306 float r2 = logf(a2);
307 return __floats2half2_rn(r1, r2);
308}
309
310template<> __device__ EIGEN_STRONG_INLINE half2 pexp<half2>(const half2& a) {
311 float a1 = __low2float(a);
312 float a2 = __high2float(a);
313 float r1 = expf(a1);
314 float r2 = expf(a2);
315 return __floats2half2_rn(r1, r2);
316}
317
318template<> __device__ EIGEN_STRONG_INLINE half2 psqrt<half2>(const half2& a) {
319 float a1 = __low2float(a);
320 float a2 = __high2float(a);
321 float r1 = sqrtf(a1);
322 float r2 = sqrtf(a2);
323 return __floats2half2_rn(r1, r2);
324}
325
326template<> __device__ EIGEN_STRONG_INLINE half2 prsqrt<half2>(const half2& a) {
327 float a1 = __low2float(a);
328 float a2 = __high2float(a);
329 float r1 = rsqrtf(a1);
330 float r2 = rsqrtf(a2);
331 return __floats2half2_rn(r1, r2);
332}
333
334#endif
335
336#elif defined EIGEN_VECTORIZE_AVX512
337
338typedef struct {
339 __m256i x;
340} Packet16h;
341
342
343template<> struct is_arithmetic<Packet16h> { enum { value = true }; };
344
345template <>
346struct packet_traits<half> : default_packet_traits {
347 typedef Packet16h type;
348 // There is no half-size packet for Packet16h.
349 typedef Packet16h half;
350 enum {
351 Vectorizable = 1,
352 AlignedOnScalar = 1,
353 size = 16,
354 HasHalfPacket = 0,
355 HasAdd = 0,
356 HasSub = 0,
357 HasMul = 0,
358 HasNegate = 0,
359 HasAbs = 0,
360 HasAbs2 = 0,
361 HasMin = 0,
362 HasMax = 0,
363 HasConj = 0,
364 HasSetLinear = 0,
365 HasDiv = 0,
366 HasSqrt = 0,
367 HasRsqrt = 0,
368 HasExp = 0,
369 HasLog = 0,
370 HasBlend = 0
371 };
372};
373
374
375template<> struct unpacket_traits<Packet16h> { typedef Eigen::half type; enum {size=16, alignment=Aligned32}; typedef Packet16h half; };
376
377template<> EIGEN_STRONG_INLINE Packet16h pset1<Packet16h>(const Eigen::half& from) {
378 Packet16h result;
379 result.x = _mm256_set1_epi16(from.x);
380 return result;
381}
382
383template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet16h>(const Packet16h& from) {
384 return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm256_extract_epi16(from.x, 0)));
385}
386
387template<> EIGEN_STRONG_INLINE Packet16h pload<Packet16h>(const Eigen::half* from) {
388 Packet16h result;
389 result.x = _mm256_load_si256(reinterpret_cast<const __m256i*>(from));
390 return result;
391}
392
393template<> EIGEN_STRONG_INLINE Packet16h ploadu<Packet16h>(const Eigen::half* from) {
394 Packet16h result;
395 result.x = _mm256_loadu_si256(reinterpret_cast<const __m256i*>(from));
396 return result;
397}
398
399template<> EIGEN_STRONG_INLINE void pstore<half>(Eigen::half* to, const Packet16h& from) {
400 _mm256_store_si256((__m256i*)to, from.x);
401}
402
403template<> EIGEN_STRONG_INLINE void pstoreu<half>(Eigen::half* to, const Packet16h& from) {
404 _mm256_storeu_si256((__m256i*)to, from.x);
405}
406
407template<> EIGEN_STRONG_INLINE Packet16h
408ploadquad(const Eigen::half* from) {
409 Packet16h result;
410 unsigned short a = from[0].x;
411 unsigned short b = from[1].x;
412 unsigned short c = from[2].x;
413 unsigned short d = from[3].x;
414 result.x = _mm256_set_epi16(d, d, d, d, c, c, c, c, b, b, b, b, a, a, a, a);
415 return result;
416}
417
418EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) {
419#ifdef EIGEN_HAS_FP16_C
420 return _mm512_cvtph_ps(a.x);
421#else
422 EIGEN_ALIGN64 half aux[16];
423 pstore(aux, a);
424 float f0(aux[0]);
425 float f1(aux[1]);
426 float f2(aux[2]);
427 float f3(aux[3]);
428 float f4(aux[4]);
429 float f5(aux[5]);
430 float f6(aux[6]);
431 float f7(aux[7]);
432 float f8(aux[8]);
433 float f9(aux[9]);
434 float fa(aux[10]);
435 float fb(aux[11]);
436 float fc(aux[12]);
437 float fd(aux[13]);
438 float fe(aux[14]);
439 float ff(aux[15]);
440
441 return _mm512_set_ps(
442 ff, fe, fd, fc, fb, fa, f9, f8, f7, f6, f5, f4, f3, f2, f1, f0);
443#endif
444}
445
446EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) {
447#ifdef EIGEN_HAS_FP16_C
448 Packet16h result;
449 result.x = _mm512_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC);
450 return result;
451#else
452 EIGEN_ALIGN64 float aux[16];
453 pstore(aux, a);
454 half h0(aux[0]);
455 half h1(aux[1]);
456 half h2(aux[2]);
457 half h3(aux[3]);
458 half h4(aux[4]);
459 half h5(aux[5]);
460 half h6(aux[6]);
461 half h7(aux[7]);
462 half h8(aux[8]);
463 half h9(aux[9]);
464 half ha(aux[10]);
465 half hb(aux[11]);
466 half hc(aux[12]);
467 half hd(aux[13]);
468 half he(aux[14]);
469 half hf(aux[15]);
470
471 Packet16h result;
472 result.x = _mm256_set_epi16(
473 hf.x, he.x, hd.x, hc.x, hb.x, ha.x, h9.x, h8.x,
474 h7.x, h6.x, h5.x, h4.x, h3.x, h2.x, h1.x, h0.x);
475 return result;
476#endif
477}
478
479template<> EIGEN_STRONG_INLINE Packet16h padd<Packet16h>(const Packet16h& a, const Packet16h& b) {
480 Packet16f af = half2float(a);
481 Packet16f bf = half2float(b);
482 Packet16f rf = padd(af, bf);
483 return float2half(rf);
484}
485
486template<> EIGEN_STRONG_INLINE Packet16h pmul<Packet16h>(const Packet16h& a, const Packet16h& b) {
487 Packet16f af = half2float(a);
488 Packet16f bf = half2float(b);
489 Packet16f rf = pmul(af, bf);
490 return float2half(rf);
491}
492
493template<> EIGEN_STRONG_INLINE half predux<Packet16h>(const Packet16h& from) {
494 Packet16f from_float = half2float(from);
495 return half(predux(from_float));
496}
497
498template<> EIGEN_STRONG_INLINE Packet16h pgather<Eigen::half, Packet16h>(const Eigen::half* from, Index stride)
499{
500 Packet16h result;
501 result.x = _mm256_set_epi16(
502 from[15*stride].x, from[14*stride].x, from[13*stride].x, from[12*stride].x,
503 from[11*stride].x, from[10*stride].x, from[9*stride].x, from[8*stride].x,
504 from[7*stride].x, from[6*stride].x, from[5*stride].x, from[4*stride].x,
505 from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x);
506 return result;
507}
508
509template<> EIGEN_STRONG_INLINE void pscatter<half, Packet16h>(half* to, const Packet16h& from, Index stride)
510{
511 EIGEN_ALIGN64 half aux[16];
512 pstore(aux, from);
513 to[stride*0].x = aux[0].x;
514 to[stride*1].x = aux[1].x;
515 to[stride*2].x = aux[2].x;
516 to[stride*3].x = aux[3].x;
517 to[stride*4].x = aux[4].x;
518 to[stride*5].x = aux[5].x;
519 to[stride*6].x = aux[6].x;
520 to[stride*7].x = aux[7].x;
521 to[stride*8].x = aux[8].x;
522 to[stride*9].x = aux[9].x;
523 to[stride*10].x = aux[10].x;
524 to[stride*11].x = aux[11].x;
525 to[stride*12].x = aux[12].x;
526 to[stride*13].x = aux[13].x;
527 to[stride*14].x = aux[14].x;
528 to[stride*15].x = aux[15].x;
529}
530
531EIGEN_STRONG_INLINE void
532ptranspose(PacketBlock<Packet16h,16>& kernel) {
533 __m256i a = kernel.packet[0].x;
534 __m256i b = kernel.packet[1].x;
535 __m256i c = kernel.packet[2].x;
536 __m256i d = kernel.packet[3].x;
537 __m256i e = kernel.packet[4].x;
538 __m256i f = kernel.packet[5].x;
539 __m256i g = kernel.packet[6].x;
540 __m256i h = kernel.packet[7].x;
541 __m256i i = kernel.packet[8].x;
542 __m256i j = kernel.packet[9].x;
543 __m256i k = kernel.packet[10].x;
544 __m256i l = kernel.packet[11].x;
545 __m256i m = kernel.packet[12].x;
546 __m256i n = kernel.packet[13].x;
547 __m256i o = kernel.packet[14].x;
548 __m256i p = kernel.packet[15].x;
549
550 __m256i ab_07 = _mm256_unpacklo_epi16(a, b);
551 __m256i cd_07 = _mm256_unpacklo_epi16(c, d);
552 __m256i ef_07 = _mm256_unpacklo_epi16(e, f);
553 __m256i gh_07 = _mm256_unpacklo_epi16(g, h);
554 __m256i ij_07 = _mm256_unpacklo_epi16(i, j);
555 __m256i kl_07 = _mm256_unpacklo_epi16(k, l);
556 __m256i mn_07 = _mm256_unpacklo_epi16(m, n);
557 __m256i op_07 = _mm256_unpacklo_epi16(o, p);
558
559 __m256i ab_8f = _mm256_unpackhi_epi16(a, b);
560 __m256i cd_8f = _mm256_unpackhi_epi16(c, d);
561 __m256i ef_8f = _mm256_unpackhi_epi16(e, f);
562 __m256i gh_8f = _mm256_unpackhi_epi16(g, h);
563 __m256i ij_8f = _mm256_unpackhi_epi16(i, j);
564 __m256i kl_8f = _mm256_unpackhi_epi16(k, l);
565 __m256i mn_8f = _mm256_unpackhi_epi16(m, n);
566 __m256i op_8f = _mm256_unpackhi_epi16(o, p);
567
568 __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07);
569 __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07);
570 __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07);
571 __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07);
572 __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07);
573 __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07);
574 __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07);
575 __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07);
576
577 __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f);
578 __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f);
579 __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f);
580 __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f);
581 __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f);
582 __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f);
583 __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f);
584 __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f);
585
586 __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03);
587 __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03);
588 __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03);
589 __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03);
590 __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47);
591 __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47);
592 __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47);
593 __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47);
594 __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b);
595 __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b);
596 __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b);
597 __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b);
598 __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf);
599 __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf);
600 __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf);
601 __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf);
602
603 // NOTE: no unpacklo/hi instr in this case, so using permute instr.
604 __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20);
605 __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31);
606 __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20);
607 __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31);
608 __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20);
609 __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31);
610 __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20);
611 __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31);
612 __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20);
613 __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31);
614 __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20);
615 __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31);
616 __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20);
617 __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31);
618 __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20);
619 __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31);
620
621 kernel.packet[0].x = a_p_0;
622 kernel.packet[1].x = a_p_1;
623 kernel.packet[2].x = a_p_2;
624 kernel.packet[3].x = a_p_3;
625 kernel.packet[4].x = a_p_4;
626 kernel.packet[5].x = a_p_5;
627 kernel.packet[6].x = a_p_6;
628 kernel.packet[7].x = a_p_7;
629 kernel.packet[8].x = a_p_8;
630 kernel.packet[9].x = a_p_9;
631 kernel.packet[10].x = a_p_a;
632 kernel.packet[11].x = a_p_b;
633 kernel.packet[12].x = a_p_c;
634 kernel.packet[13].x = a_p_d;
635 kernel.packet[14].x = a_p_e;
636 kernel.packet[15].x = a_p_f;
637}
638
639EIGEN_STRONG_INLINE void
640ptranspose(PacketBlock<Packet16h,8>& kernel) {
641 EIGEN_ALIGN64 half in[8][16];
642 pstore<half>(in[0], kernel.packet[0]);
643 pstore<half>(in[1], kernel.packet[1]);
644 pstore<half>(in[2], kernel.packet[2]);
645 pstore<half>(in[3], kernel.packet[3]);
646 pstore<half>(in[4], kernel.packet[4]);
647 pstore<half>(in[5], kernel.packet[5]);
648 pstore<half>(in[6], kernel.packet[6]);
649 pstore<half>(in[7], kernel.packet[7]);
650
651 EIGEN_ALIGN64 half out[8][16];
652
653 for (int i = 0; i < 8; ++i) {
654 for (int j = 0; j < 8; ++j) {
655 out[i][j] = in[j][2*i];
656 }
657 for (int j = 0; j < 8; ++j) {
658 out[i][j+8] = in[j][2*i+1];
659 }
660 }
661
662 kernel.packet[0] = pload<Packet16h>(out[0]);
663 kernel.packet[1] = pload<Packet16h>(out[1]);
664 kernel.packet[2] = pload<Packet16h>(out[2]);
665 kernel.packet[3] = pload<Packet16h>(out[3]);
666 kernel.packet[4] = pload<Packet16h>(out[4]);
667 kernel.packet[5] = pload<Packet16h>(out[5]);
668 kernel.packet[6] = pload<Packet16h>(out[6]);
669 kernel.packet[7] = pload<Packet16h>(out[7]);
670}
671
672EIGEN_STRONG_INLINE void
673ptranspose(PacketBlock<Packet16h,4>& kernel) {
674 EIGEN_ALIGN64 half in[4][16];
675 pstore<half>(in[0], kernel.packet[0]);
676 pstore<half>(in[1], kernel.packet[1]);
677 pstore<half>(in[2], kernel.packet[2]);
678 pstore<half>(in[3], kernel.packet[3]);
679
680 EIGEN_ALIGN64 half out[4][16];
681
682 for (int i = 0; i < 4; ++i) {
683 for (int j = 0; j < 4; ++j) {
684 out[i][j] = in[j][4*i];
685 }
686 for (int j = 0; j < 4; ++j) {
687 out[i][j+4] = in[j][4*i+1];
688 }
689 for (int j = 0; j < 4; ++j) {
690 out[i][j+8] = in[j][4*i+2];
691 }
692 for (int j = 0; j < 4; ++j) {
693 out[i][j+12] = in[j][4*i+3];
694 }
695 }
696
697 kernel.packet[0] = pload<Packet16h>(out[0]);
698 kernel.packet[1] = pload<Packet16h>(out[1]);
699 kernel.packet[2] = pload<Packet16h>(out[2]);
700 kernel.packet[3] = pload<Packet16h>(out[3]);
701}
702
703
704#elif defined EIGEN_VECTORIZE_AVX
705
706typedef struct {
707 __m128i x;
708} Packet8h;
709
710
711template<> struct is_arithmetic<Packet8h> { enum { value = true }; };
712
713template <>
714struct packet_traits<Eigen::half> : default_packet_traits {
715 typedef Packet8h type;
716 // There is no half-size packet for Packet8h.
717 typedef Packet8h half;
718 enum {
719 Vectorizable = 1,
720 AlignedOnScalar = 1,
721 size = 8,
722 HasHalfPacket = 0,
723 HasAdd = 0,
724 HasSub = 0,
725 HasMul = 0,
726 HasNegate = 0,
727 HasAbs = 0,
728 HasAbs2 = 0,
729 HasMin = 0,
730 HasMax = 0,
731 HasConj = 0,
732 HasSetLinear = 0,
733 HasDiv = 0,
734 HasSqrt = 0,
735 HasRsqrt = 0,
736 HasExp = 0,
737 HasLog = 0,
738 HasBlend = 0
739 };
740};
741
742
743template<> struct unpacket_traits<Packet8h> { typedef Eigen::half type; enum {size=8, alignment=Aligned16}; typedef Packet8h half; };
744
745template<> EIGEN_STRONG_INLINE Packet8h pset1<Packet8h>(const Eigen::half& from) {
746 Packet8h result;
747 result.x = _mm_set1_epi16(from.x);
748 return result;
749}
750
751template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet8h>(const Packet8h& from) {
752 return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm_extract_epi16(from.x, 0)));
753}
754
755template<> EIGEN_STRONG_INLINE Packet8h pload<Packet8h>(const Eigen::half* from) {
756 Packet8h result;
757 result.x = _mm_load_si128(reinterpret_cast<const __m128i*>(from));
758 return result;
759}
760
761template<> EIGEN_STRONG_INLINE Packet8h ploadu<Packet8h>(const Eigen::half* from) {
762 Packet8h result;
763 result.x = _mm_loadu_si128(reinterpret_cast<const __m128i*>(from));
764 return result;
765}
766
767template<> EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet8h& from) {
768 _mm_store_si128(reinterpret_cast<__m128i*>(to), from.x);
769}
770
771template<> EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet8h& from) {
772 _mm_storeu_si128(reinterpret_cast<__m128i*>(to), from.x);
773}
774
775template<> EIGEN_STRONG_INLINE Packet8h
776ploadquad<Packet8h>(const Eigen::half* from) {
777 Packet8h result;
778 unsigned short a = from[0].x;
779 unsigned short b = from[1].x;
780 result.x = _mm_set_epi16(b, b, b, b, a, a, a, a);
781 return result;
782}
783
784EIGEN_STRONG_INLINE Packet8f half2float(const Packet8h& a) {
785#ifdef EIGEN_HAS_FP16_C
786 return _mm256_cvtph_ps(a.x);
787#else
788 EIGEN_ALIGN32 Eigen::half aux[8];
789 pstore(aux, a);
790 float f0(aux[0]);
791 float f1(aux[1]);
792 float f2(aux[2]);
793 float f3(aux[3]);
794 float f4(aux[4]);
795 float f5(aux[5]);
796 float f6(aux[6]);
797 float f7(aux[7]);
798
799 return _mm256_set_ps(f7, f6, f5, f4, f3, f2, f1, f0);
800#endif
801}
802
803EIGEN_STRONG_INLINE Packet8h float2half(const Packet8f& a) {
804#ifdef EIGEN_HAS_FP16_C
805 Packet8h result;
806 result.x = _mm256_cvtps_ph(a, _MM_FROUND_TO_NEAREST_INT|_MM_FROUND_NO_EXC);
807 return result;
808#else
809 EIGEN_ALIGN32 float aux[8];
810 pstore(aux, a);
811 Eigen::half h0(aux[0]);
812 Eigen::half h1(aux[1]);
813 Eigen::half h2(aux[2]);
814 Eigen::half h3(aux[3]);
815 Eigen::half h4(aux[4]);
816 Eigen::half h5(aux[5]);
817 Eigen::half h6(aux[6]);
818 Eigen::half h7(aux[7]);
819
820 Packet8h result;
821 result.x = _mm_set_epi16(h7.x, h6.x, h5.x, h4.x, h3.x, h2.x, h1.x, h0.x);
822 return result;
823#endif
824}
825
826template<> EIGEN_STRONG_INLINE Packet8h pconj(const Packet8h& a) { return a; }
827
828template<> EIGEN_STRONG_INLINE Packet8h padd<Packet8h>(const Packet8h& a, const Packet8h& b) {
829 Packet8f af = half2float(a);
830 Packet8f bf = half2float(b);
831 Packet8f rf = padd(af, bf);
832 return float2half(rf);
833}
834
835template<> EIGEN_STRONG_INLINE Packet8h pmul<Packet8h>(const Packet8h& a, const Packet8h& b) {
836 Packet8f af = half2float(a);
837 Packet8f bf = half2float(b);
838 Packet8f rf = pmul(af, bf);
839 return float2half(rf);
840}
841
842template<> EIGEN_STRONG_INLINE Packet8h pgather<Eigen::half, Packet8h>(const Eigen::half* from, Index stride)
843{
844 Packet8h result;
845 result.x = _mm_set_epi16(from[7*stride].x, from[6*stride].x, from[5*stride].x, from[4*stride].x, from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x);
846 return result;
847}
848
849template<> EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet8h>(Eigen::half* to, const Packet8h& from, Index stride)
850{
851 EIGEN_ALIGN32 Eigen::half aux[8];
852 pstore(aux, from);
853 to[stride*0].x = aux[0].x;
854 to[stride*1].x = aux[1].x;
855 to[stride*2].x = aux[2].x;
856 to[stride*3].x = aux[3].x;
857 to[stride*4].x = aux[4].x;
858 to[stride*5].x = aux[5].x;
859 to[stride*6].x = aux[6].x;
860 to[stride*7].x = aux[7].x;
861}
862
863template<> EIGEN_STRONG_INLINE Eigen::half predux<Packet8h>(const Packet8h& a) {
864 Packet8f af = half2float(a);
865 float reduced = predux<Packet8f>(af);
866 return Eigen::half(reduced);
867}
868
869template<> EIGEN_STRONG_INLINE Eigen::half predux_max<Packet8h>(const Packet8h& a) {
870 Packet8f af = half2float(a);
871 float reduced = predux_max<Packet8f>(af);
872 return Eigen::half(reduced);
873}
874
875template<> EIGEN_STRONG_INLINE Eigen::half predux_min<Packet8h>(const Packet8h& a) {
876 Packet8f af = half2float(a);
877 float reduced = predux_min<Packet8f>(af);
878 return Eigen::half(reduced);
879}
880
881template<> EIGEN_STRONG_INLINE Eigen::half predux_mul<Packet8h>(const Packet8h& a) {
882 Packet8f af = half2float(a);
883 float reduced = predux_mul<Packet8f>(af);
884 return Eigen::half(reduced);
885}
886
887EIGEN_STRONG_INLINE void
888ptranspose(PacketBlock<Packet8h,8>& kernel) {
889 __m128i a = kernel.packet[0].x;
890 __m128i b = kernel.packet[1].x;
891 __m128i c = kernel.packet[2].x;
892 __m128i d = kernel.packet[3].x;
893 __m128i e = kernel.packet[4].x;
894 __m128i f = kernel.packet[5].x;
895 __m128i g = kernel.packet[6].x;
896 __m128i h = kernel.packet[7].x;
897
898 __m128i a03b03 = _mm_unpacklo_epi16(a, b);
899 __m128i c03d03 = _mm_unpacklo_epi16(c, d);
900 __m128i e03f03 = _mm_unpacklo_epi16(e, f);
901 __m128i g03h03 = _mm_unpacklo_epi16(g, h);
902 __m128i a47b47 = _mm_unpackhi_epi16(a, b);
903 __m128i c47d47 = _mm_unpackhi_epi16(c, d);
904 __m128i e47f47 = _mm_unpackhi_epi16(e, f);
905 __m128i g47h47 = _mm_unpackhi_epi16(g, h);
906
907 __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03);
908 __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03);
909 __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03);
910 __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03);
911 __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47);
912 __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47);
913 __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47);
914 __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47);
915
916 __m128i a0b0c0d0e0f0g0h0 = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01);
917 __m128i a1b1c1d1e1f1g1h1 = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01);
918 __m128i a2b2c2d2e2f2g2h2 = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23);
919 __m128i a3b3c3d3e3f3g3h3 = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23);
920 __m128i a4b4c4d4e4f4g4h4 = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45);
921 __m128i a5b5c5d5e5f5g5h5 = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45);
922 __m128i a6b6c6d6e6f6g6h6 = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67);
923 __m128i a7b7c7d7e7f7g7h7 = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67);
924
925 kernel.packet[0].x = a0b0c0d0e0f0g0h0;
926 kernel.packet[1].x = a1b1c1d1e1f1g1h1;
927 kernel.packet[2].x = a2b2c2d2e2f2g2h2;
928 kernel.packet[3].x = a3b3c3d3e3f3g3h3;
929 kernel.packet[4].x = a4b4c4d4e4f4g4h4;
930 kernel.packet[5].x = a5b5c5d5e5f5g5h5;
931 kernel.packet[6].x = a6b6c6d6e6f6g6h6;
932 kernel.packet[7].x = a7b7c7d7e7f7g7h7;
933}
934
935EIGEN_STRONG_INLINE void
936ptranspose(PacketBlock<Packet8h,4>& kernel) {
937 EIGEN_ALIGN32 Eigen::half in[4][8];
938 pstore<Eigen::half>(in[0], kernel.packet[0]);
939 pstore<Eigen::half>(in[1], kernel.packet[1]);
940 pstore<Eigen::half>(in[2], kernel.packet[2]);
941 pstore<Eigen::half>(in[3], kernel.packet[3]);
942
943 EIGEN_ALIGN32 Eigen::half out[4][8];
944
945 for (int i = 0; i < 4; ++i) {
946 for (int j = 0; j < 4; ++j) {
947 out[i][j] = in[j][2*i];
948 }
949 for (int j = 0; j < 4; ++j) {
950 out[i][j+4] = in[j][2*i+1];
951 }
952 }
953
954 kernel.packet[0] = pload<Packet8h>(out[0]);
955 kernel.packet[1] = pload<Packet8h>(out[1]);
956 kernel.packet[2] = pload<Packet8h>(out[2]);
957 kernel.packet[3] = pload<Packet8h>(out[3]);
958}
959
960
961// Disable the following code since it's broken on too many platforms / compilers.
962//#elif defined(EIGEN_VECTORIZE_SSE) && (!EIGEN_ARCH_x86_64) && (!EIGEN_COMP_MSVC)
963#elif 0
964
965typedef struct {
966 __m64 x;
967} Packet4h;
968
969
970template<> struct is_arithmetic<Packet4h> { enum { value = true }; };
971
972template <>
973struct packet_traits<Eigen::half> : default_packet_traits {
974 typedef Packet4h type;
975 // There is no half-size packet for Packet4h.
976 typedef Packet4h half;
977 enum {
978 Vectorizable = 1,
979 AlignedOnScalar = 1,
980 size = 4,
981 HasHalfPacket = 0,
982 HasAdd = 0,
983 HasSub = 0,
984 HasMul = 0,
985 HasNegate = 0,
986 HasAbs = 0,
987 HasAbs2 = 0,
988 HasMin = 0,
989 HasMax = 0,
990 HasConj = 0,
991 HasSetLinear = 0,
992 HasDiv = 0,
993 HasSqrt = 0,
994 HasRsqrt = 0,
995 HasExp = 0,
996 HasLog = 0,
997 HasBlend = 0
998 };
999};
1000
1001
1002template<> struct unpacket_traits<Packet4h> { typedef Eigen::half type; enum {size=4, alignment=Aligned16}; typedef Packet4h half; };
1003
1004template<> EIGEN_STRONG_INLINE Packet4h pset1<Packet4h>(const Eigen::half& from) {
1005 Packet4h result;
1006 result.x = _mm_set1_pi16(from.x);
1007 return result;
1008}
1009
1010template<> EIGEN_STRONG_INLINE Eigen::half pfirst<Packet4h>(const Packet4h& from) {
1011 return half_impl::raw_uint16_to_half(static_cast<unsigned short>(_mm_cvtsi64_si32(from.x)));
1012}
1013
1014template<> EIGEN_STRONG_INLINE Packet4h pconj(const Packet4h& a) { return a; }
1015
1016template<> EIGEN_STRONG_INLINE Packet4h padd<Packet4h>(const Packet4h& a, const Packet4h& b) {
1017 __int64_t a64 = _mm_cvtm64_si64(a.x);
1018 __int64_t b64 = _mm_cvtm64_si64(b.x);
1019
1020 Eigen::half h[4];
1021
1022 Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64));
1023 Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64));
1024 h[0] = ha + hb;
1025 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16));
1026 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16));
1027 h[1] = ha + hb;
1028 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32));
1029 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32));
1030 h[2] = ha + hb;
1031 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48));
1032 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48));
1033 h[3] = ha + hb;
1034 Packet4h result;
1035 result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x);
1036 return result;
1037}
1038
1039template<> EIGEN_STRONG_INLINE Packet4h pmul<Packet4h>(const Packet4h& a, const Packet4h& b) {
1040 __int64_t a64 = _mm_cvtm64_si64(a.x);
1041 __int64_t b64 = _mm_cvtm64_si64(b.x);
1042
1043 Eigen::half h[4];
1044
1045 Eigen::half ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64));
1046 Eigen::half hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64));
1047 h[0] = ha * hb;
1048 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 16));
1049 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 16));
1050 h[1] = ha * hb;
1051 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 32));
1052 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 32));
1053 h[2] = ha * hb;
1054 ha = half_impl::raw_uint16_to_half(static_cast<unsigned short>(a64 >> 48));
1055 hb = half_impl::raw_uint16_to_half(static_cast<unsigned short>(b64 >> 48));
1056 h[3] = ha * hb;
1057 Packet4h result;
1058 result.x = _mm_set_pi16(h[3].x, h[2].x, h[1].x, h[0].x);
1059 return result;
1060}
1061
1062template<> EIGEN_STRONG_INLINE Packet4h pload<Packet4h>(const Eigen::half* from) {
1063 Packet4h result;
1064 result.x = _mm_cvtsi64_m64(*reinterpret_cast<const __int64_t*>(from));
1065 return result;
1066}
1067
1068template<> EIGEN_STRONG_INLINE Packet4h ploadu<Packet4h>(const Eigen::half* from) {
1069 Packet4h result;
1070 result.x = _mm_cvtsi64_m64(*reinterpret_cast<const __int64_t*>(from));
1071 return result;
1072}
1073
1074template<> EIGEN_STRONG_INLINE void pstore<Eigen::half>(Eigen::half* to, const Packet4h& from) {
1075 __int64_t r = _mm_cvtm64_si64(from.x);
1076 *(reinterpret_cast<__int64_t*>(to)) = r;
1077}
1078
1079template<> EIGEN_STRONG_INLINE void pstoreu<Eigen::half>(Eigen::half* to, const Packet4h& from) {
1080 __int64_t r = _mm_cvtm64_si64(from.x);
1081 *(reinterpret_cast<__int64_t*>(to)) = r;
1082}
1083
1084template<> EIGEN_STRONG_INLINE Packet4h
1085ploadquad<Packet4h>(const Eigen::half* from) {
1086 return pset1<Packet4h>(*from);
1087}
1088
1089template<> EIGEN_STRONG_INLINE Packet4h pgather<Eigen::half, Packet4h>(const Eigen::half* from, Index stride)
1090{
1091 Packet4h result;
1092 result.x = _mm_set_pi16(from[3*stride].x, from[2*stride].x, from[1*stride].x, from[0*stride].x);
1093 return result;
1094}
1095
1096template<> EIGEN_STRONG_INLINE void pscatter<Eigen::half, Packet4h>(Eigen::half* to, const Packet4h& from, Index stride)
1097{
1098 __int64_t a = _mm_cvtm64_si64(from.x);
1099 to[stride*0].x = static_cast<unsigned short>(a);
1100 to[stride*1].x = static_cast<unsigned short>(a >> 16);
1101 to[stride*2].x = static_cast<unsigned short>(a >> 32);
1102 to[stride*3].x = static_cast<unsigned short>(a >> 48);
1103}
1104
1105EIGEN_STRONG_INLINE void
1106ptranspose(PacketBlock<Packet4h,4>& kernel) {
1107 __m64 T0 = _mm_unpacklo_pi16(kernel.packet[0].x, kernel.packet[1].x);
1108 __m64 T1 = _mm_unpacklo_pi16(kernel.packet[2].x, kernel.packet[3].x);
1109 __m64 T2 = _mm_unpackhi_pi16(kernel.packet[0].x, kernel.packet[1].x);
1110 __m64 T3 = _mm_unpackhi_pi16(kernel.packet[2].x, kernel.packet[3].x);
1111
1112 kernel.packet[0].x = _mm_unpacklo_pi32(T0, T1);
1113 kernel.packet[1].x = _mm_unpackhi_pi32(T0, T1);
1114 kernel.packet[2].x = _mm_unpacklo_pi32(T2, T3);
1115 kernel.packet[3].x = _mm_unpackhi_pi32(T2, T3);
1116}
1117
1118#endif
1119
1120}
1121}
1122
1123#endif // EIGEN_PACKET_MATH_HALF_CUDA_H
@ Aligned32
Definition: Constants.h:231
@ Aligned16
Definition: Constants.h:230
Namespace containing all symbols from the Eigen library.
Definition: Core:287
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:33