Eigen  3.3.0
 
Loading...
Searching...
No Matches
AVX512/PacketMath.h
1// This file is part of Eigen, a lightweight C++ template library
2// for linear algebra.
3//
4// Copyright (C) 2016 Benoit Steiner (benoit.steiner.goog@gmail.com)
5//
6// This Source Code Form is subject to the terms of the Mozilla
7// Public License v. 2.0. If a copy of the MPL was not distributed
8// with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
9
10#ifndef EIGEN_PACKET_MATH_AVX512_H
11#define EIGEN_PACKET_MATH_AVX512_H
12
13namespace Eigen {
14
15namespace internal {
16
17#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
18#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8
19#endif
20
21#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
22#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS (2*sizeof(void*))
23#endif
24
25#ifdef __FMA__
26#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
27#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
28#endif
29#endif
30
31typedef __m512 Packet16f;
32typedef __m512i Packet16i;
33typedef __m512d Packet8d;
34
35template <>
36struct is_arithmetic<__m512> {
37 enum { value = true };
38};
39template <>
40struct is_arithmetic<__m512i> {
41 enum { value = true };
42};
43template <>
44struct is_arithmetic<__m512d> {
45 enum { value = true };
46};
47
48template<> struct packet_traits<float> : default_packet_traits
49{
50 typedef Packet16f type;
51 typedef Packet8f half;
52 enum {
53 Vectorizable = 1,
54 AlignedOnScalar = 1,
55 size = 16,
56 HasHalfPacket = 1,
57#if EIGEN_GNUC_AT_LEAST(5, 3)
58#ifdef EIGEN_VECTORIZE_AVX512DQ
59 HasLog = 1,
60#endif
61 HasExp = 1,
62 HasSqrt = 1,
63 HasRsqrt = 1,
64#endif
65 HasDiv = 1
66 };
67 };
68template<> struct packet_traits<double> : default_packet_traits
69{
70 typedef Packet8d type;
71 typedef Packet4d half;
72 enum {
73 Vectorizable = 1,
74 AlignedOnScalar = 1,
75 size = 8,
76 HasHalfPacket = 1,
77#if EIGEN_GNUC_AT_LEAST(5, 3)
78 HasSqrt = 1,
79 HasRsqrt = EIGEN_FAST_MATH,
80#endif
81 HasDiv = 1
82 };
83};
84
85/* TODO Implement AVX512 for integers
86template<> struct packet_traits<int> : default_packet_traits
87{
88 typedef Packet16i type;
89 enum {
90 Vectorizable = 1,
91 AlignedOnScalar = 1,
92 size=8
93 };
94};
95*/
96
97template <>
98struct unpacket_traits<Packet16f> {
99 typedef float type;
100 typedef Packet8f half;
101 enum { size = 16, alignment=Aligned64 };
102};
103template <>
104struct unpacket_traits<Packet8d> {
105 typedef double type;
106 typedef Packet4d half;
107 enum { size = 8, alignment=Aligned64 };
108};
109template <>
110struct unpacket_traits<Packet16i> {
111 typedef int type;
112 typedef Packet8i half;
113 enum { size = 16, alignment=Aligned64 };
114};
115
116template <>
117EIGEN_STRONG_INLINE Packet16f pset1<Packet16f>(const float& from) {
118 return _mm512_set1_ps(from);
119}
120template <>
121EIGEN_STRONG_INLINE Packet8d pset1<Packet8d>(const double& from) {
122 return _mm512_set1_pd(from);
123}
124template <>
125EIGEN_STRONG_INLINE Packet16i pset1<Packet16i>(const int& from) {
126 return _mm512_set1_epi32(from);
127}
128
129template <>
130EIGEN_STRONG_INLINE Packet16f pload1<Packet16f>(const float* from) {
131 return _mm512_broadcastss_ps(_mm_load_ps1(from));
132}
133template <>
134EIGEN_STRONG_INLINE Packet8d pload1<Packet8d>(const double* from) {
135 return _mm512_broadcastsd_pd(_mm_load_pd1(from));
136}
137
138template <>
139EIGEN_STRONG_INLINE Packet16f plset<Packet16f>(const float& a) {
140 return _mm512_add_ps(
141 _mm512_set1_ps(a),
142 _mm512_set_ps(15.0f, 14.0f, 13.0f, 12.0f, 11.0f, 10.0f, 9.0f, 8.0f, 7.0f, 6.0f, 5.0f,
143 4.0f, 3.0f, 2.0f, 1.0f, 0.0f));
144}
145template <>
146EIGEN_STRONG_INLINE Packet8d plset<Packet8d>(const double& a) {
147 return _mm512_add_pd(_mm512_set1_pd(a),
148 _mm512_set_pd(7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, 0.0));
149}
150
151template <>
152EIGEN_STRONG_INLINE Packet16f padd<Packet16f>(const Packet16f& a,
153 const Packet16f& b) {
154 return _mm512_add_ps(a, b);
155}
156template <>
157EIGEN_STRONG_INLINE Packet8d padd<Packet8d>(const Packet8d& a,
158 const Packet8d& b) {
159 return _mm512_add_pd(a, b);
160}
161
162template <>
163EIGEN_STRONG_INLINE Packet16f psub<Packet16f>(const Packet16f& a,
164 const Packet16f& b) {
165 return _mm512_sub_ps(a, b);
166}
167template <>
168EIGEN_STRONG_INLINE Packet8d psub<Packet8d>(const Packet8d& a,
169 const Packet8d& b) {
170 return _mm512_sub_pd(a, b);
171}
172
173template <>
174EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) {
175 return _mm512_sub_ps(_mm512_set1_ps(0.0), a);
176}
177template <>
178EIGEN_STRONG_INLINE Packet8d pnegate(const Packet8d& a) {
179 return _mm512_sub_pd(_mm512_set1_pd(0.0), a);
180}
181
182template <>
183EIGEN_STRONG_INLINE Packet16f pconj(const Packet16f& a) {
184 return a;
185}
186template <>
187EIGEN_STRONG_INLINE Packet8d pconj(const Packet8d& a) {
188 return a;
189}
190template <>
191EIGEN_STRONG_INLINE Packet16i pconj(const Packet16i& a) {
192 return a;
193}
194
195template <>
196EIGEN_STRONG_INLINE Packet16f pmul<Packet16f>(const Packet16f& a,
197 const Packet16f& b) {
198 return _mm512_mul_ps(a, b);
199}
200template <>
201EIGEN_STRONG_INLINE Packet8d pmul<Packet8d>(const Packet8d& a,
202 const Packet8d& b) {
203 return _mm512_mul_pd(a, b);
204}
205
206template <>
207EIGEN_STRONG_INLINE Packet16f pdiv<Packet16f>(const Packet16f& a,
208 const Packet16f& b) {
209 return _mm512_div_ps(a, b);
210}
211template <>
212EIGEN_STRONG_INLINE Packet8d pdiv<Packet8d>(const Packet8d& a,
213 const Packet8d& b) {
214 return _mm512_div_pd(a, b);
215}
216
217#ifdef __FMA__
218template <>
219EIGEN_STRONG_INLINE Packet16f pmadd(const Packet16f& a, const Packet16f& b,
220 const Packet16f& c) {
221 return _mm512_fmadd_ps(a, b, c);
222}
223template <>
224EIGEN_STRONG_INLINE Packet8d pmadd(const Packet8d& a, const Packet8d& b,
225 const Packet8d& c) {
226 return _mm512_fmadd_pd(a, b, c);
227}
228#endif
229
230template <>
231EIGEN_STRONG_INLINE Packet16f pmin<Packet16f>(const Packet16f& a,
232 const Packet16f& b) {
233 return _mm512_min_ps(a, b);
234}
235template <>
236EIGEN_STRONG_INLINE Packet8d pmin<Packet8d>(const Packet8d& a,
237 const Packet8d& b) {
238 return _mm512_min_pd(a, b);
239}
240
241template <>
242EIGEN_STRONG_INLINE Packet16f pmax<Packet16f>(const Packet16f& a,
243 const Packet16f& b) {
244 return _mm512_max_ps(a, b);
245}
246template <>
247EIGEN_STRONG_INLINE Packet8d pmax<Packet8d>(const Packet8d& a,
248 const Packet8d& b) {
249 return _mm512_max_pd(a, b);
250}
251
252template <>
253EIGEN_STRONG_INLINE Packet16f pand<Packet16f>(const Packet16f& a,
254 const Packet16f& b) {
255#ifdef EIGEN_VECTORIZE_AVX512DQ
256 return _mm512_and_ps(a, b);
257#else
258 Packet16f res = _mm512_undefined_ps();
259 Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0);
260 Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0);
261 res = _mm512_insertf32x4(res, _mm_and_ps(lane0_a, lane0_b), 0);
262
263 Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1);
264 Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1);
265 res = _mm512_insertf32x4(res, _mm_and_ps(lane1_a, lane1_b), 1);
266
267 Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2);
268 Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2);
269 res = _mm512_insertf32x4(res, _mm_and_ps(lane2_a, lane2_b), 2);
270
271 Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3);
272 Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3);
273 res = _mm512_insertf32x4(res, _mm_and_ps(lane3_a, lane3_b), 3);
274
275 return res;
276#endif
277}
278template <>
279EIGEN_STRONG_INLINE Packet8d pand<Packet8d>(const Packet8d& a,
280 const Packet8d& b) {
281#ifdef EIGEN_VECTORIZE_AVX512DQ
282 return _mm512_and_pd(a, b);
283#else
284 Packet8d res = _mm512_undefined_pd();
285 Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
286 Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
287 res = _mm512_insertf64x4(res, _mm256_and_pd(lane0_a, lane0_b), 0);
288
289 Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
290 Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
291 res = _mm512_insertf64x4(res, _mm256_and_pd(lane1_a, lane1_b), 1);
292
293 return res;
294#endif
295}
296template <>
297EIGEN_STRONG_INLINE Packet16f por<Packet16f>(const Packet16f& a,
298 const Packet16f& b) {
299#ifdef EIGEN_VECTORIZE_AVX512DQ
300 return _mm512_or_ps(a, b);
301#else
302 Packet16f res = _mm512_undefined_ps();
303 Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0);
304 Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0);
305 res = _mm512_insertf32x4(res, _mm_or_ps(lane0_a, lane0_b), 0);
306
307 Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1);
308 Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1);
309 res = _mm512_insertf32x4(res, _mm_or_ps(lane1_a, lane1_b), 1);
310
311 Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2);
312 Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2);
313 res = _mm512_insertf32x4(res, _mm_or_ps(lane2_a, lane2_b), 2);
314
315 Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3);
316 Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3);
317 res = _mm512_insertf32x4(res, _mm_or_ps(lane3_a, lane3_b), 3);
318
319 return res;
320#endif
321}
322
323template <>
324EIGEN_STRONG_INLINE Packet8d por<Packet8d>(const Packet8d& a,
325 const Packet8d& b) {
326#ifdef EIGEN_VECTORIZE_AVX512DQ
327 return _mm512_or_pd(a, b);
328#else
329 Packet8d res = _mm512_undefined_pd();
330 Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
331 Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
332 res = _mm512_insertf64x4(res, _mm256_or_pd(lane0_a, lane0_b), 0);
333
334 Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
335 Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
336 res = _mm512_insertf64x4(res, _mm256_or_pd(lane1_a, lane1_b), 1);
337
338 return res;
339#endif
340}
341
342template <>
343EIGEN_STRONG_INLINE Packet16f pxor<Packet16f>(const Packet16f& a,
344 const Packet16f& b) {
345#ifdef EIGEN_VECTORIZE_AVX512DQ
346 return _mm512_xor_ps(a, b);
347#else
348 Packet16f res = _mm512_undefined_ps();
349 Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0);
350 Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0);
351 res = _mm512_insertf32x4(res, _mm_xor_ps(lane0_a, lane0_b), 0);
352
353 Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1);
354 Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1);
355 res = _mm512_insertf32x4(res, _mm_xor_ps(lane1_a, lane1_b), 1);
356
357 Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2);
358 Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2);
359 res = _mm512_insertf32x4(res, _mm_xor_ps(lane2_a, lane2_b), 2);
360
361 Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3);
362 Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3);
363 res = _mm512_insertf32x4(res, _mm_xor_ps(lane3_a, lane3_b), 3);
364
365 return res;
366#endif
367}
368template <>
369EIGEN_STRONG_INLINE Packet8d pxor<Packet8d>(const Packet8d& a,
370 const Packet8d& b) {
371#ifdef EIGEN_VECTORIZE_AVX512DQ
372 return _mm512_xor_pd(a, b);
373#else
374 Packet8d res = _mm512_undefined_pd();
375 Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
376 Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
377 res = _mm512_insertf64x4(res, _mm256_xor_pd(lane0_a, lane0_b), 0);
378
379 Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
380 Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
381 res = _mm512_insertf64x4(res, _mm256_xor_pd(lane1_a, lane1_b), 1);
382
383 return res;
384#endif
385}
386
387template <>
388EIGEN_STRONG_INLINE Packet16f pandnot<Packet16f>(const Packet16f& a,
389 const Packet16f& b) {
390#ifdef EIGEN_VECTORIZE_AVX512DQ
391 return _mm512_andnot_ps(a, b);
392#else
393 Packet16f res = _mm512_undefined_ps();
394 Packet4f lane0_a = _mm512_extractf32x4_ps(a, 0);
395 Packet4f lane0_b = _mm512_extractf32x4_ps(b, 0);
396 res = _mm512_insertf32x4(res, _mm_andnot_ps(lane0_a, lane0_b), 0);
397
398 Packet4f lane1_a = _mm512_extractf32x4_ps(a, 1);
399 Packet4f lane1_b = _mm512_extractf32x4_ps(b, 1);
400 res = _mm512_insertf32x4(res, _mm_andnot_ps(lane1_a, lane1_b), 1);
401
402 Packet4f lane2_a = _mm512_extractf32x4_ps(a, 2);
403 Packet4f lane2_b = _mm512_extractf32x4_ps(b, 2);
404 res = _mm512_insertf32x4(res, _mm_andnot_ps(lane2_a, lane2_b), 2);
405
406 Packet4f lane3_a = _mm512_extractf32x4_ps(a, 3);
407 Packet4f lane3_b = _mm512_extractf32x4_ps(b, 3);
408 res = _mm512_insertf32x4(res, _mm_andnot_ps(lane3_a, lane3_b), 3);
409
410 return res;
411#endif
412}
413template <>
414EIGEN_STRONG_INLINE Packet8d pandnot<Packet8d>(const Packet8d& a,
415 const Packet8d& b) {
416#ifdef EIGEN_VECTORIZE_AVX512DQ
417 return _mm512_andnot_pd(a, b);
418#else
419 Packet8d res = _mm512_undefined_pd();
420 Packet4d lane0_a = _mm512_extractf64x4_pd(a, 0);
421 Packet4d lane0_b = _mm512_extractf64x4_pd(b, 0);
422 res = _mm512_insertf64x4(res, _mm256_andnot_pd(lane0_a, lane0_b), 0);
423
424 Packet4d lane1_a = _mm512_extractf64x4_pd(a, 1);
425 Packet4d lane1_b = _mm512_extractf64x4_pd(b, 1);
426 res = _mm512_insertf64x4(res, _mm256_andnot_pd(lane1_a, lane1_b), 1);
427
428 return res;
429#endif
430}
431
432template <>
433EIGEN_STRONG_INLINE Packet16f pload<Packet16f>(const float* from) {
434 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ps(from);
435}
436template <>
437EIGEN_STRONG_INLINE Packet8d pload<Packet8d>(const double* from) {
438 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_pd(from);
439}
440template <>
441EIGEN_STRONG_INLINE Packet16i pload<Packet16i>(const int* from) {
442 EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_si512(
443 reinterpret_cast<const __m512i*>(from));
444}
445
446template <>
447EIGEN_STRONG_INLINE Packet16f ploadu<Packet16f>(const float* from) {
448 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ps(from);
449}
450template <>
451EIGEN_STRONG_INLINE Packet8d ploadu<Packet8d>(const double* from) {
452 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_pd(from);
453}
454template <>
455EIGEN_STRONG_INLINE Packet16i ploadu<Packet16i>(const int* from) {
456 EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_si512(
457 reinterpret_cast<const __m512i*>(from));
458}
459
460// Loads 8 floats from memory a returns the packet
461// {a0, a0 a1, a1, a2, a2, a3, a3, a4, a4, a5, a5, a6, a6, a7, a7}
462template <>
463EIGEN_STRONG_INLINE Packet16f ploaddup<Packet16f>(const float* from) {
464 Packet8f lane0 = _mm256_broadcast_ps((const __m128*)(const void*)from);
465 // mimic an "inplace" permutation of the lower 128bits using a blend
466 lane0 = _mm256_blend_ps(
467 lane0, _mm256_castps128_ps256(_mm_permute_ps(
468 _mm256_castps256_ps128(lane0), _MM_SHUFFLE(1, 0, 1, 0))),
469 15);
470 // then we can perform a consistent permutation on the global register to get
471 // everything in shape:
472 lane0 = _mm256_permute_ps(lane0, _MM_SHUFFLE(3, 3, 2, 2));
473
474 Packet8f lane1 = _mm256_broadcast_ps((const __m128*)(const void*)(from + 4));
475 // mimic an "inplace" permutation of the lower 128bits using a blend
476 lane1 = _mm256_blend_ps(
477 lane1, _mm256_castps128_ps256(_mm_permute_ps(
478 _mm256_castps256_ps128(lane1), _MM_SHUFFLE(1, 0, 1, 0))),
479 15);
480 // then we can perform a consistent permutation on the global register to get
481 // everything in shape:
482 lane1 = _mm256_permute_ps(lane1, _MM_SHUFFLE(3, 3, 2, 2));
483
484#ifdef EIGEN_VECTORIZE_AVX512DQ
485 Packet16f res = _mm512_undefined_ps();
486 return _mm512_insertf32x8(res, lane0, 0);
487 return _mm512_insertf32x8(res, lane1, 1);
488 return res;
489#else
490 Packet16f res = _mm512_undefined_ps();
491 res = _mm512_insertf32x4(res, _mm256_extractf128_ps(lane0, 0), 0);
492 res = _mm512_insertf32x4(res, _mm256_extractf128_ps(lane0, 1), 1);
493 res = _mm512_insertf32x4(res, _mm256_extractf128_ps(lane1, 0), 2);
494 res = _mm512_insertf32x4(res, _mm256_extractf128_ps(lane1, 1), 3);
495 return res;
496#endif
497}
498// Loads 4 doubles from memory a returns the packet {a0, a0 a1, a1, a2, a2, a3,
499// a3}
500template <>
501EIGEN_STRONG_INLINE Packet8d ploaddup<Packet8d>(const double* from) {
502 Packet4d lane0 = _mm256_broadcast_pd((const __m128d*)(const void*)from);
503 lane0 = _mm256_permute_pd(lane0, 3 << 2);
504
505 Packet4d lane1 = _mm256_broadcast_pd((const __m128d*)(const void*)(from + 2));
506 lane1 = _mm256_permute_pd(lane1, 3 << 2);
507
508 Packet8d res = _mm512_undefined_pd();
509 res = _mm512_insertf64x4(res, lane0, 0);
510 return _mm512_insertf64x4(res, lane1, 1);
511}
512
513// Loads 4 floats from memory a returns the packet
514// {a0, a0 a0, a0, a1, a1, a1, a1, a2, a2, a2, a2, a3, a3, a3, a3}
515template <>
516EIGEN_STRONG_INLINE Packet16f ploadquad<Packet16f>(const float* from) {
517 Packet16f tmp = _mm512_undefined_ps();
518 tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from), 0);
519 tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 1), 1);
520 tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 2), 2);
521 tmp = _mm512_insertf32x4(tmp, _mm_load_ps1(from + 3), 3);
522 return tmp;
523}
524// Loads 2 doubles from memory a returns the packet
525// {a0, a0 a0, a0, a1, a1, a1, a1}
526template <>
527EIGEN_STRONG_INLINE Packet8d ploadquad<Packet8d>(const double* from) {
528 Packet8d tmp = _mm512_undefined_pd();
529 Packet2d tmp0 = _mm_load_pd1(from);
530 Packet2d tmp1 = _mm_load_pd1(from + 1);
531 Packet4d lane0 = _mm256_broadcastsd_pd(tmp0);
532 Packet4d lane1 = _mm256_broadcastsd_pd(tmp1);
533 tmp = _mm512_insertf64x4(tmp, lane0, 0);
534 return _mm512_insertf64x4(tmp, lane1, 1);
535}
536
537template <>
538EIGEN_STRONG_INLINE void pstore<float>(float* to, const Packet16f& from) {
539 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ps(to, from);
540}
541template <>
542EIGEN_STRONG_INLINE void pstore<double>(double* to, const Packet8d& from) {
543 EIGEN_DEBUG_ALIGNED_STORE _mm512_store_pd(to, from);
544}
545template <>
546EIGEN_STRONG_INLINE void pstore<int>(int* to, const Packet16i& from) {
547 EIGEN_DEBUG_ALIGNED_STORE _mm512_storeu_si512(reinterpret_cast<__m512i*>(to),
548 from);
549}
550
551template <>
552EIGEN_STRONG_INLINE void pstoreu<float>(float* to, const Packet16f& from) {
553 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ps(to, from);
554}
555template <>
556EIGEN_STRONG_INLINE void pstoreu<double>(double* to, const Packet8d& from) {
557 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_pd(to, from);
558}
559template <>
560EIGEN_STRONG_INLINE void pstoreu<int>(int* to, const Packet16i& from) {
561 EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_si512(
562 reinterpret_cast<__m512i*>(to), from);
563}
564
565template <>
566EIGEN_DEVICE_FUNC inline Packet16f pgather<float, Packet16f>(const float* from,
567 Index stride) {
568 Packet16i stride_vector = _mm512_set1_epi32(stride);
569 Packet16i stride_multiplier =
570 _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
571 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
572
573 return _mm512_i32gather_ps(indices, from, 4);
574}
575template <>
576EIGEN_DEVICE_FUNC inline Packet8d pgather<double, Packet8d>(const double* from,
577 Index stride) {
578 Packet8i stride_vector = _mm256_set1_epi32(stride);
579 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
580 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
581
582 return _mm512_i32gather_pd(indices, from, 8);
583}
584
585template <>
586EIGEN_DEVICE_FUNC inline void pscatter<float, Packet16f>(float* to,
587 const Packet16f& from,
588 Index stride) {
589 Packet16i stride_vector = _mm512_set1_epi32(stride);
590 Packet16i stride_multiplier =
591 _mm512_set_epi32(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
592 Packet16i indices = _mm512_mullo_epi32(stride_vector, stride_multiplier);
593 _mm512_i32scatter_ps(to, indices, from, 4);
594}
595template <>
596EIGEN_DEVICE_FUNC inline void pscatter<double, Packet8d>(double* to,
597 const Packet8d& from,
598 Index stride) {
599 Packet8i stride_vector = _mm256_set1_epi32(stride);
600 Packet8i stride_multiplier = _mm256_set_epi32(7, 6, 5, 4, 3, 2, 1, 0);
601 Packet8i indices = _mm256_mullo_epi32(stride_vector, stride_multiplier);
602 _mm512_i32scatter_pd(to, indices, from, 8);
603}
604
605template <>
606EIGEN_STRONG_INLINE void pstore1<Packet16f>(float* to, const float& a) {
607 Packet16f pa = pset1<Packet16f>(a);
608 pstore(to, pa);
609}
610template <>
611EIGEN_STRONG_INLINE void pstore1<Packet8d>(double* to, const double& a) {
612 Packet8d pa = pset1<Packet8d>(a);
613 pstore(to, pa);
614}
615template <>
616EIGEN_STRONG_INLINE void pstore1<Packet16i>(int* to, const int& a) {
617 Packet16i pa = pset1<Packet16i>(a);
618 pstore(to, pa);
619}
620
621template<> EIGEN_STRONG_INLINE void prefetch<float>(const float* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
622template<> EIGEN_STRONG_INLINE void prefetch<double>(const double* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
623template<> EIGEN_STRONG_INLINE void prefetch<int>(const int* addr) { _mm_prefetch((const char*)(addr), _MM_HINT_T0); }
624
625template <>
626EIGEN_STRONG_INLINE float pfirst<Packet16f>(const Packet16f& a) {
627 return _mm_cvtss_f32(_mm512_extractf32x4_ps(a, 0));
628}
629template <>
630EIGEN_STRONG_INLINE double pfirst<Packet8d>(const Packet8d& a) {
631 return _mm_cvtsd_f64(_mm256_extractf128_pd(_mm512_extractf64x4_pd(a, 0), 0));
632}
633template <>
634EIGEN_STRONG_INLINE int pfirst<Packet16i>(const Packet16i& a) {
635 return _mm_extract_epi32(_mm512_extracti32x4_epi32(a, 0), 0);
636}
637
638template<> EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a)
639{
640 return _mm512_permutexvar_ps(_mm512_set_epi32(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15), a);
641}
642
643template<> EIGEN_STRONG_INLINE Packet8d preverse(const Packet8d& a)
644{
645 return _mm512_permutexvar_pd(_mm512_set_epi32(0, 0, 0, 1, 0, 2, 0, 3, 0, 4, 0, 5, 0, 6, 0, 7), a);
646}
647
648template<> EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a)
649{
650 // _mm512_abs_ps intrinsic not found, so hack around it
651 return (__m512)_mm512_and_si512((__m512i)a, _mm512_set1_epi32(0x7fffffff));
652}
653template <>
654EIGEN_STRONG_INLINE Packet8d pabs(const Packet8d& a) {
655 // _mm512_abs_ps intrinsic not found, so hack around it
656 return (__m512d)_mm512_and_si512((__m512i)a,
657 _mm512_set1_epi64(0x7fffffffffffffff));
658}
659
660#ifdef EIGEN_VECTORIZE_AVX512DQ
661// AVX512F does not define _mm512_extractf32x8_ps to extract _m256 from _m512
662#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
663 __m256 OUTPUT##_0 = _mm512_extractf32x8_ps(INPUT, 0) __m256 OUTPUT##_1 = \
664 _mm512_extractf32x8_ps(INPUT, 1)
665#else
666#define EIGEN_EXTRACT_8f_FROM_16f(INPUT, OUTPUT) \
667 __m256 OUTPUT##_0 = _mm256_insertf128_ps( \
668 _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 0)), \
669 _mm512_extractf32x4_ps(INPUT, 1), 1); \
670 __m256 OUTPUT##_1 = _mm256_insertf128_ps( \
671 _mm256_castps128_ps256(_mm512_extractf32x4_ps(INPUT, 2)), \
672 _mm512_extractf32x4_ps(INPUT, 3), 1);
673#endif
674
675#ifdef EIGEN_VECTORIZE_AVX512DQ
676#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
677 OUTPUT = _mm512_insertf32x8(OUTPUT, INPUTA, 0); \
678 OUTPUT = _mm512_insertf32x8(OUTPUT, INPUTB, 1);
679#else
680#define EIGEN_INSERT_8f_INTO_16f(OUTPUT, INPUTA, INPUTB) \
681 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 0), 0); \
682 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTA, 1), 1); \
683 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 0), 2); \
684 OUTPUT = _mm512_insertf32x4(OUTPUT, _mm256_extractf128_ps(INPUTB, 1), 3);
685#endif
686template<> EIGEN_STRONG_INLINE Packet16f preduxp<Packet16f>(const Packet16f*
687vecs)
688{
689 EIGEN_EXTRACT_8f_FROM_16f(vecs[0], vecs0);
690 EIGEN_EXTRACT_8f_FROM_16f(vecs[1], vecs1);
691 EIGEN_EXTRACT_8f_FROM_16f(vecs[2], vecs2);
692 EIGEN_EXTRACT_8f_FROM_16f(vecs[3], vecs3);
693 EIGEN_EXTRACT_8f_FROM_16f(vecs[4], vecs4);
694 EIGEN_EXTRACT_8f_FROM_16f(vecs[5], vecs5);
695 EIGEN_EXTRACT_8f_FROM_16f(vecs[6], vecs6);
696 EIGEN_EXTRACT_8f_FROM_16f(vecs[7], vecs7);
697 EIGEN_EXTRACT_8f_FROM_16f(vecs[8], vecs8);
698 EIGEN_EXTRACT_8f_FROM_16f(vecs[9], vecs9);
699 EIGEN_EXTRACT_8f_FROM_16f(vecs[10], vecs10);
700 EIGEN_EXTRACT_8f_FROM_16f(vecs[11], vecs11);
701 EIGEN_EXTRACT_8f_FROM_16f(vecs[12], vecs12);
702 EIGEN_EXTRACT_8f_FROM_16f(vecs[13], vecs13);
703 EIGEN_EXTRACT_8f_FROM_16f(vecs[14], vecs14);
704 EIGEN_EXTRACT_8f_FROM_16f(vecs[15], vecs15);
705
706 __m256 hsum1 = _mm256_hadd_ps(vecs0_0, vecs1_0);
707 __m256 hsum2 = _mm256_hadd_ps(vecs2_0, vecs3_0);
708 __m256 hsum3 = _mm256_hadd_ps(vecs4_0, vecs5_0);
709 __m256 hsum4 = _mm256_hadd_ps(vecs6_0, vecs7_0);
710
711 __m256 hsum5 = _mm256_hadd_ps(hsum1, hsum1);
712 __m256 hsum6 = _mm256_hadd_ps(hsum2, hsum2);
713 __m256 hsum7 = _mm256_hadd_ps(hsum3, hsum3);
714 __m256 hsum8 = _mm256_hadd_ps(hsum4, hsum4);
715
716 __m256 perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23);
717 __m256 perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23);
718 __m256 perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23);
719 __m256 perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23);
720
721 __m256 sum1 = _mm256_add_ps(perm1, hsum5);
722 __m256 sum2 = _mm256_add_ps(perm2, hsum6);
723 __m256 sum3 = _mm256_add_ps(perm3, hsum7);
724 __m256 sum4 = _mm256_add_ps(perm4, hsum8);
725
726 __m256 blend1 = _mm256_blend_ps(sum1, sum2, 0xcc);
727 __m256 blend2 = _mm256_blend_ps(sum3, sum4, 0xcc);
728
729 __m256 final = _mm256_blend_ps(blend1, blend2, 0xf0);
730
731 hsum1 = _mm256_hadd_ps(vecs0_1, vecs1_1);
732 hsum2 = _mm256_hadd_ps(vecs2_1, vecs3_1);
733 hsum3 = _mm256_hadd_ps(vecs4_1, vecs5_1);
734 hsum4 = _mm256_hadd_ps(vecs6_1, vecs7_1);
735
736 hsum5 = _mm256_hadd_ps(hsum1, hsum1);
737 hsum6 = _mm256_hadd_ps(hsum2, hsum2);
738 hsum7 = _mm256_hadd_ps(hsum3, hsum3);
739 hsum8 = _mm256_hadd_ps(hsum4, hsum4);
740
741 perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23);
742 perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23);
743 perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23);
744 perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23);
745
746 sum1 = _mm256_add_ps(perm1, hsum5);
747 sum2 = _mm256_add_ps(perm2, hsum6);
748 sum3 = _mm256_add_ps(perm3, hsum7);
749 sum4 = _mm256_add_ps(perm4, hsum8);
750
751 blend1 = _mm256_blend_ps(sum1, sum2, 0xcc);
752 blend2 = _mm256_blend_ps(sum3, sum4, 0xcc);
753
754 final = padd(final, _mm256_blend_ps(blend1, blend2, 0xf0));
755
756 hsum1 = _mm256_hadd_ps(vecs8_0, vecs9_0);
757 hsum2 = _mm256_hadd_ps(vecs10_0, vecs11_0);
758 hsum3 = _mm256_hadd_ps(vecs12_0, vecs13_0);
759 hsum4 = _mm256_hadd_ps(vecs14_0, vecs15_0);
760
761 hsum5 = _mm256_hadd_ps(hsum1, hsum1);
762 hsum6 = _mm256_hadd_ps(hsum2, hsum2);
763 hsum7 = _mm256_hadd_ps(hsum3, hsum3);
764 hsum8 = _mm256_hadd_ps(hsum4, hsum4);
765
766 perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23);
767 perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23);
768 perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23);
769 perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23);
770
771 sum1 = _mm256_add_ps(perm1, hsum5);
772 sum2 = _mm256_add_ps(perm2, hsum6);
773 sum3 = _mm256_add_ps(perm3, hsum7);
774 sum4 = _mm256_add_ps(perm4, hsum8);
775
776 blend1 = _mm256_blend_ps(sum1, sum2, 0xcc);
777 blend2 = _mm256_blend_ps(sum3, sum4, 0xcc);
778
779 __m256 final_1 = _mm256_blend_ps(blend1, blend2, 0xf0);
780
781 hsum1 = _mm256_hadd_ps(vecs8_1, vecs9_1);
782 hsum2 = _mm256_hadd_ps(vecs10_1, vecs11_1);
783 hsum3 = _mm256_hadd_ps(vecs12_1, vecs13_1);
784 hsum4 = _mm256_hadd_ps(vecs14_1, vecs15_1);
785
786 hsum5 = _mm256_hadd_ps(hsum1, hsum1);
787 hsum6 = _mm256_hadd_ps(hsum2, hsum2);
788 hsum7 = _mm256_hadd_ps(hsum3, hsum3);
789 hsum8 = _mm256_hadd_ps(hsum4, hsum4);
790
791 perm1 = _mm256_permute2f128_ps(hsum5, hsum5, 0x23);
792 perm2 = _mm256_permute2f128_ps(hsum6, hsum6, 0x23);
793 perm3 = _mm256_permute2f128_ps(hsum7, hsum7, 0x23);
794 perm4 = _mm256_permute2f128_ps(hsum8, hsum8, 0x23);
795
796 sum1 = _mm256_add_ps(perm1, hsum5);
797 sum2 = _mm256_add_ps(perm2, hsum6);
798 sum3 = _mm256_add_ps(perm3, hsum7);
799 sum4 = _mm256_add_ps(perm4, hsum8);
800
801 blend1 = _mm256_blend_ps(sum1, sum2, 0xcc);
802 blend2 = _mm256_blend_ps(sum3, sum4, 0xcc);
803
804 final_1 = padd(final_1, _mm256_blend_ps(blend1, blend2, 0xf0));
805
806 __m512 final_output;
807
808 EIGEN_INSERT_8f_INTO_16f(final_output, final, final_1);
809 return final_output;
810}
811
812template<> EIGEN_STRONG_INLINE Packet8d preduxp<Packet8d>(const Packet8d* vecs)
813{
814 Packet4d vecs0_0 = _mm512_extractf64x4_pd(vecs[0], 0);
815 Packet4d vecs0_1 = _mm512_extractf64x4_pd(vecs[0], 1);
816
817 Packet4d vecs1_0 = _mm512_extractf64x4_pd(vecs[1], 0);
818 Packet4d vecs1_1 = _mm512_extractf64x4_pd(vecs[1], 1);
819
820 Packet4d vecs2_0 = _mm512_extractf64x4_pd(vecs[2], 0);
821 Packet4d vecs2_1 = _mm512_extractf64x4_pd(vecs[2], 1);
822
823 Packet4d vecs3_0 = _mm512_extractf64x4_pd(vecs[3], 0);
824 Packet4d vecs3_1 = _mm512_extractf64x4_pd(vecs[3], 1);
825
826 Packet4d vecs4_0 = _mm512_extractf64x4_pd(vecs[4], 0);
827 Packet4d vecs4_1 = _mm512_extractf64x4_pd(vecs[4], 1);
828
829 Packet4d vecs5_0 = _mm512_extractf64x4_pd(vecs[5], 0);
830 Packet4d vecs5_1 = _mm512_extractf64x4_pd(vecs[5], 1);
831
832 Packet4d vecs6_0 = _mm512_extractf64x4_pd(vecs[6], 0);
833 Packet4d vecs6_1 = _mm512_extractf64x4_pd(vecs[6], 1);
834
835 Packet4d vecs7_0 = _mm512_extractf64x4_pd(vecs[7], 0);
836 Packet4d vecs7_1 = _mm512_extractf64x4_pd(vecs[7], 1);
837
838 Packet4d tmp0, tmp1;
839
840 tmp0 = _mm256_hadd_pd(vecs0_0, vecs1_0);
841 tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1));
842
843 tmp1 = _mm256_hadd_pd(vecs2_0, vecs3_0);
844 tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1));
845
846 __m256d final_0 = _mm256_blend_pd(tmp0, tmp1, 0xC);
847
848 tmp0 = _mm256_hadd_pd(vecs0_1, vecs1_1);
849 tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1));
850
851 tmp1 = _mm256_hadd_pd(vecs2_1, vecs3_1);
852 tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1));
853
854 final_0 = padd(final_0, _mm256_blend_pd(tmp0, tmp1, 0xC));
855
856 tmp0 = _mm256_hadd_pd(vecs4_0, vecs5_0);
857 tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1));
858
859 tmp1 = _mm256_hadd_pd(vecs6_0, vecs7_0);
860 tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1));
861
862 __m256d final_1 = _mm256_blend_pd(tmp0, tmp1, 0xC);
863
864 tmp0 = _mm256_hadd_pd(vecs4_1, vecs5_1);
865 tmp0 = _mm256_add_pd(tmp0, _mm256_permute2f128_pd(tmp0, tmp0, 1));
866
867 tmp1 = _mm256_hadd_pd(vecs6_1, vecs7_1);
868 tmp1 = _mm256_add_pd(tmp1, _mm256_permute2f128_pd(tmp1, tmp1, 1));
869
870 final_1 = padd(final_1, _mm256_blend_pd(tmp0, tmp1, 0xC));
871
872 __m512d final_output = _mm512_insertf64x4(final_output, final_0, 0);
873
874 return _mm512_insertf64x4(final_output, final_1, 1);
875}
876
877template <>
878EIGEN_STRONG_INLINE float predux<Packet16f>(const Packet16f& a) {
879 //#ifdef EIGEN_VECTORIZE_AVX512DQ
880#if 0
881 Packet8f lane0 = _mm512_extractf32x8_ps(a, 0);
882 Packet8f lane1 = _mm512_extractf32x8_ps(a, 1);
883 Packet8f sum = padd(lane0, lane1);
884 Packet8f tmp0 = _mm256_hadd_ps(sum, _mm256_permute2f128_ps(a, a, 1));
885 tmp0 = _mm256_hadd_ps(tmp0, tmp0);
886 return pfirst(_mm256_hadd_ps(tmp0, tmp0));
887#else
888 Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
889 Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
890 Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
891 Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
892 Packet4f sum = padd(padd(lane0, lane1), padd(lane2, lane3));
893 sum = _mm_hadd_ps(sum, sum);
894 sum = _mm_hadd_ps(sum, _mm_permute_ps(sum, 1));
895 return pfirst(sum);
896#endif
897}
898template <>
899EIGEN_STRONG_INLINE double predux<Packet8d>(const Packet8d& a) {
900 Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
901 Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
902 Packet4d sum = padd(lane0, lane1);
903 Packet4d tmp0 = _mm256_hadd_pd(sum, _mm256_permute2f128_pd(sum, sum, 1));
904 return pfirst(_mm256_hadd_pd(tmp0, tmp0));
905}
906
907template <>
908EIGEN_STRONG_INLINE Packet8f predux_downto4<Packet16f>(const Packet16f& a) {
909#ifdef EIGEN_VECTORIZE_AVX512DQ
910 Packet8f lane0 = _mm512_extractf32x8_ps(a, 0);
911 Packet8f lane1 = _mm512_extractf32x8_ps(a, 1);
912 return padd(lane0, lane1);
913#else
914 Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
915 Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
916 Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
917 Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
918 Packet4f sum0 = padd(lane0, lane2);
919 Packet4f sum1 = padd(lane1, lane3);
920 return _mm256_insertf128_ps(_mm256_castps128_ps256(sum0), sum1, 1);
921#endif
922}
923template <>
924EIGEN_STRONG_INLINE Packet4d predux_downto4<Packet8d>(const Packet8d& a) {
925 Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
926 Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
927 Packet4d res = padd(lane0, lane1);
928 return res;
929}
930
931template <>
932EIGEN_STRONG_INLINE float predux_mul<Packet16f>(const Packet16f& a) {
933//#ifdef EIGEN_VECTORIZE_AVX512DQ
934#if 0
935 Packet8f lane0 = _mm512_extractf32x8_ps(a, 0);
936 Packet8f lane1 = _mm512_extractf32x8_ps(a, 1);
937 Packet8f res = pmul(lane0, lane1);
938 res = pmul(res, _mm256_permute2f128_ps(res, res, 1));
939 res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
940 return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
941#else
942 Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
943 Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
944 Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
945 Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
946 Packet4f res = pmul(pmul(lane0, lane1), pmul(lane2, lane3));
947 res = pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
948 return pfirst(pmul(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
949#endif
950}
951template <>
952EIGEN_STRONG_INLINE double predux_mul<Packet8d>(const Packet8d& a) {
953 Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
954 Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
955 Packet4d res = pmul(lane0, lane1);
956 res = pmul(res, _mm256_permute2f128_pd(res, res, 1));
957 return pfirst(pmul(res, _mm256_shuffle_pd(res, res, 1)));
958}
959
960template <>
961EIGEN_STRONG_INLINE float predux_min<Packet16f>(const Packet16f& a) {
962 Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
963 Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
964 Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
965 Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
966 Packet4f res = _mm_min_ps(_mm_min_ps(lane0, lane1), _mm_min_ps(lane2, lane3));
967 res = _mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
968 return pfirst(_mm_min_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
969}
970template <>
971EIGEN_STRONG_INLINE double predux_min<Packet8d>(const Packet8d& a) {
972 Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
973 Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
974 Packet4d res = _mm256_min_pd(lane0, lane1);
975 res = _mm256_min_pd(res, _mm256_permute2f128_pd(res, res, 1));
976 return pfirst(_mm256_min_pd(res, _mm256_shuffle_pd(res, res, 1)));
977}
978
979template <>
980EIGEN_STRONG_INLINE float predux_max<Packet16f>(const Packet16f& a) {
981 Packet4f lane0 = _mm512_extractf32x4_ps(a, 0);
982 Packet4f lane1 = _mm512_extractf32x4_ps(a, 1);
983 Packet4f lane2 = _mm512_extractf32x4_ps(a, 2);
984 Packet4f lane3 = _mm512_extractf32x4_ps(a, 3);
985 Packet4f res = _mm_max_ps(_mm_max_ps(lane0, lane1), _mm_max_ps(lane2, lane3));
986 res = _mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 3, 2)));
987 return pfirst(_mm_max_ps(res, _mm_permute_ps(res, _MM_SHUFFLE(0, 0, 0, 1))));
988}
989template <>
990EIGEN_STRONG_INLINE double predux_max<Packet8d>(const Packet8d& a) {
991 Packet4d lane0 = _mm512_extractf64x4_pd(a, 0);
992 Packet4d lane1 = _mm512_extractf64x4_pd(a, 1);
993 Packet4d res = _mm256_max_pd(lane0, lane1);
994 res = _mm256_max_pd(res, _mm256_permute2f128_pd(res, res, 1));
995 return pfirst(_mm256_max_pd(res, _mm256_shuffle_pd(res, res, 1)));
996}
997
998template <int Offset>
999struct palign_impl<Offset, Packet16f> {
1000 static EIGEN_STRONG_INLINE void run(Packet16f& first,
1001 const Packet16f& second) {
1002 if (Offset != 0) {
1003 __m512i first_idx = _mm512_set_epi32(
1004 Offset + 15, Offset + 14, Offset + 13, Offset + 12, Offset + 11,
1005 Offset + 10, Offset + 9, Offset + 8, Offset + 7, Offset + 6,
1006 Offset + 5, Offset + 4, Offset + 3, Offset + 2, Offset + 1, Offset);
1007
1008 __m512i second_idx =
1009 _mm512_set_epi32(Offset - 1, Offset - 2, Offset - 3, Offset - 4,
1010 Offset - 5, Offset - 6, Offset - 7, Offset - 8,
1011 Offset - 9, Offset - 10, Offset - 11, Offset - 12,
1012 Offset - 13, Offset - 14, Offset - 15, Offset - 16);
1013
1014 unsigned short mask = 0xFFFF;
1015 mask <<= (16 - Offset);
1016
1017 first = _mm512_permutexvar_ps(first_idx, first);
1018 Packet16f tmp = _mm512_permutexvar_ps(second_idx, second);
1019 first = _mm512_mask_blend_ps(mask, first, tmp);
1020 }
1021 }
1022};
1023template <int Offset>
1024struct palign_impl<Offset, Packet8d> {
1025 static EIGEN_STRONG_INLINE void run(Packet8d& first, const Packet8d& second) {
1026 if (Offset != 0) {
1027 __m512i first_idx = _mm512_set_epi32(
1028 0, Offset + 7, 0, Offset + 6, 0, Offset + 5, 0, Offset + 4, 0,
1029 Offset + 3, 0, Offset + 2, 0, Offset + 1, 0, Offset);
1030
1031 __m512i second_idx = _mm512_set_epi32(
1032 0, Offset - 1, 0, Offset - 2, 0, Offset - 3, 0, Offset - 4, 0,
1033 Offset - 5, 0, Offset - 6, 0, Offset - 7, 0, Offset - 8);
1034
1035 unsigned char mask = 0xFF;
1036 mask <<= (8 - Offset);
1037
1038 first = _mm512_permutexvar_pd(first_idx, first);
1039 Packet8d tmp = _mm512_permutexvar_pd(second_idx, second);
1040 first = _mm512_mask_blend_pd(mask, first, tmp);
1041 }
1042 }
1043};
1044
1045
1046#define PACK_OUTPUT(OUTPUT, INPUT, INDEX, STRIDE) \
1047 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[INDEX], INPUT[INDEX + STRIDE]);
1048
1049EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 16>& kernel) {
1050 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1051 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1052 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1053 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1054 __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]);
1055 __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]);
1056 __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]);
1057 __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]);
1058 __m512 T8 = _mm512_unpacklo_ps(kernel.packet[8], kernel.packet[9]);
1059 __m512 T9 = _mm512_unpackhi_ps(kernel.packet[8], kernel.packet[9]);
1060 __m512 T10 = _mm512_unpacklo_ps(kernel.packet[10], kernel.packet[11]);
1061 __m512 T11 = _mm512_unpackhi_ps(kernel.packet[10], kernel.packet[11]);
1062 __m512 T12 = _mm512_unpacklo_ps(kernel.packet[12], kernel.packet[13]);
1063 __m512 T13 = _mm512_unpackhi_ps(kernel.packet[12], kernel.packet[13]);
1064 __m512 T14 = _mm512_unpacklo_ps(kernel.packet[14], kernel.packet[15]);
1065 __m512 T15 = _mm512_unpackhi_ps(kernel.packet[14], kernel.packet[15]);
1066 __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1067 __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1068 __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1069 __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1070 __m512 S4 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(1, 0, 1, 0));
1071 __m512 S5 = _mm512_shuffle_ps(T4, T6, _MM_SHUFFLE(3, 2, 3, 2));
1072 __m512 S6 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(1, 0, 1, 0));
1073 __m512 S7 = _mm512_shuffle_ps(T5, T7, _MM_SHUFFLE(3, 2, 3, 2));
1074 __m512 S8 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(1, 0, 1, 0));
1075 __m512 S9 = _mm512_shuffle_ps(T8, T10, _MM_SHUFFLE(3, 2, 3, 2));
1076 __m512 S10 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(1, 0, 1, 0));
1077 __m512 S11 = _mm512_shuffle_ps(T9, T11, _MM_SHUFFLE(3, 2, 3, 2));
1078 __m512 S12 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(1, 0, 1, 0));
1079 __m512 S13 = _mm512_shuffle_ps(T12, T14, _MM_SHUFFLE(3, 2, 3, 2));
1080 __m512 S14 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(1, 0, 1, 0));
1081 __m512 S15 = _mm512_shuffle_ps(T13, T15, _MM_SHUFFLE(3, 2, 3, 2));
1082
1083 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1084 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1085 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1086 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1087 EIGEN_EXTRACT_8f_FROM_16f(S4, S4);
1088 EIGEN_EXTRACT_8f_FROM_16f(S5, S5);
1089 EIGEN_EXTRACT_8f_FROM_16f(S6, S6);
1090 EIGEN_EXTRACT_8f_FROM_16f(S7, S7);
1091 EIGEN_EXTRACT_8f_FROM_16f(S8, S8);
1092 EIGEN_EXTRACT_8f_FROM_16f(S9, S9);
1093 EIGEN_EXTRACT_8f_FROM_16f(S10, S10);
1094 EIGEN_EXTRACT_8f_FROM_16f(S11, S11);
1095 EIGEN_EXTRACT_8f_FROM_16f(S12, S12);
1096 EIGEN_EXTRACT_8f_FROM_16f(S13, S13);
1097 EIGEN_EXTRACT_8f_FROM_16f(S14, S14);
1098 EIGEN_EXTRACT_8f_FROM_16f(S15, S15);
1099
1100 PacketBlock<Packet8f, 32> tmp;
1101
1102 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S4_0, 0x20);
1103 tmp.packet[1] = _mm256_permute2f128_ps(S1_0, S5_0, 0x20);
1104 tmp.packet[2] = _mm256_permute2f128_ps(S2_0, S6_0, 0x20);
1105 tmp.packet[3] = _mm256_permute2f128_ps(S3_0, S7_0, 0x20);
1106 tmp.packet[4] = _mm256_permute2f128_ps(S0_0, S4_0, 0x31);
1107 tmp.packet[5] = _mm256_permute2f128_ps(S1_0, S5_0, 0x31);
1108 tmp.packet[6] = _mm256_permute2f128_ps(S2_0, S6_0, 0x31);
1109 tmp.packet[7] = _mm256_permute2f128_ps(S3_0, S7_0, 0x31);
1110
1111 tmp.packet[8] = _mm256_permute2f128_ps(S0_1, S4_1, 0x20);
1112 tmp.packet[9] = _mm256_permute2f128_ps(S1_1, S5_1, 0x20);
1113 tmp.packet[10] = _mm256_permute2f128_ps(S2_1, S6_1, 0x20);
1114 tmp.packet[11] = _mm256_permute2f128_ps(S3_1, S7_1, 0x20);
1115 tmp.packet[12] = _mm256_permute2f128_ps(S0_1, S4_1, 0x31);
1116 tmp.packet[13] = _mm256_permute2f128_ps(S1_1, S5_1, 0x31);
1117 tmp.packet[14] = _mm256_permute2f128_ps(S2_1, S6_1, 0x31);
1118 tmp.packet[15] = _mm256_permute2f128_ps(S3_1, S7_1, 0x31);
1119
1120 // Second set of _m256 outputs
1121 tmp.packet[16] = _mm256_permute2f128_ps(S8_0, S12_0, 0x20);
1122 tmp.packet[17] = _mm256_permute2f128_ps(S9_0, S13_0, 0x20);
1123 tmp.packet[18] = _mm256_permute2f128_ps(S10_0, S14_0, 0x20);
1124 tmp.packet[19] = _mm256_permute2f128_ps(S11_0, S15_0, 0x20);
1125 tmp.packet[20] = _mm256_permute2f128_ps(S8_0, S12_0, 0x31);
1126 tmp.packet[21] = _mm256_permute2f128_ps(S9_0, S13_0, 0x31);
1127 tmp.packet[22] = _mm256_permute2f128_ps(S10_0, S14_0, 0x31);
1128 tmp.packet[23] = _mm256_permute2f128_ps(S11_0, S15_0, 0x31);
1129
1130 tmp.packet[24] = _mm256_permute2f128_ps(S8_1, S12_1, 0x20);
1131 tmp.packet[25] = _mm256_permute2f128_ps(S9_1, S13_1, 0x20);
1132 tmp.packet[26] = _mm256_permute2f128_ps(S10_1, S14_1, 0x20);
1133 tmp.packet[27] = _mm256_permute2f128_ps(S11_1, S15_1, 0x20);
1134 tmp.packet[28] = _mm256_permute2f128_ps(S8_1, S12_1, 0x31);
1135 tmp.packet[29] = _mm256_permute2f128_ps(S9_1, S13_1, 0x31);
1136 tmp.packet[30] = _mm256_permute2f128_ps(S10_1, S14_1, 0x31);
1137 tmp.packet[31] = _mm256_permute2f128_ps(S11_1, S15_1, 0x31);
1138
1139 // Pack them into the output
1140 PACK_OUTPUT(kernel.packet, tmp.packet, 0, 16);
1141 PACK_OUTPUT(kernel.packet, tmp.packet, 1, 16);
1142 PACK_OUTPUT(kernel.packet, tmp.packet, 2, 16);
1143 PACK_OUTPUT(kernel.packet, tmp.packet, 3, 16);
1144
1145 PACK_OUTPUT(kernel.packet, tmp.packet, 4, 16);
1146 PACK_OUTPUT(kernel.packet, tmp.packet, 5, 16);
1147 PACK_OUTPUT(kernel.packet, tmp.packet, 6, 16);
1148 PACK_OUTPUT(kernel.packet, tmp.packet, 7, 16);
1149
1150 PACK_OUTPUT(kernel.packet, tmp.packet, 8, 16);
1151 PACK_OUTPUT(kernel.packet, tmp.packet, 9, 16);
1152 PACK_OUTPUT(kernel.packet, tmp.packet, 10, 16);
1153 PACK_OUTPUT(kernel.packet, tmp.packet, 11, 16);
1154
1155 PACK_OUTPUT(kernel.packet, tmp.packet, 12, 16);
1156 PACK_OUTPUT(kernel.packet, tmp.packet, 13, 16);
1157 PACK_OUTPUT(kernel.packet, tmp.packet, 14, 16);
1158 PACK_OUTPUT(kernel.packet, tmp.packet, 15, 16);
1159}
1160#define PACK_OUTPUT_2(OUTPUT, INPUT, INDEX, STRIDE) \
1161 EIGEN_INSERT_8f_INTO_16f(OUTPUT[INDEX], INPUT[2 * INDEX], \
1162 INPUT[2 * INDEX + STRIDE]);
1163
1164EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet16f, 4>& kernel) {
1165 __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]);
1166 __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]);
1167 __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]);
1168 __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]);
1169
1170 __m512 S0 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(1, 0, 1, 0));
1171 __m512 S1 = _mm512_shuffle_ps(T0, T2, _MM_SHUFFLE(3, 2, 3, 2));
1172 __m512 S2 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(1, 0, 1, 0));
1173 __m512 S3 = _mm512_shuffle_ps(T1, T3, _MM_SHUFFLE(3, 2, 3, 2));
1174
1175 EIGEN_EXTRACT_8f_FROM_16f(S0, S0);
1176 EIGEN_EXTRACT_8f_FROM_16f(S1, S1);
1177 EIGEN_EXTRACT_8f_FROM_16f(S2, S2);
1178 EIGEN_EXTRACT_8f_FROM_16f(S3, S3);
1179
1180 PacketBlock<Packet8f, 8> tmp;
1181
1182 tmp.packet[0] = _mm256_permute2f128_ps(S0_0, S1_0, 0x20);
1183 tmp.packet[1] = _mm256_permute2f128_ps(S2_0, S3_0, 0x20);
1184 tmp.packet[2] = _mm256_permute2f128_ps(S0_0, S1_0, 0x31);
1185 tmp.packet[3] = _mm256_permute2f128_ps(S2_0, S3_0, 0x31);
1186
1187 tmp.packet[4] = _mm256_permute2f128_ps(S0_1, S1_1, 0x20);
1188 tmp.packet[5] = _mm256_permute2f128_ps(S2_1, S3_1, 0x20);
1189 tmp.packet[6] = _mm256_permute2f128_ps(S0_1, S1_1, 0x31);
1190 tmp.packet[7] = _mm256_permute2f128_ps(S2_1, S3_1, 0x31);
1191
1192 PACK_OUTPUT_2(kernel.packet, tmp.packet, 0, 1);
1193 PACK_OUTPUT_2(kernel.packet, tmp.packet, 1, 1);
1194 PACK_OUTPUT_2(kernel.packet, tmp.packet, 2, 1);
1195 PACK_OUTPUT_2(kernel.packet, tmp.packet, 3, 1);
1196}
1197
1198#define PACK_OUTPUT_SQ_D(OUTPUT, INPUT, INDEX, STRIDE) \
1199 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX], 0); \
1200 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[INDEX + STRIDE], 1);
1201
1202#define PACK_OUTPUT_D(OUTPUT, INPUT, INDEX, STRIDE) \
1203 OUTPUT[INDEX] = _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX)], 0); \
1204 OUTPUT[INDEX] = \
1205 _mm512_insertf64x4(OUTPUT[INDEX], INPUT[(2 * INDEX) + STRIDE], 1);
1206
1207EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 4>& kernel) {
1208 __m512d T0 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0);
1209 __m512d T1 = _mm512_shuffle_pd(kernel.packet[0], kernel.packet[1], 0xff);
1210 __m512d T2 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0);
1211 __m512d T3 = _mm512_shuffle_pd(kernel.packet[2], kernel.packet[3], 0xff);
1212
1213 PacketBlock<Packet4d, 8> tmp;
1214
1215 tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1216 _mm512_extractf64x4_pd(T2, 0), 0x20);
1217 tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1218 _mm512_extractf64x4_pd(T3, 0), 0x20);
1219 tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1220 _mm512_extractf64x4_pd(T2, 0), 0x31);
1221 tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1222 _mm512_extractf64x4_pd(T3, 0), 0x31);
1223
1224 tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1225 _mm512_extractf64x4_pd(T2, 1), 0x20);
1226 tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1227 _mm512_extractf64x4_pd(T3, 1), 0x20);
1228 tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1229 _mm512_extractf64x4_pd(T2, 1), 0x31);
1230 tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1231 _mm512_extractf64x4_pd(T3, 1), 0x31);
1232
1233 PACK_OUTPUT_D(kernel.packet, tmp.packet, 0, 1);
1234 PACK_OUTPUT_D(kernel.packet, tmp.packet, 1, 1);
1235 PACK_OUTPUT_D(kernel.packet, tmp.packet, 2, 1);
1236 PACK_OUTPUT_D(kernel.packet, tmp.packet, 3, 1);
1237}
1238
1239EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock<Packet8d, 8>& kernel) {
1240 __m512d T0 = _mm512_unpacklo_pd(kernel.packet[0], kernel.packet[1]);
1241 __m512d T1 = _mm512_unpackhi_pd(kernel.packet[0], kernel.packet[1]);
1242 __m512d T2 = _mm512_unpacklo_pd(kernel.packet[2], kernel.packet[3]);
1243 __m512d T3 = _mm512_unpackhi_pd(kernel.packet[2], kernel.packet[3]);
1244 __m512d T4 = _mm512_unpacklo_pd(kernel.packet[4], kernel.packet[5]);
1245 __m512d T5 = _mm512_unpackhi_pd(kernel.packet[4], kernel.packet[5]);
1246 __m512d T6 = _mm512_unpacklo_pd(kernel.packet[6], kernel.packet[7]);
1247 __m512d T7 = _mm512_unpackhi_pd(kernel.packet[6], kernel.packet[7]);
1248
1249 PacketBlock<Packet4d, 16> tmp;
1250
1251 tmp.packet[0] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1252 _mm512_extractf64x4_pd(T2, 0), 0x20);
1253 tmp.packet[1] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1254 _mm512_extractf64x4_pd(T3, 0), 0x20);
1255 tmp.packet[2] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 0),
1256 _mm512_extractf64x4_pd(T2, 0), 0x31);
1257 tmp.packet[3] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 0),
1258 _mm512_extractf64x4_pd(T3, 0), 0x31);
1259
1260 tmp.packet[4] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1261 _mm512_extractf64x4_pd(T2, 1), 0x20);
1262 tmp.packet[5] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1263 _mm512_extractf64x4_pd(T3, 1), 0x20);
1264 tmp.packet[6] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T0, 1),
1265 _mm512_extractf64x4_pd(T2, 1), 0x31);
1266 tmp.packet[7] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T1, 1),
1267 _mm512_extractf64x4_pd(T3, 1), 0x31);
1268
1269 tmp.packet[8] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0),
1270 _mm512_extractf64x4_pd(T6, 0), 0x20);
1271 tmp.packet[9] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0),
1272 _mm512_extractf64x4_pd(T7, 0), 0x20);
1273 tmp.packet[10] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 0),
1274 _mm512_extractf64x4_pd(T6, 0), 0x31);
1275 tmp.packet[11] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 0),
1276 _mm512_extractf64x4_pd(T7, 0), 0x31);
1277
1278 tmp.packet[12] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1),
1279 _mm512_extractf64x4_pd(T6, 1), 0x20);
1280 tmp.packet[13] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1),
1281 _mm512_extractf64x4_pd(T7, 1), 0x20);
1282 tmp.packet[14] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T4, 1),
1283 _mm512_extractf64x4_pd(T6, 1), 0x31);
1284 tmp.packet[15] = _mm256_permute2f128_pd(_mm512_extractf64x4_pd(T5, 1),
1285 _mm512_extractf64x4_pd(T7, 1), 0x31);
1286
1287 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 0, 8);
1288 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 1, 8);
1289 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 2, 8);
1290 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 3, 8);
1291
1292 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 4, 8);
1293 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 5, 8);
1294 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 6, 8);
1295 PACK_OUTPUT_SQ_D(kernel.packet, tmp.packet, 7, 8);
1296}
1297template <>
1298EIGEN_STRONG_INLINE Packet16f pblend(const Selector<16>& /*ifPacket*/,
1299 const Packet16f& /*thenPacket*/,
1300 const Packet16f& /*elsePacket*/) {
1301 assert(false && "To be implemented");
1302 return Packet16f();
1303}
1304template <>
1305EIGEN_STRONG_INLINE Packet8d pblend(const Selector<8>& /*ifPacket*/,
1306 const Packet8d& /*thenPacket*/,
1307 const Packet8d& /*elsePacket*/) {
1308 assert(false && "To be implemented");
1309 return Packet8d();
1310}
1311
1312} // end namespace internal
1313
1314} // end namespace Eigen
1315
1316#endif // EIGEN_PACKET_MATH_AVX512_H
Eigen::Aligned64
@ Aligned64
Definition: Constants.h:232
Eigen
Namespace containing all symbols from the Eigen library.
Definition: Core:287
Eigen::Index
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:33