10#ifndef EIGEN_PACKET_MATH_ALTIVEC_H
11#define EIGEN_PACKET_MATH_ALTIVEC_H
17#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD
18#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 4
21#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD
22#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD
25#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD
26#define EIGEN_HAS_SINGLE_INSTRUCTION_CJMADD
30#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS
31#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32
34typedef __vector
float Packet4f;
35typedef __vector
int Packet4i;
36typedef __vector
unsigned int Packet4ui;
37typedef __vector __bool
int Packet4bi;
38typedef __vector
short int Packet8i;
39typedef __vector
unsigned char Packet16uc;
44#define _EIGEN_DECLARE_CONST_FAST_Packet4f(NAME,X) \
45 Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(vec_splat_s32(X))
47#define _EIGEN_DECLARE_CONST_FAST_Packet4i(NAME,X) \
48 Packet4i p4i_##NAME = vec_splat_s32(X)
50#define _EIGEN_DECLARE_CONST_Packet4f(NAME,X) \
51 Packet4f p4f_##NAME = pset1<Packet4f>(X)
53#define _EIGEN_DECLARE_CONST_Packet4i(NAME,X) \
54 Packet4i p4i_##NAME = pset1<Packet4i>(X)
56#define _EIGEN_DECLARE_CONST_Packet2d(NAME,X) \
57 Packet2d p2d_##NAME = pset1<Packet2d>(X)
59#define _EIGEN_DECLARE_CONST_Packet2l(NAME,X) \
60 Packet2l p2l_##NAME = pset1<Packet2l>(X)
62#define _EIGEN_DECLARE_CONST_Packet4f_FROM_INT(NAME,X) \
63 const Packet4f p4f_##NAME = reinterpret_cast<Packet4f>(pset1<Packet4i>(X))
66#define DST_CTRL(size, count, stride) (((size) << 24) | ((count) << 16) | (stride))
70static _EIGEN_DECLARE_CONST_FAST_Packet4f(ZERO, 0);
71static _EIGEN_DECLARE_CONST_FAST_Packet4i(ZERO, 0);
72static _EIGEN_DECLARE_CONST_FAST_Packet4i(ONE,1);
73static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS16,-16);
74static _EIGEN_DECLARE_CONST_FAST_Packet4i(MINUS1,-1);
75static Packet4f p4f_ZERO_ = (Packet4f) vec_sl((Packet4ui)p4i_MINUS1, (Packet4ui)p4i_MINUS1);
77static Packet4f p4f_ONE = vec_ctf(p4i_ONE, 0);
80static Packet4f p4f_COUNTDOWN = { 0.0, 1.0, 2.0, 3.0 };
81static Packet4i p4i_COUNTDOWN = { 0, 1, 2, 3 };
83static Packet16uc p16uc_REVERSE32 = { 12,13,14,15, 8,9,10,11, 4,5,6,7, 0,1,2,3 };
84static Packet16uc p16uc_DUPLICATE32_HI = { 0,1,2,3, 0,1,2,3, 4,5,6,7, 4,5,6,7 };
88#define _EIGEN_MASK_ALIGNMENT 0xfffffffffffffff0
90#define _EIGEN_MASK_ALIGNMENT 0xfffffff0
93#define _EIGEN_ALIGNED_PTR(x) ((ptrdiff_t)(x) & _EIGEN_MASK_ALIGNMENT)
98static Packet16uc p16uc_FORWARD = vec_lvsl(0, (
float*)0);
100static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
102static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);
103static Packet16uc p16uc_PSET32_WEVEN = vec_sld(p16uc_DUPLICATE32_HI, (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);
104static Packet16uc p16uc_HALF64_0_16 = vec_sld((Packet16uc)p4i_ZERO, vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 3), 8);
106static Packet16uc p16uc_FORWARD = p16uc_REVERSE32;
107static Packet16uc p16uc_REVERSE64 = { 8,9,10,11, 12,13,14,15, 0,1,2,3, 4,5,6,7 };
108static Packet16uc p16uc_PSET32_WODD = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 1), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 3), 8);
109static Packet16uc p16uc_PSET32_WEVEN = vec_sld((Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 0), (Packet16uc) vec_splat((Packet4ui)p16uc_FORWARD, 2), 8);
110static Packet16uc p16uc_HALF64_0_16 = vec_sld(vec_splat((Packet16uc) vec_abs(p4i_MINUS16), 0), (Packet16uc)p4i_ZERO, 8);
113static Packet16uc p16uc_PSET64_HI = (Packet16uc) vec_mergeh((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN);
114static Packet16uc p16uc_PSET64_LO = (Packet16uc) vec_mergel((Packet4ui)p16uc_PSET32_WODD, (Packet4ui)p16uc_PSET32_WEVEN);
115static Packet16uc p16uc_TRANSPOSE64_HI = p16uc_PSET64_HI + p16uc_HALF64_0_16;
116static Packet16uc p16uc_TRANSPOSE64_LO = p16uc_PSET64_LO + p16uc_HALF64_0_16;
118static Packet16uc p16uc_COMPLEX32_REV = vec_sld(p16uc_REVERSE32, p16uc_REVERSE32, 8);
121static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_FORWARD, p16uc_FORWARD, 8);
123static Packet16uc p16uc_COMPLEX32_REV2 = vec_sld(p16uc_PSET64_HI, p16uc_PSET64_LO, 8);
126#if EIGEN_HAS_BUILTIN(__builtin_prefetch) || EIGEN_COMP_GNUC
127 #define EIGEN_PPC_PREFETCH(ADDR) __builtin_prefetch(ADDR);
129 #define EIGEN_PPC_PREFETCH(ADDR) asm( " dcbt [%[addr]]\n" :: [addr] "r" (ADDR) : "cc" );
132template<>
struct packet_traits<float> : default_packet_traits
134 typedef Packet4f type;
135 typedef Packet4f half;
171template<>
struct packet_traits<int> : default_packet_traits
173 typedef Packet4i type;
174 typedef Packet4i half;
190template<>
struct unpacket_traits<Packet4f> {
typedef float type;
enum {size=4, alignment=
Aligned16};
typedef Packet4f half; };
191template<>
struct unpacket_traits<Packet4i> {
typedef int type;
enum {size=4, alignment=
Aligned16};
typedef Packet4i half; };
193inline std::ostream & operator <<(std::ostream & s,
const Packet16uc & v)
200 for (
int i=0; i< 16; i++)
201 s << (
int)vt.n[i] <<
", ";
205inline std::ostream & operator <<(std::ostream & s,
const Packet4f & v)
212 s << vt.n[0] <<
", " << vt.n[1] <<
", " << vt.n[2] <<
", " << vt.n[3];
216inline std::ostream & operator <<(std::ostream & s,
const Packet4i & v)
223 s << vt.n[0] <<
", " << vt.n[1] <<
", " << vt.n[2] <<
", " << vt.n[3];
227inline std::ostream & operator <<(std::ostream & s,
const Packet4ui & v)
234 s << vt.n[0] <<
", " << vt.n[1] <<
", " << vt.n[2] <<
", " << vt.n[3];
239template<> EIGEN_STRONG_INLINE Packet4f pload<Packet4f>(
const float* from)
241 EIGEN_DEBUG_ALIGNED_LOAD
243 return vec_vsx_ld(0, from);
245 return vec_ld(0, from);
249template<> EIGEN_STRONG_INLINE Packet4i pload<Packet4i>(
const int* from)
251 EIGEN_DEBUG_ALIGNED_LOAD
253 return vec_vsx_ld(0, from);
255 return vec_ld(0, from);
259template<> EIGEN_STRONG_INLINE
void pstore<float>(
float* to,
const Packet4f& from)
261 EIGEN_DEBUG_ALIGNED_STORE
263 vec_vsx_st(from, 0, to);
269template<> EIGEN_STRONG_INLINE
void pstore<int>(
int* to,
const Packet4i& from)
271 EIGEN_DEBUG_ALIGNED_STORE
273 vec_vsx_st(from, 0, to);
279template<> EIGEN_STRONG_INLINE Packet4f pset1<Packet4f>(
const float& from) {
280 Packet4f v = {from, from, from, from};
284template<> EIGEN_STRONG_INLINE Packet4i pset1<Packet4i>(
const int& from) {
285 Packet4i v = {from, from, from, from};
288template<> EIGEN_STRONG_INLINE
void
289pbroadcast4<Packet4f>(
const float *a,
290 Packet4f& a0, Packet4f& a1, Packet4f& a2, Packet4f& a3)
292 a3 = pload<Packet4f>(a);
293 a0 = vec_splat(a3, 0);
294 a1 = vec_splat(a3, 1);
295 a2 = vec_splat(a3, 2);
296 a3 = vec_splat(a3, 3);
298template<> EIGEN_STRONG_INLINE
void
299pbroadcast4<Packet4i>(
const int *a,
300 Packet4i& a0, Packet4i& a1, Packet4i& a2, Packet4i& a3)
302 a3 = pload<Packet4i>(a);
303 a0 = vec_splat(a3, 0);
304 a1 = vec_splat(a3, 1);
305 a2 = vec_splat(a3, 2);
306 a3 = vec_splat(a3, 3);
309template<> EIGEN_DEVICE_FUNC
inline Packet4f pgather<float, Packet4f>(
const float* from,
Index stride)
311 float EIGEN_ALIGN16 af[4];
312 af[0] = from[0*stride];
313 af[1] = from[1*stride];
314 af[2] = from[2*stride];
315 af[3] = from[3*stride];
316 return pload<Packet4f>(af);
318template<> EIGEN_DEVICE_FUNC
inline Packet4i pgather<int, Packet4i>(
const int* from,
Index stride)
320 int EIGEN_ALIGN16 ai[4];
321 ai[0] = from[0*stride];
322 ai[1] = from[1*stride];
323 ai[2] = from[2*stride];
324 ai[3] = from[3*stride];
325 return pload<Packet4i>(ai);
327template<> EIGEN_DEVICE_FUNC
inline void pscatter<float, Packet4f>(
float* to,
const Packet4f& from,
Index stride)
329 float EIGEN_ALIGN16 af[4];
330 pstore<float>(af, from);
331 to[0*stride] = af[0];
332 to[1*stride] = af[1];
333 to[2*stride] = af[2];
334 to[3*stride] = af[3];
336template<> EIGEN_DEVICE_FUNC
inline void pscatter<int, Packet4i>(
int* to,
const Packet4i& from,
Index stride)
338 int EIGEN_ALIGN16 ai[4];
339 pstore<int>((
int *)ai, from);
340 to[0*stride] = ai[0];
341 to[1*stride] = ai[1];
342 to[2*stride] = ai[2];
343 to[3*stride] = ai[3];
346template<> EIGEN_STRONG_INLINE Packet4f plset<Packet4f>(
const float& a) {
return pset1<Packet4f>(a) + p4f_COUNTDOWN; }
347template<> EIGEN_STRONG_INLINE Packet4i plset<Packet4i>(
const int& a) {
return pset1<Packet4i>(a) + p4i_COUNTDOWN; }
349template<> EIGEN_STRONG_INLINE Packet4f padd<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return a + b; }
350template<> EIGEN_STRONG_INLINE Packet4i padd<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return a + b; }
352template<> EIGEN_STRONG_INLINE Packet4f psub<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return a - b; }
353template<> EIGEN_STRONG_INLINE Packet4i psub<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return a - b; }
355template<> EIGEN_STRONG_INLINE Packet4f pnegate(
const Packet4f& a) {
return p4f_ZERO - a; }
356template<> EIGEN_STRONG_INLINE Packet4i pnegate(
const Packet4i& a) {
return p4i_ZERO - a; }
358template<> EIGEN_STRONG_INLINE Packet4f pconj(
const Packet4f& a) {
return a; }
359template<> EIGEN_STRONG_INLINE Packet4i pconj(
const Packet4i& a) {
return a; }
361template<> EIGEN_STRONG_INLINE Packet4f pmul<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_madd(a,b, p4f_ZERO); }
362template<> EIGEN_STRONG_INLINE Packet4i pmul<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return a * b; }
364template<> EIGEN_STRONG_INLINE Packet4f pdiv<Packet4f>(
const Packet4f& a,
const Packet4f& b)
367 Packet4f t, y_0, y_1;
373 t = vec_nmsub(y_0, b, p4f_ONE);
374 y_1 = vec_madd(y_0, t, y_0);
376 return vec_madd(a, y_1, p4f_ZERO);
378 return vec_div(a, b);
382template<> EIGEN_STRONG_INLINE Packet4i pdiv<Packet4i>(
const Packet4i& ,
const Packet4i& )
383{ eigen_assert(
false &&
"packet integer division are not supported by AltiVec");
384 return pset1<Packet4i>(0);
388template<> EIGEN_STRONG_INLINE Packet4f pmadd(
const Packet4f& a,
const Packet4f& b,
const Packet4f& c) {
return vec_madd(a,b,c); }
389template<> EIGEN_STRONG_INLINE Packet4i pmadd(
const Packet4i& a,
const Packet4i& b,
const Packet4i& c) {
return a*b + c; }
391template<> EIGEN_STRONG_INLINE Packet4f pmin<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_min(a, b); }
392template<> EIGEN_STRONG_INLINE Packet4i pmin<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_min(a, b); }
394template<> EIGEN_STRONG_INLINE Packet4f pmax<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_max(a, b); }
395template<> EIGEN_STRONG_INLINE Packet4i pmax<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_max(a, b); }
397template<> EIGEN_STRONG_INLINE Packet4f pand<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_and(a, b); }
398template<> EIGEN_STRONG_INLINE Packet4i pand<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_and(a, b); }
400template<> EIGEN_STRONG_INLINE Packet4f por<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_or(a, b); }
401template<> EIGEN_STRONG_INLINE Packet4i por<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_or(a, b); }
403template<> EIGEN_STRONG_INLINE Packet4f pxor<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_xor(a, b); }
404template<> EIGEN_STRONG_INLINE Packet4i pxor<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_xor(a, b); }
406template<> EIGEN_STRONG_INLINE Packet4f pandnot<Packet4f>(
const Packet4f& a,
const Packet4f& b) {
return vec_and(a, vec_nor(b, b)); }
407template<> EIGEN_STRONG_INLINE Packet4i pandnot<Packet4i>(
const Packet4i& a,
const Packet4i& b) {
return vec_and(a, vec_nor(b, b)); }
409template<> EIGEN_STRONG_INLINE Packet4f pround<Packet4f>(
const Packet4f& a) {
return vec_round(a); }
410template<> EIGEN_STRONG_INLINE Packet4f pceil<Packet4f>(
const Packet4f& a) {
return vec_ceil(a); }
411template<> EIGEN_STRONG_INLINE Packet4f pfloor<Packet4f>(
const Packet4f& a) {
return vec_floor(a); }
414template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from)
416 EIGEN_DEBUG_ALIGNED_LOAD
419 MSQ = vec_ld(0, (
unsigned char *)from);
420 LSQ = vec_ld(15, (
unsigned char *)from);
421 mask = vec_lvsl(0, from);
422 return (Packet4f) vec_perm(MSQ, LSQ, mask);
425template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int* from)
427 EIGEN_DEBUG_ALIGNED_LOAD
431 MSQ = vec_ld(0, (
unsigned char *)from);
432 LSQ = vec_ld(15, (
unsigned char *)from);
433 mask = vec_lvsl(0, from);
434 return (Packet4i) vec_perm(MSQ, LSQ, mask);
438template<> EIGEN_STRONG_INLINE Packet4i ploadu<Packet4i>(
const int* from)
440 EIGEN_DEBUG_UNALIGNED_LOAD
441 return (Packet4i) vec_vsx_ld((
long)from & 15, (
const int*) _EIGEN_ALIGNED_PTR(from));
443template<> EIGEN_STRONG_INLINE Packet4f ploadu<Packet4f>(
const float* from)
445 EIGEN_DEBUG_UNALIGNED_LOAD
446 return (Packet4f) vec_vsx_ld((
long)from & 15, (
const float*) _EIGEN_ALIGNED_PTR(from));
450template<> EIGEN_STRONG_INLINE Packet4f ploaddup<Packet4f>(
const float* from)
453 if((ptrdiff_t(from) % 16) == 0) p = pload<Packet4f>(from);
454 else p = ploadu<Packet4f>(from);
455 return vec_perm(p, p, p16uc_DUPLICATE32_HI);
457template<> EIGEN_STRONG_INLINE Packet4i ploaddup<Packet4i>(
const int* from)
460 if((ptrdiff_t(from) % 16) == 0) p = pload<Packet4i>(from);
461 else p = ploadu<Packet4i>(from);
462 return vec_perm(p, p, p16uc_DUPLICATE32_HI);
466template<> EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from)
468 EIGEN_DEBUG_UNALIGNED_STORE
471 Packet16uc MSQ, LSQ, edges;
472 Packet16uc edgeAlign, align;
474 MSQ = vec_ld(0, (
unsigned char *)to);
475 LSQ = vec_ld(15, (
unsigned char *)to);
476 edgeAlign = vec_lvsl(0, to);
477 edges=vec_perm(LSQ,MSQ,edgeAlign);
478 align = vec_lvsr( 0, to );
479 MSQ = vec_perm(edges,(Packet16uc)from,align);
480 LSQ = vec_perm((Packet16uc)from,edges,align);
481 vec_st( LSQ, 15, (
unsigned char *)to );
482 vec_st( MSQ, 0, (
unsigned char *)to );
484template<> EIGEN_STRONG_INLINE
void pstoreu<int>(
int* to,
const Packet4i& from)
486 EIGEN_DEBUG_UNALIGNED_STORE
489 Packet16uc MSQ, LSQ, edges;
490 Packet16uc edgeAlign, align;
492 MSQ = vec_ld(0, (
unsigned char *)to);
493 LSQ = vec_ld(15, (
unsigned char *)to);
494 edgeAlign = vec_lvsl(0, to);
495 edges=vec_perm(LSQ, MSQ, edgeAlign);
496 align = vec_lvsr( 0, to );
497 MSQ = vec_perm(edges, (Packet16uc) from, align);
498 LSQ = vec_perm((Packet16uc) from, edges, align);
499 vec_st( LSQ, 15, (
unsigned char *)to );
500 vec_st( MSQ, 0, (
unsigned char *)to );
504template<> EIGEN_STRONG_INLINE
void pstoreu<int>(
int* to,
const Packet4i& from)
506 EIGEN_DEBUG_ALIGNED_STORE
507 vec_vsx_st(from, (
long)to & 15, (
int*) _EIGEN_ALIGNED_PTR(to));
509template<> EIGEN_STRONG_INLINE
void pstoreu<float>(
float* to,
const Packet4f& from)
511 EIGEN_DEBUG_ALIGNED_STORE
512 vec_vsx_st(from, (
long)to & 15, (
float*) _EIGEN_ALIGNED_PTR(to));
516template<> EIGEN_STRONG_INLINE
void prefetch<float>(
const float* addr) { EIGEN_PPC_PREFETCH(addr); }
517template<> EIGEN_STRONG_INLINE
void prefetch<int>(
const int* addr) { EIGEN_PPC_PREFETCH(addr); }
519template<> EIGEN_STRONG_INLINE
float pfirst<Packet4f>(
const Packet4f& a) {
float EIGEN_ALIGN16 x; vec_ste(a, 0, &x);
return x; }
520template<> EIGEN_STRONG_INLINE
int pfirst<Packet4i>(
const Packet4i& a) {
int EIGEN_ALIGN16 x; vec_ste(a, 0, &x);
return x; }
522template<> EIGEN_STRONG_INLINE Packet4f preverse(
const Packet4f& a)
524 return reinterpret_cast<Packet4f
>(vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE32));
526template<> EIGEN_STRONG_INLINE Packet4i preverse(
const Packet4i& a)
528 return reinterpret_cast<Packet4i
>(vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE32)); }
530template<> EIGEN_STRONG_INLINE Packet4f pabs(
const Packet4f& a) {
return vec_abs(a); }
531template<> EIGEN_STRONG_INLINE Packet4i pabs(
const Packet4i& a) {
return vec_abs(a); }
533template<> EIGEN_STRONG_INLINE
float predux<Packet4f>(
const Packet4f& a)
536 b = vec_sld(a, a, 8);
538 b = vec_sld(sum, sum, 4);
543template<> EIGEN_STRONG_INLINE Packet4f preduxp<Packet4f>(
const Packet4f* vecs)
545 Packet4f v[4], sum[4];
550 v[0] = vec_mergeh(vecs[0], vecs[2]);
551 v[1] = vec_mergel(vecs[0], vecs[2]);
552 v[2] = vec_mergeh(vecs[1], vecs[3]);
553 v[3] = vec_mergel(vecs[1], vecs[3]);
555 sum[0] = vec_mergeh(v[0], v[2]);
556 sum[1] = vec_mergel(v[0], v[2]);
557 sum[2] = vec_mergeh(v[1], v[3]);
558 sum[3] = vec_mergel(v[1], v[3]);
562 sum[0] = sum[0] + sum[1];
564 sum[1] = sum[2] + sum[3];
566 sum[0] = sum[0] + sum[1];
571template<> EIGEN_STRONG_INLINE
int predux<Packet4i>(
const Packet4i& a)
574 sum = vec_sums(a, p4i_ZERO);
576 sum = vec_sld(sum, p4i_ZERO, 12);
578 sum = vec_sld(p4i_ZERO, sum, 4);
583template<> EIGEN_STRONG_INLINE Packet4i preduxp<Packet4i>(
const Packet4i* vecs)
585 Packet4i v[4], sum[4];
590 v[0] = vec_mergeh(vecs[0], vecs[2]);
591 v[1] = vec_mergel(vecs[0], vecs[2]);
592 v[2] = vec_mergeh(vecs[1], vecs[3]);
593 v[3] = vec_mergel(vecs[1], vecs[3]);
595 sum[0] = vec_mergeh(v[0], v[2]);
596 sum[1] = vec_mergel(v[0], v[2]);
597 sum[2] = vec_mergeh(v[1], v[3]);
598 sum[3] = vec_mergel(v[1], v[3]);
602 sum[0] = sum[0] + sum[1];
604 sum[1] = sum[2] + sum[3];
606 sum[0] = sum[0] + sum[1];
613template<> EIGEN_STRONG_INLINE
float predux_mul<Packet4f>(
const Packet4f& a)
616 prod = pmul(a, vec_sld(a, a, 8));
617 return pfirst(pmul(prod, vec_sld(prod, prod, 4)));
620template<> EIGEN_STRONG_INLINE
int predux_mul<Packet4i>(
const Packet4i& a)
622 EIGEN_ALIGN16
int aux[4];
624 return aux[0] * aux[1] * aux[2] * aux[3];
628template<> EIGEN_STRONG_INLINE
float predux_min<Packet4f>(
const Packet4f& a)
631 b = vec_min(a, vec_sld(a, a, 8));
632 res = vec_min(b, vec_sld(b, b, 4));
636template<> EIGEN_STRONG_INLINE
int predux_min<Packet4i>(
const Packet4i& a)
639 b = vec_min(a, vec_sld(a, a, 8));
640 res = vec_min(b, vec_sld(b, b, 4));
645template<> EIGEN_STRONG_INLINE
float predux_max<Packet4f>(
const Packet4f& a)
648 b = vec_max(a, vec_sld(a, a, 8));
649 res = vec_max(b, vec_sld(b, b, 4));
653template<> EIGEN_STRONG_INLINE
int predux_max<Packet4i>(
const Packet4i& a)
656 b = vec_max(a, vec_sld(a, a, 8));
657 res = vec_max(b, vec_sld(b, b, 4));
662struct palign_impl<Offset,Packet4f>
664 static EIGEN_STRONG_INLINE
void run(Packet4f& first,
const Packet4f& second)
667 switch (Offset % 4) {
669 first = vec_sld(first, second, 4);
break;
671 first = vec_sld(first, second, 8);
break;
673 first = vec_sld(first, second, 12);
break;
676 switch (Offset % 4) {
678 first = vec_sld(second, first, 12);
break;
680 first = vec_sld(second, first, 8);
break;
682 first = vec_sld(second, first, 4);
break;
689struct palign_impl<Offset,Packet4i>
691 static EIGEN_STRONG_INLINE
void run(Packet4i& first,
const Packet4i& second)
694 switch (Offset % 4) {
696 first = vec_sld(first, second, 4);
break;
698 first = vec_sld(first, second, 8);
break;
700 first = vec_sld(first, second, 12);
break;
703 switch (Offset % 4) {
705 first = vec_sld(second, first, 12);
break;
707 first = vec_sld(second, first, 8);
break;
709 first = vec_sld(second, first, 4);
break;
715EIGEN_DEVICE_FUNC
inline void
716ptranspose(PacketBlock<Packet4f,4>& kernel) {
717 Packet4f t0, t1, t2, t3;
718 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
719 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
720 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
721 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
722 kernel.packet[0] = vec_mergeh(t0, t2);
723 kernel.packet[1] = vec_mergel(t0, t2);
724 kernel.packet[2] = vec_mergeh(t1, t3);
725 kernel.packet[3] = vec_mergel(t1, t3);
728EIGEN_DEVICE_FUNC
inline void
729ptranspose(PacketBlock<Packet4i,4>& kernel) {
730 Packet4i t0, t1, t2, t3;
731 t0 = vec_mergeh(kernel.packet[0], kernel.packet[2]);
732 t1 = vec_mergel(kernel.packet[0], kernel.packet[2]);
733 t2 = vec_mergeh(kernel.packet[1], kernel.packet[3]);
734 t3 = vec_mergel(kernel.packet[1], kernel.packet[3]);
735 kernel.packet[0] = vec_mergeh(t0, t2);
736 kernel.packet[1] = vec_mergel(t0, t2);
737 kernel.packet[2] = vec_mergeh(t1, t3);
738 kernel.packet[3] = vec_mergel(t1, t3);
741template<> EIGEN_STRONG_INLINE Packet4i pblend(
const Selector<4>& ifPacket,
const Packet4i& thenPacket,
const Packet4i& elsePacket) {
742 Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] };
743 Packet4ui mask =
reinterpret_cast<Packet4ui
>(vec_cmpeq(
reinterpret_cast<Packet4ui
>(select),
reinterpret_cast<Packet4ui
>(p4i_ONE)));
744 return vec_sel(elsePacket, thenPacket, mask);
747template<> EIGEN_STRONG_INLINE Packet4f pblend(
const Selector<4>& ifPacket,
const Packet4f& thenPacket,
const Packet4f& elsePacket) {
748 Packet4ui select = { ifPacket.select[0], ifPacket.select[1], ifPacket.select[2], ifPacket.select[3] };
749 Packet4ui mask =
reinterpret_cast<Packet4ui
>(vec_cmpeq(
reinterpret_cast<Packet4ui
>(select),
reinterpret_cast<Packet4ui
>(p4i_ONE)));
750 return vec_sel(elsePacket, thenPacket, mask);
756typedef __vector
double Packet2d;
757typedef __vector
unsigned long long Packet2ul;
758typedef __vector
long long Packet2l;
760typedef Packet2ul Packet2bl;
762typedef __vector __bool
long Packet2bl;
765static Packet2l p2l_ONE = { 1, 1 };
766static Packet2l p2l_ZERO =
reinterpret_cast<Packet2l
>(p4i_ZERO);
767static Packet2d p2d_ONE = { 1.0, 1.0 };
768static Packet2d p2d_ZERO =
reinterpret_cast<Packet2d
>(p4f_ZERO);
769static Packet2d p2d_ZERO_ = { -0.0, -0.0 };
772static Packet2d p2d_COUNTDOWN =
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(p2d_ZERO),
reinterpret_cast<Packet4f
>(p2d_ONE), 8));
774static Packet2d p2d_COUNTDOWN =
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(p2d_ONE),
reinterpret_cast<Packet4f
>(p2d_ZERO), 8));
777template<
int index> Packet2d vec_splat_dbl(Packet2d& a);
779template<> EIGEN_STRONG_INLINE Packet2d vec_splat_dbl<0>(Packet2d& a)
781 return reinterpret_cast<Packet2d
>(vec_perm(a, a, p16uc_PSET64_HI));
784template<> EIGEN_STRONG_INLINE Packet2d vec_splat_dbl<1>(Packet2d& a)
786 return reinterpret_cast<Packet2d
>(vec_perm(a, a, p16uc_PSET64_LO));
789template<>
struct packet_traits<double> : default_packet_traits
791 typedef Packet2d type;
792 typedef Packet2d half;
820template<>
struct unpacket_traits<Packet2d> {
typedef double type;
enum {size=2, alignment=
Aligned16};
typedef Packet2d half; };
822inline std::ostream & operator <<(std::ostream & s,
const Packet2l & v)
829 s << vt.n[0] <<
", " << vt.n[1];
833inline std::ostream & operator <<(std::ostream & s,
const Packet2d & v)
840 s << vt.n[0] <<
", " << vt.n[1];
845template<> EIGEN_STRONG_INLINE Packet2d pload<Packet2d>(
const double* from)
847 EIGEN_DEBUG_ALIGNED_LOAD
849 return vec_vsx_ld(0, from);
851 return vec_ld(0, from);
855template<> EIGEN_STRONG_INLINE
void pstore<double>(
double* to,
const Packet2d& from)
857 EIGEN_DEBUG_ALIGNED_STORE
859 vec_vsx_st(from, 0, to);
865template<> EIGEN_STRONG_INLINE Packet2d pset1<Packet2d>(
const double& from) {
866 Packet2d v = {from, from};
870template<> EIGEN_STRONG_INLINE
void
871pbroadcast4<Packet2d>(
const double *a,
872 Packet2d& a0, Packet2d& a1, Packet2d& a2, Packet2d& a3)
874 a1 = pload<Packet2d>(a);
875 a0 = vec_splat_dbl<0>(a1);
876 a1 = vec_splat_dbl<1>(a1);
877 a3 = pload<Packet2d>(a+2);
878 a2 = vec_splat_dbl<0>(a3);
879 a3 = vec_splat_dbl<1>(a3);
882template<> EIGEN_DEVICE_FUNC
inline Packet2d pgather<double, Packet2d>(
const double* from,
Index stride)
884 double EIGEN_ALIGN16 af[2];
885 af[0] = from[0*stride];
886 af[1] = from[1*stride];
887 return pload<Packet2d>(af);
889template<> EIGEN_DEVICE_FUNC
inline void pscatter<double, Packet2d>(
double* to,
const Packet2d& from,
Index stride)
891 double EIGEN_ALIGN16 af[2];
892 pstore<double>(af, from);
893 to[0*stride] = af[0];
894 to[1*stride] = af[1];
897template<> EIGEN_STRONG_INLINE Packet2d plset<Packet2d>(
const double& a) {
return pset1<Packet2d>(a) + p2d_COUNTDOWN; }
899template<> EIGEN_STRONG_INLINE Packet2d padd<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return a + b; }
901template<> EIGEN_STRONG_INLINE Packet2d psub<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return a - b; }
903template<> EIGEN_STRONG_INLINE Packet2d pnegate(
const Packet2d& a) {
return p2d_ZERO - a; }
905template<> EIGEN_STRONG_INLINE Packet2d pconj(
const Packet2d& a) {
return a; }
907template<> EIGEN_STRONG_INLINE Packet2d pmul<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_madd(a,b,p2d_ZERO); }
908template<> EIGEN_STRONG_INLINE Packet2d pdiv<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_div(a,b); }
911template<> EIGEN_STRONG_INLINE Packet2d pmadd(
const Packet2d& a,
const Packet2d& b,
const Packet2d& c) {
return vec_madd(a, b, c); }
913template<> EIGEN_STRONG_INLINE Packet2d pmin<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_min(a, b); }
915template<> EIGEN_STRONG_INLINE Packet2d pmax<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_max(a, b); }
917template<> EIGEN_STRONG_INLINE Packet2d pand<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_and(a, b); }
919template<> EIGEN_STRONG_INLINE Packet2d por<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_or(a, b); }
921template<> EIGEN_STRONG_INLINE Packet2d pxor<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_xor(a, b); }
923template<> EIGEN_STRONG_INLINE Packet2d pandnot<Packet2d>(
const Packet2d& a,
const Packet2d& b) {
return vec_and(a, vec_nor(b, b)); }
925template<> EIGEN_STRONG_INLINE Packet2d pround<Packet2d>(
const Packet2d& a) {
return vec_round(a); }
926template<> EIGEN_STRONG_INLINE Packet2d pceil<Packet2d>(
const Packet2d& a) {
return vec_ceil(a); }
927template<> EIGEN_STRONG_INLINE Packet2d pfloor<Packet2d>(
const Packet2d& a) {
return vec_floor(a); }
929template<> EIGEN_STRONG_INLINE Packet2d ploadu<Packet2d>(
const double* from)
931 EIGEN_DEBUG_ALIGNED_LOAD
932 return (Packet2d) vec_vsx_ld((
long)from & 15, (
const double*) _EIGEN_ALIGNED_PTR(from));
935template<> EIGEN_STRONG_INLINE Packet2d ploaddup<Packet2d>(
const double* from)
938 if((ptrdiff_t(from) % 16) == 0) p = pload<Packet2d>(from);
939 else p = ploadu<Packet2d>(from);
940 return vec_splat_dbl<0>(p);
943template<> EIGEN_STRONG_INLINE
void pstoreu<double>(
double* to,
const Packet2d& from)
945 EIGEN_DEBUG_ALIGNED_STORE
946 vec_vsx_st((Packet4f)from, (
long)to & 15, (
float*) _EIGEN_ALIGNED_PTR(to));
949template<> EIGEN_STRONG_INLINE
void prefetch<double>(
const double* addr) { EIGEN_PPC_PREFETCH(addr); }
951template<> EIGEN_STRONG_INLINE
double pfirst<Packet2d>(
const Packet2d& a) {
double EIGEN_ALIGN16 x[2]; pstore<double>(x, a);
return x[0]; }
953template<> EIGEN_STRONG_INLINE Packet2d preverse(
const Packet2d& a)
955 return reinterpret_cast<Packet2d
>(vec_perm(
reinterpret_cast<Packet16uc
>(a),
reinterpret_cast<Packet16uc
>(a), p16uc_REVERSE64));
957template<> EIGEN_STRONG_INLINE Packet2d pabs(
const Packet2d& a) {
return vec_abs(a); }
959template<> EIGEN_STRONG_INLINE
double predux<Packet2d>(
const Packet2d& a)
962 b =
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(a),
reinterpret_cast<Packet4f
>(a), 8));
964 return pfirst<Packet2d>(sum);
967template<> EIGEN_STRONG_INLINE Packet2d preduxp<Packet2d>(
const Packet2d* vecs)
970 v[0] = vecs[0] +
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(vecs[0]),
reinterpret_cast<Packet4f
>(vecs[0]), 8));
971 v[1] = vecs[1] +
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(vecs[1]),
reinterpret_cast<Packet4f
>(vecs[1]), 8));
974 sum =
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(v[0]),
reinterpret_cast<Packet4f
>(v[1]), 8));
976 sum =
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4f
>(v[1]),
reinterpret_cast<Packet4f
>(v[0]), 8));
983template<> EIGEN_STRONG_INLINE
double predux_mul<Packet2d>(
const Packet2d& a)
985 return pfirst(pmul(a,
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4ui
>(a),
reinterpret_cast<Packet4ui
>(a), 8))));
989template<> EIGEN_STRONG_INLINE
double predux_min<Packet2d>(
const Packet2d& a)
991 return pfirst(pmin(a,
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4ui
>(a),
reinterpret_cast<Packet4ui
>(a), 8))));
995template<> EIGEN_STRONG_INLINE
double predux_max<Packet2d>(
const Packet2d& a)
997 return pfirst(pmax(a,
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4ui
>(a),
reinterpret_cast<Packet4ui
>(a), 8))));
1001struct palign_impl<Offset,Packet2d>
1003 static EIGEN_STRONG_INLINE
void run(Packet2d& first,
const Packet2d& second)
1007 first =
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4ui
>(first),
reinterpret_cast<Packet4ui
>(second), 8));
1009 first =
reinterpret_cast<Packet2d
>(vec_sld(
reinterpret_cast<Packet4ui
>(second),
reinterpret_cast<Packet4ui
>(first), 8));
1014EIGEN_DEVICE_FUNC
inline void
1015ptranspose(PacketBlock<Packet2d,2>& kernel) {
1017 t0 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_HI);
1018 t1 = vec_perm(kernel.packet[0], kernel.packet[1], p16uc_TRANSPOSE64_LO);
1019 kernel.packet[0] = t0;
1020 kernel.packet[1] = t1;
1023template<> EIGEN_STRONG_INLINE Packet2d pblend(
const Selector<2>& ifPacket,
const Packet2d& thenPacket,
const Packet2d& elsePacket) {
1024 Packet2l select = { ifPacket.select[0], ifPacket.select[1] };
1025 Packet2bl mask = vec_cmpeq(
reinterpret_cast<Packet2d
>(select),
reinterpret_cast<Packet2d
>(p2l_ONE));
1026 return vec_sel(elsePacket, thenPacket, mask);
@ Aligned16
Definition: Constants.h:230
Namespace containing all symbols from the Eigen library.
Definition: Core:287
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:33