11 #ifndef EIGEN_GENERAL_PRODUCT_H
12 #define EIGEN_GENERAL_PRODUCT_H
14 #include "./InternalHeaderCheck.h"
28 #ifndef EIGEN_GEMM_TO_COEFFBASED_THRESHOLD
30 #define EIGEN_GEMM_TO_COEFFBASED_THRESHOLD 20
35 template<
int Rows,
int Cols,
int Depth>
struct product_type_selector;
37 template<
int Size,
int MaxSize>
struct product_size_category
40 #ifndef EIGEN_GPU_COMPILE_PHASE
41 is_large = MaxSize ==
Dynamic ||
42 Size >= EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD ||
43 (Size==
Dynamic && MaxSize>=EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD),
47 value = is_large ? Large
53 template<
typename Lhs,
typename Rhs>
struct product_type
55 typedef remove_all_t<Lhs> Lhs_;
56 typedef remove_all_t<Rhs> Rhs_;
58 MaxRows = traits<Lhs_>::MaxRowsAtCompileTime,
59 Rows = traits<Lhs_>::RowsAtCompileTime,
60 MaxCols = traits<Rhs_>::MaxColsAtCompileTime,
61 Cols = traits<Rhs_>::ColsAtCompileTime,
62 MaxDepth = min_size_prefer_fixed(traits<Lhs_>::MaxColsAtCompileTime,
63 traits<Rhs_>::MaxRowsAtCompileTime),
64 Depth = min_size_prefer_fixed(traits<Lhs_>::ColsAtCompileTime,
65 traits<Rhs_>::RowsAtCompileTime)
72 rows_select = product_size_category<Rows,MaxRows>::value,
73 cols_select = product_size_category<Cols,MaxCols>::value,
74 depth_select = product_size_category<Depth,MaxDepth>::value
76 typedef product_type_selector<rows_select, cols_select, depth_select> selector;
80 value = selector::ret,
83 #ifdef EIGEN_DEBUG_PRODUCT
86 EIGEN_DEBUG_VAR(Rows);
87 EIGEN_DEBUG_VAR(Cols);
88 EIGEN_DEBUG_VAR(Depth);
89 EIGEN_DEBUG_VAR(rows_select);
90 EIGEN_DEBUG_VAR(cols_select);
91 EIGEN_DEBUG_VAR(depth_select);
92 EIGEN_DEBUG_VAR(value);
101 template<
int M,
int N>
struct product_type_selector<M,N,1> {
enum { ret = OuterProduct }; };
102 template<
int M>
struct product_type_selector<M, 1, 1> {
enum { ret = LazyCoeffBasedProductMode }; };
103 template<
int N>
struct product_type_selector<1, N, 1> {
enum { ret = LazyCoeffBasedProductMode }; };
104 template<
int Depth>
struct product_type_selector<1, 1, Depth> {
enum { ret = InnerProduct }; };
105 template<>
struct product_type_selector<1, 1, 1> {
enum { ret = InnerProduct }; };
106 template<>
struct product_type_selector<Small,1, Small> {
enum { ret = CoeffBasedProductMode }; };
107 template<>
struct product_type_selector<1, Small,Small> {
enum { ret = CoeffBasedProductMode }; };
108 template<>
struct product_type_selector<Small,Small,Small> {
enum { ret = CoeffBasedProductMode }; };
109 template<>
struct product_type_selector<Small, Small, 1> {
enum { ret = LazyCoeffBasedProductMode }; };
110 template<>
struct product_type_selector<Small, Large, 1> {
enum { ret = LazyCoeffBasedProductMode }; };
111 template<>
struct product_type_selector<Large, Small, 1> {
enum { ret = LazyCoeffBasedProductMode }; };
112 template<>
struct product_type_selector<1, Large,Small> {
enum { ret = CoeffBasedProductMode }; };
113 template<>
struct product_type_selector<1, Large,Large> {
enum { ret = GemvProduct }; };
114 template<>
struct product_type_selector<1, Small,Large> {
enum { ret = CoeffBasedProductMode }; };
115 template<>
struct product_type_selector<Large,1, Small> {
enum { ret = CoeffBasedProductMode }; };
116 template<>
struct product_type_selector<Large,1, Large> {
enum { ret = GemvProduct }; };
117 template<>
struct product_type_selector<Small,1, Large> {
enum { ret = CoeffBasedProductMode }; };
118 template<>
struct product_type_selector<Small,Small,Large> {
enum { ret = GemmProduct }; };
119 template<>
struct product_type_selector<Large,Small,Large> {
enum { ret = GemmProduct }; };
120 template<>
struct product_type_selector<Small,Large,Large> {
enum { ret = GemmProduct }; };
121 template<>
struct product_type_selector<Large,Large,Large> {
enum { ret = GemmProduct }; };
122 template<>
struct product_type_selector<Large,Small,Small> {
enum { ret = CoeffBasedProductMode }; };
123 template<>
struct product_type_selector<Small,Large,Small> {
enum { ret = CoeffBasedProductMode }; };
124 template<>
struct product_type_selector<Large,Large,Small> {
enum { ret = GemmProduct }; };
156 template<
int S
ide,
int StorageOrder,
bool BlasCompatible>
157 struct gemv_dense_selector;
163 template<
typename Scalar,
int Size,
int MaxSize,
bool Cond>
struct gemv_static_vector_if;
165 template<
typename Scalar,
int Size,
int MaxSize>
166 struct gemv_static_vector_if<Scalar,Size,MaxSize,false>
168 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
Scalar* data() { eigen_internal_assert(
false &&
"should never be called");
return 0; }
171 template<
typename Scalar,
int Size>
172 struct gemv_static_vector_if<Scalar,Size,
Dynamic,true>
174 EIGEN_STRONG_INLINE EIGEN_DEVICE_FUNC
Scalar* data() {
return 0; }
177 template<
typename Scalar,
int Size,
int MaxSize>
178 struct gemv_static_vector_if<Scalar,Size,MaxSize,true>
181 ForceAlignment = internal::packet_traits<Scalar>::Vectorizable,
182 PacketSize = internal::packet_traits<Scalar>::size
184 #if EIGEN_MAX_STATIC_ALIGN_BYTES!=0
185 internal::plain_array<Scalar, internal::min_size_prefer_fixed(Size, MaxSize), 0,
186 internal::plain_enum_min(AlignedMax, PacketSize)> m_data;
187 EIGEN_STRONG_INLINE Scalar* data() {
return m_data.array; }
191 internal::plain_array<Scalar, internal::min_size_prefer_fixed(Size, MaxSize)+(ForceAlignment?EIGEN_MAX_ALIGN_BYTES:0),0> m_data;
192 EIGEN_STRONG_INLINE Scalar* data() {
193 return ForceAlignment
194 ?
reinterpret_cast<Scalar*
>((internal::UIntPtr(m_data.array) & ~(std::size_t(EIGEN_MAX_ALIGN_BYTES-1))) + EIGEN_MAX_ALIGN_BYTES)
201 template<
int StorageOrder,
bool BlasCompatible>
202 struct gemv_dense_selector<
OnTheLeft,StorageOrder,BlasCompatible>
204 template<
typename Lhs,
typename Rhs,
typename Dest>
205 static void run(
const Lhs &lhs,
const Rhs &rhs, Dest& dest,
const typename Dest::Scalar& alpha)
207 Transpose<Dest> destT(dest);
209 gemv_dense_selector<OnTheRight,OtherStorageOrder,BlasCompatible>
210 ::run(rhs.transpose(), lhs.transpose(), destT, alpha);
216 template<
typename Lhs,
typename Rhs,
typename Dest>
217 static inline void run(
const Lhs &lhs,
const Rhs &rhs, Dest& dest,
const typename Dest::Scalar& alpha)
219 typedef typename Lhs::Scalar LhsScalar;
220 typedef typename Rhs::Scalar RhsScalar;
221 typedef typename Dest::Scalar ResScalar;
223 typedef internal::blas_traits<Lhs> LhsBlasTraits;
224 typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType;
225 typedef internal::blas_traits<Rhs> RhsBlasTraits;
226 typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType;
228 typedef Map<Matrix<ResScalar,Dynamic,1>, plain_enum_min(AlignedMax, internal::packet_traits<ResScalar>::size)> MappedDest;
230 ActualLhsType actualLhs = LhsBlasTraits::extract(lhs);
231 ActualRhsType actualRhs = RhsBlasTraits::extract(rhs);
233 ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs);
236 typedef std::conditional_t<Dest::IsVectorAtCompileTime, Dest, typename Dest::ColXpr> ActualDest;
241 EvalToDestAtCompileTime = (ActualDest::InnerStrideAtCompileTime==1),
242 ComplexByReal = (NumTraits<LhsScalar>::IsComplex) && (!NumTraits<RhsScalar>::IsComplex),
243 MightCannotUseDest = ((!EvalToDestAtCompileTime) || ComplexByReal) && (ActualDest::MaxSizeAtCompileTime!=0)
246 typedef const_blas_data_mapper<LhsScalar,Index,ColMajor> LhsMapper;
247 typedef const_blas_data_mapper<RhsScalar,Index,RowMajor> RhsMapper;
248 RhsScalar compatibleAlpha = get_factor<ResScalar,RhsScalar>::run(actualAlpha);
250 if(!MightCannotUseDest)
254 general_matrix_vector_product
255 <
Index,LhsScalar,LhsMapper,
ColMajor,LhsBlasTraits::NeedToConjugate,RhsScalar,RhsMapper,RhsBlasTraits::NeedToConjugate>::run(
256 actualLhs.rows(), actualLhs.cols(),
257 LhsMapper(actualLhs.data(), actualLhs.outerStride()),
258 RhsMapper(actualRhs.data(), actualRhs.innerStride()),
264 gemv_static_vector_if<ResScalar,ActualDest::SizeAtCompileTime,ActualDest::MaxSizeAtCompileTime,MightCannotUseDest> static_dest;
266 const bool alphaIsCompatible = (!ComplexByReal) || (numext::is_exactly_zero(numext::imag(actualAlpha)));
267 const bool evalToDest = EvalToDestAtCompileTime && alphaIsCompatible;
269 ei_declare_aligned_stack_constructed_variable(ResScalar,actualDestPtr,dest.size(),
270 evalToDest ? dest.data() : static_dest.data());
274 #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
275 Index size = dest.size();
276 EIGEN_DENSE_STORAGE_CTOR_PLUGIN
278 if(!alphaIsCompatible)
280 MappedDest(actualDestPtr, dest.size()).setZero();
281 compatibleAlpha = RhsScalar(1);
284 MappedDest(actualDestPtr, dest.size()) = dest;
287 general_matrix_vector_product
288 <
Index,LhsScalar,LhsMapper,
ColMajor,LhsBlasTraits::NeedToConjugate,RhsScalar,RhsMapper,RhsBlasTraits::NeedToConjugate>::run(
289 actualLhs.rows(), actualLhs.cols(),
290 LhsMapper(actualLhs.data(), actualLhs.outerStride()),
291 RhsMapper(actualRhs.data(), actualRhs.innerStride()),
297 if(!alphaIsCompatible)
298 dest.matrix() += actualAlpha * MappedDest(actualDestPtr, dest.size());
300 dest = MappedDest(actualDestPtr, dest.size());
308 template<
typename Lhs,
typename Rhs,
typename Dest>
309 static void run(
const Lhs &lhs,
const Rhs &rhs, Dest& dest,
const typename Dest::Scalar& alpha)
311 typedef typename Lhs::Scalar LhsScalar;
312 typedef typename Rhs::Scalar RhsScalar;
313 typedef typename Dest::Scalar ResScalar;
315 typedef internal::blas_traits<Lhs> LhsBlasTraits;
316 typedef typename LhsBlasTraits::DirectLinearAccessType ActualLhsType;
317 typedef internal::blas_traits<Rhs> RhsBlasTraits;
318 typedef typename RhsBlasTraits::DirectLinearAccessType ActualRhsType;
319 typedef internal::remove_all_t<ActualRhsType> ActualRhsTypeCleaned;
321 std::add_const_t<ActualLhsType> actualLhs = LhsBlasTraits::extract(lhs);
322 std::add_const_t<ActualRhsType> actualRhs = RhsBlasTraits::extract(rhs);
324 ResScalar actualAlpha = combine_scalar_factors(alpha, lhs, rhs);
329 DirectlyUseRhs = ActualRhsTypeCleaned::InnerStrideAtCompileTime==1 || ActualRhsTypeCleaned::MaxSizeAtCompileTime==0
332 gemv_static_vector_if<RhsScalar,ActualRhsTypeCleaned::SizeAtCompileTime,ActualRhsTypeCleaned::MaxSizeAtCompileTime,!DirectlyUseRhs> static_rhs;
334 ei_declare_aligned_stack_constructed_variable(RhsScalar,actualRhsPtr,actualRhs.size(),
335 DirectlyUseRhs ?
const_cast<RhsScalar*
>(actualRhs.data()) : static_rhs.data());
339 #ifdef EIGEN_DENSE_STORAGE_CTOR_PLUGIN
340 Index size = actualRhs.size();
341 EIGEN_DENSE_STORAGE_CTOR_PLUGIN
343 Map<typename ActualRhsTypeCleaned::PlainObject>(actualRhsPtr, actualRhs.size()) = actualRhs;
346 typedef const_blas_data_mapper<LhsScalar,Index,RowMajor> LhsMapper;
347 typedef const_blas_data_mapper<RhsScalar,Index,ColMajor> RhsMapper;
348 general_matrix_vector_product
349 <
Index,LhsScalar,LhsMapper,
RowMajor,LhsBlasTraits::NeedToConjugate,RhsScalar,RhsMapper,RhsBlasTraits::NeedToConjugate>::run(
350 actualLhs.rows(), actualLhs.cols(),
351 LhsMapper(actualLhs.data(), actualLhs.outerStride()),
352 RhsMapper(actualRhsPtr, 1),
353 dest.data(), dest.col(0).innerStride(),
360 template<
typename Lhs,
typename Rhs,
typename Dest>
361 static void run(
const Lhs &lhs,
const Rhs &rhs, Dest& dest,
const typename Dest::Scalar& alpha)
363 EIGEN_STATIC_ASSERT((!nested_eval<Lhs,1>::Evaluate),EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE);
365 typename nested_eval<Rhs,1>::type actual_rhs(rhs);
366 const Index size = rhs.rows();
367 for(
Index k=0; k<size; ++k)
368 dest += (alpha*actual_rhs.coeff(k)) * lhs.col(k);
374 template<
typename Lhs,
typename Rhs,
typename Dest>
375 static void run(
const Lhs &lhs,
const Rhs &rhs, Dest& dest,
const typename Dest::Scalar& alpha)
377 EIGEN_STATIC_ASSERT((!nested_eval<Lhs,1>::Evaluate),EIGEN_INTERNAL_COMPILATION_ERROR_OR_YOU_MADE_A_PROGRAMMING_MISTAKE);
378 typename nested_eval<Rhs,Lhs::RowsAtCompileTime>::type actual_rhs(rhs);
379 const Index rows = dest.rows();
380 for(
Index i=0; i<rows; ++i)
381 dest.coeffRef(i) += alpha * (lhs.row(i).cwiseProduct(actual_rhs.transpose())).sum();
397 template<
typename Derived>
398 template<
typename OtherDerived>
399 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
400 const Product<Derived, OtherDerived>
408 ProductIsValid = Derived::ColsAtCompileTime==
Dynamic
409 || OtherDerived::RowsAtCompileTime==
Dynamic
410 || int(Derived::ColsAtCompileTime)==int(OtherDerived::RowsAtCompileTime),
411 AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime,
412 SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived,OtherDerived)
417 EIGEN_STATIC_ASSERT(ProductIsValid || !(AreVectors && SameSizes),
418 INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS)
419 EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors),
420 INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION)
421 EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT)
422 #ifdef EIGEN_DEBUG_PRODUCT
423 internal::product_type<Derived,OtherDerived>::debug();
440 template<
typename Derived>
441 template<
typename OtherDerived>
442 EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
447 ProductIsValid = Derived::ColsAtCompileTime==
Dynamic
448 || OtherDerived::RowsAtCompileTime==
Dynamic
449 || int(Derived::ColsAtCompileTime)==int(OtherDerived::RowsAtCompileTime),
450 AreVectors = Derived::IsVectorAtCompileTime && OtherDerived::IsVectorAtCompileTime,
451 SameSizes = EIGEN_PREDICATE_SAME_MATRIX_SIZE(Derived,OtherDerived)
456 EIGEN_STATIC_ASSERT(ProductIsValid || !(AreVectors && SameSizes),
457 INVALID_VECTOR_VECTOR_PRODUCT__IF_YOU_WANTED_A_DOT_OR_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTIONS)
458 EIGEN_STATIC_ASSERT(ProductIsValid || !(SameSizes && !AreVectors),
459 INVALID_MATRIX_PRODUCT__IF_YOU_WANTED_A_COEFF_WISE_PRODUCT_YOU_MUST_USE_THE_EXPLICIT_FUNCTION)
460 EIGEN_STATIC_ASSERT(ProductIsValid || SameSizes, INVALID_MATRIX_PRODUCT)
internal::traits< Homogeneous< MatrixType, Direction_ > >::Scalar Scalar
Definition: DenseBase.h:61
Derived & derived()
Definition: EigenBase.h:48
Base class for all dense matrices, vectors, and expressions.
Definition: MatrixBase.h:52
const Product< Derived, OtherDerived, LazyProduct > lazyProduct(const MatrixBase< OtherDerived > &other) const
Definition: GeneralProduct.h:444
const Product< Derived, OtherDerived > operator*(const MatrixBase< OtherDerived > &other) const
Definition: GeneralProduct.h:401
Expression of the product of two arbitrary matrices or vectors.
Definition: Product.h:77
@ ColMajor
Definition: Constants.h:321
@ RowMajor
Definition: Constants.h:323
@ OnTheLeft
Definition: Constants.h:334
@ OnTheRight
Definition: Constants.h:336
Namespace containing all symbols from the Eigen library.
Definition: Core:139
EIGEN_DEFAULT_DENSE_INDEX_TYPE Index
The Index type as used for the API.
Definition: Meta.h:59
const int Dynamic
Definition: Constants.h:24