123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935 |
- // This file is part of Eigen, a lightweight C++ template library
- // for linear algebra.
- //
- // Copyright (C) 2011 Benoit Jacob <jacob.benoit.1@gmail.com>
- // Copyright (C) 2011-2014 Gael Guennebaud <gael.guennebaud@inria.fr>
- // Copyright (C) 2011-2012 Jitse Niesen <jitse@maths.leeds.ac.uk>
- //
- // This Source Code Form is subject to the terms of the Mozilla
- // Public License v. 2.0. If a copy of the MPL was not distributed
- // with this file, You can obtain one at http://mozilla.org/MPL/2.0/.
- #ifndef EIGEN_ASSIGN_EVALUATOR_H
- #define EIGEN_ASSIGN_EVALUATOR_H
- namespace Eigen {
- // This implementation is based on Assign.h
- namespace internal {
-
- /***************************************************************************
- * Part 1 : the logic deciding a strategy for traversal and unrolling *
- ***************************************************************************/
- // copy_using_evaluator_traits is based on assign_traits
- template <typename DstEvaluator, typename SrcEvaluator, typename AssignFunc>
- struct copy_using_evaluator_traits
- {
- typedef typename DstEvaluator::XprType Dst;
- typedef typename Dst::Scalar DstScalar;
-
- enum {
- DstFlags = DstEvaluator::Flags,
- SrcFlags = SrcEvaluator::Flags
- };
-
- public:
- enum {
- DstAlignment = DstEvaluator::Alignment,
- SrcAlignment = SrcEvaluator::Alignment,
- DstHasDirectAccess = (DstFlags & DirectAccessBit) == DirectAccessBit,
- JointAlignment = EIGEN_PLAIN_ENUM_MIN(DstAlignment,SrcAlignment)
- };
- private:
- enum {
- InnerSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::SizeAtCompileTime)
- : int(DstFlags)&RowMajorBit ? int(Dst::ColsAtCompileTime)
- : int(Dst::RowsAtCompileTime),
- InnerMaxSize = int(Dst::IsVectorAtCompileTime) ? int(Dst::MaxSizeAtCompileTime)
- : int(DstFlags)&RowMajorBit ? int(Dst::MaxColsAtCompileTime)
- : int(Dst::MaxRowsAtCompileTime),
- OuterStride = int(outer_stride_at_compile_time<Dst>::ret),
- MaxSizeAtCompileTime = Dst::SizeAtCompileTime
- };
- // TODO distinguish between linear traversal and inner-traversals
- typedef typename find_best_packet<DstScalar,Dst::SizeAtCompileTime>::type LinearPacketType;
- typedef typename find_best_packet<DstScalar,InnerSize>::type InnerPacketType;
- enum {
- LinearPacketSize = unpacket_traits<LinearPacketType>::size,
- InnerPacketSize = unpacket_traits<InnerPacketType>::size
- };
- public:
- enum {
- LinearRequiredAlignment = unpacket_traits<LinearPacketType>::alignment,
- InnerRequiredAlignment = unpacket_traits<InnerPacketType>::alignment
- };
- private:
- enum {
- DstIsRowMajor = DstFlags&RowMajorBit,
- SrcIsRowMajor = SrcFlags&RowMajorBit,
- StorageOrdersAgree = (int(DstIsRowMajor) == int(SrcIsRowMajor)),
- MightVectorize = bool(StorageOrdersAgree)
- && (int(DstFlags) & int(SrcFlags) & ActualPacketAccessBit)
- && bool(functor_traits<AssignFunc>::PacketAccess),
- MayInnerVectorize = MightVectorize
- && int(InnerSize)!=Dynamic && int(InnerSize)%int(InnerPacketSize)==0
- && int(OuterStride)!=Dynamic && int(OuterStride)%int(InnerPacketSize)==0
- && (EIGEN_UNALIGNED_VECTORIZE || int(JointAlignment)>=int(InnerRequiredAlignment)),
- MayLinearize = bool(StorageOrdersAgree) && (int(DstFlags) & int(SrcFlags) & LinearAccessBit),
- MayLinearVectorize = bool(MightVectorize) && bool(MayLinearize) && bool(DstHasDirectAccess)
- && (EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment)) || MaxSizeAtCompileTime == Dynamic),
- /* If the destination isn't aligned, we have to do runtime checks and we don't unroll,
- so it's only good for large enough sizes. */
- MaySliceVectorize = bool(MightVectorize) && bool(DstHasDirectAccess)
- && (int(InnerMaxSize)==Dynamic || int(InnerMaxSize)>=(EIGEN_UNALIGNED_VECTORIZE?InnerPacketSize:(3*InnerPacketSize)))
- /* slice vectorization can be slow, so we only want it if the slices are big, which is
- indicated by InnerMaxSize rather than InnerSize, think of the case of a dynamic block
- in a fixed-size matrix
- However, with EIGEN_UNALIGNED_VECTORIZE and unrolling, slice vectorization is still worth it */
- };
- public:
- enum {
- Traversal = int(MayLinearVectorize) && (LinearPacketSize>InnerPacketSize) ? int(LinearVectorizedTraversal)
- : int(MayInnerVectorize) ? int(InnerVectorizedTraversal)
- : int(MayLinearVectorize) ? int(LinearVectorizedTraversal)
- : int(MaySliceVectorize) ? int(SliceVectorizedTraversal)
- : int(MayLinearize) ? int(LinearTraversal)
- : int(DefaultTraversal),
- Vectorized = int(Traversal) == InnerVectorizedTraversal
- || int(Traversal) == LinearVectorizedTraversal
- || int(Traversal) == SliceVectorizedTraversal
- };
- typedef typename conditional<int(Traversal)==LinearVectorizedTraversal, LinearPacketType, InnerPacketType>::type PacketType;
- private:
- enum {
- ActualPacketSize = int(Traversal)==LinearVectorizedTraversal ? LinearPacketSize
- : Vectorized ? InnerPacketSize
- : 1,
- UnrollingLimit = EIGEN_UNROLLING_LIMIT * ActualPacketSize,
- MayUnrollCompletely = int(Dst::SizeAtCompileTime) != Dynamic
- && int(Dst::SizeAtCompileTime) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <= int(UnrollingLimit),
- MayUnrollInner = int(InnerSize) != Dynamic
- && int(InnerSize) * (int(DstEvaluator::CoeffReadCost)+int(SrcEvaluator::CoeffReadCost)) <= int(UnrollingLimit)
- };
- public:
- enum {
- Unrolling = (int(Traversal) == int(InnerVectorizedTraversal) || int(Traversal) == int(DefaultTraversal))
- ? (
- int(MayUnrollCompletely) ? int(CompleteUnrolling)
- : int(MayUnrollInner) ? int(InnerUnrolling)
- : int(NoUnrolling)
- )
- : int(Traversal) == int(LinearVectorizedTraversal)
- ? ( bool(MayUnrollCompletely) && ( EIGEN_UNALIGNED_VECTORIZE || (int(DstAlignment)>=int(LinearRequiredAlignment)))
- ? int(CompleteUnrolling)
- : int(NoUnrolling) )
- : int(Traversal) == int(LinearTraversal)
- ? ( bool(MayUnrollCompletely) ? int(CompleteUnrolling)
- : int(NoUnrolling) )
- #if EIGEN_UNALIGNED_VECTORIZE
- : int(Traversal) == int(SliceVectorizedTraversal)
- ? ( bool(MayUnrollInner) ? int(InnerUnrolling)
- : int(NoUnrolling) )
- #endif
- : int(NoUnrolling)
- };
- #ifdef EIGEN_DEBUG_ASSIGN
- static void debug()
- {
- std::cerr << "DstXpr: " << typeid(typename DstEvaluator::XprType).name() << std::endl;
- std::cerr << "SrcXpr: " << typeid(typename SrcEvaluator::XprType).name() << std::endl;
- std::cerr.setf(std::ios::hex, std::ios::basefield);
- std::cerr << "DstFlags" << " = " << DstFlags << " (" << demangle_flags(DstFlags) << " )" << std::endl;
- std::cerr << "SrcFlags" << " = " << SrcFlags << " (" << demangle_flags(SrcFlags) << " )" << std::endl;
- std::cerr.unsetf(std::ios::hex);
- EIGEN_DEBUG_VAR(DstAlignment)
- EIGEN_DEBUG_VAR(SrcAlignment)
- EIGEN_DEBUG_VAR(LinearRequiredAlignment)
- EIGEN_DEBUG_VAR(InnerRequiredAlignment)
- EIGEN_DEBUG_VAR(JointAlignment)
- EIGEN_DEBUG_VAR(InnerSize)
- EIGEN_DEBUG_VAR(InnerMaxSize)
- EIGEN_DEBUG_VAR(LinearPacketSize)
- EIGEN_DEBUG_VAR(InnerPacketSize)
- EIGEN_DEBUG_VAR(ActualPacketSize)
- EIGEN_DEBUG_VAR(StorageOrdersAgree)
- EIGEN_DEBUG_VAR(MightVectorize)
- EIGEN_DEBUG_VAR(MayLinearize)
- EIGEN_DEBUG_VAR(MayInnerVectorize)
- EIGEN_DEBUG_VAR(MayLinearVectorize)
- EIGEN_DEBUG_VAR(MaySliceVectorize)
- std::cerr << "Traversal" << " = " << Traversal << " (" << demangle_traversal(Traversal) << ")" << std::endl;
- EIGEN_DEBUG_VAR(SrcEvaluator::CoeffReadCost)
- EIGEN_DEBUG_VAR(UnrollingLimit)
- EIGEN_DEBUG_VAR(MayUnrollCompletely)
- EIGEN_DEBUG_VAR(MayUnrollInner)
- std::cerr << "Unrolling" << " = " << Unrolling << " (" << demangle_unrolling(Unrolling) << ")" << std::endl;
- std::cerr << std::endl;
- }
- #endif
- };
- /***************************************************************************
- * Part 2 : meta-unrollers
- ***************************************************************************/
- /************************
- *** Default traversal ***
- ************************/
- template<typename Kernel, int Index, int Stop>
- struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling
- {
- // FIXME: this is not very clean, perhaps this information should be provided by the kernel?
- typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
- typedef typename DstEvaluatorType::XprType DstXprType;
-
- enum {
- outer = Index / DstXprType::InnerSizeAtCompileTime,
- inner = Index % DstXprType::InnerSizeAtCompileTime
- };
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- kernel.assignCoeffByOuterInner(outer, inner);
- copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
- }
- };
- template<typename Kernel, int Stop>
- struct copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, Stop, Stop>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) { }
- };
- template<typename Kernel, int Index_, int Stop>
- struct copy_using_evaluator_DefaultTraversal_InnerUnrolling
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel, Index outer)
- {
- kernel.assignCoeffByOuterInner(outer, Index_);
- copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Index_+1, Stop>::run(kernel, outer);
- }
- };
- template<typename Kernel, int Stop>
- struct copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, Stop, Stop>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&, Index) { }
- };
- /***********************
- *** Linear traversal ***
- ***********************/
- template<typename Kernel, int Index, int Stop>
- struct copy_using_evaluator_LinearTraversal_CompleteUnrolling
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel& kernel)
- {
- kernel.assignCoeff(Index);
- copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Index+1, Stop>::run(kernel);
- }
- };
- template<typename Kernel, int Stop>
- struct copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, Stop, Stop>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) { }
- };
- /**************************
- *** Inner vectorization ***
- **************************/
- template<typename Kernel, int Index, int Stop>
- struct copy_using_evaluator_innervec_CompleteUnrolling
- {
- // FIXME: this is not very clean, perhaps this information should be provided by the kernel?
- typedef typename Kernel::DstEvaluatorType DstEvaluatorType;
- typedef typename DstEvaluatorType::XprType DstXprType;
- typedef typename Kernel::PacketType PacketType;
-
- enum {
- outer = Index / DstXprType::InnerSizeAtCompileTime,
- inner = Index % DstXprType::InnerSizeAtCompileTime,
- SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
- DstAlignment = Kernel::AssignmentTraits::DstAlignment
- };
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
- enum { NextIndex = Index + unpacket_traits<PacketType>::size };
- copy_using_evaluator_innervec_CompleteUnrolling<Kernel, NextIndex, Stop>::run(kernel);
- }
- };
- template<typename Kernel, int Stop>
- struct copy_using_evaluator_innervec_CompleteUnrolling<Kernel, Stop, Stop>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&) { }
- };
- template<typename Kernel, int Index_, int Stop, int SrcAlignment, int DstAlignment>
- struct copy_using_evaluator_innervec_InnerUnrolling
- {
- typedef typename Kernel::PacketType PacketType;
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel, Index outer)
- {
- kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, Index_);
- enum { NextIndex = Index_ + unpacket_traits<PacketType>::size };
- copy_using_evaluator_innervec_InnerUnrolling<Kernel, NextIndex, Stop, SrcAlignment, DstAlignment>::run(kernel, outer);
- }
- };
- template<typename Kernel, int Stop, int SrcAlignment, int DstAlignment>
- struct copy_using_evaluator_innervec_InnerUnrolling<Kernel, Stop, Stop, SrcAlignment, DstAlignment>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &, Index) { }
- };
- /***************************************************************************
- * Part 3 : implementation of all cases
- ***************************************************************************/
- // dense_assignment_loop is based on assign_impl
- template<typename Kernel,
- int Traversal = Kernel::AssignmentTraits::Traversal,
- int Unrolling = Kernel::AssignmentTraits::Unrolling>
- struct dense_assignment_loop;
- /************************
- *** Default traversal ***
- ************************/
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, DefaultTraversal, NoUnrolling>
- {
- EIGEN_DEVICE_FUNC static void EIGEN_STRONG_INLINE run(Kernel &kernel)
- {
- for(Index outer = 0; outer < kernel.outerSize(); ++outer) {
- for(Index inner = 0; inner < kernel.innerSize(); ++inner) {
- kernel.assignCoeffByOuterInner(outer, inner);
- }
- }
- }
- };
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, DefaultTraversal, CompleteUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
- copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
- }
- };
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, DefaultTraversal, InnerUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
- const Index outerSize = kernel.outerSize();
- for(Index outer = 0; outer < outerSize; ++outer)
- copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime>::run(kernel, outer);
- }
- };
- /***************************
- *** Linear vectorization ***
- ***************************/
- // The goal of unaligned_dense_assignment_loop is simply to factorize the handling
- // of the non vectorizable beginning and ending parts
- template <bool IsAligned = false>
- struct unaligned_dense_assignment_loop
- {
- // if IsAligned = true, then do nothing
- template <typename Kernel>
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel&, Index, Index) {}
- };
- template <>
- struct unaligned_dense_assignment_loop<false>
- {
- // MSVC must not inline this functions. If it does, it fails to optimize the
- // packet access path.
- // FIXME check which version exhibits this issue
- #if EIGEN_COMP_MSVC
- template <typename Kernel>
- static EIGEN_DONT_INLINE void run(Kernel &kernel,
- Index start,
- Index end)
- #else
- template <typename Kernel>
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel,
- Index start,
- Index end)
- #endif
- {
- for (Index index = start; index < end; ++index)
- kernel.assignCoeff(index);
- }
- };
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, NoUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- const Index size = kernel.size();
- typedef typename Kernel::Scalar Scalar;
- typedef typename Kernel::PacketType PacketType;
- enum {
- requestedAlignment = Kernel::AssignmentTraits::LinearRequiredAlignment,
- packetSize = unpacket_traits<PacketType>::size,
- dstIsAligned = int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
- dstAlignment = packet_traits<Scalar>::AlignedOnScalar ? int(requestedAlignment)
- : int(Kernel::AssignmentTraits::DstAlignment),
- srcAlignment = Kernel::AssignmentTraits::JointAlignment
- };
- const Index alignedStart = dstIsAligned ? 0 : internal::first_aligned<requestedAlignment>(kernel.dstDataPtr(), size);
- const Index alignedEnd = alignedStart + ((size-alignedStart)/packetSize)*packetSize;
- unaligned_dense_assignment_loop<dstIsAligned!=0>::run(kernel, 0, alignedStart);
- for(Index index = alignedStart; index < alignedEnd; index += packetSize)
- kernel.template assignPacket<dstAlignment, srcAlignment, PacketType>(index);
- unaligned_dense_assignment_loop<>::run(kernel, alignedEnd, size);
- }
- };
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, LinearVectorizedTraversal, CompleteUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
- typedef typename Kernel::PacketType PacketType;
-
- enum { size = DstXprType::SizeAtCompileTime,
- packetSize =unpacket_traits<PacketType>::size,
- alignedSize = (size/packetSize)*packetSize };
- copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, alignedSize>::run(kernel);
- copy_using_evaluator_DefaultTraversal_CompleteUnrolling<Kernel, alignedSize, size>::run(kernel);
- }
- };
- /**************************
- *** Inner vectorization ***
- **************************/
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, NoUnrolling>
- {
- typedef typename Kernel::PacketType PacketType;
- enum {
- SrcAlignment = Kernel::AssignmentTraits::SrcAlignment,
- DstAlignment = Kernel::AssignmentTraits::DstAlignment
- };
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- const Index innerSize = kernel.innerSize();
- const Index outerSize = kernel.outerSize();
- const Index packetSize = unpacket_traits<PacketType>::size;
- for(Index outer = 0; outer < outerSize; ++outer)
- for(Index inner = 0; inner < innerSize; inner+=packetSize)
- kernel.template assignPacketByOuterInner<DstAlignment, SrcAlignment, PacketType>(outer, inner);
- }
- };
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, CompleteUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
- copy_using_evaluator_innervec_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
- }
- };
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, InnerVectorizedTraversal, InnerUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
- typedef typename Kernel::AssignmentTraits Traits;
- const Index outerSize = kernel.outerSize();
- for(Index outer = 0; outer < outerSize; ++outer)
- copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, DstXprType::InnerSizeAtCompileTime,
- Traits::SrcAlignment, Traits::DstAlignment>::run(kernel, outer);
- }
- };
- /***********************
- *** Linear traversal ***
- ***********************/
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, LinearTraversal, NoUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- const Index size = kernel.size();
- for(Index i = 0; i < size; ++i)
- kernel.assignCoeff(i);
- }
- };
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, LinearTraversal, CompleteUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
- copy_using_evaluator_LinearTraversal_CompleteUnrolling<Kernel, 0, DstXprType::SizeAtCompileTime>::run(kernel);
- }
- };
- /**************************
- *** Slice vectorization ***
- ***************************/
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, NoUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- typedef typename Kernel::Scalar Scalar;
- typedef typename Kernel::PacketType PacketType;
- enum {
- packetSize = unpacket_traits<PacketType>::size,
- requestedAlignment = int(Kernel::AssignmentTraits::InnerRequiredAlignment),
- alignable = packet_traits<Scalar>::AlignedOnScalar || int(Kernel::AssignmentTraits::DstAlignment)>=sizeof(Scalar),
- dstIsAligned = int(Kernel::AssignmentTraits::DstAlignment)>=int(requestedAlignment),
- dstAlignment = alignable ? int(requestedAlignment)
- : int(Kernel::AssignmentTraits::DstAlignment)
- };
- const Scalar *dst_ptr = kernel.dstDataPtr();
- if((!bool(dstIsAligned)) && (UIntPtr(dst_ptr) % sizeof(Scalar))>0)
- {
- // the pointer is not aligend-on scalar, so alignment is not possible
- return dense_assignment_loop<Kernel,DefaultTraversal,NoUnrolling>::run(kernel);
- }
- const Index packetAlignedMask = packetSize - 1;
- const Index innerSize = kernel.innerSize();
- const Index outerSize = kernel.outerSize();
- const Index alignedStep = alignable ? (packetSize - kernel.outerStride() % packetSize) & packetAlignedMask : 0;
- Index alignedStart = ((!alignable) || bool(dstIsAligned)) ? 0 : internal::first_aligned<requestedAlignment>(dst_ptr, innerSize);
- for(Index outer = 0; outer < outerSize; ++outer)
- {
- const Index alignedEnd = alignedStart + ((innerSize-alignedStart) & ~packetAlignedMask);
- // do the non-vectorizable part of the assignment
- for(Index inner = 0; inner<alignedStart ; ++inner)
- kernel.assignCoeffByOuterInner(outer, inner);
- // do the vectorizable part of the assignment
- for(Index inner = alignedStart; inner<alignedEnd; inner+=packetSize)
- kernel.template assignPacketByOuterInner<dstAlignment, Unaligned, PacketType>(outer, inner);
- // do the non-vectorizable part of the assignment
- for(Index inner = alignedEnd; inner<innerSize ; ++inner)
- kernel.assignCoeffByOuterInner(outer, inner);
- alignedStart = numext::mini((alignedStart+alignedStep)%packetSize, innerSize);
- }
- }
- };
- #if EIGEN_UNALIGNED_VECTORIZE
- template<typename Kernel>
- struct dense_assignment_loop<Kernel, SliceVectorizedTraversal, InnerUnrolling>
- {
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE void run(Kernel &kernel)
- {
- typedef typename Kernel::DstEvaluatorType::XprType DstXprType;
- typedef typename Kernel::PacketType PacketType;
- enum { size = DstXprType::InnerSizeAtCompileTime,
- packetSize =unpacket_traits<PacketType>::size,
- vectorizableSize = (size/packetSize)*packetSize };
- for(Index outer = 0; outer < kernel.outerSize(); ++outer)
- {
- copy_using_evaluator_innervec_InnerUnrolling<Kernel, 0, vectorizableSize, 0, 0>::run(kernel, outer);
- copy_using_evaluator_DefaultTraversal_InnerUnrolling<Kernel, vectorizableSize, size>::run(kernel, outer);
- }
- }
- };
- #endif
- /***************************************************************************
- * Part 4 : Generic dense assignment kernel
- ***************************************************************************/
- // This class generalize the assignment of a coefficient (or packet) from one dense evaluator
- // to another dense writable evaluator.
- // It is parametrized by the two evaluators, and the actual assignment functor.
- // This abstraction level permits to keep the evaluation loops as simple and as generic as possible.
- // One can customize the assignment using this generic dense_assignment_kernel with different
- // functors, or by completely overloading it, by-passing a functor.
- template<typename DstEvaluatorTypeT, typename SrcEvaluatorTypeT, typename Functor, int Version = Specialized>
- class generic_dense_assignment_kernel
- {
- protected:
- typedef typename DstEvaluatorTypeT::XprType DstXprType;
- typedef typename SrcEvaluatorTypeT::XprType SrcXprType;
- public:
-
- typedef DstEvaluatorTypeT DstEvaluatorType;
- typedef SrcEvaluatorTypeT SrcEvaluatorType;
- typedef typename DstEvaluatorType::Scalar Scalar;
- typedef copy_using_evaluator_traits<DstEvaluatorTypeT, SrcEvaluatorTypeT, Functor> AssignmentTraits;
- typedef typename AssignmentTraits::PacketType PacketType;
-
-
- EIGEN_DEVICE_FUNC generic_dense_assignment_kernel(DstEvaluatorType &dst, const SrcEvaluatorType &src, const Functor &func, DstXprType& dstExpr)
- : m_dst(dst), m_src(src), m_functor(func), m_dstExpr(dstExpr)
- {
- #ifdef EIGEN_DEBUG_ASSIGN
- AssignmentTraits::debug();
- #endif
- }
-
- EIGEN_DEVICE_FUNC Index size() const { return m_dstExpr.size(); }
- EIGEN_DEVICE_FUNC Index innerSize() const { return m_dstExpr.innerSize(); }
- EIGEN_DEVICE_FUNC Index outerSize() const { return m_dstExpr.outerSize(); }
- EIGEN_DEVICE_FUNC Index rows() const { return m_dstExpr.rows(); }
- EIGEN_DEVICE_FUNC Index cols() const { return m_dstExpr.cols(); }
- EIGEN_DEVICE_FUNC Index outerStride() const { return m_dstExpr.outerStride(); }
-
- EIGEN_DEVICE_FUNC DstEvaluatorType& dstEvaluator() { return m_dst; }
- EIGEN_DEVICE_FUNC const SrcEvaluatorType& srcEvaluator() const { return m_src; }
-
- /// Assign src(row,col) to dst(row,col) through the assignment functor.
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Index row, Index col)
- {
- m_functor.assignCoeff(m_dst.coeffRef(row,col), m_src.coeff(row,col));
- }
-
- /// \sa assignCoeff(Index,Index)
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeff(Index index)
- {
- m_functor.assignCoeff(m_dst.coeffRef(index), m_src.coeff(index));
- }
-
- /// \sa assignCoeff(Index,Index)
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignCoeffByOuterInner(Index outer, Index inner)
- {
- Index row = rowIndexByOuterInner(outer, inner);
- Index col = colIndexByOuterInner(outer, inner);
- assignCoeff(row, col);
- }
-
-
- template<int StoreMode, int LoadMode, typename PacketType>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index row, Index col)
- {
- m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(row,col), m_src.template packet<LoadMode,PacketType>(row,col));
- }
-
- template<int StoreMode, int LoadMode, typename PacketType>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacket(Index index)
- {
- m_functor.template assignPacket<StoreMode>(&m_dst.coeffRef(index), m_src.template packet<LoadMode,PacketType>(index));
- }
-
- template<int StoreMode, int LoadMode, typename PacketType>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void assignPacketByOuterInner(Index outer, Index inner)
- {
- Index row = rowIndexByOuterInner(outer, inner);
- Index col = colIndexByOuterInner(outer, inner);
- assignPacket<StoreMode,LoadMode,PacketType>(row, col);
- }
-
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index rowIndexByOuterInner(Index outer, Index inner)
- {
- typedef typename DstEvaluatorType::ExpressionTraits Traits;
- return int(Traits::RowsAtCompileTime) == 1 ? 0
- : int(Traits::ColsAtCompileTime) == 1 ? inner
- : int(DstEvaluatorType::Flags)&RowMajorBit ? outer
- : inner;
- }
- EIGEN_DEVICE_FUNC static EIGEN_STRONG_INLINE Index colIndexByOuterInner(Index outer, Index inner)
- {
- typedef typename DstEvaluatorType::ExpressionTraits Traits;
- return int(Traits::ColsAtCompileTime) == 1 ? 0
- : int(Traits::RowsAtCompileTime) == 1 ? inner
- : int(DstEvaluatorType::Flags)&RowMajorBit ? inner
- : outer;
- }
- EIGEN_DEVICE_FUNC const Scalar* dstDataPtr() const
- {
- return m_dstExpr.data();
- }
-
- protected:
- DstEvaluatorType& m_dst;
- const SrcEvaluatorType& m_src;
- const Functor &m_functor;
- // TODO find a way to avoid the needs of the original expression
- DstXprType& m_dstExpr;
- };
- /***************************************************************************
- * Part 5 : Entry point for dense rectangular assignment
- ***************************************************************************/
- template<typename DstXprType,typename SrcXprType, typename Functor>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void resize_if_allowed(DstXprType &dst, const SrcXprType& src, const Functor &/*func*/)
- {
- EIGEN_ONLY_USED_FOR_DEBUG(dst);
- EIGEN_ONLY_USED_FOR_DEBUG(src);
- eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
- }
- template<typename DstXprType,typename SrcXprType, typename T1, typename T2>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void resize_if_allowed(DstXprType &dst, const SrcXprType& src, const internal::assign_op<T1,T2> &/*func*/)
- {
- Index dstRows = src.rows();
- Index dstCols = src.cols();
- if(((dst.rows()!=dstRows) || (dst.cols()!=dstCols)))
- dst.resize(dstRows, dstCols);
- eigen_assert(dst.rows() == dstRows && dst.cols() == dstCols);
- }
- template<typename DstXprType, typename SrcXprType, typename Functor>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src, const Functor &func)
- {
- typedef evaluator<DstXprType> DstEvaluatorType;
- typedef evaluator<SrcXprType> SrcEvaluatorType;
- SrcEvaluatorType srcEvaluator(src);
- // NOTE To properly handle A = (A*A.transpose())/s with A rectangular,
- // we need to resize the destination after the source evaluator has been created.
- resize_if_allowed(dst, src, func);
- DstEvaluatorType dstEvaluator(dst);
-
- typedef generic_dense_assignment_kernel<DstEvaluatorType,SrcEvaluatorType,Functor> Kernel;
- Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived());
- dense_assignment_loop<Kernel>::run(kernel);
- }
- template<typename DstXprType, typename SrcXprType>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src)
- {
- call_dense_assignment_loop(dst, src, internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar>());
- }
- /***************************************************************************
- * Part 6 : Generic assignment
- ***************************************************************************/
- // Based on the respective shapes of the destination and source,
- // the class AssignmentKind determine the kind of assignment mechanism.
- // AssignmentKind must define a Kind typedef.
- template<typename DstShape, typename SrcShape> struct AssignmentKind;
- // Assignement kind defined in this file:
- struct Dense2Dense {};
- struct EigenBase2EigenBase {};
- template<typename,typename> struct AssignmentKind { typedef EigenBase2EigenBase Kind; };
- template<> struct AssignmentKind<DenseShape,DenseShape> { typedef Dense2Dense Kind; };
-
- // This is the main assignment class
- template< typename DstXprType, typename SrcXprType, typename Functor,
- typename Kind = typename AssignmentKind< typename evaluator_traits<DstXprType>::Shape , typename evaluator_traits<SrcXprType>::Shape >::Kind,
- typename EnableIf = void>
- struct Assignment;
- // The only purpose of this call_assignment() function is to deal with noalias() / "assume-aliasing" and automatic transposition.
- // Indeed, I (Gael) think that this concept of "assume-aliasing" was a mistake, and it makes thing quite complicated.
- // So this intermediate function removes everything related to "assume-aliasing" such that Assignment
- // does not has to bother about these annoying details.
- template<typename Dst, typename Src>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void call_assignment(Dst& dst, const Src& src)
- {
- call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
- }
- template<typename Dst, typename Src>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void call_assignment(const Dst& dst, const Src& src)
- {
- call_assignment(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
- }
-
- // Deal with "assume-aliasing"
- template<typename Dst, typename Src, typename Func>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if< evaluator_assume_aliasing<Src>::value, void*>::type = 0)
- {
- typename plain_matrix_type<Src>::type tmp(src);
- call_assignment_no_alias(dst, tmp, func);
- }
- template<typename Dst, typename Src, typename Func>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void call_assignment(Dst& dst, const Src& src, const Func& func, typename enable_if<!evaluator_assume_aliasing<Src>::value, void*>::type = 0)
- {
- call_assignment_no_alias(dst, src, func);
- }
- // by-pass "assume-aliasing"
- // When there is no aliasing, we require that 'dst' has been properly resized
- template<typename Dst, template <typename> class StorageBase, typename Src, typename Func>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void call_assignment(NoAlias<Dst,StorageBase>& dst, const Src& src, const Func& func)
- {
- call_assignment_no_alias(dst.expression(), src, func);
- }
- template<typename Dst, typename Src, typename Func>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void call_assignment_no_alias(Dst& dst, const Src& src, const Func& func)
- {
- enum {
- NeedToTranspose = ( (int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1)
- || (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)
- ) && int(Dst::SizeAtCompileTime) != 1
- };
- typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst>::type ActualDstTypeCleaned;
- typedef typename internal::conditional<NeedToTranspose, Transpose<Dst>, Dst&>::type ActualDstType;
- ActualDstType actualDst(dst);
-
- // TODO check whether this is the right place to perform these checks:
- EIGEN_STATIC_ASSERT_LVALUE(Dst)
- EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned,Src)
- EIGEN_CHECK_BINARY_COMPATIBILIY(Func,typename ActualDstTypeCleaned::Scalar,typename Src::Scalar);
-
- Assignment<ActualDstTypeCleaned,Src,Func>::run(actualDst, src, func);
- }
- template<typename Dst, typename Src>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void call_assignment_no_alias(Dst& dst, const Src& src)
- {
- call_assignment_no_alias(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
- }
- template<typename Dst, typename Src, typename Func>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src, const Func& func)
- {
- // TODO check whether this is the right place to perform these checks:
- EIGEN_STATIC_ASSERT_LVALUE(Dst)
- EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(Dst,Src)
- EIGEN_CHECK_BINARY_COMPATIBILIY(Func,typename Dst::Scalar,typename Src::Scalar);
- Assignment<Dst,Src,Func>::run(dst, src, func);
- }
- template<typename Dst, typename Src>
- EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE
- void call_assignment_no_alias_no_transpose(Dst& dst, const Src& src)
- {
- call_assignment_no_alias_no_transpose(dst, src, internal::assign_op<typename Dst::Scalar,typename Src::Scalar>());
- }
- // forward declaration
- template<typename Dst, typename Src> void check_for_aliasing(const Dst &dst, const Src &src);
- // Generic Dense to Dense assignment
- // Note that the last template argument "Weak" is needed to make it possible to perform
- // both partial specialization+SFINAE without ambiguous specialization
- template< typename DstXprType, typename SrcXprType, typename Functor, typename Weak>
- struct Assignment<DstXprType, SrcXprType, Functor, Dense2Dense, Weak>
- {
- EIGEN_DEVICE_FUNC
- static EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const Functor &func)
- {
- #ifndef EIGEN_NO_DEBUG
- internal::check_for_aliasing(dst, src);
- #endif
-
- call_dense_assignment_loop(dst, src, func);
- }
- };
- // Generic assignment through evalTo.
- // TODO: not sure we have to keep that one, but it helps porting current code to new evaluator mechanism.
- // Note that the last template argument "Weak" is needed to make it possible to perform
- // both partial specialization+SFINAE without ambiguous specialization
- template< typename DstXprType, typename SrcXprType, typename Functor, typename Weak>
- struct Assignment<DstXprType, SrcXprType, Functor, EigenBase2EigenBase, Weak>
- {
- EIGEN_DEVICE_FUNC
- static EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::assign_op<typename DstXprType::Scalar,typename SrcXprType::Scalar> &/*func*/)
- {
- Index dstRows = src.rows();
- Index dstCols = src.cols();
- if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
- dst.resize(dstRows, dstCols);
- eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
- src.evalTo(dst);
- }
- // NOTE The following two functions are templated to avoid their instanciation if not needed
- // This is needed because some expressions supports evalTo only and/or have 'void' as scalar type.
- template<typename SrcScalarType>
- EIGEN_DEVICE_FUNC
- static EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::add_assign_op<typename DstXprType::Scalar,SrcScalarType> &/*func*/)
- {
- Index dstRows = src.rows();
- Index dstCols = src.cols();
- if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
- dst.resize(dstRows, dstCols);
- eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
- src.addTo(dst);
- }
- template<typename SrcScalarType>
- EIGEN_DEVICE_FUNC
- static EIGEN_STRONG_INLINE void run(DstXprType &dst, const SrcXprType &src, const internal::sub_assign_op<typename DstXprType::Scalar,SrcScalarType> &/*func*/)
- {
- Index dstRows = src.rows();
- Index dstCols = src.cols();
- if((dst.rows()!=dstRows) || (dst.cols()!=dstCols))
- dst.resize(dstRows, dstCols);
- eigen_assert(dst.rows() == src.rows() && dst.cols() == src.cols());
- src.subTo(dst);
- }
- };
- } // namespace internal
- } // end namespace Eigen
- #endif // EIGEN_ASSIGN_EVALUATOR_H
|