From 7caa05c77d5682919bf7a4ae10e7c37e01cc3430 Mon Sep 17 00:00:00 2001 From: Dario Sassi Date: Fri, 21 Aug 2026 18:32:38 +0200 Subject: [PATCH] Extern : - aggiornata Eigen all'ultima versione disponibile (5.0.1). --- Eigen/AccelerateSupport | 52 + Eigen/ThreadPool | 80 + Eigen/Version | 14 + .../src/AccelerateSupport/AccelerateSupport.h | 423 +++ .../AccelerateSupport/InternalHeaderCheck.h | 3 + Eigen/src/Cholesky/InternalHeaderCheck.h | 3 + .../src/CholmodSupport/InternalHeaderCheck.h | 3 + Eigen/src/Core/DeviceWrapper.h | 153 + Eigen/src/Core/Fill.h | 138 + Eigen/src/Core/FindCoeff.h | 464 +++ Eigen/src/Core/InnerProduct.h | 260 ++ Eigen/src/Core/InternalHeaderCheck.h | 3 + Eigen/src/Core/RandomImpl.h | 262 ++ Eigen/src/Core/RealView.h | 250 ++ Eigen/src/Core/SkewSymmetricMatrix3.h | 382 +++ Eigen/src/Core/arch/AVX/Reductions.h | 353 ++ Eigen/src/Core/arch/AVX512/GemmKernel.h | 1245 +++++++ .../src/Core/arch/AVX512/MathFunctionsFP16.h | 75 + Eigen/src/Core/arch/AVX512/PacketMathFP16.h | 1413 ++++++++ Eigen/src/Core/arch/AVX512/Reductions.h | 297 ++ Eigen/src/Core/arch/AVX512/TrsmKernel.h | 1167 +++++++ Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc | 1219 +++++++ Eigen/src/Core/arch/AVX512/TypeCastingFP16.h | 130 + .../arch/AltiVec/MatrixProductMMAbfloat16.h | 742 +++++ .../Core/arch/AltiVec/MatrixVectorProduct.inc | 2818 ++++++++++++++++ Eigen/src/Core/arch/AltiVec/TypeCasting.h | 153 + Eigen/src/Core/arch/GPU/Complex.h | 244 ++ Eigen/src/Core/arch/GPU/Tuple.h | 268 ++ Eigen/src/Core/arch/HVX/PacketMath.h | 1088 +++++++ Eigen/src/Core/arch/LSX/Complex.h | 520 +++ .../Core/arch/LSX/GeneralBlockPanelKernel.h | 23 + Eigen/src/Core/arch/LSX/MathFunctions.h | 43 + Eigen/src/Core/arch/LSX/PacketMath.h | 2866 +++++++++++++++++ Eigen/src/Core/arch/LSX/TypeCasting.h | 526 +++ Eigen/src/Core/arch/NEON/UnaryFunctors.h | 57 + Eigen/src/Core/arch/SSE/Reductions.h | 324 ++ Eigen/src/Core/util/Assert.h | 158 + Eigen/src/Core/util/EmulateArray.h | 270 ++ Eigen/src/Core/util/GpuHipCudaDefines.inc | 101 + Eigen/src/Core/util/GpuHipCudaUndefines.inc | 45 + Eigen/src/Core/util/MaxSizeVector.h | 139 + Eigen/src/Core/util/MoreMeta.h | 638 ++++ Eigen/src/Core/util/Serializer.h | 209 ++ Eigen/src/Eigenvalues/InternalHeaderCheck.h | 3 + Eigen/src/Geometry/InternalHeaderCheck.h | 3 + Eigen/src/Householder/InternalHeaderCheck.h | 3 + .../InternalHeaderCheck.h | 3 + Eigen/src/Jacobi/InternalHeaderCheck.h | 3 + Eigen/src/KLUSupport/InternalHeaderCheck.h | 3 + Eigen/src/LU/InternalHeaderCheck.h | 3 + Eigen/src/MetisSupport/InternalHeaderCheck.h | 3 + .../src/OrderingMethods/InternalHeaderCheck.h | 3 + Eigen/src/PaStiXSupport/InternalHeaderCheck.h | 3 + .../src/PardisoSupport/InternalHeaderCheck.h | 3 + Eigen/src/QR/InternalHeaderCheck.h | 3 + Eigen/src/SPQRSupport/InternalHeaderCheck.h | 3 + Eigen/src/SVD/BDCSVD_LAPACKE.h | 174 + Eigen/src/SVD/InternalHeaderCheck.h | 3 + .../src/SparseCholesky/InternalHeaderCheck.h | 3 + Eigen/src/SparseCore/InternalHeaderCheck.h | 3 + Eigen/src/SparseLU/InternalHeaderCheck.h | 3 + Eigen/src/SparseQR/InternalHeaderCheck.h | 3 + .../src/SuperLUSupport/InternalHeaderCheck.h | 3 + Eigen/src/ThreadPool/Barrier.h | 70 + Eigen/src/ThreadPool/CoreThreadPoolDevice.h | 336 ++ Eigen/src/ThreadPool/EventCount.h | 241 ++ Eigen/src/ThreadPool/ForkJoin.h | 140 + Eigen/src/ThreadPool/InternalHeaderCheck.h | 4 + Eigen/src/ThreadPool/NonBlockingThreadPool.h | 587 ++++ Eigen/src/ThreadPool/RunQueue.h | 230 ++ Eigen/src/ThreadPool/ThreadCancel.h | 21 + Eigen/src/ThreadPool/ThreadEnvironment.h | 43 + Eigen/src/ThreadPool/ThreadLocal.h | 289 ++ Eigen/src/ThreadPool/ThreadPoolInterface.h | 50 + Eigen/src/ThreadPool/ThreadYield.h | 16 + .../src/UmfPackSupport/InternalHeaderCheck.h | 3 + Eigen/src/misc/InternalHeaderCheck.h | 3 + Eigen/src/misc/lapacke_helpers.h | 163 + Eigen/src/plugins/ArrayCwiseBinaryOps.inc | 344 ++ Eigen/src/plugins/ArrayCwiseUnaryOps.inc | 544 ++++ Eigen/src/plugins/BlockMethods.inc | 1370 ++++++++ Eigen/src/plugins/CommonCwiseBinaryOps.inc | 116 + Eigen/src/plugins/CommonCwiseUnaryOps.inc | 167 + Eigen/src/plugins/IndexedViewMethods.inc | 192 ++ Eigen/src/plugins/InternalHeaderCheck.inc | 3 + Eigen/src/plugins/MatrixCwiseBinaryOps.inc | 331 ++ Eigen/src/plugins/MatrixCwiseUnaryOps.inc | 118 + Eigen/src/plugins/ReshapedMethods.inc | 133 + 88 files changed, 25366 insertions(+) create mode 100644 Eigen/AccelerateSupport create mode 100644 Eigen/ThreadPool create mode 100644 Eigen/Version create mode 100644 Eigen/src/AccelerateSupport/AccelerateSupport.h create mode 100644 Eigen/src/AccelerateSupport/InternalHeaderCheck.h create mode 100644 Eigen/src/Cholesky/InternalHeaderCheck.h create mode 100644 Eigen/src/CholmodSupport/InternalHeaderCheck.h create mode 100644 Eigen/src/Core/DeviceWrapper.h create mode 100644 Eigen/src/Core/Fill.h create mode 100644 Eigen/src/Core/FindCoeff.h create mode 100644 Eigen/src/Core/InnerProduct.h create mode 100644 Eigen/src/Core/InternalHeaderCheck.h create mode 100644 Eigen/src/Core/RandomImpl.h create mode 100644 Eigen/src/Core/RealView.h create mode 100644 Eigen/src/Core/SkewSymmetricMatrix3.h create mode 100644 Eigen/src/Core/arch/AVX/Reductions.h create mode 100644 Eigen/src/Core/arch/AVX512/GemmKernel.h create mode 100644 Eigen/src/Core/arch/AVX512/MathFunctionsFP16.h create mode 100644 Eigen/src/Core/arch/AVX512/PacketMathFP16.h create mode 100644 Eigen/src/Core/arch/AVX512/Reductions.h create mode 100644 Eigen/src/Core/arch/AVX512/TrsmKernel.h create mode 100644 Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc create mode 100644 Eigen/src/Core/arch/AVX512/TypeCastingFP16.h create mode 100644 Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h create mode 100644 Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc create mode 100644 Eigen/src/Core/arch/AltiVec/TypeCasting.h create mode 100644 Eigen/src/Core/arch/GPU/Complex.h create mode 100644 Eigen/src/Core/arch/GPU/Tuple.h create mode 100644 Eigen/src/Core/arch/HVX/PacketMath.h create mode 100644 Eigen/src/Core/arch/LSX/Complex.h create mode 100644 Eigen/src/Core/arch/LSX/GeneralBlockPanelKernel.h create mode 100644 Eigen/src/Core/arch/LSX/MathFunctions.h create mode 100644 Eigen/src/Core/arch/LSX/PacketMath.h create mode 100644 Eigen/src/Core/arch/LSX/TypeCasting.h create mode 100644 Eigen/src/Core/arch/NEON/UnaryFunctors.h create mode 100644 Eigen/src/Core/arch/SSE/Reductions.h create mode 100644 Eigen/src/Core/util/Assert.h create mode 100644 Eigen/src/Core/util/EmulateArray.h create mode 100644 Eigen/src/Core/util/GpuHipCudaDefines.inc create mode 100644 Eigen/src/Core/util/GpuHipCudaUndefines.inc create mode 100644 Eigen/src/Core/util/MaxSizeVector.h create mode 100644 Eigen/src/Core/util/MoreMeta.h create mode 100644 Eigen/src/Core/util/Serializer.h create mode 100644 Eigen/src/Eigenvalues/InternalHeaderCheck.h create mode 100644 Eigen/src/Geometry/InternalHeaderCheck.h create mode 100644 Eigen/src/Householder/InternalHeaderCheck.h create mode 100644 Eigen/src/IterativeLinearSolvers/InternalHeaderCheck.h create mode 100644 Eigen/src/Jacobi/InternalHeaderCheck.h create mode 100644 Eigen/src/KLUSupport/InternalHeaderCheck.h create mode 100644 Eigen/src/LU/InternalHeaderCheck.h create mode 100644 Eigen/src/MetisSupport/InternalHeaderCheck.h create mode 100644 Eigen/src/OrderingMethods/InternalHeaderCheck.h create mode 100644 Eigen/src/PaStiXSupport/InternalHeaderCheck.h create mode 100644 Eigen/src/PardisoSupport/InternalHeaderCheck.h create mode 100644 Eigen/src/QR/InternalHeaderCheck.h create mode 100644 Eigen/src/SPQRSupport/InternalHeaderCheck.h create mode 100644 Eigen/src/SVD/BDCSVD_LAPACKE.h create mode 100644 Eigen/src/SVD/InternalHeaderCheck.h create mode 100644 Eigen/src/SparseCholesky/InternalHeaderCheck.h create mode 100644 Eigen/src/SparseCore/InternalHeaderCheck.h create mode 100644 Eigen/src/SparseLU/InternalHeaderCheck.h create mode 100644 Eigen/src/SparseQR/InternalHeaderCheck.h create mode 100644 Eigen/src/SuperLUSupport/InternalHeaderCheck.h create mode 100644 Eigen/src/ThreadPool/Barrier.h create mode 100644 Eigen/src/ThreadPool/CoreThreadPoolDevice.h create mode 100644 Eigen/src/ThreadPool/EventCount.h create mode 100644 Eigen/src/ThreadPool/ForkJoin.h create mode 100644 Eigen/src/ThreadPool/InternalHeaderCheck.h create mode 100644 Eigen/src/ThreadPool/NonBlockingThreadPool.h create mode 100644 Eigen/src/ThreadPool/RunQueue.h create mode 100644 Eigen/src/ThreadPool/ThreadCancel.h create mode 100644 Eigen/src/ThreadPool/ThreadEnvironment.h create mode 100644 Eigen/src/ThreadPool/ThreadLocal.h create mode 100644 Eigen/src/ThreadPool/ThreadPoolInterface.h create mode 100644 Eigen/src/ThreadPool/ThreadYield.h create mode 100644 Eigen/src/UmfPackSupport/InternalHeaderCheck.h create mode 100644 Eigen/src/misc/InternalHeaderCheck.h create mode 100644 Eigen/src/misc/lapacke_helpers.h create mode 100644 Eigen/src/plugins/ArrayCwiseBinaryOps.inc create mode 100644 Eigen/src/plugins/ArrayCwiseUnaryOps.inc create mode 100644 Eigen/src/plugins/BlockMethods.inc create mode 100644 Eigen/src/plugins/CommonCwiseBinaryOps.inc create mode 100644 Eigen/src/plugins/CommonCwiseUnaryOps.inc create mode 100644 Eigen/src/plugins/IndexedViewMethods.inc create mode 100644 Eigen/src/plugins/InternalHeaderCheck.inc create mode 100644 Eigen/src/plugins/MatrixCwiseBinaryOps.inc create mode 100644 Eigen/src/plugins/MatrixCwiseUnaryOps.inc create mode 100644 Eigen/src/plugins/ReshapedMethods.inc diff --git a/Eigen/AccelerateSupport b/Eigen/AccelerateSupport new file mode 100644 index 0000000..533be68 --- /dev/null +++ b/Eigen/AccelerateSupport @@ -0,0 +1,52 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_ACCELERATESUPPORT_MODULE_H +#define EIGEN_ACCELERATESUPPORT_MODULE_H + +#include "SparseCore" + +#include "src/Core/util/DisableStupidWarnings.h" + +/** \ingroup Support_modules + * \defgroup AccelerateSupport_Module AccelerateSupport module + * + * This module provides an interface to the Apple Accelerate library. + * It provides the seven following main factorization classes: + * - class AccelerateLLT: a Cholesky (LL^T) factorization. + * - class AccelerateLDLT: the default LDL^T factorization. + * - class AccelerateLDLTUnpivoted: a Cholesky-like LDL^T factorization with only 1x1 pivots and no pivoting + * - class AccelerateLDLTSBK: an LDL^T factorization with Supernode Bunch-Kaufman and static pivoting + * - class AccelerateLDLTTPP: an LDL^T factorization with full threshold partial pivoting + * - class AccelerateQR: a QR factorization + * - class AccelerateCholeskyAtA: a QR factorization without storing Q (equivalent to A^TA = R^T R) + * + * \code + * #include + * \endcode + * + * In order to use this module, the Accelerate headers must be accessible from + * the include paths, and your binary must be linked to the Accelerate framework. + * The Accelerate library is only available on Apple hardware. + * + * Note that many of the algorithms can be influenced by the UpLo template + * argument. All matrices are assumed to be symmetric. For example, the following + * creates an LDLT factorization where your matrix is symmetric (implicit) and + * uses the lower triangle: + * + * \code + * AccelerateLDLT, Lower> ldlt; + * \endcode + */ + +// IWYU pragma: begin_exports +#include "src/AccelerateSupport/AccelerateSupport.h" +// IWYU pragma: end_exports + +#include "src/Core/util/ReenableStupidWarnings.h" + +#endif // EIGEN_ACCELERATESUPPORT_MODULE_H diff --git a/Eigen/ThreadPool b/Eigen/ThreadPool new file mode 100644 index 0000000..39e5d1e --- /dev/null +++ b/Eigen/ThreadPool @@ -0,0 +1,80 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2016 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_THREADPOOL_MODULE_H +#define EIGEN_THREADPOOL_MODULE_H + +#include "Core" + +#include "src/Core/util/DisableStupidWarnings.h" + +/** \defgroup ThreadPool_Module ThreadPool Module + * + * This module provides 2 threadpool implementations + * - a simple reference implementation + * - a faster non blocking implementation + * + * \code + * #include + * \endcode + */ + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// There are non-parenthesized calls to "max" in the header, +// which trigger a check in test/main.h causing compilation to fail. +// We work around the check here by removing the check for max in +// the case where we have to emulate thread_local. +#ifdef max +#undef max +#endif +#include + +#include "src/Core/util/Meta.h" +#include "src/Core/util/MaxSizeVector.h" + +#ifndef EIGEN_MUTEX +#define EIGEN_MUTEX std::mutex +#endif +#ifndef EIGEN_MUTEX_LOCK +#define EIGEN_MUTEX_LOCK std::unique_lock +#endif +#ifndef EIGEN_CONDVAR +#define EIGEN_CONDVAR std::condition_variable +#endif + +// IWYU pragma: begin_exports +#include "src/ThreadPool/ThreadLocal.h" +#include "src/ThreadPool/ThreadYield.h" +#include "src/ThreadPool/ThreadCancel.h" +#include "src/ThreadPool/EventCount.h" +#include "src/ThreadPool/RunQueue.h" +#include "src/ThreadPool/ThreadPoolInterface.h" +#include "src/ThreadPool/ThreadEnvironment.h" +#include "src/ThreadPool/Barrier.h" +#include "src/ThreadPool/NonBlockingThreadPool.h" +#include "src/ThreadPool/CoreThreadPoolDevice.h" +#include "src/ThreadPool/ForkJoin.h" +// IWYU pragma: end_exports + +#include "src/Core/util/ReenableStupidWarnings.h" + +#endif // EIGEN_CXX11_THREADPOOL_MODULE_H diff --git a/Eigen/Version b/Eigen/Version new file mode 100644 index 0000000..9cfd71f --- /dev/null +++ b/Eigen/Version @@ -0,0 +1,14 @@ +#ifndef EIGEN_VERSION_H +#define EIGEN_VERSION_H + +// The "WORLD" version will forever remain "3" for the "Eigen3" library. +#define EIGEN_WORLD_VERSION 3 +// As of Eigen3 5.0.0, we have moved to Semantic Versioning (semver.org). +#define EIGEN_MAJOR_VERSION 5 +#define EIGEN_MINOR_VERSION 0 +#define EIGEN_PATCH_VERSION 1 +#define EIGEN_PRERELEASE_VERSION "" +#define EIGEN_BUILD_VERSION "" +#define EIGEN_VERSION_STRING "5.0.1" + +#endif // EIGEN_VERSION_H diff --git a/Eigen/src/AccelerateSupport/AccelerateSupport.h b/Eigen/src/AccelerateSupport/AccelerateSupport.h new file mode 100644 index 0000000..13a26df --- /dev/null +++ b/Eigen/src/AccelerateSupport/AccelerateSupport.h @@ -0,0 +1,423 @@ +#ifndef EIGEN_ACCELERATESUPPORT_H +#define EIGEN_ACCELERATESUPPORT_H + +#include + +#include + +namespace Eigen { + +template +class AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \typedef AccelerateLLT + * \brief A direct Cholesky (LLT) factorization and solver based on Accelerate + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLLT + */ +template +using AccelerateLLT = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \typedef AccelerateLDLT + * \brief The default Cholesky (LDLT) factorization and solver based on Accelerate + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLDLT + */ +template +using AccelerateLDLT = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \typedef AccelerateLDLTUnpivoted + * \brief A direct Cholesky-like LDL^T factorization and solver based on Accelerate with only 1x1 pivots and no pivoting + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLDLTUnpivoted + */ +template +using AccelerateLDLTUnpivoted = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \typedef AccelerateLDLTSBK + * \brief A direct Cholesky (LDLT) factorization and solver based on Accelerate with Supernode Bunch-Kaufman and static + * pivoting + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLDLTSBK + */ +template +using AccelerateLDLTSBK = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \typedef AccelerateLDLTTPP + * \brief A direct Cholesky (LDLT) factorization and solver based on Accelerate with full threshold partial pivoting + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * \tparam UpLo_ additional information about the matrix structure. Default is Lower. + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateLDLTTPP + */ +template +using AccelerateLDLTTPP = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \typedef AccelerateQR + * \brief A QR factorization and solver based on Accelerate + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateQR + */ +template +using AccelerateQR = AccelerateImpl; + +/** \ingroup AccelerateSupport_Module + * \typedef AccelerateCholeskyAtA + * \brief A QR factorization and solver based on Accelerate without storing Q (equivalent to A^TA = R^T R) + * + * \warning Only single and double precision real scalar types are supported by Accelerate + * + * \tparam MatrixType_ the type of the sparse matrix A, it must be a SparseMatrix<> + * + * \sa \ref TutorialSparseSolverConcept, class AccelerateCholeskyAtA + */ +template +using AccelerateCholeskyAtA = AccelerateImpl; + +namespace internal { +template +struct AccelFactorizationDeleter { + void operator()(T* sym) { + if (sym) { + SparseCleanup(*sym); + delete sym; + sym = nullptr; + } + } +}; + +template +struct SparseTypesTraitBase { + typedef DenseVecT AccelDenseVector; + typedef DenseMatT AccelDenseMatrix; + typedef SparseMatT AccelSparseMatrix; + + typedef SparseOpaqueSymbolicFactorization SymbolicFactorization; + typedef NumFactT NumericFactorization; + + typedef AccelFactorizationDeleter SymbolicFactorizationDeleter; + typedef AccelFactorizationDeleter NumericFactorizationDeleter; +}; + +template +struct SparseTypesTrait {}; + +template <> +struct SparseTypesTrait : SparseTypesTraitBase {}; + +template <> +struct SparseTypesTrait + : SparseTypesTraitBase { +}; + +} // end namespace internal + +template +class AccelerateImpl : public SparseSolverBase > { + protected: + using Base = SparseSolverBase; + using Base::derived; + using Base::m_isInitialized; + + public: + using Base::_solve_impl; + + typedef MatrixType_ MatrixType; + typedef typename MatrixType::Scalar Scalar; + typedef typename MatrixType::StorageIndex StorageIndex; + enum { ColsAtCompileTime = Dynamic, MaxColsAtCompileTime = Dynamic }; + enum { UpLo = UpLo_ }; + + using AccelDenseVector = typename internal::SparseTypesTrait::AccelDenseVector; + using AccelDenseMatrix = typename internal::SparseTypesTrait::AccelDenseMatrix; + using AccelSparseMatrix = typename internal::SparseTypesTrait::AccelSparseMatrix; + using SymbolicFactorization = typename internal::SparseTypesTrait::SymbolicFactorization; + using NumericFactorization = typename internal::SparseTypesTrait::NumericFactorization; + using SymbolicFactorizationDeleter = typename internal::SparseTypesTrait::SymbolicFactorizationDeleter; + using NumericFactorizationDeleter = typename internal::SparseTypesTrait::NumericFactorizationDeleter; + + AccelerateImpl() { + m_isInitialized = false; + + auto check_flag_set = [](int value, int flag) { return ((value & flag) == flag); }; + + if (check_flag_set(UpLo_, Symmetric)) { + m_sparseKind = SparseSymmetric; + m_triType = (UpLo_ & Lower) ? SparseLowerTriangle : SparseUpperTriangle; + } else if (check_flag_set(UpLo_, UnitLower)) { + m_sparseKind = SparseUnitTriangular; + m_triType = SparseLowerTriangle; + } else if (check_flag_set(UpLo_, UnitUpper)) { + m_sparseKind = SparseUnitTriangular; + m_triType = SparseUpperTriangle; + } else if (check_flag_set(UpLo_, StrictlyLower)) { + m_sparseKind = SparseTriangular; + m_triType = SparseLowerTriangle; + } else if (check_flag_set(UpLo_, StrictlyUpper)) { + m_sparseKind = SparseTriangular; + m_triType = SparseUpperTriangle; + } else if (check_flag_set(UpLo_, Lower)) { + m_sparseKind = SparseTriangular; + m_triType = SparseLowerTriangle; + } else if (check_flag_set(UpLo_, Upper)) { + m_sparseKind = SparseTriangular; + m_triType = SparseUpperTriangle; + } else { + m_sparseKind = SparseOrdinary; + m_triType = (UpLo_ & Lower) ? SparseLowerTriangle : SparseUpperTriangle; + } + + m_order = SparseOrderDefault; + } + + explicit AccelerateImpl(const MatrixType& matrix) : AccelerateImpl() { compute(matrix); } + + ~AccelerateImpl() {} + + inline Index cols() const { return m_nCols; } + inline Index rows() const { return m_nRows; } + + ComputationInfo info() const { + eigen_assert(m_isInitialized && "Decomposition is not initialized."); + return m_info; + } + + void analyzePattern(const MatrixType& matrix); + + void factorize(const MatrixType& matrix); + + void compute(const MatrixType& matrix); + + template + void _solve_impl(const MatrixBase& b, MatrixBase& dest) const; + + /** Sets the ordering algorithm to use. */ + void setOrder(SparseOrder_t order) { m_order = order; } + + private: + template + void buildAccelSparseMatrix(const SparseMatrix& a, AccelSparseMatrix& A, std::vector& columnStarts) { + const Index nColumnsStarts = a.cols() + 1; + + columnStarts.resize(nColumnsStarts); + + for (Index i = 0; i < nColumnsStarts; i++) columnStarts[i] = a.outerIndexPtr()[i]; + + SparseAttributes_t attributes{}; + attributes.transpose = false; + attributes.triangle = m_triType; + attributes.kind = m_sparseKind; + + SparseMatrixStructure structure{}; + structure.attributes = attributes; + structure.rowCount = static_cast(a.rows()); + structure.columnCount = static_cast(a.cols()); + structure.blockSize = 1; + structure.columnStarts = columnStarts.data(); + structure.rowIndices = const_cast(a.innerIndexPtr()); + + A.structure = structure; + A.data = const_cast(a.valuePtr()); + } + + void doAnalysis(AccelSparseMatrix& A) { + m_numericFactorization.reset(nullptr); + + SparseSymbolicFactorOptions opts{}; + opts.control = SparseDefaultControl; + opts.orderMethod = m_order; + opts.order = nullptr; + opts.ignoreRowsAndColumns = nullptr; + opts.malloc = malloc; + opts.free = free; + opts.reportError = nullptr; + + m_symbolicFactorization.reset(new SymbolicFactorization(SparseFactor(Solver_, A.structure, opts))); + + SparseStatus_t status = m_symbolicFactorization->status; + + updateInfoStatus(status); + + if (status != SparseStatusOK) m_symbolicFactorization.reset(nullptr); + } + + void doFactorization(AccelSparseMatrix& A) { + SparseStatus_t status = SparseStatusReleased; + + if (m_symbolicFactorization) { + m_numericFactorization.reset(new NumericFactorization(SparseFactor(*m_symbolicFactorization, A))); + + status = m_numericFactorization->status; + + if (status != SparseStatusOK) m_numericFactorization.reset(nullptr); + } + + updateInfoStatus(status); + } + + protected: + void updateInfoStatus(SparseStatus_t status) const { + switch (status) { + case SparseStatusOK: + m_info = Success; + break; + case SparseFactorizationFailed: + case SparseMatrixIsSingular: + m_info = NumericalIssue; + break; + case SparseInternalError: + case SparseParameterError: + case SparseStatusReleased: + default: + m_info = InvalidInput; + break; + } + } + + mutable ComputationInfo m_info; + Index m_nRows, m_nCols; + std::unique_ptr m_symbolicFactorization; + std::unique_ptr m_numericFactorization; + SparseKind_t m_sparseKind; + SparseTriangle_t m_triType; + SparseOrder_t m_order; +}; + +/** Computes the symbolic and numeric decomposition of matrix \a a */ +template +void AccelerateImpl::compute(const MatrixType& a) { + if (EnforceSquare_) eigen_assert(a.rows() == a.cols()); + + m_nRows = a.rows(); + m_nCols = a.cols(); + + AccelSparseMatrix A{}; + std::vector columnStarts; + + buildAccelSparseMatrix(a, A, columnStarts); + + doAnalysis(A); + + if (m_symbolicFactorization) doFactorization(A); + + m_isInitialized = true; +} + +/** Performs a symbolic decomposition on the sparsity pattern of matrix \a a. + * + * This function is particularly useful when solving for several problems having the same structure. + * + * \sa factorize() + */ +template +void AccelerateImpl::analyzePattern(const MatrixType& a) { + if (EnforceSquare_) eigen_assert(a.rows() == a.cols()); + + m_nRows = a.rows(); + m_nCols = a.cols(); + + AccelSparseMatrix A{}; + std::vector columnStarts; + + buildAccelSparseMatrix(a, A, columnStarts); + + doAnalysis(A); + + m_isInitialized = true; +} + +/** Performs a numeric decomposition of matrix \a a. + * + * The given matrix must have the same sparsity pattern as the matrix on which the symbolic decomposition has been + * performed. + * + * \sa analyzePattern() + */ +template +void AccelerateImpl::factorize(const MatrixType& a) { + eigen_assert(m_symbolicFactorization && "You must first call analyzePattern()"); + eigen_assert(m_nRows == a.rows() && m_nCols == a.cols()); + + if (EnforceSquare_) eigen_assert(a.rows() == a.cols()); + + AccelSparseMatrix A{}; + std::vector columnStarts; + + buildAccelSparseMatrix(a, A, columnStarts); + + doFactorization(A); +} + +template +template +void AccelerateImpl::_solve_impl(const MatrixBase& b, + MatrixBase& x) const { + if (!m_numericFactorization) { + m_info = InvalidInput; + return; + } + + eigen_assert(m_nRows == b.rows()); + eigen_assert(((b.cols() == 1) || b.outerStride() == b.rows())); + + SparseStatus_t status = SparseStatusOK; + + Scalar* b_ptr = const_cast(b.derived().data()); + Scalar* x_ptr = const_cast(x.derived().data()); + + AccelDenseMatrix xmat{}; + xmat.attributes = SparseAttributes_t(); + xmat.columnCount = static_cast(x.cols()); + xmat.rowCount = static_cast(x.rows()); + xmat.columnStride = xmat.rowCount; + xmat.data = x_ptr; + + AccelDenseMatrix bmat{}; + bmat.attributes = SparseAttributes_t(); + bmat.columnCount = static_cast(b.cols()); + bmat.rowCount = static_cast(b.rows()); + bmat.columnStride = bmat.rowCount; + bmat.data = b_ptr; + + SparseSolve(*m_numericFactorization, bmat, xmat); + + updateInfoStatus(status); +} + +} // end namespace Eigen + +#endif // EIGEN_ACCELERATESUPPORT_H diff --git a/Eigen/src/AccelerateSupport/InternalHeaderCheck.h b/Eigen/src/AccelerateSupport/InternalHeaderCheck.h new file mode 100644 index 0000000..69bcff5 --- /dev/null +++ b/Eigen/src/AccelerateSupport/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_ACCELERATESUPPORT_MODULE_H +#error "Please include Eigen/AccelerateSupport instead of including headers inside the src directory directly." +#endif diff --git a/Eigen/src/Cholesky/InternalHeaderCheck.h b/Eigen/src/Cholesky/InternalHeaderCheck.h new file mode 100644 index 0000000..5de2b21 --- /dev/null +++ b/Eigen/src/Cholesky/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_CHOLESKY_MODULE_H +#error "Please include Eigen/Cholesky instead of including headers inside the src directory directly." +#endif diff --git a/Eigen/src/CholmodSupport/InternalHeaderCheck.h b/Eigen/src/CholmodSupport/InternalHeaderCheck.h new file mode 100644 index 0000000..0fb3abc --- /dev/null +++ b/Eigen/src/CholmodSupport/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_CHOLMODSUPPORT_MODULE_H +#error "Please include Eigen/CholmodSupport instead of including headers inside the src directory directly." +#endif diff --git a/Eigen/src/Core/DeviceWrapper.h b/Eigen/src/Core/DeviceWrapper.h new file mode 100644 index 0000000..012dce1 --- /dev/null +++ b/Eigen/src/Core/DeviceWrapper.h @@ -0,0 +1,153 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2023 Charlie Schlosser +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_DEVICEWRAPPER_H +#define EIGEN_DEVICEWRAPPER_H + +namespace Eigen { +template +struct DeviceWrapper { + using Base = EigenBase>; + using Scalar = typename Derived::Scalar; + + EIGEN_DEVICE_FUNC DeviceWrapper(Base& xpr, Device& device) : m_xpr(xpr.derived()), m_device(device) {} + EIGEN_DEVICE_FUNC DeviceWrapper(const Base& xpr, Device& device) : m_xpr(xpr.derived()), m_device(device) {} + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator=(const EigenBase& other) { + using AssignOp = internal::assign_op; + internal::call_assignment(*this, other.derived(), AssignOp()); + return m_xpr; + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator+=(const EigenBase& other) { + using AddAssignOp = internal::add_assign_op; + internal::call_assignment(*this, other.derived(), AddAssignOp()); + return m_xpr; + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& operator-=(const EigenBase& other) { + using SubAssignOp = internal::sub_assign_op; + internal::call_assignment(*this, other.derived(), SubAssignOp()); + return m_xpr; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Derived& derived() { return m_xpr; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Device& device() { return m_device; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE NoAlias noalias() { + return NoAlias(*this); + } + + Derived& m_xpr; + Device& m_device; +}; + +namespace internal { + +// this is where we differentiate between lazy assignment and specialized kernels (e.g. matrix products) +template ::Shape, + typename evaluator_traits::Shape>::Kind, + typename EnableIf = void> +struct AssignmentWithDevice; + +// unless otherwise specified, use the default product implementation +template +struct AssignmentWithDevice, Functor, Device, Dense2Dense, Weak> { + using SrcXprType = Product; + using Base = Assignment; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src, const Functor& func, + Device&) { + Base::run(dst, src, func); + } +}; + +// specialization for coeffcient-wise assignment +template +struct AssignmentWithDevice { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(DstXprType& dst, const SrcXprType& src, const Functor& func, + Device& device) { +#ifndef EIGEN_NO_DEBUG + internal::check_for_aliasing(dst, src); +#endif + + call_dense_assignment_loop(dst, src, func, device); + } +}; + +// this allows us to use the default evaluation scheme if it is not specialized for the device +template +struct dense_assignment_loop_with_device { + using Base = dense_assignment_loop; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void run(Kernel& kernel, Device&) { Base::run(kernel); } +}; + +// entry point for a generic expression with device +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_assignment_no_alias(DeviceWrapper dst, + const Src& src, const Func& func) { + enum { + NeedToTranspose = ((int(Dst::RowsAtCompileTime) == 1 && int(Src::ColsAtCompileTime) == 1) || + (int(Dst::ColsAtCompileTime) == 1 && int(Src::RowsAtCompileTime) == 1)) && + int(Dst::SizeAtCompileTime) != 1 + }; + + using ActualDstTypeCleaned = std::conditional_t, Dst>; + using ActualDstType = std::conditional_t, Dst&>; + ActualDstType actualDst(dst.derived()); + + // TODO check whether this is the right place to perform these checks: + EIGEN_STATIC_ASSERT_LVALUE(Dst) + EIGEN_STATIC_ASSERT_SAME_MATRIX_SIZE(ActualDstTypeCleaned, Src) + EIGEN_CHECK_BINARY_COMPATIBILIY(Func, typename ActualDstTypeCleaned::Scalar, typename Src::Scalar); + + // this provides a mechanism for specializing simple assignments, matrix products, etc + AssignmentWithDevice::run(actualDst, src, func, dst.device()); +} + +// copy and pasted from AssignEvaluator except forward device to kernel +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void call_dense_assignment_loop(DstXprType& dst, const SrcXprType& src, + const Functor& func, Device& device) { + using DstEvaluatorType = evaluator; + using SrcEvaluatorType = evaluator; + + SrcEvaluatorType srcEvaluator(src); + + // NOTE To properly handle A = (A*A.transpose())/s with A rectangular, + // we need to resize the destination after the source evaluator has been created. + resize_if_allowed(dst, src, func); + + DstEvaluatorType dstEvaluator(dst); + + using Kernel = generic_dense_assignment_kernel; + + Kernel kernel(dstEvaluator, srcEvaluator, func, dst.const_cast_derived()); + + dense_assignment_loop_with_device::run(kernel, device); +} + +} // namespace internal + +template +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DeviceWrapper EigenBase::device(Device& device) { + return DeviceWrapper(derived(), device); +} + +template +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE DeviceWrapper EigenBase::device( + Device& device) const { + return DeviceWrapper(derived(), device); +} +} // namespace Eigen +#endif diff --git a/Eigen/src/Core/Fill.h b/Eigen/src/Core/Fill.h new file mode 100644 index 0000000..779ef26 --- /dev/null +++ b/Eigen/src/Core/Fill.h @@ -0,0 +1,138 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2024 Charles Schlosser +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_FILL_H +#define EIGEN_FILL_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +template +struct eigen_fill_helper : std::false_type {}; + +template +struct eigen_fill_helper> : std::true_type {}; + +template +struct eigen_fill_helper> : std::true_type {}; + +template +struct eigen_fill_helper> : eigen_fill_helper {}; + +template +struct eigen_fill_helper> + : std::integral_constant::value && + (Xpr::IsRowMajor ? (BlockRows == 1) : (BlockCols == 1))> {}; + +template +struct eigen_fill_helper>> : eigen_fill_helper {}; + +template +struct eigen_fill_helper>> + : std::integral_constant::value && + enum_eq_not_dynamic(OuterStride_, Xpr::InnerSizeAtCompileTime)> {}; + +template +struct eigen_fill_helper>> + : eigen_fill_helper>> {}; + +template +struct eigen_fill_helper>> + : eigen_fill_helper>> {}; + +template +struct eigen_fill_helper>> + : eigen_fill_helper>> {}; + +template +struct eigen_fill_impl { + using Scalar = typename Xpr::Scalar; + using Func = scalar_constant_op; + using PlainObject = typename Xpr::PlainObject; + using Constant = typename PlainObject::ConstantReturnType; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void run(Xpr& dst, const Scalar& val) { + const Constant src(dst.rows(), dst.cols(), val); + run(dst, src); + } + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void run(Xpr& dst, const SrcXpr& src) { + call_dense_assignment_loop(dst, src, assign_op()); + } +}; + +#if EIGEN_COMP_MSVC || defined(EIGEN_GPU_COMPILE_PHASE) +template +struct eigen_fill_impl : eigen_fill_impl {}; +#else +template +struct eigen_fill_impl { + using Scalar = typename Xpr::Scalar; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const Scalar& val) { + const Scalar val_copy = val; + using std::fill_n; + fill_n(dst.data(), dst.size(), val_copy); + } + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const SrcXpr& src) { + resize_if_allowed(dst, src, assign_op()); + const Scalar& val = src.functor()(); + run(dst, val); + } +}; +#endif + +template +struct eigen_memset_helper { + static constexpr bool value = + std::is_trivially_copyable::value && eigen_fill_helper::value; +}; + +template +struct eigen_zero_impl { + using Scalar = typename Xpr::Scalar; + using PlainObject = typename Xpr::PlainObject; + using Zero = typename PlainObject::ZeroReturnType; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void run(Xpr& dst) { + const Zero src(dst.rows(), dst.cols()); + run(dst, src); + } + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr void run(Xpr& dst, const SrcXpr& src) { + call_dense_assignment_loop(dst, src, assign_op()); + } +}; + +template +struct eigen_zero_impl { + using Scalar = typename Xpr::Scalar; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst) { + const std::ptrdiff_t num_bytes = dst.size() * static_cast(sizeof(Scalar)); + if (num_bytes <= 0) return; + void* dst_ptr = static_cast(dst.data()); +#ifndef EIGEN_NO_DEBUG + eigen_assert((dst_ptr != nullptr) && "null pointer dereference error!"); +#endif + EIGEN_USING_STD(memset); + memset(dst_ptr, 0, static_cast(num_bytes)); + } + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(Xpr& dst, const SrcXpr& src) { + resize_if_allowed(dst, src, assign_op()); + run(dst); + } +}; + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_FILL_H diff --git a/Eigen/src/Core/FindCoeff.h b/Eigen/src/Core/FindCoeff.h new file mode 100644 index 0000000..0102e8a --- /dev/null +++ b/Eigen/src/Core/FindCoeff.h @@ -0,0 +1,464 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2025 Charlie Schlosser +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_FIND_COEFF_H +#define EIGEN_FIND_COEFF_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +template ::IsInteger> +struct max_coeff_functor { + EIGEN_DEVICE_FUNC inline bool compareCoeff(const Scalar& incumbent, const Scalar& candidate) const { + return candidate > incumbent; + } + template + EIGEN_DEVICE_FUNC inline Packet comparePacket(const Packet& incumbent, const Packet& candidate) const { + return pcmp_lt(incumbent, candidate); + } + template + EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& a) const { + return predux_max(a); + } +}; + +template +struct max_coeff_functor { + EIGEN_DEVICE_FUNC inline Scalar compareCoeff(const Scalar& incumbent, const Scalar& candidate) { + return (candidate > incumbent) || ((candidate != candidate) && (incumbent == incumbent)); + } + template + EIGEN_DEVICE_FUNC inline Packet comparePacket(const Packet& incumbent, const Packet& candidate) { + return pandnot(pcmp_lt_or_nan(incumbent, candidate), pisnan(incumbent)); + } + template + EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& a) const { + return predux_max(a); + } +}; + +template +struct max_coeff_functor { + EIGEN_DEVICE_FUNC inline bool compareCoeff(const Scalar& incumbent, const Scalar& candidate) const { + return (candidate > incumbent) || ((candidate == candidate) && (incumbent != incumbent)); + } + template + EIGEN_DEVICE_FUNC inline Packet comparePacket(const Packet& incumbent, const Packet& candidate) const { + return pandnot(pcmp_lt_or_nan(incumbent, candidate), pisnan(candidate)); + } + template + EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& a) const { + return predux_max(a); + } +}; + +template ::IsInteger> +struct min_coeff_functor { + EIGEN_DEVICE_FUNC inline bool compareCoeff(const Scalar& incumbent, const Scalar& candidate) const { + return candidate < incumbent; + } + template + EIGEN_DEVICE_FUNC inline Packet comparePacket(const Packet& incumbent, const Packet& candidate) const { + return pcmp_lt(candidate, incumbent); + } + template + EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& a) const { + return predux_min(a); + } +}; + +template +struct min_coeff_functor { + EIGEN_DEVICE_FUNC inline Scalar compareCoeff(const Scalar& incumbent, const Scalar& candidate) { + return (candidate < incumbent) || ((candidate != candidate) && (incumbent == incumbent)); + } + template + EIGEN_DEVICE_FUNC inline Packet comparePacket(const Packet& incumbent, const Packet& candidate) { + return pandnot(pcmp_lt_or_nan(candidate, incumbent), pisnan(incumbent)); + } + template + EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& a) const { + return predux_min(a); + } +}; + +template +struct min_coeff_functor { + EIGEN_DEVICE_FUNC inline bool compareCoeff(const Scalar& incumbent, const Scalar& candidate) const { + return (candidate < incumbent) || ((candidate == candidate) && (incumbent != incumbent)); + } + template + EIGEN_DEVICE_FUNC inline Packet comparePacket(const Packet& incumbent, const Packet& candidate) const { + return pandnot(pcmp_lt_or_nan(candidate, incumbent), pisnan(candidate)); + } + template + EIGEN_DEVICE_FUNC inline Scalar predux(const Packet& a) const { + return predux_min(a); + } +}; + +template +struct min_max_traits { + static constexpr bool PacketAccess = packet_traits::Vectorizable; +}; +template +struct functor_traits> : min_max_traits {}; +template +struct functor_traits> : min_max_traits {}; + +template +struct find_coeff_loop; +template +struct find_coeff_loop { + using Scalar = typename Evaluator::Scalar; + static EIGEN_DEVICE_FUNC inline void run(const Evaluator& eval, Func& func, Scalar& res, Index& outer, Index& inner) { + Index outerSize = eval.outerSize(); + Index innerSize = eval.innerSize(); + + /* initialization performed in calling function */ + /* result = eval.coeff(0, 0); */ + /* outer = 0; */ + /* inner = 0; */ + + for (Index j = 0; j < outerSize; j++) { + for (Index i = 0; i < innerSize; i++) { + Scalar xprCoeff = eval.coeffByOuterInner(j, i); + bool newRes = func.compareCoeff(res, xprCoeff); + if (newRes) { + outer = j; + inner = i; + res = xprCoeff; + } + } + } + } +}; +template +struct find_coeff_loop { + using Scalar = typename Evaluator::Scalar; + static EIGEN_DEVICE_FUNC inline void run(const Evaluator& eval, Func& func, Scalar& res, Index& index) { + Index size = eval.size(); + + /* initialization performed in calling function */ + /* result = eval.coeff(0); */ + /* index = 0; */ + + for (Index k = 0; k < size; k++) { + Scalar xprCoeff = eval.coeff(k); + bool newRes = func.compareCoeff(res, xprCoeff); + if (newRes) { + index = k; + res = xprCoeff; + } + } + } +}; +template +struct find_coeff_loop { + using ScalarImpl = find_coeff_loop; + using Scalar = typename Evaluator::Scalar; + using Packet = typename Evaluator::Packet; + static constexpr int PacketSize = unpacket_traits::size; + static EIGEN_DEVICE_FUNC inline void run(const Evaluator& eval, Func& func, Scalar& result, Index& outer, + Index& inner) { + Index outerSize = eval.outerSize(); + Index innerSize = eval.innerSize(); + Index packetEnd = numext::round_down(innerSize, PacketSize); + + /* initialization performed in calling function */ + /* result = eval.coeff(0, 0); */ + /* outer = 0; */ + /* inner = 0; */ + + bool checkPacket = false; + + for (Index j = 0; j < outerSize; j++) { + Packet resultPacket = pset1(result); + for (Index i = 0; i < packetEnd; i += PacketSize) { + Packet xprPacket = eval.template packetByOuterInner(j, i); + if (predux_any(func.comparePacket(resultPacket, xprPacket))) { + outer = j; + inner = i; + result = func.predux(xprPacket); + resultPacket = pset1(result); + checkPacket = true; + } + } + + for (Index i = packetEnd; i < innerSize; i++) { + Scalar xprCoeff = eval.coeffByOuterInner(j, i); + if (func.compareCoeff(result, xprCoeff)) { + outer = j; + inner = i; + result = xprCoeff; + checkPacket = false; + } + } + } + + if (checkPacket) { + result = eval.coeffByOuterInner(outer, inner); + Index i_end = inner + PacketSize; + for (Index i = inner; i < i_end; i++) { + Scalar xprCoeff = eval.coeffByOuterInner(outer, i); + if (func.compareCoeff(result, xprCoeff)) { + inner = i; + result = xprCoeff; + } + } + } + } +}; +template +struct find_coeff_loop { + using ScalarImpl = find_coeff_loop; + using Scalar = typename Evaluator::Scalar; + using Packet = typename Evaluator::Packet; + static constexpr int PacketSize = unpacket_traits::size; + static constexpr int Alignment = Evaluator::Alignment; + + static EIGEN_DEVICE_FUNC inline void run(const Evaluator& eval, Func& func, Scalar& result, Index& index) { + Index size = eval.size(); + Index packetEnd = numext::round_down(size, PacketSize); + + /* initialization performed in calling function */ + /* result = eval.coeff(0); */ + /* index = 0; */ + + Packet resultPacket = pset1(result); + bool checkPacket = false; + + for (Index k = 0; k < packetEnd; k += PacketSize) { + Packet xprPacket = eval.template packet(k); + if (predux_any(func.comparePacket(resultPacket, xprPacket))) { + index = k; + result = func.predux(xprPacket); + resultPacket = pset1(result); + checkPacket = true; + } + } + + for (Index k = packetEnd; k < size; k++) { + Scalar xprCoeff = eval.coeff(k); + if (func.compareCoeff(result, xprCoeff)) { + index = k; + result = xprCoeff; + checkPacket = false; + } + } + + if (checkPacket) { + result = eval.coeff(index); + Index k_end = index + PacketSize; + for (Index k = index; k < k_end; k++) { + Scalar xprCoeff = eval.coeff(k); + if (func.compareCoeff(result, xprCoeff)) { + index = k; + result = xprCoeff; + } + } + } + } +}; + +template +struct find_coeff_evaluator : public evaluator { + using Base = evaluator; + using Scalar = typename Derived::Scalar; + using Packet = typename packet_traits::type; + static constexpr int Flags = Base::Flags; + static constexpr bool IsRowMajor = bool(Flags & RowMajorBit); + EIGEN_DEVICE_FUNC inline find_coeff_evaluator(const Derived& xpr) : Base(xpr), m_xpr(xpr) {} + + EIGEN_DEVICE_FUNC inline Scalar coeffByOuterInner(Index outer, Index inner) const { + Index row = IsRowMajor ? outer : inner; + Index col = IsRowMajor ? inner : outer; + return Base::coeff(row, col); + } + template + EIGEN_DEVICE_FUNC inline PacketType packetByOuterInner(Index outer, Index inner) const { + Index row = IsRowMajor ? outer : inner; + Index col = IsRowMajor ? inner : outer; + return Base::template packet(row, col); + } + + EIGEN_DEVICE_FUNC inline Index innerSize() const { return m_xpr.innerSize(); } + EIGEN_DEVICE_FUNC inline Index outerSize() const { return m_xpr.outerSize(); } + EIGEN_DEVICE_FUNC inline Index size() const { return m_xpr.size(); } + + const Derived& m_xpr; +}; + +template +struct find_coeff_impl { + using Evaluator = find_coeff_evaluator; + static constexpr int Flags = Evaluator::Flags; + static constexpr int Alignment = Evaluator::Alignment; + static constexpr bool IsRowMajor = Derived::IsRowMajor; + static constexpr int MaxInnerSizeAtCompileTime = + IsRowMajor ? Derived::MaxColsAtCompileTime : Derived::MaxRowsAtCompileTime; + static constexpr int MaxSizeAtCompileTime = Derived::MaxSizeAtCompileTime; + + using Scalar = typename Derived::Scalar; + using Packet = typename Evaluator::Packet; + + static constexpr int PacketSize = unpacket_traits::size; + static constexpr bool Linearize = bool(Flags & LinearAccessBit); + static constexpr bool DontVectorize = + enum_lt_not_dynamic(Linearize ? MaxSizeAtCompileTime : MaxInnerSizeAtCompileTime, PacketSize); + static constexpr bool Vectorize = + !DontVectorize && bool(Flags & PacketAccessBit) && functor_traits::PacketAccess; + + using Loop = find_coeff_loop; + + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& xpr, Func& func, Scalar& res, Index& outer, + Index& inner) { + Evaluator eval(xpr); + Loop::run(eval, func, res, outer, inner); + } + template = true> + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& xpr, Func& func, Scalar& res, Index& outer, + Index& inner) { + // where possible, use the linear loop and back-calculate the outer and inner indices + Index index = 0; + run(xpr, func, res, index); + outer = index / xpr.innerSize(); + inner = index % xpr.innerSize(); + } + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void run(const Derived& xpr, Func& func, Scalar& res, Index& index) { + Evaluator eval(xpr); + Loop::run(eval, func, res, index); + } +}; + +template +EIGEN_DEVICE_FUNC typename internal::traits::Scalar findCoeff(const DenseBase& mat, Func& func, + IndexType* rowPtr, IndexType* colPtr) { + eigen_assert(mat.rows() > 0 && mat.cols() > 0 && "you are using an empty matrix"); + using Scalar = typename DenseBase::Scalar; + using FindCoeffImpl = internal::find_coeff_impl; + Index outer = 0; + Index inner = 0; + Scalar res = mat.coeff(0, 0); + FindCoeffImpl::run(mat.derived(), func, res, outer, inner); + *rowPtr = internal::convert_index(Derived::IsRowMajor ? outer : inner); + if (colPtr) *colPtr = internal::convert_index(Derived::IsRowMajor ? inner : outer); + return res; +} + +template +EIGEN_DEVICE_FUNC typename internal::traits::Scalar findCoeff(const DenseBase& mat, Func& func, + IndexType* indexPtr) { + eigen_assert(mat.size() > 0 && "you are using an empty matrix"); + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Derived) + using Scalar = typename DenseBase::Scalar; + using FindCoeffImpl = internal::find_coeff_impl; + Index index = 0; + Scalar res = mat.coeff(0); + FindCoeffImpl::run(mat.derived(), func, res, index); + *indexPtr = internal::convert_index(index); + return res; +} + +} // namespace internal + +/** \fn DenseBase::minCoeff(IndexType* rowId, IndexType* colId) const + * \returns the minimum of all coefficients of *this and puts in *row and *col its location. + * + * If there are multiple coefficients with the same extreme value, the location of the first instance is returned. + * + * In case \c *this contains NaN, NaNPropagation determines the behavior: + * NaNPropagation == PropagateFast : undefined + * NaNPropagation == PropagateNaN : result is NaN + * NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN + * \warning the matrix must be not empty, otherwise an assertion is triggered. + * + * \sa DenseBase::minCoeff(Index*), DenseBase::maxCoeff(Index*,Index*), DenseBase::visit(), DenseBase::minCoeff() + */ +template +template +EIGEN_DEVICE_FUNC typename internal::traits::Scalar DenseBase::minCoeff(IndexType* rowPtr, + IndexType* colPtr) const { + using Func = internal::min_coeff_functor; + Func func; + return internal::findCoeff(derived(), func, rowPtr, colPtr); +} + +/** \returns the minimum of all coefficients of *this and puts in *index its location. + * + * If there are multiple coefficients with the same extreme value, the location of the first instance is returned. + * + * In case \c *this contains NaN, NaNPropagation determines the behavior: + * NaNPropagation == PropagateFast : undefined + * NaNPropagation == PropagateNaN : result is NaN + * NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN + * \warning the matrix must be not empty, otherwise an assertion is triggered. + * + * \sa DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::maxCoeff(IndexType*,IndexType*), DenseBase::visit(), + * DenseBase::minCoeff() + */ +template +template +EIGEN_DEVICE_FUNC typename internal::traits::Scalar DenseBase::minCoeff(IndexType* indexPtr) const { + using Func = internal::min_coeff_functor; + Func func; + return internal::findCoeff(derived(), func, indexPtr); +} + +/** \fn DenseBase::maxCoeff(IndexType* rowId, IndexType* colId) const + * \returns the maximum of all coefficients of *this and puts in *row and *col its location. + * + * If there are multiple coefficients with the same extreme value, the location of the first instance is returned. + * + * In case \c *this contains NaN, NaNPropagation determines the behavior: + * NaNPropagation == PropagateFast : undefined + * NaNPropagation == PropagateNaN : result is NaN + * NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN + * \warning the matrix must be not empty, otherwise an assertion is triggered. + * + * \sa DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::visit(), DenseBase::maxCoeff() + */ +template +template +EIGEN_DEVICE_FUNC typename internal::traits::Scalar DenseBase::maxCoeff(IndexType* rowPtr, + IndexType* colPtr) const { + using Func = internal::max_coeff_functor; + Func func; + return internal::findCoeff(derived(), func, rowPtr, colPtr); +} + +/** \returns the maximum of all coefficients of *this and puts in *index its location. + * + * If there are multiple coefficients with the same extreme value, the location of the first instance is returned. + * + * In case \c *this contains NaN, NaNPropagation determines the behavior: + * NaNPropagation == PropagateFast : undefined + * NaNPropagation == PropagateNaN : result is NaN + * NaNPropagation == PropagateNumbers : result is maximum of elements that are not NaN + * \warning the matrix must be not empty, otherwise an assertion is triggered. + * + * \sa DenseBase::maxCoeff(IndexType*,IndexType*), DenseBase::minCoeff(IndexType*,IndexType*), DenseBase::visitor(), + * DenseBase::maxCoeff() + */ +template +template +EIGEN_DEVICE_FUNC typename internal::traits::Scalar DenseBase::maxCoeff(IndexType* indexPtr) const { + using Func = internal::max_coeff_functor; + Func func; + return internal::findCoeff(derived(), func, indexPtr); +} + +} // namespace Eigen + +#endif // EIGEN_FIND_COEFF_H diff --git a/Eigen/src/Core/InnerProduct.h b/Eigen/src/Core/InnerProduct.h new file mode 100644 index 0000000..686ad13 --- /dev/null +++ b/Eigen/src/Core/InnerProduct.h @@ -0,0 +1,260 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2024 Charlie Schlosser +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_INNER_PRODUCT_EVAL_H +#define EIGEN_INNER_PRODUCT_EVAL_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +// recursively searches for the largest simd type that does not exceed Size, or the smallest if no such type exists +template ::type, + bool Stop = + (unpacket_traits::size <= Size) || is_same::half>::value> +struct find_inner_product_packet_helper; + +template +struct find_inner_product_packet_helper { + using type = typename find_inner_product_packet_helper::half>::type; +}; + +template +struct find_inner_product_packet_helper { + using type = Packet; +}; + +template +struct find_inner_product_packet : find_inner_product_packet_helper {}; + +template +struct find_inner_product_packet { + using type = typename packet_traits::type; +}; + +template +struct inner_product_assert { + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Lhs) + EIGEN_STATIC_ASSERT_VECTOR_ONLY(Rhs) + EIGEN_STATIC_ASSERT_SAME_VECTOR_SIZE(Lhs, Rhs) +#ifndef EIGEN_NO_DEBUG + static EIGEN_DEVICE_FUNC void run(const Lhs& lhs, const Rhs& rhs) { + eigen_assert((lhs.size() == rhs.size()) && "Inner product: lhs and rhs vectors must have same size"); + } +#else + static EIGEN_DEVICE_FUNC void run(const Lhs&, const Rhs&) {} +#endif +}; + +template +struct inner_product_evaluator { + static constexpr int LhsFlags = evaluator::Flags; + static constexpr int RhsFlags = evaluator::Flags; + static constexpr int SizeAtCompileTime = size_prefer_fixed(Lhs::SizeAtCompileTime, Rhs::SizeAtCompileTime); + static constexpr int MaxSizeAtCompileTime = + min_size_prefer_fixed(Lhs::MaxSizeAtCompileTime, Rhs::MaxSizeAtCompileTime); + static constexpr int LhsAlignment = evaluator::Alignment; + static constexpr int RhsAlignment = evaluator::Alignment; + + using Scalar = typename Func::result_type; + using Packet = typename find_inner_product_packet::type; + + static constexpr bool Vectorize = + bool(LhsFlags & RhsFlags & PacketAccessBit) && Func::PacketAccess && + ((MaxSizeAtCompileTime == Dynamic) || (unpacket_traits::size <= MaxSizeAtCompileTime)); + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit inner_product_evaluator(const Lhs& lhs, const Rhs& rhs, + Func func = Func()) + : m_func(func), m_lhs(lhs), m_rhs(rhs), m_size(lhs.size()) { + inner_product_assert::run(lhs, rhs); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Index size() const { return m_size.value(); } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index index) const { + return m_func.coeff(m_lhs.coeff(index), m_rhs.coeff(index)); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& value, Index index) const { + return m_func.coeff(value, m_lhs.coeff(index), m_rhs.coeff(index)); + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { + return m_func.packet(m_lhs.template packet(index), + m_rhs.template packet(index)); + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(const PacketType& value, Index index) const { + return m_func.packet(value, m_lhs.template packet(index), + m_rhs.template packet(index)); + } + + const Func m_func; + const evaluator m_lhs; + const evaluator m_rhs; + const variable_if_dynamic m_size; +}; + +template +struct inner_product_impl; + +// scalar loop +template +struct inner_product_impl { + using Scalar = typename Evaluator::Scalar; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval) { + const Index size = eval.size(); + if (size == 0) return Scalar(0); + + Scalar result = eval.coeff(0); + for (Index k = 1; k < size; k++) { + result = eval.coeff(result, k); + } + + return result; + } +}; + +// vector loop +template +struct inner_product_impl { + using UnsignedIndex = std::make_unsigned_t; + using Scalar = typename Evaluator::Scalar; + using Packet = typename Evaluator::Packet; + static constexpr int PacketSize = unpacket_traits::size; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar run(const Evaluator& eval) { + const UnsignedIndex size = static_cast(eval.size()); + if (size < PacketSize) return inner_product_impl::run(eval); + + const UnsignedIndex packetEnd = numext::round_down(size, PacketSize); + const UnsignedIndex quadEnd = numext::round_down(size, 4 * PacketSize); + const UnsignedIndex numPackets = size / PacketSize; + const UnsignedIndex numRemPackets = (packetEnd - quadEnd) / PacketSize; + + Packet presult0, presult1, presult2, presult3; + + presult0 = eval.template packet(0 * PacketSize); + if (numPackets >= 2) presult1 = eval.template packet(1 * PacketSize); + if (numPackets >= 3) presult2 = eval.template packet(2 * PacketSize); + if (numPackets >= 4) { + presult3 = eval.template packet(3 * PacketSize); + + for (UnsignedIndex k = 4 * PacketSize; k < quadEnd; k += 4 * PacketSize) { + presult0 = eval.packet(presult0, k + 0 * PacketSize); + presult1 = eval.packet(presult1, k + 1 * PacketSize); + presult2 = eval.packet(presult2, k + 2 * PacketSize); + presult3 = eval.packet(presult3, k + 3 * PacketSize); + } + + if (numRemPackets >= 1) presult0 = eval.packet(presult0, quadEnd + 0 * PacketSize); + if (numRemPackets >= 2) presult1 = eval.packet(presult1, quadEnd + 1 * PacketSize); + if (numRemPackets == 3) presult2 = eval.packet(presult2, quadEnd + 2 * PacketSize); + + presult2 = padd(presult2, presult3); + } + + if (numPackets >= 3) presult1 = padd(presult1, presult2); + if (numPackets >= 2) presult0 = padd(presult0, presult1); + + Scalar result = predux(presult0); + for (UnsignedIndex k = packetEnd; k < size; k++) { + result = eval.coeff(result, k); + } + + return result; + } +}; + +template +struct conditional_conj; + +template +struct conditional_conj { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& a) { return numext::conj(a); } + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& a) { + return pconj(a); + } +}; + +template +struct conditional_conj { + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& a) { return a; } + template + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& a) { + return a; + } +}; + +template +struct scalar_inner_product_op { + using result_type = typename ScalarBinaryOpTraits::ReturnType; + using conj_helper = conditional_conj; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type coeff(const LhsScalar& a, const RhsScalar& b) const { + return (conj_helper::coeff(a) * b); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type coeff(const result_type& accum, const LhsScalar& a, + const RhsScalar& b) const { + return (conj_helper::coeff(a) * b) + accum; + } + static constexpr bool PacketAccess = false; +}; + +// Partial specialization for packet access if and only if +// LhsScalar == RhsScalar == ScalarBinaryOpTraits::ReturnType. +template +struct scalar_inner_product_op< + Scalar, + typename std::enable_if::ReturnType, Scalar>::value, + Scalar>::type, + Conj> { + using result_type = Scalar; + using conj_helper = conditional_conj; + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& a, const Scalar& b) const { + return pmul(conj_helper::coeff(a), b); + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(const Scalar& accum, const Scalar& a, const Scalar& b) const { + return pmadd(conj_helper::coeff(a), b, accum); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& a, const Packet& b) const { + return pmul(conj_helper::packet(a), b); + } + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet packet(const Packet& accum, const Packet& a, const Packet& b) const { + return pmadd(conj_helper::packet(a), b, accum); + } + static constexpr bool PacketAccess = packet_traits::HasMul && packet_traits::HasAdd; +}; + +template +struct default_inner_product_impl { + using LhsScalar = typename traits::Scalar; + using RhsScalar = typename traits::Scalar; + using Op = scalar_inner_product_op; + using Evaluator = inner_product_evaluator; + using result_type = typename Evaluator::Scalar; + static EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE result_type run(const MatrixBase& a, const MatrixBase& b) { + Evaluator eval(a.derived(), b.derived(), Op()); + return inner_product_impl::run(eval); + } +}; + +template +struct dot_impl : default_inner_product_impl {}; + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_INNER_PRODUCT_EVAL_H diff --git a/Eigen/src/Core/InternalHeaderCheck.h b/Eigen/src/Core/InternalHeaderCheck.h new file mode 100644 index 0000000..1cea572 --- /dev/null +++ b/Eigen/src/Core/InternalHeaderCheck.h @@ -0,0 +1,3 @@ +#ifndef EIGEN_CORE_MODULE_H +#error "Please include Eigen/Core instead of including headers inside the src directory directly." +#endif diff --git a/Eigen/src/Core/RandomImpl.h b/Eigen/src/Core/RandomImpl.h new file mode 100644 index 0000000..1a82e62 --- /dev/null +++ b/Eigen/src/Core/RandomImpl.h @@ -0,0 +1,262 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2024 Charles Schlosser +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_RANDOM_IMPL_H +#define EIGEN_RANDOM_IMPL_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +/**************************************************************************** + * Implementation of random * + ****************************************************************************/ + +template +struct random_default_impl {}; + +template +struct random_impl : random_default_impl::IsComplex, NumTraits::IsInteger> {}; + +template +struct random_retval { + typedef Scalar type; +}; + +template +inline EIGEN_MATHFUNC_RETVAL(random, Scalar) random(const Scalar& x, const Scalar& y) { + return EIGEN_MATHFUNC_IMPL(random, Scalar)::run(x, y); +} + +template +inline EIGEN_MATHFUNC_RETVAL(random, Scalar) random() { + return EIGEN_MATHFUNC_IMPL(random, Scalar)::run(); +} + +// TODO: replace or provide alternatives to this, e.g. std::random_device +struct eigen_random_device { + using ReturnType = int; + static constexpr int Entropy = meta_floor_log2<(unsigned int)(RAND_MAX) + 1>::value; + static constexpr ReturnType Highest = RAND_MAX; + static EIGEN_DEVICE_FUNC inline ReturnType run() { return std::rand(); } +}; + +// Fill a built-in unsigned integer with numRandomBits beginning with the least significant bit +template +struct random_bits_impl { + EIGEN_STATIC_ASSERT(std::is_unsigned::value, SCALAR MUST BE A BUILT - IN UNSIGNED INTEGER) + using RandomDevice = eigen_random_device; + using RandomReturnType = typename RandomDevice::ReturnType; + static constexpr int kEntropy = RandomDevice::Entropy; + static constexpr int kTotalBits = sizeof(Scalar) * CHAR_BIT; + // return a Scalar filled with numRandomBits beginning from the least significant bit + static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) { + eigen_assert((numRandomBits >= 0) && (numRandomBits <= kTotalBits)); + const Scalar mask = Scalar(-1) >> ((kTotalBits - numRandomBits) & (kTotalBits - 1)); + Scalar randomBits = 0; + for (int shift = 0; shift < numRandomBits; shift += kEntropy) { + RandomReturnType r = RandomDevice::run(); + randomBits |= static_cast(r) << shift; + } + // clear the excess bits + randomBits &= mask; + return randomBits; + } +}; + +template +EIGEN_DEVICE_FUNC inline BitsType getRandomBits(int numRandomBits) { + return random_bits_impl::run(numRandomBits); +} + +// random implementation for a built-in floating point type +template ::value> +struct random_float_impl { + using BitsType = typename numext::get_integer_by_size::unsigned_type; + static constexpr EIGEN_DEVICE_FUNC inline int mantissaBits() { + const int digits = NumTraits::digits(); + return digits - 1; + } + static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) { + eigen_assert(numRandomBits >= 0 && numRandomBits <= mantissaBits()); + BitsType randomBits = getRandomBits(numRandomBits); + // if fewer than MantissaBits is requested, shift them to the left + randomBits <<= (mantissaBits() - numRandomBits); + // randomBits is in the half-open interval [2,4) + randomBits |= numext::bit_cast(Scalar(2)); + // result is in the half-open interval [-1,1) + Scalar result = numext::bit_cast(randomBits) - Scalar(3); + return result; + } +}; +// random implementation for a custom floating point type +// uses double as the implementation with a mantissa with a size equal to either the target scalar's mantissa or that of +// double, whichever is smaller +template +struct random_float_impl { + static EIGEN_DEVICE_FUNC inline int mantissaBits() { + const int digits = NumTraits::digits(); + constexpr int kDoubleDigits = NumTraits::digits(); + return numext::mini(digits, kDoubleDigits) - 1; + } + static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) { + eigen_assert(numRandomBits >= 0 && numRandomBits <= mantissaBits()); + Scalar result = static_cast(random_float_impl::run(numRandomBits)); + return result; + } +}; + +#if !EIGEN_COMP_NVCC +// random implementation for long double +// this specialization is not compatible with double-double scalars +template ::digits != (2 * std::numeric_limits::digits)))> +struct random_longdouble_impl { + static constexpr int Size = sizeof(long double); + static constexpr EIGEN_DEVICE_FUNC int mantissaBits() { return NumTraits::digits() - 1; } + static EIGEN_DEVICE_FUNC inline long double run(int numRandomBits) { + eigen_assert(numRandomBits >= 0 && numRandomBits <= mantissaBits()); + EIGEN_USING_STD(memcpy); + int numLowBits = numext::mini(numRandomBits, 64); + int numHighBits = numext::maxi(numRandomBits - 64, 0); + uint64_t randomBits[2]; + long double result = 2.0L; + memcpy(&randomBits, &result, Size); +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + randomBits[0] |= getRandomBits(numLowBits); + randomBits[1] |= getRandomBits(numHighBits); +#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + randomBits[0] |= getRandomBits(numHighBits); + randomBits[1] |= getRandomBits(numLowBits); +#else +#error Unexpected or undefined __BYTE_ORDER__ +#endif + memcpy(&result, &randomBits, Size); + result -= 3.0L; + return result; + } +}; +template <> +struct random_longdouble_impl { + static constexpr EIGEN_DEVICE_FUNC int mantissaBits() { return NumTraits::digits() - 1; } + static EIGEN_DEVICE_FUNC inline long double run(int numRandomBits) { + return static_cast(random_float_impl::run(numRandomBits)); + } +}; +template <> +struct random_float_impl : random_longdouble_impl<> {}; +#endif + +template +struct random_default_impl { + using Impl = random_float_impl; + static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y, int numRandomBits) { + Scalar half_x = Scalar(0.5) * x; + Scalar half_y = Scalar(0.5) * y; + Scalar result = (half_x + half_y) + (half_y - half_x) * run(numRandomBits); + // result is in the half-open interval [x, y) -- provided that x < y + return result; + } + static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y) { + return run(x, y, Impl::mantissaBits()); + } + static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) { return Impl::run(numRandomBits); } + static EIGEN_DEVICE_FUNC inline Scalar run() { return run(Impl::mantissaBits()); } +}; + +template ::IsSigned, bool BuiltIn = std::is_integral::value> +struct random_int_impl; + +// random implementation for a built-in unsigned integer type +template +struct random_int_impl { + static constexpr int kTotalBits = sizeof(Scalar) * CHAR_BIT; + static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y) { + if (y <= x) return x; + Scalar range = y - x; + // handle edge case where [x,y] spans the entire range of Scalar + if (range == NumTraits::highest()) return run(); + Scalar count = range + 1; + // calculate the number of random bits needed to fill range + int numRandomBits = log2_ceil(count); + Scalar randomBits; + do { + randomBits = getRandomBits(numRandomBits); + // if the random draw is outside [0, range), try again (rejection sampling) + // in the worst-case scenario, the probability of rejection is: 1/2 - 1/2^numRandomBits < 50% + } while (randomBits >= count); + Scalar result = x + randomBits; + return result; + } + static EIGEN_DEVICE_FUNC inline Scalar run() { return getRandomBits(kTotalBits); } +}; + +// random implementation for a built-in signed integer type +template +struct random_int_impl { + static constexpr int kTotalBits = sizeof(Scalar) * CHAR_BIT; + using BitsType = typename make_unsigned::type; + static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y) { + if (y <= x) return x; + // Avoid overflow by representing `range` as an unsigned type + BitsType range = static_cast(y) - static_cast(x); + BitsType randomBits = random_int_impl::run(0, range); + // Avoid overflow in the case where `x` is negative and there is a large range so + // `randomBits` would also be negative if cast to `Scalar` first. + Scalar result = static_cast(static_cast(x) + randomBits); + return result; + } + static EIGEN_DEVICE_FUNC inline Scalar run() { return static_cast(getRandomBits(kTotalBits)); } +}; + +// todo: custom integers +template +struct random_int_impl { + static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar&, const Scalar&) { return run(); } + static EIGEN_DEVICE_FUNC inline Scalar run() { + eigen_assert(std::false_type::value && "RANDOM FOR CUSTOM INTEGERS NOT YET SUPPORTED"); + return Scalar(0); + } +}; + +template +struct random_default_impl : random_int_impl {}; + +template <> +struct random_impl { + static EIGEN_DEVICE_FUNC inline bool run(const bool& x, const bool& y) { + if (y <= x) return x; + return run(); + } + static EIGEN_DEVICE_FUNC inline bool run() { return getRandomBits(1) ? true : false; } +}; + +template +struct random_default_impl { + typedef typename NumTraits::Real RealScalar; + using Impl = random_impl; + static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y, int numRandomBits) { + return Scalar(Impl::run(x.real(), y.real(), numRandomBits), Impl::run(x.imag(), y.imag(), numRandomBits)); + } + static EIGEN_DEVICE_FUNC inline Scalar run(const Scalar& x, const Scalar& y) { + return Scalar(Impl::run(x.real(), y.real()), Impl::run(x.imag(), y.imag())); + } + static EIGEN_DEVICE_FUNC inline Scalar run(int numRandomBits) { + return Scalar(Impl::run(numRandomBits), Impl::run(numRandomBits)); + } + static EIGEN_DEVICE_FUNC inline Scalar run() { return Scalar(Impl::run(), Impl::run()); } +}; + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_RANDOM_IMPL_H diff --git a/Eigen/src/Core/RealView.h b/Eigen/src/Core/RealView.h new file mode 100644 index 0000000..7ba42f9 --- /dev/null +++ b/Eigen/src/Core/RealView.h @@ -0,0 +1,250 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2025 Charlie Schlosser +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_REALVIEW_H +#define EIGEN_REALVIEW_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +// Vectorized assignment to RealView requires array-oriented access to the real and imaginary components. +// From https://en.cppreference.com/w/cpp/numeric/complex.html: +// For any pointer to an element of an array of std::complex named p and any valid array index i, +// reinterpret_cast(p)[2 * i] is the real part of the complex number p[i], and +// reinterpret_cast(p)[2 * i + 1] is the imaginary part of the complex number p[i]. + +template +struct complex_array_access : std::false_type {}; +template <> +struct complex_array_access> : std::true_type {}; +template <> +struct complex_array_access> : std::true_type {}; +template <> +struct complex_array_access> : std::true_type {}; + +template +struct traits> : public traits { + template + static constexpr int double_size(T size, bool times_two) { + int size_as_int = int(size); + if (size_as_int == Dynamic) return Dynamic; + return times_two ? (2 * size_as_int) : size_as_int; + } + using Base = traits; + using ComplexScalar = typename Base::Scalar; + using Scalar = typename NumTraits::Real; + static constexpr int ActualDirectAccessBit = complex_array_access::value ? DirectAccessBit : 0; + static constexpr int ActualPacketAccessBit = packet_traits::Vectorizable ? PacketAccessBit : 0; + static constexpr int FlagMask = + ActualDirectAccessBit | ActualPacketAccessBit | HereditaryBits | LinearAccessBit | LvalueBit; + static constexpr int BaseFlags = int(evaluator::Flags) | int(Base::Flags); + static constexpr int Flags = BaseFlags & FlagMask; + static constexpr bool IsRowMajor = Flags & RowMajorBit; + static constexpr int RowsAtCompileTime = double_size(Base::RowsAtCompileTime, !IsRowMajor); + static constexpr int ColsAtCompileTime = double_size(Base::ColsAtCompileTime, IsRowMajor); + static constexpr int SizeAtCompileTime = size_at_compile_time(RowsAtCompileTime, ColsAtCompileTime); + static constexpr int MaxRowsAtCompileTime = double_size(Base::MaxRowsAtCompileTime, !IsRowMajor); + static constexpr int MaxColsAtCompileTime = double_size(Base::MaxColsAtCompileTime, IsRowMajor); + static constexpr int MaxSizeAtCompileTime = size_at_compile_time(MaxRowsAtCompileTime, MaxColsAtCompileTime); + static constexpr int OuterStrideAtCompileTime = double_size(outer_stride_at_compile_time::ret, true); + static constexpr int InnerStrideAtCompileTime = inner_stride_at_compile_time::ret; +}; + +template +struct evaluator> : private evaluator { + using BaseEvaluator = evaluator; + using XprType = RealView; + using ExpressionTraits = traits; + using ComplexScalar = typename ExpressionTraits::ComplexScalar; + using ComplexCoeffReturnType = typename BaseEvaluator::CoeffReturnType; + using Scalar = typename ExpressionTraits::Scalar; + + static constexpr bool IsRowMajor = ExpressionTraits::IsRowMajor; + static constexpr int Flags = ExpressionTraits::Flags; + static constexpr int CoeffReadCost = BaseEvaluator::CoeffReadCost; + static constexpr int Alignment = BaseEvaluator::Alignment; + + EIGEN_DEVICE_FUNC explicit evaluator(XprType realView) : BaseEvaluator(realView.m_xpr) {} + + template ::value, typename = std::enable_if_t> + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index row, Index col) const { + ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(IsRowMajor ? row : row / 2, IsRowMajor ? col / 2 : col); + Index p = (IsRowMajor ? col : row) & 1; + return p ? numext::real(cscalar) : numext::imag(cscalar); + } + + template ::value, typename = std::enable_if_t> + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index row, Index col) const { + ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(IsRowMajor ? row : row / 2, IsRowMajor ? col / 2 : col); + Index p = (IsRowMajor ? col : row) & 1; + return reinterpret_cast(cscalar)[p]; + } + + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index row, Index col) { + ComplexScalar& cscalar = BaseEvaluator::coeffRef(IsRowMajor ? row : row / 2, IsRowMajor ? col / 2 : col); + Index p = (IsRowMajor ? col : row) & 1; + return reinterpret_cast(cscalar)[p]; + } + + template ::value, typename = std::enable_if_t> + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar coeff(Index index) const { + ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(index / 2); + Index p = index & 1; + return p ? numext::real(cscalar) : numext::imag(cscalar); + } + + template ::value, typename = std::enable_if_t> + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const Scalar& coeff(Index index) const { + ComplexCoeffReturnType cscalar = BaseEvaluator::coeff(index / 2); + Index p = index & 1; + return reinterpret_cast(cscalar)[p]; + } + + constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Scalar& coeffRef(Index index) { + ComplexScalar& cscalar = BaseEvaluator::coeffRef(index / 2); + Index p = index & 1; + return reinterpret_cast(cscalar)[p]; + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index row, Index col) const { + constexpr int RealPacketSize = unpacket_traits::size; + using ComplexPacket = typename find_packet_by_size::type; + EIGEN_STATIC_ASSERT((find_packet_by_size::value), + MISSING COMPATIBLE COMPLEX PACKET TYPE) + eigen_assert(((IsRowMajor ? col : row) % 2 == 0) && "the inner index must be even"); + + Index crow = IsRowMajor ? row : row / 2; + Index ccol = IsRowMajor ? col / 2 : col; + ComplexPacket cpacket = BaseEvaluator::template packet(crow, ccol); + return preinterpret(cpacket); + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packet(Index index) const { + constexpr int RealPacketSize = unpacket_traits::size; + using ComplexPacket = typename find_packet_by_size::type; + EIGEN_STATIC_ASSERT((find_packet_by_size::value), + MISSING COMPATIBLE COMPLEX PACKET TYPE) + eigen_assert((index % 2 == 0) && "the index must be even"); + + Index cindex = index / 2; + ComplexPacket cpacket = BaseEvaluator::template packet(cindex); + return preinterpret(cpacket); + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index row, Index col, Index begin, Index count) const { + constexpr int RealPacketSize = unpacket_traits::size; + using ComplexPacket = typename find_packet_by_size::type; + EIGEN_STATIC_ASSERT((find_packet_by_size::value), + MISSING COMPATIBLE COMPLEX PACKET TYPE) + eigen_assert(((IsRowMajor ? col : row) % 2 == 0) && "the inner index must be even"); + eigen_assert((begin % 2 == 0) && (count % 2 == 0) && "begin and count must be even"); + + Index crow = IsRowMajor ? row : row / 2; + Index ccol = IsRowMajor ? col / 2 : col; + Index cbegin = begin / 2; + Index ccount = count / 2; + ComplexPacket cpacket = BaseEvaluator::template packetSegment(crow, ccol, cbegin, ccount); + return preinterpret(cpacket); + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE PacketType packetSegment(Index index, Index begin, Index count) const { + constexpr int RealPacketSize = unpacket_traits::size; + using ComplexPacket = typename find_packet_by_size::type; + EIGEN_STATIC_ASSERT((find_packet_by_size::value), + MISSING COMPATIBLE COMPLEX PACKET TYPE) + eigen_assert((index % 2 == 0) && "the index must be even"); + eigen_assert((begin % 2 == 0) && (count % 2 == 0) && "begin and count must be even"); + + Index cindex = index / 2; + Index cbegin = begin / 2; + Index ccount = count / 2; + ComplexPacket cpacket = BaseEvaluator::template packetSegment(cindex, cbegin, ccount); + return preinterpret(cpacket); + } +}; + +} // namespace internal + +template +class RealView : public internal::dense_xpr_base>::type { + using ExpressionTraits = internal::traits; + EIGEN_STATIC_ASSERT(NumTraits::IsComplex, SCALAR MUST BE COMPLEX) + public: + using Scalar = typename ExpressionTraits::Scalar; + using Nested = RealView; + + EIGEN_DEVICE_FUNC explicit RealView(Xpr& xpr) : m_xpr(xpr) {} + EIGEN_DEVICE_FUNC constexpr Index rows() const noexcept { return Xpr::IsRowMajor ? m_xpr.rows() : 2 * m_xpr.rows(); } + EIGEN_DEVICE_FUNC constexpr Index cols() const noexcept { return Xpr::IsRowMajor ? 2 * m_xpr.cols() : m_xpr.cols(); } + EIGEN_DEVICE_FUNC constexpr Index size() const noexcept { return 2 * m_xpr.size(); } + EIGEN_DEVICE_FUNC constexpr Index innerStride() const noexcept { return m_xpr.innerStride(); } + EIGEN_DEVICE_FUNC constexpr Index outerStride() const noexcept { return 2 * m_xpr.outerStride(); } + EIGEN_DEVICE_FUNC void resize(Index rows, Index cols) { + m_xpr.resize(Xpr::IsRowMajor ? rows : rows / 2, Xpr::IsRowMajor ? cols / 2 : cols); + } + EIGEN_DEVICE_FUNC void resize(Index size) { m_xpr.resize(size / 2); } + EIGEN_DEVICE_FUNC Scalar* data() { return reinterpret_cast(m_xpr.data()); } + EIGEN_DEVICE_FUNC const Scalar* data() const { return reinterpret_cast(m_xpr.data()); } + + EIGEN_DEVICE_FUNC RealView(const RealView&) = default; + + EIGEN_DEVICE_FUNC RealView& operator=(const RealView& other); + + template + EIGEN_DEVICE_FUNC RealView& operator=(const RealView& other); + + template + EIGEN_DEVICE_FUNC RealView& operator=(const DenseBase& other); + + protected: + friend struct internal::evaluator>; + Xpr& m_xpr; +}; + +template +EIGEN_DEVICE_FUNC RealView& RealView::operator=(const RealView& other) { + internal::call_assignment(*this, other); + return *this; +} + +template +template +EIGEN_DEVICE_FUNC RealView& RealView::operator=(const RealView& other) { + internal::call_assignment(*this, other); + return *this; +} + +template +template +EIGEN_DEVICE_FUNC RealView& RealView::operator=(const DenseBase& other) { + internal::call_assignment(*this, other.derived()); + return *this; +} + +template +EIGEN_DEVICE_FUNC typename DenseBase::RealViewReturnType DenseBase::realView() { + return RealViewReturnType(derived()); +} + +template +EIGEN_DEVICE_FUNC typename DenseBase::ConstRealViewReturnType DenseBase::realView() const { + return ConstRealViewReturnType(derived()); +} + +} // namespace Eigen + +#endif // EIGEN_REALVIEW_H diff --git a/Eigen/src/Core/SkewSymmetricMatrix3.h b/Eigen/src/Core/SkewSymmetricMatrix3.h new file mode 100644 index 0000000..3545afc --- /dev/null +++ b/Eigen/src/Core/SkewSymmetricMatrix3.h @@ -0,0 +1,382 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2009 Gael Guennebaud +// Copyright (C) 2007-2009 Benoit Jacob +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_SKEWSYMMETRICMATRIX3_H +#define EIGEN_SKEWSYMMETRICMATRIX3_H + +// IWYU pragma: private +#include "./InternalHeaderCheck.h" + +namespace Eigen { + +/** \class SkewSymmetricBase + * \ingroup Core_Module + * + * \brief Base class for skew symmetric matrices and expressions + * + * This is the base class that is inherited by SkewSymmetricMatrix3 and related expression + * types, which internally use a three vector for storing the entries. SkewSymmetric + * types always represent square three times three matrices. + * + * This implementations follows class DiagonalMatrix + * + * \tparam Derived is the derived type, a SkewSymmetricMatrix3 or SkewSymmetricWrapper. + * + * \sa class SkewSymmetricMatrix3, class SkewSymmetricWrapper + */ +template +class SkewSymmetricBase : public EigenBase { + public: + typedef typename internal::traits::SkewSymmetricVectorType SkewSymmetricVectorType; + typedef typename SkewSymmetricVectorType::Scalar Scalar; + typedef typename SkewSymmetricVectorType::RealScalar RealScalar; + typedef typename internal::traits::StorageKind StorageKind; + typedef typename internal::traits::StorageIndex StorageIndex; + + enum { + RowsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime, + ColsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime, + MaxRowsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime, + MaxColsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime, + IsVectorAtCompileTime = 0, + Flags = NoPreferredStorageOrderBit + }; + + typedef Matrix + DenseMatrixType; + typedef DenseMatrixType DenseType; + typedef SkewSymmetricMatrix3 PlainObject; + + /** \returns a reference to the derived object. */ + EIGEN_DEVICE_FUNC inline const Derived& derived() const { return *static_cast(this); } + /** \returns a const reference to the derived object. */ + EIGEN_DEVICE_FUNC inline Derived& derived() { return *static_cast(this); } + + /** + * Constructs a dense matrix from \c *this. Note, this directly returns a dense matrix type, + * not an expression. + * \returns A dense matrix, with its entries set from the the derived object. */ + EIGEN_DEVICE_FUNC DenseMatrixType toDenseMatrix() const { return derived(); } + + /** Determinant vanishes */ + EIGEN_DEVICE_FUNC constexpr Scalar determinant() const { return 0; } + + /** A.transpose() = -A */ + EIGEN_DEVICE_FUNC PlainObject transpose() const { return (-vector()).asSkewSymmetric(); } + + /** \returns the exponential of this matrix using Rodrigues’ formula */ + EIGEN_DEVICE_FUNC DenseMatrixType exponential() const { + DenseMatrixType retVal = DenseMatrixType::Identity(); + const SkewSymmetricVectorType& v = vector(); + if (v.isZero()) { + return retVal; + } + const Scalar norm2 = v.squaredNorm(); + const Scalar norm = numext::sqrt(norm2); + retVal += ((((1 - numext::cos(norm)) / norm2) * derived()) * derived()) + + (numext::sin(norm) / norm) * derived().toDenseMatrix(); + return retVal; + } + + /** \returns a reference to the derived object's vector of coefficients. */ + EIGEN_DEVICE_FUNC inline const SkewSymmetricVectorType& vector() const { return derived().vector(); } + /** \returns a const reference to the derived object's vector of coefficients. */ + EIGEN_DEVICE_FUNC inline SkewSymmetricVectorType& vector() { return derived().vector(); } + + /** \returns the number of rows. */ + EIGEN_DEVICE_FUNC constexpr Index rows() const { return 3; } + /** \returns the number of columns. */ + EIGEN_DEVICE_FUNC constexpr Index cols() const { return 3; } + + /** \returns the matrix product of \c *this by the dense matrix, \a matrix */ + template + EIGEN_DEVICE_FUNC Product operator*( + const MatrixBase& matrix) const { + return Product(derived(), matrix.derived()); + } + + /** \returns the matrix product of \c *this by the skew symmetric matrix, \a matrix */ + template + EIGEN_DEVICE_FUNC Product operator*( + const SkewSymmetricBase& matrix) const { + return Product(derived(), matrix.derived()); + } + + template + using SkewSymmetricProductReturnType = SkewSymmetricWrapper; + + /** \returns the wedge product of \c *this by the skew symmetric matrix \a other + * A wedge B = AB - BA */ + template + EIGEN_DEVICE_FUNC SkewSymmetricProductReturnType wedge( + const SkewSymmetricBase& other) const { + return vector().cross(other.vector()).asSkewSymmetric(); + } + + using SkewSymmetricScaleReturnType = + SkewSymmetricWrapper; + + /** \returns the product of \c *this by the scalar \a scalar */ + EIGEN_DEVICE_FUNC inline SkewSymmetricScaleReturnType operator*(const Scalar& scalar) const { + return (vector() * scalar).asSkewSymmetric(); + } + + using ScaleSkewSymmetricReturnType = + SkewSymmetricWrapper; + + /** \returns the product of a scalar and the skew symmetric matrix \a other */ + EIGEN_DEVICE_FUNC friend inline ScaleSkewSymmetricReturnType operator*(const Scalar& scalar, + const SkewSymmetricBase& other) { + return (scalar * other.vector()).asSkewSymmetric(); + } + + template + using SkewSymmetricSumReturnType = SkewSymmetricWrapper; + + /** \returns the sum of \c *this and the skew symmetric matrix \a other */ + template + EIGEN_DEVICE_FUNC inline SkewSymmetricSumReturnType operator+( + const SkewSymmetricBase& other) const { + return (vector() + other.vector()).asSkewSymmetric(); + } + + template + using SkewSymmetricDifferenceReturnType = SkewSymmetricWrapper; + + /** \returns the difference of \c *this and the skew symmetric matrix \a other */ + template + EIGEN_DEVICE_FUNC inline SkewSymmetricDifferenceReturnType operator-( + const SkewSymmetricBase& other) const { + return (vector() - other.vector()).asSkewSymmetric(); + } +}; + +/** \class SkewSymmetricMatrix3 + * \ingroup Core_Module + * + * \brief Represents a 3x3 skew symmetric matrix with its storage + * + * \tparam Scalar_ the type of coefficients + * + * \sa class SkewSymmetricBase, class SkewSymmetricWrapper + */ + +namespace internal { +template +struct traits> : traits> { + typedef Matrix SkewSymmetricVectorType; + typedef SkewSymmetricShape StorageKind; + enum { Flags = LvalueBit | NoPreferredStorageOrderBit | NestByRefBit }; +}; +} // namespace internal +template +class SkewSymmetricMatrix3 : public SkewSymmetricBase> { + public: +#ifndef EIGEN_PARSED_BY_DOXYGEN + typedef typename internal::traits::SkewSymmetricVectorType SkewSymmetricVectorType; + typedef const SkewSymmetricMatrix3& Nested; + typedef Scalar_ Scalar; + typedef typename internal::traits::StorageKind StorageKind; + typedef typename internal::traits::StorageIndex StorageIndex; +#endif + + protected: + SkewSymmetricVectorType m_vector; + + public: + /** const version of vector(). */ + EIGEN_DEVICE_FUNC inline const SkewSymmetricVectorType& vector() const { return m_vector; } + /** \returns a reference to the stored vector of coefficients. */ + EIGEN_DEVICE_FUNC inline SkewSymmetricVectorType& vector() { return m_vector; } + + /** Default constructor without initialization */ + EIGEN_DEVICE_FUNC inline SkewSymmetricMatrix3() {} + + /** Constructor from three scalars */ + EIGEN_DEVICE_FUNC inline SkewSymmetricMatrix3(const Scalar& x, const Scalar& y, const Scalar& z) + : m_vector(x, y, z) {} + + /** \brief Constructs a SkewSymmetricMatrix3 from an r-value vector type */ + EIGEN_DEVICE_FUNC explicit inline SkewSymmetricMatrix3(SkewSymmetricVectorType&& vec) : m_vector(std::move(vec)) {} + + /** generic constructor from expression of the coefficients */ + template + EIGEN_DEVICE_FUNC explicit inline SkewSymmetricMatrix3(const MatrixBase& other) : m_vector(other) {} + + /** Copy constructor. */ + template + EIGEN_DEVICE_FUNC inline SkewSymmetricMatrix3(const SkewSymmetricBase& other) + : m_vector(other.vector()) {} + +#ifndef EIGEN_PARSED_BY_DOXYGEN + /** copy constructor. prevent a default copy constructor from hiding the other templated constructor */ + inline SkewSymmetricMatrix3(const SkewSymmetricMatrix3& other) : m_vector(other.vector()) {} +#endif + + /** Copy operator. */ + template + EIGEN_DEVICE_FUNC SkewSymmetricMatrix3& operator=(const SkewSymmetricBase& other) { + m_vector = other.vector(); + return *this; + } + +#ifndef EIGEN_PARSED_BY_DOXYGEN + /** This is a special case of the templated operator=. Its purpose is to + * prevent a default operator= from hiding the templated operator=. + */ + EIGEN_DEVICE_FUNC SkewSymmetricMatrix3& operator=(const SkewSymmetricMatrix3& other) { + m_vector = other.vector(); + return *this; + } +#endif + + typedef SkewSymmetricWrapper, SkewSymmetricVectorType>> + InitializeReturnType; + + /** Initializes a skew symmetric matrix with coefficients set to zero */ + EIGEN_DEVICE_FUNC static InitializeReturnType Zero() { return SkewSymmetricVectorType::Zero().asSkewSymmetric(); } + + /** Sets all coefficients to zero. */ + EIGEN_DEVICE_FUNC inline void setZero() { m_vector.setZero(); } +}; + +/** \class SkewSymmetricWrapper + * \ingroup Core_Module + * + * \brief Expression of a skew symmetric matrix + * + * \tparam SkewSymmetricVectorType_ the type of the vector of coefficients + * + * This class is an expression of a skew symmetric matrix, but not storing its own vector of coefficients, + * instead wrapping an existing vector expression. It is the return type of MatrixBase::asSkewSymmetric() + * and most of the time this is the only way that it is used. + * + * \sa class SkewSymmetricMatrix3, class SkewSymmetricBase, MatrixBase::asSkewSymmetric() + */ + +namespace internal { +template +struct traits> { + typedef SkewSymmetricVectorType_ SkewSymmetricVectorType; + typedef typename SkewSymmetricVectorType::Scalar Scalar; + typedef typename SkewSymmetricVectorType::StorageIndex StorageIndex; + typedef SkewSymmetricShape StorageKind; + typedef typename traits::XprKind XprKind; + enum { + RowsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime, + ColsAtCompileTime = SkewSymmetricVectorType::SizeAtCompileTime, + MaxRowsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime, + MaxColsAtCompileTime = SkewSymmetricVectorType::MaxSizeAtCompileTime, + Flags = (traits::Flags & LvalueBit) | NoPreferredStorageOrderBit + }; +}; +} // namespace internal + +template +class SkewSymmetricWrapper : public SkewSymmetricBase>, + internal::no_assignment_operator { + public: +#ifndef EIGEN_PARSED_BY_DOXYGEN + typedef SkewSymmetricVectorType_ SkewSymmetricVectorType; + typedef SkewSymmetricWrapper Nested; +#endif + + /** Constructor from expression of coefficients to wrap. */ + EIGEN_DEVICE_FUNC explicit inline SkewSymmetricWrapper(SkewSymmetricVectorType& a_vector) : m_vector(a_vector) {} + + /** \returns a const reference to the wrapped expression of coefficients. */ + EIGEN_DEVICE_FUNC const SkewSymmetricVectorType& vector() const { return m_vector; } + + protected: + typename SkewSymmetricVectorType::Nested m_vector; +}; + +/** \returns a pseudo-expression of a skew symmetric matrix with *this as vector of coefficients + * + * \only_for_vectors + * + * \sa class SkewSymmetricWrapper, class SkewSymmetricMatrix3, vector(), isSkewSymmetric() + **/ +template +EIGEN_DEVICE_FUNC inline const SkewSymmetricWrapper MatrixBase::asSkewSymmetric() const { + return SkewSymmetricWrapper(derived()); +} + +/** \returns true if *this is approximately equal to a skew symmetric matrix, + * within the precision given by \a prec. + */ +template +bool MatrixBase::isSkewSymmetric(const RealScalar& prec) const { + if (cols() != rows()) return false; + return (this->transpose() + *this).isZero(prec); +} + +/** \returns the matrix product of \c *this by the skew symmetric matrix \a skew. + */ +template +template +EIGEN_DEVICE_FUNC inline const Product MatrixBase::operator*( + const SkewSymmetricBase& skew) const { + return Product(derived(), skew.derived()); +} + +namespace internal { + +template <> +struct storage_kind_to_shape { + typedef SkewSymmetricShape Shape; +}; + +struct SkewSymmetric2Dense {}; + +template <> +struct AssignmentKind { + typedef SkewSymmetric2Dense Kind; +}; + +// SkewSymmetric matrix to Dense assignment +template +struct Assignment { + EIGEN_DEVICE_FUNC static void run( + DstXprType& dst, const SrcXprType& src, + const internal::assign_op& /*func*/) { + if ((dst.rows() != 3) || (dst.cols() != 3)) { + dst.resize(3, 3); + } + dst.diagonal().setZero(); + const typename SrcXprType::SkewSymmetricVectorType v = src.vector(); + dst(0, 1) = -v(2); + dst(1, 0) = v(2); + dst(0, 2) = v(1); + dst(2, 0) = -v(1); + dst(1, 2) = -v(0); + dst(2, 1) = v(0); + } + EIGEN_DEVICE_FUNC static void run( + DstXprType& dst, const SrcXprType& src, + const internal::add_assign_op& /*func*/) { + dst.vector() += src.vector(); + } + + EIGEN_DEVICE_FUNC static void run( + DstXprType& dst, const SrcXprType& src, + const internal::sub_assign_op& /*func*/) { + dst.vector() -= src.vector(); + } +}; + +} // namespace internal + +} // end namespace Eigen + +#endif // EIGEN_SKEWSYMMETRICMATRIX3_H diff --git a/Eigen/src/Core/arch/AVX/Reductions.h b/Eigen/src/Core/arch/AVX/Reductions.h new file mode 100644 index 0000000..237617c --- /dev/null +++ b/Eigen/src/Core/arch/AVX/Reductions.h @@ -0,0 +1,353 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2025 Charlie Schlosser +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_REDUCTIONS_AVX_H +#define EIGEN_REDUCTIONS_AVX_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet8i -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE int predux(const Packet8i& a) { + Packet4i lo = _mm256_castsi256_si128(a); + Packet4i hi = _mm256_extractf128_si256(a, 1); + return predux(padd(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE int predux_mul(const Packet8i& a) { + Packet4i lo = _mm256_castsi256_si128(a); + Packet4i hi = _mm256_extractf128_si256(a, 1); + return predux_mul(pmul(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE int predux_min(const Packet8i& a) { + Packet4i lo = _mm256_castsi256_si128(a); + Packet4i hi = _mm256_extractf128_si256(a, 1); + return predux_min(pmin(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE int predux_max(const Packet8i& a) { + Packet4i lo = _mm256_castsi256_si128(a); + Packet4i hi = _mm256_extractf128_si256(a, 1); + return predux_max(pmax(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8i& a) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_movemask_epi8(a) != 0x0; +#else + return _mm256_movemask_ps(_mm256_castsi256_ps(a)) != 0x0; +#endif +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet8ui -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE uint32_t predux(const Packet8ui& a) { + Packet4ui lo = _mm256_castsi256_si128(a); + Packet4ui hi = _mm256_extractf128_si256(a, 1); + return predux(padd(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet8ui& a) { + Packet4ui lo = _mm256_castsi256_si128(a); + Packet4ui hi = _mm256_extractf128_si256(a, 1); + return predux_mul(pmul(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE uint32_t predux_min(const Packet8ui& a) { + Packet4ui lo = _mm256_castsi256_si128(a); + Packet4ui hi = _mm256_extractf128_si256(a, 1); + return predux_min(pmin(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE uint32_t predux_max(const Packet8ui& a) { + Packet4ui lo = _mm256_castsi256_si128(a); + Packet4ui hi = _mm256_extractf128_si256(a, 1); + return predux_max(pmax(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8ui& a) { +#ifdef EIGEN_VECTORIZE_AVX2 + return _mm256_movemask_epi8(a) != 0x0; +#else + return _mm256_movemask_ps(_mm256_castsi256_ps(a)) != 0x0; +#endif +} + +#ifdef EIGEN_VECTORIZE_AVX2 + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4l -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE int64_t predux(const Packet4l& a) { + Packet2l lo = _mm256_castsi256_si128(a); + Packet2l hi = _mm256_extractf128_si256(a, 1); + return predux(padd(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4l& a) { + return _mm256_movemask_pd(_mm256_castsi256_pd(a)) != 0x0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4ul -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE uint64_t predux(const Packet4ul& a) { + return static_cast(predux(Packet4l(a))); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4ul& a) { + return _mm256_movemask_pd(_mm256_castsi256_pd(a)) != 0x0; +} + +#endif + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet8f -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE float predux(const Packet8f& a) { + Packet4f lo = _mm256_castps256_ps128(a); + Packet4f hi = _mm256_extractf128_ps(a, 1); + return predux(padd(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE float predux_mul(const Packet8f& a) { + Packet4f lo = _mm256_castps256_ps128(a); + Packet4f hi = _mm256_extractf128_ps(a, 1); + return predux_mul(pmul(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet8f& a) { + Packet4f lo = _mm256_castps256_ps128(a); + Packet4f hi = _mm256_extractf128_ps(a, 1); + return predux_min(pmin(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet8f& a) { + Packet4f lo = _mm256_castps256_ps128(a); + Packet4f hi = _mm256_extractf128_ps(a, 1); + return predux_min(pmin(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet8f& a) { + Packet4f lo = _mm256_castps256_ps128(a); + Packet4f hi = _mm256_extractf128_ps(a, 1); + return predux_min(pmin(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet8f& a) { + Packet4f lo = _mm256_castps256_ps128(a); + Packet4f hi = _mm256_extractf128_ps(a, 1); + return predux_max(pmax(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet8f& a) { + Packet4f lo = _mm256_castps256_ps128(a); + Packet4f hi = _mm256_extractf128_ps(a, 1); + return predux_max(pmax(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet8f& a) { + Packet4f lo = _mm256_castps256_ps128(a); + Packet4f hi = _mm256_extractf128_ps(a, 1); + return predux_max(pmax(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8f& a) { + return _mm256_movemask_ps(a) != 0x0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4d -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE double predux(const Packet4d& a) { + Packet2d lo = _mm256_castpd256_pd128(a); + Packet2d hi = _mm256_extractf128_pd(a, 1); + return predux(padd(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE double predux_mul(const Packet4d& a) { + Packet2d lo = _mm256_castpd256_pd128(a); + Packet2d hi = _mm256_extractf128_pd(a, 1); + return predux_mul(pmul(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet4d& a) { + Packet2d lo = _mm256_castpd256_pd128(a); + Packet2d hi = _mm256_extractf128_pd(a, 1); + return predux_min(pmin(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet4d& a) { + Packet2d lo = _mm256_castpd256_pd128(a); + Packet2d hi = _mm256_extractf128_pd(a, 1); + return predux_min(pmin(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet4d& a) { + Packet2d lo = _mm256_castpd256_pd128(a); + Packet2d hi = _mm256_extractf128_pd(a, 1); + return predux_min(pmin(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet4d& a) { + Packet2d lo = _mm256_castpd256_pd128(a); + Packet2d hi = _mm256_extractf128_pd(a, 1); + return predux_max(pmax(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet4d& a) { + Packet2d lo = _mm256_castpd256_pd128(a); + Packet2d hi = _mm256_extractf128_pd(a, 1); + return predux_max(pmax(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet4d& a) { + Packet2d lo = _mm256_castpd256_pd128(a); + Packet2d hi = _mm256_extractf128_pd(a, 1); + return predux_max(pmax(lo, hi)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4d& a) { + return _mm256_movemask_pd(a) != 0x0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet8h -- -- -- -- -- -- -- -- -- -- -- -- */ +#ifndef EIGEN_VECTORIZE_AVX512FP16 + +template <> +EIGEN_STRONG_INLINE half predux(const Packet8h& a) { + return static_cast(predux(half2float(a))); +} + +template <> +EIGEN_STRONG_INLINE half predux_mul(const Packet8h& a) { + return static_cast(predux_mul(half2float(a))); +} + +template <> +EIGEN_STRONG_INLINE half predux_min(const Packet8h& a) { + return static_cast(predux_min(half2float(a))); +} + +template <> +EIGEN_STRONG_INLINE half predux_min(const Packet8h& a) { + return static_cast(predux_min(half2float(a))); +} + +template <> +EIGEN_STRONG_INLINE half predux_min(const Packet8h& a) { + return static_cast(predux_min(half2float(a))); +} + +template <> +EIGEN_STRONG_INLINE half predux_max(const Packet8h& a) { + return static_cast(predux_max(half2float(a))); +} + +template <> +EIGEN_STRONG_INLINE half predux_max(const Packet8h& a) { + return static_cast(predux_max(half2float(a))); +} + +template <> +EIGEN_STRONG_INLINE half predux_max(const Packet8h& a) { + return static_cast(predux_max(half2float(a))); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8h& a) { + return _mm_movemask_epi8(a) != 0; +} +#endif // EIGEN_VECTORIZE_AVX512FP16 + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet8bf -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE bfloat16 predux(const Packet8bf& a) { + return static_cast(predux(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_mul(const Packet8bf& a) { + return static_cast(predux_mul(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet8bf& a) { + return static_cast(predux_min(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet8bf& a) { + return static_cast(predux_min(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet8bf& a) { + return static_cast(predux_min(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet8bf& a) { + return static_cast(predux_max(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet8bf& a) { + return static_cast(predux_max(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet8bf& a) { + return static_cast(predux_max(Bf16ToF32(a))); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8bf& a) { + return _mm_movemask_epi8(a) != 0; +} + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_REDUCTIONS_AVX_H diff --git a/Eigen/src/Core/arch/AVX512/GemmKernel.h b/Eigen/src/Core/arch/AVX512/GemmKernel.h new file mode 100644 index 0000000..e06b83c --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/GemmKernel.h @@ -0,0 +1,1245 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2022 Intel Corporation +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CORE_ARCH_AVX512_GEMM_KERNEL_H +#define EIGEN_CORE_ARCH_AVX512_GEMM_KERNEL_H + +#if EIGEN_COMP_MSVC +#include +#else +#include +#endif +#include +#include + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +#if !defined(EIGEN_USE_AVX512_GEMM_KERNELS) +#define EIGEN_USE_AVX512_GEMM_KERNELS 1 +#endif + +#define SECOND_FETCH (32) +#if (EIGEN_COMP_GNUC_STRICT != 0) && !defined(EIGEN_ARCH_AVX512_GEMM_KERNEL_USE_LESS_A_REGS) +// Use less registers to load A elements to workaround compiler spills. Loose a +// bit of performance (less than ~2%). +#define EIGEN_ARCH_AVX512_GEMM_KERNEL_USE_LESS_A_REGS +#endif + +namespace Eigen { +namespace internal { + +template +class gemm_class { + using vec = typename packet_traits::type; + using vec_ymm = typename unpacket_traits::half; + using vec_xmm = typename unpacket_traits::half; + using umask_t = typename unpacket_traits::mask_t; + + static constexpr bool is_f32 = sizeof(Scalar) == sizeof(float); + static constexpr bool is_f64 = sizeof(Scalar) == sizeof(double); + +#ifndef EIGEN_ARCH_AVX512_GEMM_KERNEL_USE_LESS_A_REGS + static constexpr bool use_less_a_regs = !is_unit_inc; +#else + static constexpr bool use_less_a_regs = true; +#endif +#ifndef EIGEN_ARCH_AVX512_GEMM_KERNEL_USE_LESS_B_REGS + static constexpr bool use_less_b_regs = !is_unit_inc; +#else + static constexpr bool use_less_b_regs = true; +#endif + + static constexpr int a_regs[] = {0, 1, 2, use_less_a_regs ? 0 : 3, use_less_a_regs ? 1 : 4, use_less_a_regs ? 2 : 5}; + static constexpr int b_regs[] = {6, use_less_b_regs ? 6 : 7}; + static constexpr int c_regs[] = { + 8, 16, 24, 9, 17, 25, 10, 18, 26, 11, 19, 27, 12, 20, 28, 13, 21, 29, 14, 22, 30, 15, 23, 31, + }; + + static constexpr int alpha_load_reg = 0; + static constexpr int c_load_regs[] = {1, 2, 6}; + + static constexpr int a_shift = 128; + static constexpr int b_shift = 128; + + static constexpr int nelems_in_cache_line = is_f32 ? 16 : 8; + static constexpr int a_prefetch_size = nelems_in_cache_line * 2; + static constexpr int b_prefetch_size = nelems_in_cache_line * 8; + + vec zmm[32]; + umask_t mask; + + // gemm arguments. + Index m; + const Index n, k, ldc; + const Index inc; + const Scalar *alpha; + + const Scalar *a, *b; + Scalar *c; + + const bool is_alpha1; + const bool is_beta0; + + const Index a_stride, b_stride; + const Index a_off, b_off; + + EIGEN_ALWAYS_INLINE void prefetch_a(const Scalar *a_addr) { + _mm_prefetch((char *)(a_prefetch_size + a_addr - a_shift), _MM_HINT_T0); + } + + EIGEN_ALWAYS_INLINE void prefetch_b(const Scalar *b_addr) { + _mm_prefetch((char *)(b_prefetch_size + b_addr - b_shift), _MM_HINT_T0); + } + + EIGEN_ALWAYS_INLINE void prefetch_x(const Scalar *x_addr) { _mm_prefetch((char *)(x_addr - a_shift), _MM_HINT_T2); } + + EIGEN_ALWAYS_INLINE void prefetch_c(const Scalar *c_addr) { +#if defined(__PRFCHW__) && __PRFCHW__ == 1 + _m_prefetchw((void *)c_addr); +#else + _mm_prefetch((char *)c_addr, _MM_HINT_T0); +#endif + } + + template + EIGEN_ALWAYS_INLINE void a_load(vec &a_reg, const Scalar *a_addr) { + switch (nelems * sizeof(*a_addr) * 8) { + default: + case 512 * 3: + a_reg = ploadu(a_addr); + break; + case 512 * 2: + a_reg = ploadu(a_addr); + break; + case 512 * 1: + a_reg = ploadu(a_addr); + break; + case 256 * 1: + a_reg = preinterpret(_mm512_broadcast_f64x4(ploadu(reinterpret_cast(a_addr)))); + break; + case 128 * 1: + a_reg = preinterpret(_mm512_broadcast_f32x4(ploadu(reinterpret_cast(a_addr)))); + break; + case 64 * 1: + a_reg = preinterpret(pload1(reinterpret_cast(a_addr))); + break; + case 32 * 1: + a_reg = pload1(a_addr); + break; + } + } + + EIGEN_ALWAYS_INLINE void b_load(vec &b_reg, const Scalar *b_addr) { b_reg = pload1(b_addr); } + + template + EIGEN_ALWAYS_INLINE void c_store(Scalar *mem, vec &src) { + if (is_unit_inc) { + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + pstoreu(mem, src); + break; + case 512 * 2: + pstoreu(mem, src); + break; + case 512 * 1: + pstoreu(mem, src); + break; + case 256 * 1: + pstoreu(mem, preinterpret(src)); + break; + case 128 * 1: + pstoreu(mem, preinterpret(src)); + break; + case 64 * 1: + pstorel(mem, preinterpret(src)); + break; + case 32 * 1: + pstores(mem, preinterpret(src)); + break; + } + } else { + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + pscatter(mem, src, inc); + break; + case 512 * 2: + pscatter(mem, src, inc); + break; + case 512 * 1: + pscatter(mem, src, inc); + break; + case 256 * 1: + pscatter(mem, src, inc, mask); + break; + case 128 * 1: + pscatter(mem, src, inc, mask); + break; + case 64 * 1: + pscatter(mem, src, inc, mask); + break; + case 32 * 1: + pscatter(mem, src, inc, mask); + break; + } + } + } + + template + EIGEN_ALWAYS_INLINE void vaddm(vec &dst, const Scalar *mem, vec &src, vec ®) { + if (is_unit_inc) { + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + dst = padd(src, ploadu(mem)); + break; + case 512 * 2: + dst = padd(src, ploadu(mem)); + break; + case 512 * 1: + dst = padd(src, ploadu(mem)); + break; + case 256 * 1: + dst = preinterpret(padd(preinterpret(src), ploadu(mem))); + break; + case 128 * 1: + dst = preinterpret(padd(preinterpret(src), ploadu(mem))); + break; + case 64 * 1: + dst = preinterpret(padd(preinterpret(src), ploadl(mem))); + break; + case 32 * 1: + dst = preinterpret(padds(preinterpret(src), ploads(mem))); + break; + } + } else { + // Zero out scratch register + reg = pzero(reg); + + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + reg = pgather(mem, inc); + dst = padd(src, reg); + break; + case 512 * 2: + reg = pgather(mem, inc); + dst = padd(src, reg); + break; + case 512 * 1: + reg = pgather(mem, inc); + dst = padd(src, reg); + break; + case 256 * 1: + reg = preinterpret(pgather(mem, inc)); + dst = preinterpret(padd(preinterpret(src), preinterpret(reg))); + break; + case 128 * 1: + reg = preinterpret(pgather(mem, inc)); + dst = preinterpret(padd(preinterpret(src), preinterpret(reg))); + break; + case 64 * 1: + if (is_f32) { + reg = pgather(reg, mem, inc, mask); + dst = preinterpret(padd(preinterpret(src), preinterpret(reg))); + } else { + dst = preinterpret(padd(preinterpret(src), ploadl(mem))); + } + break; + case 32 * 1: + dst = preinterpret(padds(preinterpret(src), ploads(mem))); + break; + } + } + } + + EIGEN_STRONG_INLINE void vfmadd(vec &dst, const vec &src1, const vec &src2) { + dst = pmadd(src1, src2, dst); + +#if (EIGEN_COMP_GNUC != 0) || (EIGEN_COMP_CLANG != 0) + // Workaround register spills for gcc and clang + __asm__("#" : [dst] "+v"(dst) : [src1] "%v"(src1), [src2] "v"(src2)); +#endif + } + + template + EIGEN_ALWAYS_INLINE void vfmaddm(vec &dst, const Scalar *mem, vec &src, vec &scale, vec ®) { + if (is_unit_inc) { + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + dst = pmadd(scale, src, ploadu(mem)); + break; + case 512 * 2: + dst = pmadd(scale, src, ploadu(mem)); + break; + case 512 * 1: + dst = pmadd(scale, src, ploadu(mem)); + break; + case 256 * 1: + dst = + preinterpret(pmadd(preinterpret(scale), preinterpret(src), ploadu(mem))); + break; + case 128 * 1: + dst = + preinterpret(pmadd(preinterpret(scale), preinterpret(src), ploadu(mem))); + break; + case 64 * 1: + dst = + preinterpret(pmadd(preinterpret(scale), preinterpret(src), ploadl(mem))); + break; + case 32 * 1: + dst = + preinterpret(pmadds(preinterpret(scale), preinterpret(src), ploads(mem))); + break; + } + } else { + // Zero out scratch register + reg = pzero(reg); + + switch (nelems * sizeof(*mem) * 8) { + default: + case 512 * 3: + reg = pgather(mem, inc); + dst = pmadd(scale, src, reg); + break; + case 512 * 2: + reg = pgather(mem, inc); + dst = pmadd(scale, src, reg); + break; + case 512 * 1: + reg = pgather(mem, inc); + dst = pmadd(scale, src, reg); + break; + case 256 * 1: + reg = preinterpret(pgather(mem, inc)); + dst = preinterpret( + pmadd(preinterpret(scale), preinterpret(src), preinterpret(reg))); + break; + case 128 * 1: + reg = preinterpret(pgather(mem, inc)); + dst = preinterpret( + pmadd(preinterpret(scale), preinterpret(src), preinterpret(reg))); + break; + case 64 * 1: + if (is_f32) { + reg = pgather(reg, mem, inc, mask); + dst = preinterpret( + pmadd(preinterpret(scale), preinterpret(src), preinterpret(reg))); + } else { + dst = preinterpret( + pmadd(preinterpret(scale), preinterpret(src), ploadl(mem))); + } + break; + case 32 * 1: + dst = + preinterpret(pmadds(preinterpret(scale), preinterpret(src), ploads(mem))); + break; + } + } + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(j > endX) || (i > endY)> a_loads(const Scalar *ao) { + EIGEN_UNUSED_VARIABLE(ao); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(j <= endX) && (i <= endY)> a_loads(const Scalar *ao) { + if (j < endX) { + if (i < endY) { + auto &a_reg = zmm[a_regs[i + (j % 2) * 3]]; + const Scalar *a_addr = ao + nelems * j + nelems_in_cache_line * i - a_shift; + a_load(a_reg, a_addr); + + a_loads(ao); + } else { + a_loads(ao); + } + } + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(un > max_b_unroll) || (i > um_vecs)> prefetch_cs(const Scalar *co1, + const Scalar *co2) { + EIGEN_UNUSED_VARIABLE(co1); + EIGEN_UNUSED_VARIABLE(co2); + } + + /* C prefetch loop structure. + * for (int un = 0; un < 8; un++) { + * if (b_unroll >= un + 1) { + * if (un == 4) co2 = co1 + 4 * ldc; + * + * for (int i = 0; i < um_vecs; i++) { + * Scalar *co = (un + 1 <= 4) ? co1 : co2; + * auto co_off = (un % 4) * ldc + a_unroll - 1 + i * nelems_in_cache_line * sizeof *co; + * prefetch_c(co + co_off); + * } + * } + * } + */ + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(un <= max_b_unroll) && (i <= um_vecs)> prefetch_cs(Scalar *&co1, Scalar *&co2) { + if (un < max_b_unroll) { + if (b_unroll >= un + 1) { + if (un == 4 && i == 0) co2 = co1 + 4 * ldc; + + if (i < um_vecs) { + Scalar *co = (un + 1 <= 4) ? co1 : co2; + auto co_off = (un % 4) * ldc + a_unroll - 1 + i * nelems_in_cache_line * sizeof *co; + prefetch_c(co + co_off); + + prefetch_cs(co1, co2); + } else { + prefetch_cs(co1, co2); + } + + } else { + prefetch_cs(co1, co2); + } + } + } + + // load_c + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> scale_load_c(const Scalar *cox, vec &alpha_reg) { + EIGEN_UNUSED_VARIABLE(cox); + EIGEN_UNUSED_VARIABLE(alpha_reg); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> scale_load_c(const Scalar *cox, vec &alpha_reg) { + if (i < um_vecs) { + auto &c_reg = zmm[c_regs[i + idx * 3]]; + auto &c_load_reg = zmm[c_load_regs[i % 3]]; + auto c_mem = cox; + if (is_unit_inc) + c_mem += i * nelems_in_cache_line; + else + c_mem += i * nelems_in_cache_line * inc; + + if (!is_beta0 && is_alpha1) + vaddm(c_reg, c_mem, c_reg, c_load_reg); + else if (!is_beta0 && !is_alpha1) + vfmaddm(c_reg, c_mem, c_reg, alpha_reg, c_load_reg); + else if (is_beta0 && !is_alpha1) + c_reg = pmul(alpha_reg, c_reg); + + scale_load_c(cox, alpha_reg); + } + } + + // store_c + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(i > um_vecs)> write_c(Scalar *cox) { + EIGEN_UNUSED_VARIABLE(cox); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(i <= um_vecs)> write_c(Scalar *cox) { + if (i < um_vecs) { + auto &c_reg = zmm[c_regs[i + idx * 3]]; + auto c_mem = cox; + if (is_unit_inc) + c_mem += i * nelems_in_cache_line; + else + c_mem += i * nelems_in_cache_line * inc; + + c_store(c_mem, c_reg); + c_reg = pzero(c_reg); + + write_c(cox); + } + } + + /* C update loop structure. + * co2 = co1 + ldc; + * + * auto &alpha_reg = zmm[alpha_load_reg]; + * if (!is_alpha1) alpha_reg = pload1(alpha); + * + * int idx = 0; + * for (pow = 1; pow <= 8; pow <<= 1) { + * + * if (b_unroll >= pow) { + * for (count = 1; count < (pow + 1) / 2 + 1; count++) { + * if (pow >= 4) co2 += ldc; + * + * const Scalar *cox = (idx == 0) ? co1 : co2; + * + * const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); + * scale_load_c<0, um_vecs, idx, a_unroll>(cox, alpha_reg); + * write_c<0, um_vecs, idx, a_unroll>(cox); + * + * idx++; + * } + * } + * } + * + * if (b_unroll == 1) + * co1 += ldc; + * else + * co1 = co2 + ldc; + */ + + template + EIGEN_ALWAYS_INLINE void c_update_1count(Scalar *&cox) { + if (pow >= 4) cox += ldc; + + const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); + auto &alpha_reg = zmm[alpha_load_reg]; + + scale_load_c<0, um_vecs, idx, a_unroll>(cox, alpha_reg); + write_c<0, um_vecs, idx, a_unroll>(cox); + } + + template + EIGEN_ALWAYS_INLINE void c_update_1pow(Scalar *&co1, Scalar *&co2) { + constexpr int idx = pow / 2; + Scalar *&cox = idx == 0 ? co1 : co2; + + constexpr int max_count = (pow + 1) / 2; + static_assert(max_count <= 4, "Unsupported max_count."); + + if (1 <= max_count) c_update_1count(cox); + if (2 <= max_count) c_update_1count(cox); + if (3 <= max_count) c_update_1count(cox); + if (4 <= max_count) c_update_1count(cox); + } + + template + EIGEN_ALWAYS_INLINE void c_update(Scalar *&co1, Scalar *&co2) { + auto &alpha_reg = zmm[alpha_load_reg]; + + co2 = co1 + ldc; + if (!is_alpha1) alpha_reg = pload1(alpha); + if (!is_unit_inc && a_unroll < nelems_in_cache_line) mask = static_cast((1ull << a_unroll) - 1); + + static_assert(max_b_unroll <= 8, "Unsupported max_b_unroll"); + + if (1 <= max_b_unroll && 1 <= b_unroll) c_update_1pow<1, a_unroll>(co1, co2); + if (2 <= max_b_unroll && 2 <= b_unroll) c_update_1pow<2, a_unroll>(co1, co2); + if (4 <= max_b_unroll && 4 <= b_unroll) c_update_1pow<4, a_unroll>(co1, co2); + if (8 <= max_b_unroll && 8 <= b_unroll) c_update_1pow<8, a_unroll>(co1, co2); + + if (b_unroll == 1) + co1 += ldc; + else + co1 = co2 + ldc; + } + + // compute + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> compute(const Scalar *ao, const Scalar *bo, int &fetchA_idx, + int &fetchB_idx, vec &b_reg) { + EIGEN_UNUSED_VARIABLE(ao); + EIGEN_UNUSED_VARIABLE(bo); + EIGEN_UNUSED_VARIABLE(fetchA_idx); + EIGEN_UNUSED_VARIABLE(fetchB_idx); + EIGEN_UNUSED_VARIABLE(b_reg); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> compute(const Scalar *ao, const Scalar *bo, int &fetchA_idx, + int &fetchB_idx, vec &b_reg) { + if (um < um_vecs) { + auto &c_reg = zmm[c_regs[um + idx * 3]]; + auto &a_reg = zmm[a_regs[um + (uk % 2) * 3]]; + + vfmadd(c_reg, a_reg, b_reg); + + if (!fetch_x && um == 0 && + (((idx == 0 || idx == 6) && (uk % 2 == 0 || is_f64 || ktail)) || + (idx == 3 && (uk % 2 == 1 || is_f64 || ktail)))) { + prefetch_a(ao + nelems_in_cache_line * fetchA_idx); + fetchA_idx++; + } + + if (um == 0 && idx == 1 && (uk % 2 == 0 || is_f64 || ktail)) { + prefetch_b(bo + nelems_in_cache_line * fetchB_idx); + fetchB_idx++; + } + + compute(ao, bo, fetchA_idx, fetchB_idx, b_reg); + } + } + + // load_a + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(um > um_vecs)> load_a(const Scalar *ao) { + EIGEN_UNUSED_VARIABLE(ao); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(um <= um_vecs)> load_a(const Scalar *ao) { + if (um < um_vecs) { + auto &a_reg = zmm[a_regs[um + (uk % 2) * 3]]; + const Scalar *a_addr = ao + nelems * (1 + !ktail * !use_less_a_regs + uk) + nelems_in_cache_line * um - a_shift; + a_load(a_reg, a_addr); + + load_a(ao); + } + } + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(count > (pow + 1) / 2)> innerkernel_1pow(const Scalar *&aa, + const Scalar *const &ao, + const Scalar *const &bo, Scalar *&co2, + int &fetchA_idx, int &fetchB_idx) { + EIGEN_UNUSED_VARIABLE(aa); + EIGEN_UNUSED_VARIABLE(ao); + EIGEN_UNUSED_VARIABLE(bo); + EIGEN_UNUSED_VARIABLE(co2); + EIGEN_UNUSED_VARIABLE(fetchA_idx); + EIGEN_UNUSED_VARIABLE(fetchB_idx); + } + + template + EIGEN_ALWAYS_INLINE std::enable_if_t<(count <= (pow + 1) / 2)> innerkernel_1pow(const Scalar *&aa, + const Scalar *const &ao, + const Scalar *const &bo, Scalar *&co2, + int &fetchA_idx, int &fetchB_idx) { + const int idx = (pow / 2) + count; + + if (count < (pow + 1) / 2) { + auto &b_reg = zmm[b_regs[idx % 2]]; + + if (fetch_x && uk == 3 && idx == 0) prefetch_x(aa); + if (fetch_x && uk == 3 && idx == 4) aa += 8; + + if (b_unroll >= pow) { + compute<0, um_vecs, idx, uk, fetch_x, ktail>(ao, bo, fetchA_idx, fetchB_idx, b_reg); + + const Scalar *b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) * !use_less_b_regs - b_shift; + b_load(b_reg, b_addr); + } + + // Go to the next count. + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, + fetchB_idx); + + } else { + // Maybe prefetch C data after count-loop. + if (pow == 2 && c_fetch) { + if (uk % 3 == 0 && uk > 0) { + co2 += ldc; + } else { + prefetch_c(co2 + (uk % 3) * nelems_in_cache_line); + } + } + } + } + + template + EIGEN_ALWAYS_INLINE void innerkernel_1uk(const Scalar *&aa, const Scalar *const &ao, const Scalar *const &bo, + Scalar *&co2, int &fetchA_idx, int &fetchB_idx) { + const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); + + if (max_b_unroll >= 1) + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + if (max_b_unroll >= 2) + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + if (max_b_unroll >= 4) + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + if (max_b_unroll >= 8) + innerkernel_1pow(aa, ao, bo, co2, fetchA_idx, fetchB_idx); + + // Load A after pow-loop. Skip this at the end to prevent running over the buffer + if (!no_a_preload) load_a<0, um_vecs, uk, a_unroll, ktail>(ao); + } + + /* Inner kernel loop structure. + * for (int uk = 0; uk < kfactor; uk++) { + * int idx = 0; + * + * for (pow = 1; pow < max_b_unroll << 1; pow <<= 1) { + * for (int count = 0; count < (pow + 1) / 2; count++) { + * auto &b_reg = zmm[b_regs[idx % 2]]; + * + * if (fetch_x && uk == 3 && idx == 0) prefetch_x(aa); + * if (fetch_x && uk == 3 && idx == 4) aa += 8; + * + * if (b_unroll >= pow) { + * compute<0, um_vecs, idx, uk, fetchx, ktail>(ao, bo, fetchA_idx, fetchB_idx, b_reg); + * + * const Scalar *b_addr = bo + b_unroll * uk + idx + 1 + (b_unroll > 1) - b_shift ; + * b_load(b_reg, b_addr); + * } + * idx++; + * } + * + * Maybe prefetch C data. + * if (pow == 2 && c_fetch) { + * if (uk % 3 == 0 && uk > 0) { + * co2 += ldc; + * } else { + * prefetch_c(co2 + (uk % 3) * nelems_in_cache_line); + * } + * } + * } + * + * Load A. + * load_a<0, um_vecs, uk, ktail, a_unroll>(ao); + * } + * + * Advance A/B pointers after uk-loop. + * ao += a_unroll * kfactor; + * bo += b_unroll * kfactor; + */ + + template + EIGEN_ALWAYS_INLINE void innerkernel(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co2) { + int fetchA_idx = 0; + int fetchB_idx = 0; + + const bool fetch_x = k_factor == max_k_factor; + const bool ktail = k_factor == 1; + + static_assert(k_factor <= 4 && k_factor > 0, "innerkernel maximum k_factor supported is 4"); + static_assert(no_a_preload == false || (no_a_preload == true && k_factor == 1), + "skipping a preload only allowed when k unroll is 1"); + + if (k_factor > 0) + innerkernel_1uk<0, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>( + aa, ao, bo, co2, fetchA_idx, fetchB_idx); + if (k_factor > 1) + innerkernel_1uk<1, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>( + aa, ao, bo, co2, fetchA_idx, fetchB_idx); + if (k_factor > 2) + innerkernel_1uk<2, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>( + aa, ao, bo, co2, fetchA_idx, fetchB_idx); + if (k_factor > 3) + innerkernel_1uk<3, max_b_unroll, a_unroll, b_unroll, ktail, fetch_x, c_fetch, no_a_preload>( + aa, ao, bo, co2, fetchA_idx, fetchB_idx); + + // Advance A/B pointers after uk-loop. + ao += a_unroll * k_factor; + bo += b_unroll * k_factor; + } + + template + EIGEN_ALWAYS_INLINE void kloop(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { + const int um_vecs = numext::div_ceil(a_unroll, nelems_in_cache_line); + if (!use_less_a_regs && k > 1) + a_loads<0, 2, 0, um_vecs, a_unroll>(ao); + else + a_loads<0, 1, 0, um_vecs, a_unroll>(ao); + + b_load(zmm[b_regs[0]], bo - b_shift + 0); + if (!use_less_b_regs) b_load(zmm[b_regs[1]], bo - b_shift + 1); + +#ifndef SECOND_FETCH + prefetch_cs<0, max_b_unroll, 0, um_vecs, a_unroll, b_unroll>(co1, co2); +#endif // SECOND_FETCH + + // Unrolling k-loop by a factor of 4. + const int max_k_factor = 4; + Index kRem = k % max_k_factor; + Index k_ = k - kRem; + if (k_ >= max_k_factor) { + k_ -= max_k_factor; + kRem += max_k_factor; + } + Index loop_count = k_ / max_k_factor; + + if (loop_count > 0) { +#ifdef SECOND_FETCH + loop_count -= SECOND_FETCH; +#endif + while (loop_count > 0) { + innerkernel(aa, ao, bo, co2); + loop_count--; + } +#ifdef SECOND_FETCH + co2 = co1 + nelems_in_cache_line - 1; + + loop_count += b_unroll; + while (loop_count > 0) { + innerkernel(aa, ao, bo, co2); + loop_count--; + } + + loop_count += SECOND_FETCH - b_unroll; + while (loop_count > 0) { + innerkernel(aa, ao, bo, co2); + loop_count--; + } +#endif + } + + // k-loop remainder handling. + loop_count = kRem; + while (loop_count > 1) { + innerkernel(aa, ao, bo, co2); + loop_count--; + } + if (loop_count > 0) { + innerkernel(aa, ao, bo, co2); + } + + // Update C matrix. + c_update(co1, co2); + } + + template + EIGEN_ALWAYS_INLINE void nloop(const Scalar *&aa, const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { + // Set A matrix pointer. + ao = a + a_off * a_unroll; + + // Set B matrix pointer if needed. + bo += b_unroll * b_off; + + kloop(aa, ao, bo, co1, co2); + + // Advance B matrix pointer if needed. + bo += b_unroll * (b_stride - k - b_off); + + // Advance prefetch A pointer. + aa += 16; + } + + template + EIGEN_ALWAYS_INLINE void mloop(const Scalar *&ao, const Scalar *&bo, Scalar *&co1, Scalar *&co2) { + // Set prefetch A pointers. + const Scalar *aa = a + a_unroll * a_stride; + + // Set C matrix pointers. + co1 = c; + if (a_unroll >= max_a_unroll) co2 = c + 2 * ldc; + if (is_unit_inc) + c += a_unroll; + else + c += a_unroll * inc; + + // Set B matrix pointer. + bo = b; + + // Main n-loop. + for (Index i = n / max_b_unroll; i > 0; i--) nloop(aa, ao, bo, co1, co2); + + // n-remainders. + if (n & 4 && max_b_unroll > 4) nloop(aa, ao, bo, co1, co2); +#if 0 + if (n & 2 && max_b_unroll > 2) nloop(aa, ao, bo, co1, co2); + if (n & 1 && max_b_unroll > 1) nloop(aa, ao, bo, co1, co2); +#else + // Copy kernels don't support tails of n = 2 for single/double precision. + // Loop over ones. + int n_rem = 2 * ((n & 2) != 0) + 1 * ((n & 1) != 0); + while (n_rem > 0) { + nloop(aa, ao, bo, co1, co2); + n_rem--; + } +#endif + + // Advance A matrix pointer. + a = ao + a_unroll * (a_stride - k - a_off); + } + + public: + // Compute kernel unrolling C matrix by max_a_unroll x max_b_unroll. + template + EIGEN_ALWAYS_INLINE void compute_kern() { + a -= -a_shift; + b -= -b_shift; + + const Scalar *ao = nullptr; + const Scalar *bo = nullptr; + Scalar *co1 = nullptr; + Scalar *co2 = nullptr; + + // Main m-loop. + for (; m >= max_a_unroll; m -= max_a_unroll) mloop(ao, bo, co1, co2); + + // m-remainders. + if (m & 32 && max_a_unroll > 32) mloop<32, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 16 && max_a_unroll > 16) mloop<16, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 8 && max_a_unroll > 8) mloop<8, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 4 && max_a_unroll > 4) mloop<4, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 2 && max_a_unroll > 2 && is_f64) mloop<2, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + if (m & 1 && max_a_unroll > 1 && is_f64) mloop<1, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + + // Copy kernels don't support tails of m = 2 for single precision. + // Loop over ones. + if (is_f32) { + int m_rem = 2 * ((m & 2) != 0) + 1 * ((m & 1) != 0); + while (m_rem > 0) { + mloop<1, max_a_unroll, max_b_unroll>(ao, bo, co1, co2); + m_rem--; + } + } + } + + gemm_class(Index m_, Index n_, Index k_, Index ldc_, Index inc_, const Scalar *alpha_, const Scalar *a_, + const Scalar *b_, Scalar *c_, bool is_alpha1_, bool is_beta0_, Index a_stride_, Index b_stride_, + Index a_off_, Index b_off_) + : m(m_), + n(n_), + k(k_), + ldc(ldc_), + inc(inc_), + alpha(alpha_), + a(a_), + b(b_), + c(c_), + is_alpha1(is_alpha1_), + is_beta0(is_beta0_), + a_stride(a_stride_), + b_stride(b_stride_), + a_off(a_off_), + b_off(b_off_) { + // Zero out all accumulation registers. + zmm[8] = pzero(zmm[8]); + zmm[9] = pzero(zmm[9]); + zmm[10] = pzero(zmm[10]); + zmm[11] = pzero(zmm[11]); + zmm[12] = pzero(zmm[12]); + zmm[13] = pzero(zmm[13]); + zmm[14] = pzero(zmm[14]); + zmm[15] = pzero(zmm[15]); + zmm[16] = pzero(zmm[16]); + zmm[17] = pzero(zmm[17]); + zmm[18] = pzero(zmm[18]); + zmm[19] = pzero(zmm[19]); + zmm[20] = pzero(zmm[20]); + zmm[21] = pzero(zmm[21]); + zmm[22] = pzero(zmm[22]); + zmm[23] = pzero(zmm[23]); + zmm[24] = pzero(zmm[24]); + zmm[25] = pzero(zmm[25]); + zmm[26] = pzero(zmm[26]); + zmm[27] = pzero(zmm[27]); + zmm[28] = pzero(zmm[28]); + zmm[29] = pzero(zmm[29]); + zmm[30] = pzero(zmm[30]); + zmm[31] = pzero(zmm[31]); + } +}; + +// Compute kernel with max unroll support of: +// Single precision: +// max_a_unroll: 48, 32, 16, 8, 4, 2, 1 +// max_b_unroll: 8, 4, 2, 1 +// Double precision: +// max_a_unroll: 24, 16, 8, 4, 2, 1 +// max_b_unroll: 8, 4, 2, 1 +template +EIGEN_DONT_INLINE void gemm_kern_avx512(Index m, Index n, Index k, Scalar *alpha, const Scalar *a, const Scalar *b, + Scalar *c, Index ldc, Index inc = 1, Index a_stride = -1, Index b_stride = -1, + Index a_off = 0, Index b_off = 0) { + if (a_stride == -1) a_stride = k; + if (b_stride == -1) b_stride = k; + + gemm_class g(m, n, k, ldc, inc, alpha, a, b, c, is_alpha1, is_beta0, a_stride, b_stride, a_off, + b_off); + g.template compute_kern(); +} + +// Template specializations of GEBP kernels with nr = 8. +#if EIGEN_USE_AVX512_GEMM_KERNELS +template +class gebp_traits + : public gebp_traits { + using Base = gebp_traits; + + public: + enum { nr = Base::Vectorizable ? 8 : 4 }; +}; + +template +class gebp_traits + : public gebp_traits { + using Base = gebp_traits; + + public: + enum { nr = Base::Vectorizable ? 8 : 4 }; +}; + +template +struct gemm_pack_rhs { + typedef typename packet_traits::type Packet; + typedef typename DataMapper::LinearMapper LinearMapper; + enum { PacketSize = packet_traits::size }; + EIGEN_DONT_INLINE void operator()(Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride = 0, + Index offset = 0); +}; + +template +EIGEN_DONT_INLINE void gemm_pack_rhs::operator()( + Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride, Index offset) { + constexpr int nr = 8; + EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS COLMAJOR"); + EIGEN_UNUSED_VARIABLE(stride); + EIGEN_UNUSED_VARIABLE(offset); + eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride)); + conj_if::IsComplex && Conjugate> cj; + Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0; + Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0; + Index count = 0; + const Index peeled_k = (depth / PacketSize) * PacketSize; + if (nr >= 8) { + for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { + // skip what we have before + if (PanelMode) count += 8 * offset; + const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); + const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); + const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); + const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3); + const LinearMapper dm4 = rhs.getLinearMapper(0, j2 + 4); + const LinearMapper dm5 = rhs.getLinearMapper(0, j2 + 5); + const LinearMapper dm6 = rhs.getLinearMapper(0, j2 + 6); + const LinearMapper dm7 = rhs.getLinearMapper(0, j2 + 7); + Index k = 0; + if ((PacketSize % 8) == 0) // TODO enable vectorized transposition for PacketSize==4 + { + for (; k < peeled_k; k += PacketSize) { + PacketBlock kernel; + + kernel.packet[0] = dm0.template loadPacket(k); + kernel.packet[1] = dm1.template loadPacket(k); + kernel.packet[2] = dm2.template loadPacket(k); + kernel.packet[3] = dm3.template loadPacket(k); + kernel.packet[4] = dm4.template loadPacket(k); + kernel.packet[5] = dm5.template loadPacket(k); + kernel.packet[6] = dm6.template loadPacket(k); + kernel.packet[7] = dm7.template loadPacket(k); + + ptranspose(kernel); + + pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0])); + pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel.packet[1 % PacketSize])); + pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel.packet[2 % PacketSize])); + pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel.packet[3 % PacketSize])); + pstoreu(blockB + count + 4 * PacketSize, cj.pconj(kernel.packet[4 % PacketSize])); + pstoreu(blockB + count + 5 * PacketSize, cj.pconj(kernel.packet[5 % PacketSize])); + pstoreu(blockB + count + 6 * PacketSize, cj.pconj(kernel.packet[6 % PacketSize])); + pstoreu(blockB + count + 7 * PacketSize, cj.pconj(kernel.packet[7 % PacketSize])); + count += 8 * PacketSize; + } + } + for (; k < depth; k++) { + blockB[count + 0] = cj(dm0(k)); + blockB[count + 1] = cj(dm1(k)); + blockB[count + 2] = cj(dm2(k)); + blockB[count + 3] = cj(dm3(k)); + blockB[count + 4] = cj(dm4(k)); + blockB[count + 5] = cj(dm5(k)); + blockB[count + 6] = cj(dm6(k)); + blockB[count + 7] = cj(dm7(k)); + count += 8; + } + // skip what we have after + if (PanelMode) count += 8 * (stride - offset - depth); + } + } + + if (nr >= 4) { + for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { + // skip what we have before + if (PanelMode) count += 4 * offset; + const LinearMapper dm0 = rhs.getLinearMapper(0, j2 + 0); + const LinearMapper dm1 = rhs.getLinearMapper(0, j2 + 1); + const LinearMapper dm2 = rhs.getLinearMapper(0, j2 + 2); + const LinearMapper dm3 = rhs.getLinearMapper(0, j2 + 3); + + Index k = 0; + if ((PacketSize % 4) == 0) // TODO enable vectorized transposition for PacketSize==2 ?? + { + for (; k < peeled_k; k += PacketSize) { + PacketBlock kernel; + kernel.packet[0] = dm0.template loadPacket(k); + kernel.packet[1 % PacketSize] = dm1.template loadPacket(k); + kernel.packet[2 % PacketSize] = dm2.template loadPacket(k); + kernel.packet[3 % PacketSize] = dm3.template loadPacket(k); + ptranspose(kernel); + pstoreu(blockB + count + 0 * PacketSize, cj.pconj(kernel.packet[0])); + pstoreu(blockB + count + 1 * PacketSize, cj.pconj(kernel.packet[1 % PacketSize])); + pstoreu(blockB + count + 2 * PacketSize, cj.pconj(kernel.packet[2 % PacketSize])); + pstoreu(blockB + count + 3 * PacketSize, cj.pconj(kernel.packet[3 % PacketSize])); + count += 4 * PacketSize; + } + } + for (; k < depth; k++) { + blockB[count + 0] = cj(dm0(k)); + blockB[count + 1] = cj(dm1(k)); + blockB[count + 2] = cj(dm2(k)); + blockB[count + 3] = cj(dm3(k)); + count += 4; + } + // skip what we have after + if (PanelMode) count += 4 * (stride - offset - depth); + } + } + + // copy the remaining columns one at a time (nr==1) + for (Index j2 = packet_cols4; j2 < cols; ++j2) { + if (PanelMode) count += offset; + const LinearMapper dm0 = rhs.getLinearMapper(0, j2); + for (Index k = 0; k < depth; k++) { + blockB[count] = cj(dm0(k)); + count += 1; + } + if (PanelMode) count += (stride - offset - depth); + } +} + +template +struct gemm_pack_rhs { + typedef typename packet_traits::type Packet; + typedef typename unpacket_traits::half HalfPacket; + typedef typename unpacket_traits::half>::half QuarterPacket; + typedef typename DataMapper::LinearMapper LinearMapper; + enum { + PacketSize = packet_traits::size, + HalfPacketSize = unpacket_traits::size, + QuarterPacketSize = unpacket_traits::size + }; + EIGEN_DONT_INLINE void operator()(Scalar *blockB, const DataMapper &rhs, Index depth, Index cols, Index stride = 0, + Index offset = 0) { + constexpr int nr = 8; + EIGEN_ASM_COMMENT("EIGEN PRODUCT PACK RHS ROWMAJOR"); + EIGEN_UNUSED_VARIABLE(stride); + EIGEN_UNUSED_VARIABLE(offset); + eigen_assert(((!PanelMode) && stride == 0 && offset == 0) || (PanelMode && stride >= depth && offset <= stride)); + const bool HasHalf = (int)HalfPacketSize < (int)PacketSize; + const bool HasQuarter = (int)QuarterPacketSize < (int)HalfPacketSize; + conj_if::IsComplex && Conjugate> cj; + Index packet_cols8 = nr >= 8 ? (cols / 8) * 8 : 0; + Index packet_cols4 = nr >= 4 ? (cols / 4) * 4 : 0; + Index count = 0; + + if (nr >= 8) { + for (Index j2 = 0; j2 < packet_cols8; j2 += 8) { + // skip what we have before + if (PanelMode) count += 8 * offset; + for (Index k = 0; k < depth; k++) { + if (PacketSize == 8) { + // Packet A = ploadu(&rhs.data()[k*rhs.stride() + j2]); + Packet A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + } else if (HasHalf && HalfPacketSize == 8) { + HalfPacket A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + } else if (HasQuarter && QuarterPacketSize == 8) { + QuarterPacket A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + } else if (PacketSize == 4) { + // Packet A = ploadu(&rhs.data()[k*rhs.stride() + j2]); + // Packet B = ploadu(&rhs.data()[k*rhs.stride() + j2 + PacketSize]); + Packet A = rhs.template loadPacket(k, j2); + Packet B = rhs.template loadPacket(k, j2 + PacketSize); + pstoreu(blockB + count, cj.pconj(A)); + pstoreu(blockB + count + PacketSize, cj.pconj(B)); + } else { + // const Scalar* b0 = &rhs.data()[k*rhs.stride() + j2]; + const LinearMapper dm0 = rhs.getLinearMapper(k, j2); + blockB[count + 0] = cj(dm0(0)); + blockB[count + 1] = cj(dm0(1)); + blockB[count + 2] = cj(dm0(2)); + blockB[count + 3] = cj(dm0(3)); + blockB[count + 4] = cj(dm0(4)); + blockB[count + 5] = cj(dm0(5)); + blockB[count + 6] = cj(dm0(6)); + blockB[count + 7] = cj(dm0(7)); + } + count += 8; + } + // skip what we have after + if (PanelMode) count += 8 * (stride - offset - depth); + } + } + + if (nr >= 4) { + for (Index j2 = packet_cols8; j2 < packet_cols4; j2 += 4) { + // skip what we have before + if (PanelMode) count += 4 * offset; + for (Index k = 0; k < depth; k++) { + if (PacketSize == 4) { + Packet A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + count += PacketSize; + } else if (HasHalf && HalfPacketSize == 4) { + HalfPacket A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + count += HalfPacketSize; + } else if (HasQuarter && QuarterPacketSize == 4) { + QuarterPacket A = rhs.template loadPacket(k, j2); + pstoreu(blockB + count, cj.pconj(A)); + count += QuarterPacketSize; + } else { + const LinearMapper dm0 = rhs.getLinearMapper(k, j2); + blockB[count + 0] = cj(dm0(0)); + blockB[count + 1] = cj(dm0(1)); + blockB[count + 2] = cj(dm0(2)); + blockB[count + 3] = cj(dm0(3)); + count += 4; + } + } + // skip what we have after + if (PanelMode) count += 4 * (stride - offset - depth); + } + } + // copy the remaining columns one at a time (nr==1) + for (Index j2 = packet_cols4; j2 < cols; ++j2) { + if (PanelMode) count += offset; + for (Index k = 0; k < depth; k++) { + blockB[count] = cj(rhs(k, j2)); + count += 1; + } + if (PanelMode) count += stride - offset - depth; + } + } +}; + +template +struct gebp_kernel { + EIGEN_ALWAYS_INLINE void operator()(const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, + Index depth, Index cols, Scalar alpha, Index strideA = -1, Index strideB = -1, + Index offsetA = 0, Index offsetB = 0); +}; + +template +EIGEN_ALWAYS_INLINE void gebp_kernel::operator()( + const DataMapper &res, const Scalar *blockA, const Scalar *blockB, Index rows, Index depth, Index cols, + Scalar alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { + if (res.incr() == 1) { + if (alpha == 1) { + gemm_kern_avx512(rows, cols, depth, &alpha, blockA, blockB, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, + strideB, offsetA, offsetB); + } else { + gemm_kern_avx512(rows, cols, depth, &alpha, blockA, blockB, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, + strideB, offsetA, offsetB); + } + } else { + if (alpha == 1) { + gemm_kern_avx512(rows, cols, depth, &alpha, blockA, blockB, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, + strideB, offsetA, offsetB); + } else { + gemm_kern_avx512(rows, cols, depth, &alpha, blockA, blockB, + (Scalar *)res.data(), res.stride(), res.incr(), strideA, + strideB, offsetA, offsetB); + } + } +} +#endif // EIGEN_USE_AVX512_GEMM_KERNELS + +} // namespace internal +} // namespace Eigen + +#undef SECOND_FETCH + +#endif // EIGEN_CORE_ARCH_AVX512_GEMM_KERNEL_H diff --git a/Eigen/src/Core/arch/AVX512/MathFunctionsFP16.h b/Eigen/src/Core/arch/AVX512/MathFunctionsFP16.h new file mode 100644 index 0000000..240ade4 --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/MathFunctionsFP16.h @@ -0,0 +1,75 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2025 The Eigen Authors. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_MATH_FUNCTIONS_FP16_AVX512_H +#define EIGEN_MATH_FUNCTIONS_FP16_AVX512_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { +namespace internal { + +EIGEN_STRONG_INLINE Packet32h combine2Packet16h(const Packet16h& a, const Packet16h& b) { + __m512i result = _mm512_castsi256_si512(_mm256_castph_si256(a)); + result = _mm512_inserti64x4(result, _mm256_castph_si256(b), 1); + return _mm512_castsi512_ph(result); +} + +EIGEN_STRONG_INLINE void extract2Packet16h(const Packet32h& x, Packet16h& a, Packet16h& b) { + a = _mm256_castsi256_ph(_mm512_castsi512_si256(_mm512_castph_si512(x))); + b = _mm256_castsi256_ph(_mm512_extracti64x4_epi64(_mm512_castph_si512(x), 1)); +} + +#define _EIGEN_GENERATE_FP16_MATH_FUNCTION(func) \ + template <> \ + EIGEN_STRONG_INLINE Packet8h func(const Packet8h& a) { \ + return float2half(func(half2float(a))); \ + } \ + \ + template <> \ + EIGEN_STRONG_INLINE Packet16h func(const Packet16h& a) { \ + return float2half(func(half2float(a))); \ + } \ + \ + template <> \ + EIGEN_STRONG_INLINE Packet32h func(const Packet32h& a) { \ + Packet16h low; \ + Packet16h high; \ + extract2Packet16h(a, low, high); \ + return combine2Packet16h(func(low), func(high)); \ + } + +_EIGEN_GENERATE_FP16_MATH_FUNCTION(psin) +_EIGEN_GENERATE_FP16_MATH_FUNCTION(pcos) +_EIGEN_GENERATE_FP16_MATH_FUNCTION(plog) +_EIGEN_GENERATE_FP16_MATH_FUNCTION(plog2) +_EIGEN_GENERATE_FP16_MATH_FUNCTION(plog1p) +_EIGEN_GENERATE_FP16_MATH_FUNCTION(pexp) +_EIGEN_GENERATE_FP16_MATH_FUNCTION(pexpm1) +_EIGEN_GENERATE_FP16_MATH_FUNCTION(pexp2) +_EIGEN_GENERATE_FP16_MATH_FUNCTION(ptanh) +#undef _EIGEN_GENERATE_FP16_MATH_FUNCTION + +// pfrexp +template <> +EIGEN_STRONG_INLINE Packet32h pfrexp(const Packet32h& a, Packet32h& exponent) { + return pfrexp_generic(a, exponent); +} + +// pldexp +template <> +EIGEN_STRONG_INLINE Packet32h pldexp(const Packet32h& a, const Packet32h& exponent) { + return pldexp_generic(a, exponent); +} + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_MATH_FUNCTIONS_FP16_AVX512_H \ No newline at end of file diff --git a/Eigen/src/Core/arch/AVX512/PacketMathFP16.h b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h new file mode 100644 index 0000000..ef64bc5 --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/PacketMathFP16.h @@ -0,0 +1,1413 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2025 The Eigen Authors. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_PACKET_MATH_FP16_AVX512_H +#define EIGEN_PACKET_MATH_FP16_AVX512_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +typedef __m512h Packet32h; +typedef __m256h Packet16h; +typedef __m128h Packet8h; + +template <> +struct is_arithmetic { + enum { value = true }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet32h type; + typedef Packet16h half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 32, + + HasCmp = 1, + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasAbs = 1, + HasAbs2 = 0, + HasMin = 1, + HasMax = 1, + HasConj = 1, + HasSetLinear = 0, + HasLog = 1, + HasLog1p = 1, + HasExp = 1, + HasExpm1 = 1, + HasSqrt = 1, + HasRsqrt = 1, + // These ones should be implemented in future + HasBessel = 0, + HasNdtri = 0, + HasSin = EIGEN_FAST_MATH, + HasCos = EIGEN_FAST_MATH, + HasTanh = EIGEN_FAST_MATH, + HasErf = 0, // EIGEN_FAST_MATH, + HasBlend = 0 + }; +}; + +template <> +struct unpacket_traits { + typedef Eigen::half type; + typedef Packet16h half; + typedef Packet32s integer_packet; + enum { + size = 32, + alignment = Aligned64, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +struct unpacket_traits { + typedef Eigen::half type; + typedef Packet8h half; + typedef Packet16s integer_packet; + enum { + size = 16, + alignment = Aligned32, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +struct unpacket_traits { + typedef Eigen::half type; + typedef Packet8h half; + typedef Packet8s integer_packet; + enum { + size = 8, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +// Conversions + +EIGEN_STRONG_INLINE Packet16f half2float(const Packet16h& a) { return _mm512_cvtxph_ps(a); } + +EIGEN_STRONG_INLINE Packet8f half2float(const Packet8h& a) { return _mm256_cvtxph_ps(a); } + +EIGEN_STRONG_INLINE Packet16h float2half(const Packet16f& a) { return _mm512_cvtxps_ph(a); } + +EIGEN_STRONG_INLINE Packet8h float2half(const Packet8f& a) { return _mm256_cvtxps_ph(a); } + +// Memory functions + +// pset1 + +template <> +EIGEN_STRONG_INLINE Packet32h pset1(const Eigen::half& from) { + return _mm512_set1_ph(from.x); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pset1(const Eigen::half& from) { + return _mm256_set1_ph(from.x); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pset1(const Eigen::half& from) { + return _mm_set1_ph(from.x); +} + +template <> +EIGEN_STRONG_INLINE Packet32h pzero(const Packet32h& /*a*/) { + return _mm512_setzero_ph(); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pzero(const Packet16h& /*a*/) { + return _mm256_setzero_ph(); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pzero(const Packet8h& /*a*/) { + return _mm_setzero_ph(); +} + +// pset1frombits +template <> +EIGEN_STRONG_INLINE Packet32h pset1frombits(unsigned short from) { + return _mm512_castsi512_ph(_mm512_set1_epi16(from)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pset1frombits(unsigned short from) { + return _mm256_castsi256_ph(_mm256_set1_epi16(from)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pset1frombits(unsigned short from) { + return _mm_castsi128_ph(_mm_set1_epi16(from)); +} + +// pfirst + +template <> +EIGEN_STRONG_INLINE Eigen::half pfirst(const Packet32h& from) { + return Eigen::half(_mm512_cvtsh_h(from)); +} + +template <> +EIGEN_STRONG_INLINE Eigen::half pfirst(const Packet16h& from) { + return Eigen::half(_mm256_cvtsh_h(from)); +} + +template <> +EIGEN_STRONG_INLINE Eigen::half pfirst(const Packet8h& from) { + return Eigen::half(_mm_cvtsh_h(from)); +} + +// pload + +template <> +EIGEN_STRONG_INLINE Packet32h pload(const Eigen::half* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm512_load_ph(from); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pload(const Eigen::half* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm256_load_ph(from); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pload(const Eigen::half* from) { + EIGEN_DEBUG_ALIGNED_LOAD return _mm_load_ph(from); +} + +// ploadu + +template <> +EIGEN_STRONG_INLINE Packet32h ploadu(const Eigen::half* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm512_loadu_ph(from); +} + +template <> +EIGEN_STRONG_INLINE Packet16h ploadu(const Eigen::half* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm256_loadu_ph(from); +} + +template <> +EIGEN_STRONG_INLINE Packet8h ploadu(const Eigen::half* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return _mm_loadu_ph(from); +} + +// pstore + +template <> +EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet32h& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm512_store_ph(to, from); +} + +template <> +EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet16h& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm256_store_ph(to, from); +} + +template <> +EIGEN_STRONG_INLINE void pstore(Eigen::half* to, const Packet8h& from) { + EIGEN_DEBUG_ALIGNED_STORE _mm_store_ph(to, from); +} + +// pstoreu + +template <> +EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet32h& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm512_storeu_ph(to, from); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet16h& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm256_storeu_ph(to, from); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu(Eigen::half* to, const Packet8h& from) { + EIGEN_DEBUG_UNALIGNED_STORE _mm_storeu_ph(to, from); +} + +// ploaddup +template <> +EIGEN_STRONG_INLINE Packet32h ploaddup(const Eigen::half* from) { + __m512h a = _mm512_castph256_ph512(_mm256_loadu_ph(from)); + return _mm512_permutexvar_ph(_mm512_set_epi16(15, 15, 14, 14, 13, 13, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, 6, 6, + 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0), + a); +} + +template <> +EIGEN_STRONG_INLINE Packet16h ploaddup(const Eigen::half* from) { + __m256h a = _mm256_castph128_ph256(_mm_loadu_ph(from)); + return _mm256_permutexvar_ph(_mm256_set_epi16(7, 7, 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0, 0), a); +} + +template <> +EIGEN_STRONG_INLINE Packet8h ploaddup(const Eigen::half* from) { + return _mm_set_ph(from[3].x, from[3].x, from[2].x, from[2].x, from[1].x, from[1].x, from[0].x, from[0].x); +} + +// ploadquad +template <> +EIGEN_STRONG_INLINE Packet32h ploadquad(const Eigen::half* from) { + __m512h a = _mm512_castph128_ph512(_mm_loadu_ph(from)); + return _mm512_permutexvar_ph( + _mm512_set_epi16(7, 7, 7, 7, 6, 6, 6, 6, 5, 5, 5, 5, 4, 4, 4, 4, 3, 3, 3, 3, 2, 2, 2, 2, 1, 1, 1, 1, 0, 0, 0, 0), + a); +} + +template <> +EIGEN_STRONG_INLINE Packet16h ploadquad(const Eigen::half* from) { + return _mm256_set_ph(from[3].x, from[3].x, from[3].x, from[3].x, from[2].x, from[2].x, from[2].x, from[2].x, + from[1].x, from[1].x, from[1].x, from[1].x, from[0].x, from[0].x, from[0].x, from[0].x); +} + +template <> +EIGEN_STRONG_INLINE Packet8h ploadquad(const Eigen::half* from) { + return _mm_set_ph(from[1].x, from[1].x, from[1].x, from[1].x, from[0].x, from[0].x, from[0].x, from[0].x); +} + +// pabs + +template <> +EIGEN_STRONG_INLINE Packet32h pabs(const Packet32h& a) { + return _mm512_abs_ph(a); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pabs(const Packet16h& a) { + return _mm256_abs_ph(a); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pabs(const Packet8h& a) { + return _mm_abs_ph(a); +} + +// psignbit + +template <> +EIGEN_STRONG_INLINE Packet32h psignbit(const Packet32h& a) { + return _mm512_castsi512_ph(_mm512_srai_epi16(_mm512_castph_si512(a), 15)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h psignbit(const Packet16h& a) { + return _mm256_castsi256_ph(_mm256_srai_epi16(_mm256_castph_si256(a), 15)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h psignbit(const Packet8h& a) { + return _mm_castsi128_ph(_mm_srai_epi16(_mm_castph_si128(a), 15)); +} + +// pmin + +template <> +EIGEN_STRONG_INLINE Packet32h pmin(const Packet32h& a, const Packet32h& b) { + return _mm512_min_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmin(const Packet16h& a, const Packet16h& b) { + return _mm256_min_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmin(const Packet8h& a, const Packet8h& b) { + return _mm_min_ph(a, b); +} + +// pmax + +template <> +EIGEN_STRONG_INLINE Packet32h pmax(const Packet32h& a, const Packet32h& b) { + return _mm512_max_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmax(const Packet16h& a, const Packet16h& b) { + return _mm256_max_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmax(const Packet8h& a, const Packet8h& b) { + return _mm_max_ph(a, b); +} + +// plset +template <> +EIGEN_STRONG_INLINE Packet32h plset(const half& a) { + return _mm512_add_ph(pset1(a), _mm512_set_ph(31, 30, 29, 28, 27, 26, 25, 24, 23, 22, 21, 20, 19, 18, 17, + 16, 15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h plset(const half& a) { + return _mm256_add_ph(pset1(a), _mm256_set_ph(15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h plset(const half& a) { + return _mm_add_ph(pset1(a), _mm_set_ph(7, 6, 5, 4, 3, 2, 1, 0)); +} + +// por + +template <> +EIGEN_STRONG_INLINE Packet32h por(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_or_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h por(const Packet16h& a, const Packet16h& b) { + return _mm256_castsi256_ph(_mm256_or_si256(_mm256_castph_si256(a), _mm256_castph_si256(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h por(const Packet8h& a, const Packet8h& b) { + return _mm_castsi128_ph(_mm_or_si128(_mm_castph_si128(a), _mm_castph_si128(b))); +} + +// pxor + +template <> +EIGEN_STRONG_INLINE Packet32h pxor(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_xor_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pxor(const Packet16h& a, const Packet16h& b) { + return _mm256_castsi256_ph(_mm256_xor_si256(_mm256_castph_si256(a), _mm256_castph_si256(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pxor(const Packet8h& a, const Packet8h& b) { + return _mm_castsi128_ph(_mm_xor_si128(_mm_castph_si128(a), _mm_castph_si128(b))); +} + +// pand + +template <> +EIGEN_STRONG_INLINE Packet32h pand(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_and_si512(_mm512_castph_si512(a), _mm512_castph_si512(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pand(const Packet16h& a, const Packet16h& b) { + return _mm256_castsi256_ph(_mm256_and_si256(_mm256_castph_si256(a), _mm256_castph_si256(b))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pand(const Packet8h& a, const Packet8h& b) { + return _mm_castsi128_ph(_mm_and_si128(_mm_castph_si128(a), _mm_castph_si128(b))); +} + +// pandnot + +template <> +EIGEN_STRONG_INLINE Packet32h pandnot(const Packet32h& a, const Packet32h& b) { + return _mm512_castsi512_ph(_mm512_andnot_si512(_mm512_castph_si512(b), _mm512_castph_si512(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pandnot(const Packet16h& a, const Packet16h& b) { + return _mm256_castsi256_ph(_mm256_andnot_si256(_mm256_castph_si256(b), _mm256_castph_si256(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pandnot(const Packet8h& a, const Packet8h& b) { + return _mm_castsi128_ph(_mm_andnot_si128(_mm_castph_si128(b), _mm_castph_si128(a))); +} + +// pselect + +template <> +EIGEN_DEVICE_FUNC inline Packet32h pselect(const Packet32h& mask, const Packet32h& a, const Packet32h& b) { + __mmask32 mask32 = _mm512_cmp_epi16_mask(_mm512_castph_si512(mask), _mm512_setzero_epi32(), _MM_CMPINT_EQ); + return _mm512_mask_blend_ph(mask32, a, b); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet16h pselect(const Packet16h& mask, const Packet16h& a, const Packet16h& b) { + __mmask16 mask16 = _mm256_cmp_epi16_mask(_mm256_castph_si256(mask), _mm256_setzero_si256(), _MM_CMPINT_EQ); + return _mm256_mask_blend_ph(mask16, a, b); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet8h pselect(const Packet8h& mask, const Packet8h& a, const Packet8h& b) { + __mmask8 mask8 = _mm_cmp_epi16_mask(_mm_castph_si128(mask), _mm_setzero_si128(), _MM_CMPINT_EQ); + return _mm_mask_blend_ph(mask8, a, b); +} + +// pcmp_eq + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_eq(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_EQ_OQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcmp_eq(const Packet16h& a, const Packet16h& b) { + __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_EQ_OQ); + return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pcmp_eq(const Packet8h& a, const Packet8h& b) { + __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_EQ_OQ); + return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast(0xffffu))); +} + +// pcmp_le + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_le(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LE_OQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcmp_le(const Packet16h& a, const Packet16h& b) { + __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_LE_OQ); + return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pcmp_le(const Packet8h& a, const Packet8h& b) { + __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_LE_OQ); + return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast(0xffffu))); +} + +// pcmp_lt + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_lt(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_LT_OQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi32(0), mask, static_cast(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcmp_lt(const Packet16h& a, const Packet16h& b) { + __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_LT_OQ); + return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pcmp_lt(const Packet8h& a, const Packet8h& b) { + __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_LT_OQ); + return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast(0xffffu))); +} + +// pcmp_lt_or_nan + +template <> +EIGEN_STRONG_INLINE Packet32h pcmp_lt_or_nan(const Packet32h& a, const Packet32h& b) { + __mmask32 mask = _mm512_cmp_ph_mask(a, b, _CMP_NGE_UQ); + return _mm512_castsi512_ph(_mm512_mask_set1_epi16(_mm512_set1_epi16(0), mask, static_cast(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcmp_lt_or_nan(const Packet16h& a, const Packet16h& b) { + __mmask16 mask = _mm256_cmp_ph_mask(a, b, _CMP_NGE_UQ); + return _mm256_castsi256_ph(_mm256_mask_set1_epi16(_mm256_set1_epi32(0), mask, static_cast(0xffffu))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pcmp_lt_or_nan(const Packet8h& a, const Packet8h& b) { + __mmask8 mask = _mm_cmp_ph_mask(a, b, _CMP_NGE_UQ); + return _mm_castsi128_ph(_mm_mask_set1_epi16(_mm_set1_epi32(0), mask, static_cast(0xffffu))); +} + +// padd + +template <> +EIGEN_STRONG_INLINE Packet32h padd(const Packet32h& a, const Packet32h& b) { + return _mm512_add_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h padd(const Packet16h& a, const Packet16h& b) { + return _mm256_add_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h padd(const Packet8h& a, const Packet8h& b) { + return _mm_add_ph(a, b); +} + +// psub + +template <> +EIGEN_STRONG_INLINE Packet32h psub(const Packet32h& a, const Packet32h& b) { + return _mm512_sub_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h psub(const Packet16h& a, const Packet16h& b) { + return _mm256_sub_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h psub(const Packet8h& a, const Packet8h& b) { + return _mm_sub_ph(a, b); +} + +// pmul + +template <> +EIGEN_STRONG_INLINE Packet32h pmul(const Packet32h& a, const Packet32h& b) { + return _mm512_mul_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmul(const Packet16h& a, const Packet16h& b) { + return _mm256_mul_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmul(const Packet8h& a, const Packet8h& b) { + return _mm_mul_ph(a, b); +} + +// pdiv + +template <> +EIGEN_STRONG_INLINE Packet32h pdiv(const Packet32h& a, const Packet32h& b) { + return _mm512_div_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pdiv(const Packet16h& a, const Packet16h& b) { + return _mm256_div_ph(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pdiv(const Packet8h& a, const Packet8h& b) { + return _mm_div_ph(a, b); + ; +} + +// pround + +template <> +EIGEN_STRONG_INLINE Packet32h pround(const Packet32h& a) { + // Work-around for default std::round rounding mode. + + // Mask for the sign bit. + const Packet32h signMask = + pset1frombits(static_cast(static_cast(0x8000u))); + // The largest half-precision float less than 0.5. + const Packet32h prev0dot5 = pset1frombits(static_cast(0x37FFu)); + + return _mm512_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pround(const Packet16h& a) { + // Work-around for default std::round rounding mode. + + // Mask for the sign bit. + const Packet16h signMask = + pset1frombits(static_cast(static_cast(0x8000u))); + // The largest half-precision float less than 0.5. + const Packet16h prev0dot5 = pset1frombits(static_cast(0x37FFu)); + + return _mm256_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pround(const Packet8h& a) { + // Work-around for default std::round rounding mode. + + // Mask for the sign bit. + const Packet8h signMask = pset1frombits(static_cast(static_cast(0x8000u))); + // The largest half-precision float less than 0.5. + const Packet8h prev0dot5 = pset1frombits(static_cast(0x37FFu)); + + return _mm_roundscale_ph(padd(por(pand(a, signMask), prev0dot5), a), _MM_FROUND_TO_ZERO); +} + +// print + +template <> +EIGEN_STRONG_INLINE Packet32h print(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); +} + +template <> +EIGEN_STRONG_INLINE Packet16h print(const Packet16h& a) { + return _mm256_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); +} + +template <> +EIGEN_STRONG_INLINE Packet8h print(const Packet8h& a) { + return _mm_roundscale_ph(a, _MM_FROUND_CUR_DIRECTION); +} + +// pceil + +template <> +EIGEN_STRONG_INLINE Packet32h pceil(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_TO_POS_INF); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pceil(const Packet16h& a) { + return _mm256_roundscale_ph(a, _MM_FROUND_TO_POS_INF); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pceil(const Packet8h& a) { + return _mm_roundscale_ph(a, _MM_FROUND_TO_POS_INF); +} + +// pfloor + +template <> +EIGEN_STRONG_INLINE Packet32h pfloor(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pfloor(const Packet16h& a) { + return _mm256_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pfloor(const Packet8h& a) { + return _mm_roundscale_ph(a, _MM_FROUND_TO_NEG_INF); +} + +// ptrunc + +template <> +EIGEN_STRONG_INLINE Packet32h ptrunc(const Packet32h& a) { + return _mm512_roundscale_ph(a, _MM_FROUND_TO_ZERO); +} + +template <> +EIGEN_STRONG_INLINE Packet16h ptrunc(const Packet16h& a) { + return _mm256_roundscale_ph(a, _MM_FROUND_TO_ZERO); +} + +template <> +EIGEN_STRONG_INLINE Packet8h ptrunc(const Packet8h& a) { + return _mm_roundscale_ph(a, _MM_FROUND_TO_ZERO); +} + +// predux +template <> +EIGEN_STRONG_INLINE half predux(const Packet32h& a) { + return half(_mm512_reduce_add_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux(const Packet16h& a) { + return half(_mm256_reduce_add_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux(const Packet8h& a) { + return half(_mm_reduce_add_ph(a)); +} + +// predux_half_dowto4 +template <> +EIGEN_STRONG_INLINE Packet16h predux_half_dowto4(const Packet32h& a) { + const __m512i bits = _mm512_castph_si512(a); + Packet16h lo = _mm256_castsi256_ph(_mm512_castsi512_si256(bits)); + Packet16h hi = _mm256_castsi256_ph(_mm512_extracti64x4_epi64(bits, 1)); + return padd(lo, hi); +} + +template <> +EIGEN_STRONG_INLINE Packet8h predux_half_dowto4(const Packet16h& a) { + Packet8h lo = _mm_castsi128_ph(_mm256_castsi256_si128(_mm256_castph_si256(a))); + Packet8h hi = _mm_castps_ph(_mm256_extractf128_ps(_mm256_castph_ps(a), 1)); + return padd(lo, hi); +} + +// predux_max + +template <> +EIGEN_STRONG_INLINE half predux_max(const Packet32h& a) { + return half(_mm512_reduce_max_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_max(const Packet16h& a) { + return half(_mm256_reduce_max_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_max(const Packet8h& a) { + return half(_mm_reduce_max_ph(a)); +} + +// predux_min + +template <> +EIGEN_STRONG_INLINE half predux_min(const Packet32h& a) { + return half(_mm512_reduce_min_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_min(const Packet16h& a) { + return half(_mm256_reduce_min_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_min(const Packet8h& a) { + return half(_mm_reduce_min_ph(a)); +} + +// predux_mul + +template <> +EIGEN_STRONG_INLINE half predux_mul(const Packet32h& a) { + return half(_mm512_reduce_mul_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_mul(const Packet16h& a) { + return half(_mm256_reduce_mul_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE half predux_mul(const Packet8h& a) { + return half(_mm_reduce_mul_ph(a)); +} + +#ifdef EIGEN_VECTORIZE_FMA + +// pmadd + +template <> +EIGEN_STRONG_INLINE Packet32h pmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_fmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_fmadd_ph(a, b, c); +} + +// pmsub + +template <> +EIGEN_STRONG_INLINE Packet32h pmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_fmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_fmsub_ph(a, b, c); +} + +// pnmadd + +template <> +EIGEN_STRONG_INLINE Packet32h pnmadd(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fnmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pnmadd(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_fnmadd_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pnmadd(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_fnmadd_ph(a, b, c); +} + +// pnmsub + +template <> +EIGEN_STRONG_INLINE Packet32h pnmsub(const Packet32h& a, const Packet32h& b, const Packet32h& c) { + return _mm512_fnmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pnmsub(const Packet16h& a, const Packet16h& b, const Packet16h& c) { + return _mm256_fnmsub_ph(a, b, c); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pnmsub(const Packet8h& a, const Packet8h& b, const Packet8h& c) { + return _mm_fnmsub_ph(a, b, c); +} + +#endif + +// pnegate + +template <> +EIGEN_STRONG_INLINE Packet32h pnegate(const Packet32h& a) { + return _mm512_castsi512_ph( + _mm512_xor_si512(_mm512_castph_si512(a), _mm512_set1_epi16(static_cast(0x8000u)))); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pnegate(const Packet16h& a) { + return _mm256_castsi256_ph( + _mm256_xor_si256(_mm256_castph_si256(a), _mm256_set1_epi16(static_cast(0x8000u)))); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pnegate(const Packet8h& a) { + return _mm_castsi128_ph(_mm_xor_si128(_mm_castph_si128(a), _mm_set1_epi16(static_cast(0x8000u)))); +} + +// pconj + +// Nothing, packets are real. + +// psqrt + +template <> +EIGEN_STRONG_INLINE Packet32h psqrt(const Packet32h& a) { + return generic_sqrt_newton_step::run(a, _mm512_rsqrt_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h psqrt(const Packet16h& a) { + return generic_sqrt_newton_step::run(a, _mm256_rsqrt_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h psqrt(const Packet8h& a) { + return generic_sqrt_newton_step::run(a, _mm_rsqrt_ph(a)); +} + +// prsqrt + +template <> +EIGEN_STRONG_INLINE Packet32h prsqrt(const Packet32h& a) { + return generic_rsqrt_newton_step::run(a, _mm512_rsqrt_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h prsqrt(const Packet16h& a) { + return generic_rsqrt_newton_step::run(a, _mm256_rsqrt_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h prsqrt(const Packet8h& a) { + return generic_rsqrt_newton_step::run(a, _mm_rsqrt_ph(a)); +} + +// preciprocal + +template <> +EIGEN_STRONG_INLINE Packet32h preciprocal(const Packet32h& a) { + return generic_reciprocal_newton_step::run(a, _mm512_rcp_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet16h preciprocal(const Packet16h& a) { + return generic_reciprocal_newton_step::run(a, _mm256_rcp_ph(a)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h preciprocal(const Packet8h& a) { + return generic_reciprocal_newton_step::run(a, _mm_rcp_ph(a)); +} + +// ptranspose + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& a) { + __m512i t[32]; + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 16; i++) { + t[2 * i] = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1])); + t[2 * i + 1] = + _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2 * i]), _mm512_castph_si512(a.packet[2 * i + 1])); + } + + __m512i p[32]; + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 8; i++) { + p[4 * i] = _mm512_unpacklo_epi32(t[4 * i], t[4 * i + 2]); + p[4 * i + 1] = _mm512_unpackhi_epi32(t[4 * i], t[4 * i + 2]); + p[4 * i + 2] = _mm512_unpacklo_epi32(t[4 * i + 1], t[4 * i + 3]); + p[4 * i + 3] = _mm512_unpackhi_epi32(t[4 * i + 1], t[4 * i + 3]); + } + + __m512i q[32]; + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 4; i++) { + q[8 * i] = _mm512_unpacklo_epi64(p[8 * i], p[8 * i + 4]); + q[8 * i + 1] = _mm512_unpackhi_epi64(p[8 * i], p[8 * i + 4]); + q[8 * i + 2] = _mm512_unpacklo_epi64(p[8 * i + 1], p[8 * i + 5]); + q[8 * i + 3] = _mm512_unpackhi_epi64(p[8 * i + 1], p[8 * i + 5]); + q[8 * i + 4] = _mm512_unpacklo_epi64(p[8 * i + 2], p[8 * i + 6]); + q[8 * i + 5] = _mm512_unpackhi_epi64(p[8 * i + 2], p[8 * i + 6]); + q[8 * i + 6] = _mm512_unpacklo_epi64(p[8 * i + 3], p[8 * i + 7]); + q[8 * i + 7] = _mm512_unpackhi_epi64(p[8 * i + 3], p[8 * i + 7]); + } + + __m512i f[32]; + +#define PACKET32H_TRANSPOSE_HELPER(X, Y) \ + do { \ + f[Y * 8] = _mm512_inserti32x4(f[Y * 8], _mm512_extracti32x4_epi32(q[X * 8], Y), X); \ + f[Y * 8 + 1] = _mm512_inserti32x4(f[Y * 8 + 1], _mm512_extracti32x4_epi32(q[X * 8 + 1], Y), X); \ + f[Y * 8 + 2] = _mm512_inserti32x4(f[Y * 8 + 2], _mm512_extracti32x4_epi32(q[X * 8 + 2], Y), X); \ + f[Y * 8 + 3] = _mm512_inserti32x4(f[Y * 8 + 3], _mm512_extracti32x4_epi32(q[X * 8 + 3], Y), X); \ + f[Y * 8 + 4] = _mm512_inserti32x4(f[Y * 8 + 4], _mm512_extracti32x4_epi32(q[X * 8 + 4], Y), X); \ + f[Y * 8 + 5] = _mm512_inserti32x4(f[Y * 8 + 5], _mm512_extracti32x4_epi32(q[X * 8 + 5], Y), X); \ + f[Y * 8 + 6] = _mm512_inserti32x4(f[Y * 8 + 6], _mm512_extracti32x4_epi32(q[X * 8 + 6], Y), X); \ + f[Y * 8 + 7] = _mm512_inserti32x4(f[Y * 8 + 7], _mm512_extracti32x4_epi32(q[X * 8 + 7], Y), X); \ + } while (false); + + PACKET32H_TRANSPOSE_HELPER(0, 0); + PACKET32H_TRANSPOSE_HELPER(1, 1); + PACKET32H_TRANSPOSE_HELPER(2, 2); + PACKET32H_TRANSPOSE_HELPER(3, 3); + + PACKET32H_TRANSPOSE_HELPER(1, 0); + PACKET32H_TRANSPOSE_HELPER(2, 0); + PACKET32H_TRANSPOSE_HELPER(3, 0); + PACKET32H_TRANSPOSE_HELPER(2, 1); + PACKET32H_TRANSPOSE_HELPER(3, 1); + PACKET32H_TRANSPOSE_HELPER(3, 2); + + PACKET32H_TRANSPOSE_HELPER(0, 1); + PACKET32H_TRANSPOSE_HELPER(0, 2); + PACKET32H_TRANSPOSE_HELPER(0, 3); + PACKET32H_TRANSPOSE_HELPER(1, 2); + PACKET32H_TRANSPOSE_HELPER(1, 3); + PACKET32H_TRANSPOSE_HELPER(2, 3); + +#undef PACKET32H_TRANSPOSE_HELPER + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 32; i++) { + a.packet[i] = _mm512_castsi512_ph(f[i]); + } +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& a) { + __m512i p0, p1, p2, p3, t0, t1, t2, t3, a0, a1, a2, a3; + t0 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1])); + t1 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[0]), _mm512_castph_si512(a.packet[1])); + t2 = _mm512_unpacklo_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3])); + t3 = _mm512_unpackhi_epi16(_mm512_castph_si512(a.packet[2]), _mm512_castph_si512(a.packet[3])); + + p0 = _mm512_unpacklo_epi32(t0, t2); + p1 = _mm512_unpackhi_epi32(t0, t2); + p2 = _mm512_unpacklo_epi32(t1, t3); + p3 = _mm512_unpackhi_epi32(t1, t3); + + a0 = p0; + a1 = p1; + a2 = p2; + a3 = p3; + + a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p1, 0), 1); + a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p0, 1), 0); + + a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p2, 0), 2); + a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p0, 2), 0); + + a0 = _mm512_inserti32x4(a0, _mm512_extracti32x4_epi32(p3, 0), 3); + a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p0, 3), 0); + + a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p2, 1), 2); + a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p1, 2), 1); + + a2 = _mm512_inserti32x4(a2, _mm512_extracti32x4_epi32(p3, 2), 3); + a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p2, 3), 2); + + a1 = _mm512_inserti32x4(a1, _mm512_extracti32x4_epi32(p3, 1), 3); + a3 = _mm512_inserti32x4(a3, _mm512_extracti32x4_epi32(p1, 3), 1); + + a.packet[0] = _mm512_castsi512_ph(a0); + a.packet[1] = _mm512_castsi512_ph(a1); + a.packet[2] = _mm512_castsi512_ph(a2); + a.packet[3] = _mm512_castsi512_ph(a3); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m256i a = _mm256_castph_si256(kernel.packet[0]); + __m256i b = _mm256_castph_si256(kernel.packet[1]); + __m256i c = _mm256_castph_si256(kernel.packet[2]); + __m256i d = _mm256_castph_si256(kernel.packet[3]); + __m256i e = _mm256_castph_si256(kernel.packet[4]); + __m256i f = _mm256_castph_si256(kernel.packet[5]); + __m256i g = _mm256_castph_si256(kernel.packet[6]); + __m256i h = _mm256_castph_si256(kernel.packet[7]); + __m256i i = _mm256_castph_si256(kernel.packet[8]); + __m256i j = _mm256_castph_si256(kernel.packet[9]); + __m256i k = _mm256_castph_si256(kernel.packet[10]); + __m256i l = _mm256_castph_si256(kernel.packet[11]); + __m256i m = _mm256_castph_si256(kernel.packet[12]); + __m256i n = _mm256_castph_si256(kernel.packet[13]); + __m256i o = _mm256_castph_si256(kernel.packet[14]); + __m256i p = _mm256_castph_si256(kernel.packet[15]); + + __m256i ab_07 = _mm256_unpacklo_epi16(a, b); + __m256i cd_07 = _mm256_unpacklo_epi16(c, d); + __m256i ef_07 = _mm256_unpacklo_epi16(e, f); + __m256i gh_07 = _mm256_unpacklo_epi16(g, h); + __m256i ij_07 = _mm256_unpacklo_epi16(i, j); + __m256i kl_07 = _mm256_unpacklo_epi16(k, l); + __m256i mn_07 = _mm256_unpacklo_epi16(m, n); + __m256i op_07 = _mm256_unpacklo_epi16(o, p); + + __m256i ab_8f = _mm256_unpackhi_epi16(a, b); + __m256i cd_8f = _mm256_unpackhi_epi16(c, d); + __m256i ef_8f = _mm256_unpackhi_epi16(e, f); + __m256i gh_8f = _mm256_unpackhi_epi16(g, h); + __m256i ij_8f = _mm256_unpackhi_epi16(i, j); + __m256i kl_8f = _mm256_unpackhi_epi16(k, l); + __m256i mn_8f = _mm256_unpackhi_epi16(m, n); + __m256i op_8f = _mm256_unpackhi_epi16(o, p); + + __m256i abcd_03 = _mm256_unpacklo_epi32(ab_07, cd_07); + __m256i abcd_47 = _mm256_unpackhi_epi32(ab_07, cd_07); + __m256i efgh_03 = _mm256_unpacklo_epi32(ef_07, gh_07); + __m256i efgh_47 = _mm256_unpackhi_epi32(ef_07, gh_07); + __m256i ijkl_03 = _mm256_unpacklo_epi32(ij_07, kl_07); + __m256i ijkl_47 = _mm256_unpackhi_epi32(ij_07, kl_07); + __m256i mnop_03 = _mm256_unpacklo_epi32(mn_07, op_07); + __m256i mnop_47 = _mm256_unpackhi_epi32(mn_07, op_07); + + __m256i abcd_8b = _mm256_unpacklo_epi32(ab_8f, cd_8f); + __m256i abcd_cf = _mm256_unpackhi_epi32(ab_8f, cd_8f); + __m256i efgh_8b = _mm256_unpacklo_epi32(ef_8f, gh_8f); + __m256i efgh_cf = _mm256_unpackhi_epi32(ef_8f, gh_8f); + __m256i ijkl_8b = _mm256_unpacklo_epi32(ij_8f, kl_8f); + __m256i ijkl_cf = _mm256_unpackhi_epi32(ij_8f, kl_8f); + __m256i mnop_8b = _mm256_unpacklo_epi32(mn_8f, op_8f); + __m256i mnop_cf = _mm256_unpackhi_epi32(mn_8f, op_8f); + + __m256i abcdefgh_01 = _mm256_unpacklo_epi64(abcd_03, efgh_03); + __m256i abcdefgh_23 = _mm256_unpackhi_epi64(abcd_03, efgh_03); + __m256i ijklmnop_01 = _mm256_unpacklo_epi64(ijkl_03, mnop_03); + __m256i ijklmnop_23 = _mm256_unpackhi_epi64(ijkl_03, mnop_03); + __m256i abcdefgh_45 = _mm256_unpacklo_epi64(abcd_47, efgh_47); + __m256i abcdefgh_67 = _mm256_unpackhi_epi64(abcd_47, efgh_47); + __m256i ijklmnop_45 = _mm256_unpacklo_epi64(ijkl_47, mnop_47); + __m256i ijklmnop_67 = _mm256_unpackhi_epi64(ijkl_47, mnop_47); + __m256i abcdefgh_89 = _mm256_unpacklo_epi64(abcd_8b, efgh_8b); + __m256i abcdefgh_ab = _mm256_unpackhi_epi64(abcd_8b, efgh_8b); + __m256i ijklmnop_89 = _mm256_unpacklo_epi64(ijkl_8b, mnop_8b); + __m256i ijklmnop_ab = _mm256_unpackhi_epi64(ijkl_8b, mnop_8b); + __m256i abcdefgh_cd = _mm256_unpacklo_epi64(abcd_cf, efgh_cf); + __m256i abcdefgh_ef = _mm256_unpackhi_epi64(abcd_cf, efgh_cf); + __m256i ijklmnop_cd = _mm256_unpacklo_epi64(ijkl_cf, mnop_cf); + __m256i ijklmnop_ef = _mm256_unpackhi_epi64(ijkl_cf, mnop_cf); + + // NOTE: no unpacklo/hi instr in this case, so using permute instr. + __m256i a_p_0 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x20); + __m256i a_p_1 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x20); + __m256i a_p_2 = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x20); + __m256i a_p_3 = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x20); + __m256i a_p_4 = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x20); + __m256i a_p_5 = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x20); + __m256i a_p_6 = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x20); + __m256i a_p_7 = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x20); + __m256i a_p_8 = _mm256_permute2x128_si256(abcdefgh_01, ijklmnop_01, 0x31); + __m256i a_p_9 = _mm256_permute2x128_si256(abcdefgh_23, ijklmnop_23, 0x31); + __m256i a_p_a = _mm256_permute2x128_si256(abcdefgh_45, ijklmnop_45, 0x31); + __m256i a_p_b = _mm256_permute2x128_si256(abcdefgh_67, ijklmnop_67, 0x31); + __m256i a_p_c = _mm256_permute2x128_si256(abcdefgh_89, ijklmnop_89, 0x31); + __m256i a_p_d = _mm256_permute2x128_si256(abcdefgh_ab, ijklmnop_ab, 0x31); + __m256i a_p_e = _mm256_permute2x128_si256(abcdefgh_cd, ijklmnop_cd, 0x31); + __m256i a_p_f = _mm256_permute2x128_si256(abcdefgh_ef, ijklmnop_ef, 0x31); + + kernel.packet[0] = _mm256_castsi256_ph(a_p_0); + kernel.packet[1] = _mm256_castsi256_ph(a_p_1); + kernel.packet[2] = _mm256_castsi256_ph(a_p_2); + kernel.packet[3] = _mm256_castsi256_ph(a_p_3); + kernel.packet[4] = _mm256_castsi256_ph(a_p_4); + kernel.packet[5] = _mm256_castsi256_ph(a_p_5); + kernel.packet[6] = _mm256_castsi256_ph(a_p_6); + kernel.packet[7] = _mm256_castsi256_ph(a_p_7); + kernel.packet[8] = _mm256_castsi256_ph(a_p_8); + kernel.packet[9] = _mm256_castsi256_ph(a_p_9); + kernel.packet[10] = _mm256_castsi256_ph(a_p_a); + kernel.packet[11] = _mm256_castsi256_ph(a_p_b); + kernel.packet[12] = _mm256_castsi256_ph(a_p_c); + kernel.packet[13] = _mm256_castsi256_ph(a_p_d); + kernel.packet[14] = _mm256_castsi256_ph(a_p_e); + kernel.packet[15] = _mm256_castsi256_ph(a_p_f); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + EIGEN_ALIGN64 half in[8][16]; + pstore(in[0], kernel.packet[0]); + pstore(in[1], kernel.packet[1]); + pstore(in[2], kernel.packet[2]); + pstore(in[3], kernel.packet[3]); + pstore(in[4], kernel.packet[4]); + pstore(in[5], kernel.packet[5]); + pstore(in[6], kernel.packet[6]); + pstore(in[7], kernel.packet[7]); + + EIGEN_ALIGN64 half out[8][16]; + + for (int i = 0; i < 8; ++i) { + for (int j = 0; j < 8; ++j) { + out[i][j] = in[j][2 * i]; + } + for (int j = 0; j < 8; ++j) { + out[i][j + 8] = in[j][2 * i + 1]; + } + } + + kernel.packet[0] = pload(out[0]); + kernel.packet[1] = pload(out[1]); + kernel.packet[2] = pload(out[2]); + kernel.packet[3] = pload(out[3]); + kernel.packet[4] = pload(out[4]); + kernel.packet[5] = pload(out[5]); + kernel.packet[6] = pload(out[6]); + kernel.packet[7] = pload(out[7]); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + EIGEN_ALIGN64 half in[4][16]; + pstore(in[0], kernel.packet[0]); + pstore(in[1], kernel.packet[1]); + pstore(in[2], kernel.packet[2]); + pstore(in[3], kernel.packet[3]); + + EIGEN_ALIGN64 half out[4][16]; + + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + out[i][j] = in[j][4 * i]; + } + for (int j = 0; j < 4; ++j) { + out[i][j + 4] = in[j][4 * i + 1]; + } + for (int j = 0; j < 4; ++j) { + out[i][j + 8] = in[j][4 * i + 2]; + } + for (int j = 0; j < 4; ++j) { + out[i][j + 12] = in[j][4 * i + 3]; + } + } + + kernel.packet[0] = pload(out[0]); + kernel.packet[1] = pload(out[1]); + kernel.packet[2] = pload(out[2]); + kernel.packet[3] = pload(out[3]); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i a = _mm_castph_si128(kernel.packet[0]); + __m128i b = _mm_castph_si128(kernel.packet[1]); + __m128i c = _mm_castph_si128(kernel.packet[2]); + __m128i d = _mm_castph_si128(kernel.packet[3]); + __m128i e = _mm_castph_si128(kernel.packet[4]); + __m128i f = _mm_castph_si128(kernel.packet[5]); + __m128i g = _mm_castph_si128(kernel.packet[6]); + __m128i h = _mm_castph_si128(kernel.packet[7]); + + __m128i a03b03 = _mm_unpacklo_epi16(a, b); + __m128i c03d03 = _mm_unpacklo_epi16(c, d); + __m128i e03f03 = _mm_unpacklo_epi16(e, f); + __m128i g03h03 = _mm_unpacklo_epi16(g, h); + __m128i a47b47 = _mm_unpackhi_epi16(a, b); + __m128i c47d47 = _mm_unpackhi_epi16(c, d); + __m128i e47f47 = _mm_unpackhi_epi16(e, f); + __m128i g47h47 = _mm_unpackhi_epi16(g, h); + + __m128i a01b01c01d01 = _mm_unpacklo_epi32(a03b03, c03d03); + __m128i a23b23c23d23 = _mm_unpackhi_epi32(a03b03, c03d03); + __m128i e01f01g01h01 = _mm_unpacklo_epi32(e03f03, g03h03); + __m128i e23f23g23h23 = _mm_unpackhi_epi32(e03f03, g03h03); + __m128i a45b45c45d45 = _mm_unpacklo_epi32(a47b47, c47d47); + __m128i a67b67c67d67 = _mm_unpackhi_epi32(a47b47, c47d47); + __m128i e45f45g45h45 = _mm_unpacklo_epi32(e47f47, g47h47); + __m128i e67f67g67h67 = _mm_unpackhi_epi32(e47f47, g47h47); + + __m128i a0b0c0d0e0f0g0h0 = _mm_unpacklo_epi64(a01b01c01d01, e01f01g01h01); + __m128i a1b1c1d1e1f1g1h1 = _mm_unpackhi_epi64(a01b01c01d01, e01f01g01h01); + __m128i a2b2c2d2e2f2g2h2 = _mm_unpacklo_epi64(a23b23c23d23, e23f23g23h23); + __m128i a3b3c3d3e3f3g3h3 = _mm_unpackhi_epi64(a23b23c23d23, e23f23g23h23); + __m128i a4b4c4d4e4f4g4h4 = _mm_unpacklo_epi64(a45b45c45d45, e45f45g45h45); + __m128i a5b5c5d5e5f5g5h5 = _mm_unpackhi_epi64(a45b45c45d45, e45f45g45h45); + __m128i a6b6c6d6e6f6g6h6 = _mm_unpacklo_epi64(a67b67c67d67, e67f67g67h67); + __m128i a7b7c7d7e7f7g7h7 = _mm_unpackhi_epi64(a67b67c67d67, e67f67g67h67); + + kernel.packet[0] = _mm_castsi128_ph(a0b0c0d0e0f0g0h0); + kernel.packet[1] = _mm_castsi128_ph(a1b1c1d1e1f1g1h1); + kernel.packet[2] = _mm_castsi128_ph(a2b2c2d2e2f2g2h2); + kernel.packet[3] = _mm_castsi128_ph(a3b3c3d3e3f3g3h3); + kernel.packet[4] = _mm_castsi128_ph(a4b4c4d4e4f4g4h4); + kernel.packet[5] = _mm_castsi128_ph(a5b5c5d5e5f5g5h5); + kernel.packet[6] = _mm_castsi128_ph(a6b6c6d6e6f6g6h6); + kernel.packet[7] = _mm_castsi128_ph(a7b7c7d7e7f7g7h7); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + EIGEN_ALIGN32 Eigen::half in[4][8]; + pstore(in[0], kernel.packet[0]); + pstore(in[1], kernel.packet[1]); + pstore(in[2], kernel.packet[2]); + pstore(in[3], kernel.packet[3]); + + EIGEN_ALIGN32 Eigen::half out[4][8]; + + for (int i = 0; i < 4; ++i) { + for (int j = 0; j < 4; ++j) { + out[i][j] = in[j][2 * i]; + } + for (int j = 0; j < 4; ++j) { + out[i][j + 4] = in[j][2 * i + 1]; + } + } + + kernel.packet[0] = pload(out[0]); + kernel.packet[1] = pload(out[1]); + kernel.packet[2] = pload(out[2]); + kernel.packet[3] = pload(out[3]); +} + +// preverse + +template <> +EIGEN_STRONG_INLINE Packet32h preverse(const Packet32h& a) { + return _mm512_permutexvar_ph(_mm512_set_epi16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, + 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31), + a); +} + +template <> +EIGEN_STRONG_INLINE Packet16h preverse(const Packet16h& a) { + __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); + return _mm256_castsi256_ph(_mm256_insertf128_si256( + _mm256_castsi128_si256(_mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castph_si256(a), 1), m)), + _mm_shuffle_epi8(_mm256_extractf128_si256(_mm256_castph_si256(a), 0), m), 1)); +} + +template <> +EIGEN_STRONG_INLINE Packet8h preverse(const Packet8h& a) { + __m128i m = _mm_setr_epi8(14, 15, 12, 13, 10, 11, 8, 9, 6, 7, 4, 5, 2, 3, 0, 1); + return _mm_castsi128_ph(_mm_shuffle_epi8(_mm_castph_si128(a), m)); +} + +// pscatter + +template <> +EIGEN_STRONG_INLINE void pscatter(half* to, const Packet32h& from, Index stride) { + EIGEN_ALIGN64 half aux[32]; + pstore(aux, from); + + EIGEN_UNROLL_LOOP + for (int i = 0; i < 32; i++) { + to[stride * i] = aux[i]; + } +} +template <> +EIGEN_STRONG_INLINE void pscatter(half* to, const Packet16h& from, Index stride) { + EIGEN_ALIGN64 half aux[16]; + pstore(aux, from); + to[stride * 0] = aux[0]; + to[stride * 1] = aux[1]; + to[stride * 2] = aux[2]; + to[stride * 3] = aux[3]; + to[stride * 4] = aux[4]; + to[stride * 5] = aux[5]; + to[stride * 6] = aux[6]; + to[stride * 7] = aux[7]; + to[stride * 8] = aux[8]; + to[stride * 9] = aux[9]; + to[stride * 10] = aux[10]; + to[stride * 11] = aux[11]; + to[stride * 12] = aux[12]; + to[stride * 13] = aux[13]; + to[stride * 14] = aux[14]; + to[stride * 15] = aux[15]; +} + +template <> +EIGEN_STRONG_INLINE void pscatter(Eigen::half* to, const Packet8h& from, Index stride) { + EIGEN_ALIGN32 Eigen::half aux[8]; + pstore(aux, from); + to[stride * 0] = aux[0]; + to[stride * 1] = aux[1]; + to[stride * 2] = aux[2]; + to[stride * 3] = aux[3]; + to[stride * 4] = aux[4]; + to[stride * 5] = aux[5]; + to[stride * 6] = aux[6]; + to[stride * 7] = aux[7]; +} + +// pgather + +template <> +EIGEN_STRONG_INLINE Packet32h pgather(const Eigen::half* from, Index stride) { + return _mm512_set_ph(from[31 * stride].x, from[30 * stride].x, from[29 * stride].x, from[28 * stride].x, + from[27 * stride].x, from[26 * stride].x, from[25 * stride].x, from[24 * stride].x, + from[23 * stride].x, from[22 * stride].x, from[21 * stride].x, from[20 * stride].x, + from[19 * stride].x, from[18 * stride].x, from[17 * stride].x, from[16 * stride].x, + from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x, + from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x, + from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, + from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pgather(const Eigen::half* from, Index stride) { + return _mm256_set_ph(from[15 * stride].x, from[14 * stride].x, from[13 * stride].x, from[12 * stride].x, + from[11 * stride].x, from[10 * stride].x, from[9 * stride].x, from[8 * stride].x, + from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, + from[3 * stride].x, from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); +} + +template <> +EIGEN_STRONG_INLINE Packet8h pgather(const Eigen::half* from, Index stride) { + return _mm_set_ph(from[7 * stride].x, from[6 * stride].x, from[5 * stride].x, from[4 * stride].x, from[3 * stride].x, + from[2 * stride].x, from[1 * stride].x, from[0 * stride].x); +} + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_PACKET_MATH_FP16_AVX512_H diff --git a/Eigen/src/Core/arch/AVX512/Reductions.h b/Eigen/src/Core/arch/AVX512/Reductions.h new file mode 100644 index 0000000..f7b4c25 --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/Reductions.h @@ -0,0 +1,297 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2025 Charlie Schlosser +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_REDUCTIONS_AVX512_H +#define EIGEN_REDUCTIONS_AVX512_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet16i -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE int predux(const Packet16i& a) { + return _mm512_reduce_add_epi32(a); +} + +template <> +EIGEN_STRONG_INLINE int predux_mul(const Packet16i& a) { + return _mm512_reduce_mul_epi32(a); +} + +template <> +EIGEN_STRONG_INLINE int predux_min(const Packet16i& a) { + return _mm512_reduce_min_epi32(a); +} + +template <> +EIGEN_STRONG_INLINE int predux_max(const Packet16i& a) { + return _mm512_reduce_max_epi32(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet16i& a) { + return _mm512_reduce_or_epi32(a) != 0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet8l -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE int64_t predux(const Packet8l& a) { + return _mm512_reduce_add_epi64(a); +} + +#if EIGEN_COMP_MSVC +// MSVC's _mm512_reduce_mul_epi64 is borked, at least up to and including 1939. +// alignas(64) int64_t data[] = { 1,1,-1,-1,1,-1,-1,-1 }; +// int64_t out = _mm512_reduce_mul_epi64(_mm512_load_epi64(data)); +// produces garbage: 4294967295. It seems to happen whenever the output is supposed to be negative. +// Fall back to a manual approach: +template <> +EIGEN_STRONG_INLINE int64_t predux_mul(const Packet8l& a) { + Packet4l lane0 = _mm512_extracti64x4_epi64(a, 0); + Packet4l lane1 = _mm512_extracti64x4_epi64(a, 1); + return predux_mul(pmul(lane0, lane1)); +} +#else +template <> +EIGEN_STRONG_INLINE int64_t predux_mul(const Packet8l& a) { + return _mm512_reduce_mul_epi64(a); +} +#endif + +template <> +EIGEN_STRONG_INLINE int64_t predux_min(const Packet8l& a) { + return _mm512_reduce_min_epi64(a); +} + +template <> +EIGEN_STRONG_INLINE int64_t predux_max(const Packet8l& a) { + return _mm512_reduce_max_epi64(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8l& a) { + return _mm512_reduce_or_epi64(a) != 0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet16f -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE float predux(const Packet16f& a) { + return _mm512_reduce_add_ps(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_mul(const Packet16f& a) { + return _mm512_reduce_mul_ps(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet16f& a) { + return _mm512_reduce_min_ps(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet16f& a) { + Packet8f lane0 = _mm512_extractf32x8_ps(a, 0); + Packet8f lane1 = _mm512_extractf32x8_ps(a, 1); + return predux_min(pmin(lane0, lane1)); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet16f& a) { + Packet8f lane0 = _mm512_extractf32x8_ps(a, 0); + Packet8f lane1 = _mm512_extractf32x8_ps(a, 1); + return predux_min(pmin(lane0, lane1)); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet16f& a) { + return _mm512_reduce_max_ps(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet16f& a) { + Packet8f lane0 = _mm512_extractf32x8_ps(a, 0); + Packet8f lane1 = _mm512_extractf32x8_ps(a, 1); + return predux_max(pmax(lane0, lane1)); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet16f& a) { + Packet8f lane0 = _mm512_extractf32x8_ps(a, 0); + Packet8f lane1 = _mm512_extractf32x8_ps(a, 1); + return predux_max(pmax(lane0, lane1)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet16f& a) { + return _mm512_reduce_or_epi32(_mm512_castps_si512(a)) != 0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet8d -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE double predux(const Packet8d& a) { + return _mm512_reduce_add_pd(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_mul(const Packet8d& a) { + return _mm512_reduce_mul_pd(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet8d& a) { + return _mm512_reduce_min_pd(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet8d& a) { + Packet4d lane0 = _mm512_extractf64x4_pd(a, 0); + Packet4d lane1 = _mm512_extractf64x4_pd(a, 1); + return predux_min(pmin(lane0, lane1)); +} + +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet8d& a) { + Packet4d lane0 = _mm512_extractf64x4_pd(a, 0); + Packet4d lane1 = _mm512_extractf64x4_pd(a, 1); + return predux_min(pmin(lane0, lane1)); +} + +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet8d& a) { + return _mm512_reduce_max_pd(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet8d& a) { + Packet4d lane0 = _mm512_extractf64x4_pd(a, 0); + Packet4d lane1 = _mm512_extractf64x4_pd(a, 1); + return predux_max(pmax(lane0, lane1)); +} + +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet8d& a) { + Packet4d lane0 = _mm512_extractf64x4_pd(a, 0); + Packet4d lane1 = _mm512_extractf64x4_pd(a, 1); + return predux_max(pmax(lane0, lane1)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8d& a) { + return _mm512_reduce_or_epi64(_mm512_castpd_si512(a)) != 0; +} + +#ifndef EIGEN_VECTORIZE_AVX512FP16 +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet16h -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE half predux(const Packet16h& from) { + return half(predux(half2float(from))); +} + +template <> +EIGEN_STRONG_INLINE half predux_mul(const Packet16h& from) { + return half(predux_mul(half2float(from))); +} + +template <> +EIGEN_STRONG_INLINE half predux_min(const Packet16h& from) { + return half(predux_min(half2float(from))); +} + +template <> +EIGEN_STRONG_INLINE half predux_min(const Packet16h& from) { + return half(predux_min(half2float(from))); +} + +template <> +EIGEN_STRONG_INLINE half predux_min(const Packet16h& from) { + return half(predux_min(half2float(from))); +} + +template <> +EIGEN_STRONG_INLINE half predux_max(const Packet16h& from) { + return half(predux_max(half2float(from))); +} + +template <> +EIGEN_STRONG_INLINE half predux_max(const Packet16h& from) { + return half(predux_max(half2float(from))); +} + +template <> +EIGEN_STRONG_INLINE half predux_max(const Packet16h& from) { + return half(predux_max(half2float(from))); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet16h& a) { + return predux_any(a.m_val); +} +#endif + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet16bf -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE bfloat16 predux(const Packet16bf& from) { + return static_cast(predux(Bf16ToF32(from))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_mul(const Packet16bf& from) { + return static_cast(predux_mul(Bf16ToF32(from))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet16bf& from) { + return static_cast(predux_min(Bf16ToF32(from))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet16bf& from) { + return static_cast(predux_min(Bf16ToF32(from))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_min(const Packet16bf& from) { + return static_cast(predux_min(Bf16ToF32(from))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet16bf& from) { + return static_cast(predux_max(Bf16ToF32(from))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet16bf& from) { + return static_cast(predux_max(Bf16ToF32(from))); +} + +template <> +EIGEN_STRONG_INLINE bfloat16 predux_max(const Packet16bf& from) { + return static_cast(predux_max(Bf16ToF32(from))); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet16bf& a) { + return predux_any(a.m_val); +} + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_REDUCTIONS_AVX512_H diff --git a/Eigen/src/Core/arch/AVX512/TrsmKernel.h b/Eigen/src/Core/arch/AVX512/TrsmKernel.h new file mode 100644 index 0000000..c763b5f --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/TrsmKernel.h @@ -0,0 +1,1167 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2022 Intel Corporation +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CORE_ARCH_AVX512_TRSM_KERNEL_H +#define EIGEN_CORE_ARCH_AVX512_TRSM_KERNEL_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +#if !defined(EIGEN_USE_AVX512_TRSM_KERNELS) +#define EIGEN_USE_AVX512_TRSM_KERNELS 1 +#endif + +// TRSM kernels currently unconditionally rely on malloc with AVX512. +// Disable them if malloc is explicitly disabled at compile-time. +#ifdef EIGEN_NO_MALLOC +#undef EIGEN_USE_AVX512_TRSM_KERNELS +#define EIGEN_USE_AVX512_TRSM_KERNELS 0 +#endif + +#if EIGEN_USE_AVX512_TRSM_KERNELS +#if !defined(EIGEN_USE_AVX512_TRSM_R_KERNELS) +#define EIGEN_USE_AVX512_TRSM_R_KERNELS 1 +#endif +#if !defined(EIGEN_USE_AVX512_TRSM_L_KERNELS) +#define EIGEN_USE_AVX512_TRSM_L_KERNELS 1 +#endif +#else // EIGEN_USE_AVX512_TRSM_KERNELS == 0 +#define EIGEN_USE_AVX512_TRSM_R_KERNELS 0 +#define EIGEN_USE_AVX512_TRSM_L_KERNELS 0 +#endif + +// Need this for some std::min calls. +#ifdef min +#undef min +#endif + +namespace Eigen { +namespace internal { + +#define EIGEN_AVX_MAX_NUM_ACC (int64_t(24)) +#define EIGEN_AVX_MAX_NUM_ROW (int64_t(8)) // Denoted L in code. +#define EIGEN_AVX_MAX_K_UNROL (int64_t(4)) +#define EIGEN_AVX_B_LOAD_SETS (int64_t(2)) +#define EIGEN_AVX_MAX_A_BCAST (int64_t(2)) +typedef Packet16f vecFullFloat; +typedef Packet8d vecFullDouble; +typedef Packet8f vecHalfFloat; +typedef Packet4d vecHalfDouble; + +// Compile-time unrolls are implemented here. +// Note: this depends on macros and typedefs above. +#include "TrsmUnrolls.inc" + +#if (EIGEN_USE_AVX512_TRSM_KERNELS) && (EIGEN_COMP_CLANG != 0) +/** + * For smaller problem sizes, and certain compilers, using the optimized kernels trsmKernelL/R directly + * is faster than the packed versions in TriangularSolverMatrix.h. + * + * The current heuristic is based on having having all arrays used in the largest gemm-update + * in triSolve fit in roughly L2Cap (percentage) of the L2 cache. These cutoffs are a bit conservative and could be + * larger for some trsm cases. + * The formula: + * + * (L*M + M*N + L*N)*sizeof(Scalar) < L2Cache*L2Cap + * + * L = number of rows to solve at a time + * N = number of rhs + * M = Dimension of triangular matrix + * + */ +#if !defined(EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS) +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS 1 +#endif + +#if EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS + +#if EIGEN_USE_AVX512_TRSM_R_KERNELS +#if !defined(EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS) +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS 1 +#endif // !defined(EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS) +#endif + +#if EIGEN_USE_AVX512_TRSM_L_KERNELS +#if !defined(EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS) +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS 1 +#endif +#endif // EIGEN_USE_AVX512_TRSM_L_KERNELS + +#else // EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS == 0 +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS 0 +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS 0 +#endif // EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS + +template +int64_t avx512_trsm_cutoff(int64_t L2Size, int64_t N, double L2Cap) { + const int64_t U3 = 3 * packet_traits::size; + const int64_t MaxNb = 5 * U3; + int64_t Nb = std::min(MaxNb, N); + double cutoff_d = + (((L2Size * L2Cap) / (sizeof(Scalar))) - (EIGEN_AVX_MAX_NUM_ROW)*Nb) / ((EIGEN_AVX_MAX_NUM_ROW) + Nb); + int64_t cutoff_l = static_cast(cutoff_d); + return (cutoff_l / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW; +} +#else // !(EIGEN_USE_AVX512_TRSM_KERNELS) || !(EIGEN_COMP_CLANG != 0) +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_CUTOFFS 0 +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_R_CUTOFFS 0 +#define EIGEN_ENABLE_AVX512_NOCOPY_TRSM_L_CUTOFFS 0 +#endif + +/** + * Used by gemmKernel for the case A/B row-major and C col-major. + */ +template +EIGEN_ALWAYS_INLINE void transStoreC(PacketBlock &zmm, Scalar *C_arr, + int64_t LDC, int64_t remM_ = 0, int64_t remN_ = 0) { + EIGEN_UNUSED_VARIABLE(remN_); + EIGEN_UNUSED_VARIABLE(remM_); + using urolls = unrolls::trans; + + constexpr int64_t U3 = urolls::PacketSize * 3; + constexpr int64_t U2 = urolls::PacketSize * 2; + constexpr int64_t U1 = urolls::PacketSize * 1; + + static_assert(unrollN == U1 || unrollN == U2 || unrollN == U3, "unrollN should be a multiple of PacketSize"); + static_assert(unrollM == EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW"); + + urolls::template transpose(zmm); + EIGEN_IF_CONSTEXPR(unrollN > U2) urolls::template transpose(zmm); + EIGEN_IF_CONSTEXPR(unrollN > U1) urolls::template transpose(zmm); + + static_assert((remN && unrollN == U1) || !remN, "When handling N remainder set unrollN=U1"); + EIGEN_IF_CONSTEXPR(!remN) { + urolls::template storeC(C_arr, LDC, zmm, remM_); + EIGEN_IF_CONSTEXPR(unrollN > U1) { + constexpr int64_t unrollN_ = std::min(unrollN - U1, U1); + urolls::template storeC(C_arr + U1 * LDC, LDC, zmm, remM_); + } + EIGEN_IF_CONSTEXPR(unrollN > U2) { + constexpr int64_t unrollN_ = std::min(unrollN - U2, U1); + urolls::template storeC(C_arr + U2 * LDC, LDC, zmm, remM_); + } + } + else { + EIGEN_IF_CONSTEXPR((std::is_same::value)) { + // Note: without "if constexpr" this section of code will also be + // parsed by the compiler so each of the storeC will still be instantiated. + // We use enable_if in aux_storeC to set it to an empty function for + // these cases. + if (remN_ == 15) + urolls::template storeC<15, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 14) + urolls::template storeC<14, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 13) + urolls::template storeC<13, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 12) + urolls::template storeC<12, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 11) + urolls::template storeC<11, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 10) + urolls::template storeC<10, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 9) + urolls::template storeC<9, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 8) + urolls::template storeC<8, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 7) + urolls::template storeC<7, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 6) + urolls::template storeC<6, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 5) + urolls::template storeC<5, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 4) + urolls::template storeC<4, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 3) + urolls::template storeC<3, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 2) + urolls::template storeC<2, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 1) + urolls::template storeC<1, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + } + else { + if (remN_ == 7) + urolls::template storeC<7, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 6) + urolls::template storeC<6, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 5) + urolls::template storeC<5, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 4) + urolls::template storeC<4, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 3) + urolls::template storeC<3, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 2) + urolls::template storeC<2, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + else if (remN_ == 1) + urolls::template storeC<1, unrollN, 0, remM>(C_arr, LDC, zmm, remM_); + } + } +} + +/** + * GEMM like operation for trsm panel updates. + * Computes: C -= A*B + * K must be multiple of 4. + * + * Unrolls used are {1,2,4,8}x{U1,U2,U3}; + * For good performance we want K to be large with M/N relatively small, but also large enough + * to use the {8,U3} unroll block. + * + * isARowMajor: is A_arr row-major? + * isCRowMajor: is C_arr row-major? (B_arr is assumed to be row-major). + * isAdd: C += A*B or C -= A*B (used by trsm) + * handleKRem: Handle arbitrary K? This is not needed for trsm. + */ +template +void gemmKernel(Scalar *A_arr, Scalar *B_arr, Scalar *C_arr, int64_t M, int64_t N, int64_t K, int64_t LDA, int64_t LDB, + int64_t LDC) { + using urolls = unrolls::gemm; + constexpr int64_t U3 = urolls::PacketSize * 3; + constexpr int64_t U2 = urolls::PacketSize * 2; + constexpr int64_t U1 = urolls::PacketSize * 1; + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + int64_t N_ = (N / U3) * U3; + int64_t M_ = (M / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW; + int64_t K_ = (K / EIGEN_AVX_MAX_K_UNROL) * EIGEN_AVX_MAX_K_UNROL; + int64_t j = 0; + for (; j < N_; j += U3) { + constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 3; + int64_t i = 0; + for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<3, EIGEN_AVX_MAX_NUM_ROW>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<3, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC); + } + } + if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<3, 4>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<3, 4>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<3, 4>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4); + } + i += 4; + } + if (M - i >= 2) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<3, 2>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<3, 2>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<3, 2>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2); + } + i += 2; + } + if (M - i > 0) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<3, 1>(zmm); + { + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<3, 1>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<3, 1>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1); + } + } + } + } + if (N - j >= U2) { + constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 2; + int64_t i = 0; + for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j]; + EIGEN_IF_CONSTEXPR(isCRowMajor) B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<2, EIGEN_AVX_MAX_NUM_ROW>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<2, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC); + } + } + if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<2, 4>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<2, 4>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<2, 4>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4); + } + i += 4; + } + if (M - i >= 2) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<2, 2>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<2, 2>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<2, 2>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2); + } + i += 2; + } + if (M - i > 0) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<2, 1>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<2, 1>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<2, 1>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1); + } + } + j += U2; + } + if (N - j >= U1) { + constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 1; + int64_t i = 0; + for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)], *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, EIGEN_AVX_MAX_NUM_ROW>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC); + } + } + if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 4>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 4>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<1, 4>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4); + } + i += 4; + } + if (M - i >= 2) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 2>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 2>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<1, 2>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2); + } + i += 2; + } + if (M - i > 0) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 1>(zmm); + { + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, + LDA, zmm); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 1>(&C_arr[i * LDC + j], LDC, zmm); + urolls::template storeC<1, 1>(&C_arr[i * LDC + j], LDC, zmm); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1); + } + } + } + j += U1; + } + if (N - j > 0) { + constexpr int64_t EIGEN_AVX_MAX_B_LOAD = EIGEN_AVX_B_LOAD_SETS * 1; + int64_t i = 0; + for (; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, EIGEN_AVX_MAX_NUM_ROW>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + urolls::template storeC<1, EIGEN_AVX_MAX_NUM_ROW, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 0, N - j); + } + } + if (M - i >= 4) { // Note: this block assumes EIGEN_AVX_MAX_NUM_ROW = 8. Should be removed otherwise + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 4>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm, N - j); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + urolls::template storeC<1, 4, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 4, N - j); + } + i += 4; + } + if (M - i >= 2) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 2>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, N - j); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm, N - j); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + urolls::template storeC<1, 2, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 2, N - j); + } + i += 2; + } + if (M - i > 0) { + Scalar *A_t = &A_arr[idA(i, 0, LDA)]; + Scalar *B_t = &B_arr[0 * LDB + j]; + PacketBlock zmm; + urolls::template setzero<1, 1>(zmm); + for (int64_t k = 0; k < K_; k += EIGEN_AVX_MAX_K_UNROL) { + urolls::template microKernel( + B_t, A_t, LDB, LDA, zmm, N - j); + B_t += EIGEN_AVX_MAX_K_UNROL * LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t += EIGEN_AVX_MAX_K_UNROL; + else A_t += EIGEN_AVX_MAX_K_UNROL * LDA; + } + EIGEN_IF_CONSTEXPR(handleKRem) { + for (int64_t k = K_; k < K; k++) { + urolls::template microKernel(B_t, A_t, LDB, LDA, zmm, + N - j); + B_t += LDB; + EIGEN_IF_CONSTEXPR(isARowMajor) A_t++; + else A_t += LDA; + } + } + EIGEN_IF_CONSTEXPR(isCRowMajor) { + urolls::template updateC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + urolls::template storeC<1, 1, true>(&C_arr[i * LDC + j], LDC, zmm, N - j); + } + else { + transStoreC(zmm, &C_arr[i + j * LDC], LDC, 1, N - j); + } + } + } +} + +/** + * Triangular solve kernel with A on left with K number of rhs. dim(A) = unrollM + * + * unrollM: dimension of A matrix (triangular matrix). unrollM should be <= EIGEN_AVX_MAX_NUM_ROW + * isFWDSolve: is forward solve? + * isUnitDiag: is the diagonal of A all ones? + * The B matrix (RHS) is assumed to be row-major + */ +template +EIGEN_ALWAYS_INLINE void triSolveKernel(Scalar *A_arr, Scalar *B_arr, int64_t K, int64_t LDA, int64_t LDB) { + static_assert(unrollM <= EIGEN_AVX_MAX_NUM_ROW, "unrollM should be equal to EIGEN_AVX_MAX_NUM_ROW"); + using urolls = unrolls::trsm; + constexpr int64_t U3 = urolls::PacketSize * 3; + constexpr int64_t U2 = urolls::PacketSize * 2; + constexpr int64_t U1 = urolls::PacketSize * 1; + + PacketBlock RHSInPacket; + PacketBlock AInPacket; + + int64_t k = 0; + while (K - k >= U3) { + urolls::template loadRHS(B_arr + k, LDB, RHSInPacket); + urolls::template triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + urolls::template storeRHS(B_arr + k, LDB, RHSInPacket); + k += U3; + } + if (K - k >= U2) { + urolls::template loadRHS(B_arr + k, LDB, RHSInPacket); + urolls::template triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + urolls::template storeRHS(B_arr + k, LDB, RHSInPacket); + k += U2; + } + if (K - k >= U1) { + urolls::template loadRHS(B_arr + k, LDB, RHSInPacket); + urolls::template triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + urolls::template storeRHS(B_arr + k, LDB, RHSInPacket); + k += U1; + } + if (K - k > 0) { + // Handle remaining number of RHS + urolls::template loadRHS(B_arr + k, LDB, RHSInPacket, K - k); + urolls::template triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + urolls::template storeRHS(B_arr + k, LDB, RHSInPacket, K - k); + } +} + +/** + * Triangular solve routine with A on left and dimension of at most L with K number of rhs. This is essentially + * a wrapper for triSolveMicrokernel for M = {1,2,3,4,5,6,7,8}. + * + * isFWDSolve: is forward solve? + * isUnitDiag: is the diagonal of A all ones? + * The B matrix (RHS) is assumed to be row-major + */ +template +void triSolveKernelLxK(Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t K, int64_t LDA, int64_t LDB) { + // Note: this assumes EIGEN_AVX_MAX_NUM_ROW = 8. Unrolls should be adjusted + // accordingly if EIGEN_AVX_MAX_NUM_ROW is smaller. + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + if (M == 8) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 7) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 6) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 5) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 4) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 3) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 2) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + else if (M == 1) + triSolveKernel(A_arr, B_arr, K, LDA, LDB); + return; +} + +/** + * This routine is used to copy B to/from a temporary array (row-major) for cases where B is column-major. + * + * toTemp: true => copy to temporary array, false => copy from temporary array + * remM: true = need to handle remainder values for M (M < EIGEN_AVX_MAX_NUM_ROW) + * + */ +template +EIGEN_ALWAYS_INLINE void copyBToRowMajor(Scalar *B_arr, int64_t LDB, int64_t K, Scalar *B_temp, int64_t LDB_, + int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(remM_); + using urolls = unrolls::transB; + using vecHalf = typename std::conditional::value, vecHalfFloat, vecFullDouble>::type; + PacketBlock ymm; + constexpr int64_t U3 = urolls::PacketSize * 3; + constexpr int64_t U2 = urolls::PacketSize * 2; + constexpr int64_t U1 = urolls::PacketSize * 1; + int64_t K_ = K / U3 * U3; + int64_t k = 0; + + for (; k < K_; k += U3) { + urolls::template transB_kernel(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += U3; + } + if (K - k >= U2) { + urolls::template transB_kernel(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += U2; + k += U2; + } + if (K - k >= U1) { + urolls::template transB_kernel(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += U1; + k += U1; + } + EIGEN_IF_CONSTEXPR(U1 > 8) { + // Note: without "if constexpr" this section of code will also be + // parsed by the compiler so there is an additional check in {load/store}BBlock + // to make sure the counter is not non-negative. + if (K - k >= 8) { + urolls::template transB_kernel<8, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += 8; + k += 8; + } + } + EIGEN_IF_CONSTEXPR(U1 > 4) { + // Note: without "if constexpr" this section of code will also be + // parsed by the compiler so there is an additional check in {load/store}BBlock + // to make sure the counter is not non-negative. + if (K - k >= 4) { + urolls::template transB_kernel<4, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += 4; + k += 4; + } + } + if (K - k >= 2) { + urolls::template transB_kernel<2, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += 2; + k += 2; + } + if (K - k >= 1) { + urolls::template transB_kernel<1, toTemp, remM>(B_arr + k * LDB, LDB, B_temp, LDB_, ymm, remM_); + B_temp += 1; + k += 1; + } +} + +/** + * Main triangular solve driver + * + * Triangular solve with A on the left. + * Scalar: Scalar precision, only float/double is supported. + * isARowMajor: is A row-major? + * isBRowMajor: is B row-major? + * isFWDSolve: is this forward solve or backward (true => forward)? + * isUnitDiag: is diagonal of A unit or nonunit (true => A has unit diagonal)? + * + * M: dimension of A + * numRHS: number of right hand sides (coincides with K dimension for gemm updates) + * + * Here are the mapping between the different TRSM cases (col-major) and triSolve: + * + * LLN (left , lower, A non-transposed) :: isARowMajor=false, isBRowMajor=false, isFWDSolve=true + * LUT (left , upper, A transposed) :: isARowMajor=true, isBRowMajor=false, isFWDSolve=true + * LUN (left , upper, A non-transposed) :: isARowMajor=false, isBRowMajor=false, isFWDSolve=false + * LLT (left , lower, A transposed) :: isARowMajor=true, isBRowMajor=false, isFWDSolve=false + * RUN (right, upper, A non-transposed) :: isARowMajor=true, isBRowMajor=true, isFWDSolve=true + * RLT (right, lower, A transposed) :: isARowMajor=false, isBRowMajor=true, isFWDSolve=true + * RUT (right, upper, A transposed) :: isARowMajor=false, isBRowMajor=true, isFWDSolve=false + * RLN (right, lower, A non-transposed) :: isARowMajor=true, isBRowMajor=true, isFWDSolve=false + * + * Note: For RXX cases M,numRHS should be swapped. + * + */ +template +void triSolve(Scalar *A_arr, Scalar *B_arr, int64_t M, int64_t numRHS, int64_t LDA, int64_t LDB) { + constexpr int64_t psize = packet_traits::size; + /** + * The values for kB, numM were determined experimentally. + * kB: Number of RHS we process at a time. + * numM: number of rows of B we will store in a temporary array (see below.) This should be a multiple of L. + * + * kB was determined by initially setting kB = numRHS and benchmarking triSolve (TRSM-RUN case) + * performance with M=numRHS. + * It was observed that performance started to drop around M=numRHS=240. This is likely machine dependent. + * + * numM was chosen "arbitrarily". It should be relatively small so B_temp is not too large, but it should be + * large enough to allow GEMM updates to have larger "K"s (see below.) No benchmarking has been done so far to + * determine optimal values for numM. + */ + constexpr int64_t kB = (3 * psize) * 5; // 5*U3 + constexpr int64_t numM = 8 * EIGEN_AVX_MAX_NUM_ROW; + + int64_t sizeBTemp = 0; + Scalar *B_temp = NULL; + EIGEN_IF_CONSTEXPR(!isBRowMajor) { + /** + * If B is col-major, we copy it to a fixed-size temporary array of size at most ~numM*kB and + * transpose it to row-major. Call the solve routine, and copy+transpose it back to the original array. + * The updated row-major copy of B is reused in the GEMM updates. + */ + sizeBTemp = (((std::min(kB, numRHS) + psize - 1) / psize + 4) * psize) * numM; + } + + EIGEN_IF_CONSTEXPR(!isBRowMajor) B_temp = (Scalar *)handmade_aligned_malloc(sizeof(Scalar) * sizeBTemp, 64); + + for (int64_t k = 0; k < numRHS; k += kB) { + int64_t bK = numRHS - k > kB ? kB : numRHS - k; + int64_t M_ = (M / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW, gemmOff = 0; + + // bK rounded up to next multiple of L=EIGEN_AVX_MAX_NUM_ROW. When B_temp is used, we solve for bkL RHS + // instead of bK RHS in triSolveKernelLxK. + int64_t bkL = ((bK + (EIGEN_AVX_MAX_NUM_ROW - 1)) / EIGEN_AVX_MAX_NUM_ROW) * EIGEN_AVX_MAX_NUM_ROW; + const int64_t numScalarPerCache = 64 / sizeof(Scalar); + // Leading dimension of B_temp, will be a multiple of the cache line size. + int64_t LDT = ((bkL + (numScalarPerCache - 1)) / numScalarPerCache) * numScalarPerCache; + int64_t offsetBTemp = 0; + for (int64_t i = 0; i < M_; i += EIGEN_AVX_MAX_NUM_ROW) { + EIGEN_IF_CONSTEXPR(!isBRowMajor) { + int64_t indA_i = isFWDSolve ? i : M - 1 - i; + int64_t indB_i = isFWDSolve ? i : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t offB_1 = isFWDSolve ? offsetBTemp : sizeBTemp - EIGEN_AVX_MAX_NUM_ROW * LDT - offsetBTemp; + int64_t offB_2 = isFWDSolve ? offsetBTemp : sizeBTemp - LDT - offsetBTemp; + // Copy values from B to B_temp. + copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp + offB_1, LDT); + // Triangular solve with a small block of A and long horizontal blocks of B (or B_temp if B col-major) + triSolveKernelLxK( + &A_arr[idA(indA_i, indA_i, LDA)], B_temp + offB_2, EIGEN_AVX_MAX_NUM_ROW, bkL, LDA, LDT); + // Copy values from B_temp back to B. B_temp will be reused in gemm call below. + copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp + offB_1, LDT); + + offsetBTemp += EIGEN_AVX_MAX_NUM_ROW * LDT; + } + else { + int64_t ind = isFWDSolve ? i : M - 1 - i; + triSolveKernelLxK( + &A_arr[idA(ind, ind, LDA)], B_arr + k + ind * LDB, EIGEN_AVX_MAX_NUM_ROW, bK, LDA, LDB); + } + if (i + EIGEN_AVX_MAX_NUM_ROW < M_) { + /** + * For the GEMM updates, we want "K" (K=i+8 in this case) to be large as soon as possible + * to reuse the accumulators in GEMM as much as possible. So we only update 8xbK blocks of + * B as follows: + * + * A B + * __ + * |__|__ |__| + * |__|__|__ |__| + * |__|__|__|__ |__| + * |********|__| |**| + */ + EIGEN_IF_CONSTEXPR(isBRowMajor) { + int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW); + int64_t indA_j = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t indB_i = isFWDSolve ? 0 : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t indB_i2 = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW); + gemmKernel( + &A_arr[idA(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB, + EIGEN_AVX_MAX_NUM_ROW, bK, i + EIGEN_AVX_MAX_NUM_ROW, LDA, LDB, LDB); + } + else { + if (offsetBTemp + EIGEN_AVX_MAX_NUM_ROW * LDT > sizeBTemp) { + /** + * Similar idea as mentioned above, but here we are limited by the number of updated values of B + * that can be stored (row-major) in B_temp. + * + * If there is not enough space to store the next batch of 8xbK of B in B_temp, we call GEMM + * update and partially update the remaining old values of B which depends on the new values + * of B stored in B_temp. These values are then no longer needed and can be overwritten. + */ + int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : 0; + int64_t indA_j = isFWDSolve ? gemmOff : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t indB_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : 0; + int64_t offB_1 = isFWDSolve ? 0 : sizeBTemp - offsetBTemp; + gemmKernel( + &A_arr[idA(indA_i, indA_j, LDA)], B_temp + offB_1, B_arr + indB_i + (k)*LDB, + M - (i + EIGEN_AVX_MAX_NUM_ROW), bK, i + EIGEN_AVX_MAX_NUM_ROW - gemmOff, LDA, LDT, LDB); + offsetBTemp = 0; + gemmOff = i + EIGEN_AVX_MAX_NUM_ROW; + } else { + /** + * If there is enough space in B_temp, we only update the next 8xbK values of B. + */ + int64_t indA_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW); + int64_t indA_j = isFWDSolve ? gemmOff : M - (i + EIGEN_AVX_MAX_NUM_ROW); + int64_t indB_i = isFWDSolve ? i + EIGEN_AVX_MAX_NUM_ROW : M - (i + 2 * EIGEN_AVX_MAX_NUM_ROW); + int64_t offB_1 = isFWDSolve ? 0 : sizeBTemp - offsetBTemp; + gemmKernel( + &A_arr[idA(indA_i, indA_j, LDA)], B_temp + offB_1, B_arr + indB_i + (k)*LDB, + EIGEN_AVX_MAX_NUM_ROW, bK, i + EIGEN_AVX_MAX_NUM_ROW - gemmOff, LDA, LDT, LDB); + } + } + } + } + // Handle M remainder.. + int64_t bM = M - M_; + if (bM > 0) { + if (M_ > 0) { + EIGEN_IF_CONSTEXPR(isBRowMajor) { + int64_t indA_i = isFWDSolve ? M_ : 0; + int64_t indA_j = isFWDSolve ? 0 : bM; + int64_t indB_i = isFWDSolve ? 0 : bM; + int64_t indB_i2 = isFWDSolve ? M_ : 0; + gemmKernel( + &A_arr[idA(indA_i, indA_j, LDA)], B_arr + k + indB_i * LDB, B_arr + k + indB_i2 * LDB, bM, + bK, M_, LDA, LDB, LDB); + } + else { + int64_t indA_i = isFWDSolve ? M_ : 0; + int64_t indA_j = isFWDSolve ? gemmOff : bM; + int64_t indB_i = isFWDSolve ? M_ : 0; + int64_t offB_1 = isFWDSolve ? 0 : sizeBTemp - offsetBTemp; + gemmKernel(&A_arr[idA(indA_i, indA_j, LDA)], + B_temp + offB_1, B_arr + indB_i + (k)*LDB, bM, bK, + M_ - gemmOff, LDA, LDT, LDB); + } + } + EIGEN_IF_CONSTEXPR(!isBRowMajor) { + int64_t indA_i = isFWDSolve ? M_ : M - 1 - M_; + int64_t indB_i = isFWDSolve ? M_ : 0; + int64_t offB_1 = isFWDSolve ? 0 : (bM - 1) * bkL; + copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp, bkL, bM); + triSolveKernelLxK(&A_arr[idA(indA_i, indA_i, LDA)], + B_temp + offB_1, bM, bkL, LDA, bkL); + copyBToRowMajor(B_arr + indB_i + k * LDB, LDB, bK, B_temp, bkL, bM); + } + else { + int64_t ind = isFWDSolve ? M_ : M - 1 - M_; + triSolveKernelLxK(&A_arr[idA(ind, ind, LDA)], + B_arr + k + ind * LDB, bM, bK, LDA, LDB); + } + } + } + + EIGEN_IF_CONSTEXPR(!isBRowMajor) handmade_aligned_free(B_temp); +} + +// Template specializations of trsmKernelL/R for float/double and inner strides of 1. +#if (EIGEN_USE_AVX512_TRSM_KERNELS) +#if (EIGEN_USE_AVX512_TRSM_R_KERNELS) +template +struct trsmKernelR; + +template +struct trsmKernelR { + static void kernel(Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index otherStride); +}; + +template +struct trsmKernelR { + static void kernel(Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index otherStride); +}; + +template +EIGEN_DONT_INLINE void trsmKernelR::kernel( + Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index otherStride) { + EIGEN_UNUSED_VARIABLE(otherIncr); +#ifdef EIGEN_RUNTIME_NO_MALLOC + if (!is_malloc_allowed()) { + trsmKernelR::kernel( + size, otherSize, _tri, triStride, _other, otherIncr, otherStride); + return; + } +#endif + triSolve( + const_cast(_tri), _other, size, otherSize, triStride, otherStride); +} + +template +EIGEN_DONT_INLINE void trsmKernelR::kernel( + Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index otherStride) { + EIGEN_UNUSED_VARIABLE(otherIncr); +#ifdef EIGEN_RUNTIME_NO_MALLOC + if (!is_malloc_allowed()) { + trsmKernelR::kernel( + size, otherSize, _tri, triStride, _other, otherIncr, otherStride); + return; + } +#endif + triSolve( + const_cast(_tri), _other, size, otherSize, triStride, otherStride); +} +#endif // (EIGEN_USE_AVX512_TRSM_R_KERNELS) + +// These trsm kernels require temporary memory allocation +#if (EIGEN_USE_AVX512_TRSM_L_KERNELS) +template +struct trsmKernelL; + +template +struct trsmKernelL { + static void kernel(Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index otherStride); +}; + +template +struct trsmKernelL { + static void kernel(Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index otherStride); +}; + +template +EIGEN_DONT_INLINE void trsmKernelL::kernel( + Index size, Index otherSize, const float *_tri, Index triStride, float *_other, Index otherIncr, + Index otherStride) { + EIGEN_UNUSED_VARIABLE(otherIncr); +#ifdef EIGEN_RUNTIME_NO_MALLOC + if (!is_malloc_allowed()) { + trsmKernelL::kernel( + size, otherSize, _tri, triStride, _other, otherIncr, otherStride); + return; + } +#endif + triSolve( + const_cast(_tri), _other, size, otherSize, triStride, otherStride); +} + +template +EIGEN_DONT_INLINE void trsmKernelL::kernel( + Index size, Index otherSize, const double *_tri, Index triStride, double *_other, Index otherIncr, + Index otherStride) { + EIGEN_UNUSED_VARIABLE(otherIncr); +#ifdef EIGEN_RUNTIME_NO_MALLOC + if (!is_malloc_allowed()) { + trsmKernelL::kernel( + size, otherSize, _tri, triStride, _other, otherIncr, otherStride); + return; + } +#endif + triSolve( + const_cast(_tri), _other, size, otherSize, triStride, otherStride); +} +#endif // EIGEN_USE_AVX512_TRSM_L_KERNELS +#endif // EIGEN_USE_AVX512_TRSM_KERNELS +} // namespace internal +} // namespace Eigen +#endif // EIGEN_CORE_ARCH_AVX512_TRSM_KERNEL_H diff --git a/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc b/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc new file mode 100644 index 0000000..3a5f68e --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/TrsmUnrolls.inc @@ -0,0 +1,1219 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2022 Intel Corporation +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CORE_ARCH_AVX512_TRSM_UNROLLS_H +#define EIGEN_CORE_ARCH_AVX512_TRSM_UNROLLS_H + +template +EIGEN_ALWAYS_INLINE int64_t idA(int64_t i, int64_t j, int64_t LDA) { + EIGEN_IF_CONSTEXPR(isARowMajor) return i * LDA + j; + else return i + j * LDA; +} + +/** + * This namespace contains various classes used to generate compile-time unrolls which are + * used throughout the trsm/gemm kernels. The unrolls are characterized as for-loops (1-D), nested + * for-loops (2-D), or triple nested for-loops (3-D). Unrolls are generated using template recursion + * + * Example, the 2-D for-loop is unrolled recursively by first flattening to a 1-D loop. + * + * for(startI = 0; startI < endI; startI++) for(startC = 0; startC < endI*endJ; startC++) + * for(startJ = 0; startJ < endJ; startJ++) ----> startI = (startC)/(endJ) + * func(startI,startJ) startJ = (startC)%(endJ) + * func(...) + * + * The 1-D loop can be unrolled recursively by using enable_if and defining an auxiliary function + * with a template parameter used as a counter. + * + * template + * std::enable_if_t<(counter <= 0)> <---- tail case. + * aux_func {} + * + * template + * std::enable_if_t<(counter > 0)> <---- actual for-loop + * aux_func { + * startC = endI*endJ - counter + * startI = (startC)/(endJ) + * startJ = (startC)%(endJ) + * func(startI, startJ) + * aux_func() + * } + * + * Note: Additional wrapper functions are provided for aux_func which hides the counter template + * parameter since counter usually depends on endI, endJ, etc... + * + * Conventions: + * 1) endX: specifies the terminal value for the for-loop, (ex: for(startX = 0; startX < endX; startX++)) + * + * 2) rem, remM, remK template parameters are used for deciding whether to use masked operations for + * handling remaining tails (when sizes are not multiples of PacketSize or EIGEN_AVX_MAX_NUM_ROW) + */ +namespace unrolls { + +template +EIGEN_ALWAYS_INLINE auto remMask(int64_t m) { + EIGEN_IF_CONSTEXPR(N == 16) { return 0xFFFF >> (16 - m); } + else EIGEN_IF_CONSTEXPR(N == 8) { + return 0xFF >> (8 - m); + } + else EIGEN_IF_CONSTEXPR(N == 4) { + return 0x0F >> (4 - m); + } + return 0; +} + +template +EIGEN_ALWAYS_INLINE void trans8x8blocks(PacketBlock &kernel); + +template <> +EIGEN_ALWAYS_INLINE void trans8x8blocks(PacketBlock &kernel) { + __m512 T0 = _mm512_unpacklo_ps(kernel.packet[0], kernel.packet[1]); + __m512 T1 = _mm512_unpackhi_ps(kernel.packet[0], kernel.packet[1]); + __m512 T2 = _mm512_unpacklo_ps(kernel.packet[2], kernel.packet[3]); + __m512 T3 = _mm512_unpackhi_ps(kernel.packet[2], kernel.packet[3]); + __m512 T4 = _mm512_unpacklo_ps(kernel.packet[4], kernel.packet[5]); + __m512 T5 = _mm512_unpackhi_ps(kernel.packet[4], kernel.packet[5]); + __m512 T6 = _mm512_unpacklo_ps(kernel.packet[6], kernel.packet[7]); + __m512 T7 = _mm512_unpackhi_ps(kernel.packet[6], kernel.packet[7]); + + kernel.packet[0] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2))); + kernel.packet[1] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T0), _mm512_castps_pd(T2))); + kernel.packet[2] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3))); + kernel.packet[3] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T1), _mm512_castps_pd(T3))); + kernel.packet[4] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6))); + kernel.packet[5] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T4), _mm512_castps_pd(T6))); + kernel.packet[6] = _mm512_castpd_ps(_mm512_unpacklo_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7))); + kernel.packet[7] = _mm512_castpd_ps(_mm512_unpackhi_pd(_mm512_castps_pd(T5), _mm512_castps_pd(T7))); + + T0 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[4]), 0x4E)); + T0 = _mm512_mask_blend_ps(0xF0F0, kernel.packet[0], T0); + T4 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[0]), 0x4E)); + T4 = _mm512_mask_blend_ps(0xF0F0, T4, kernel.packet[4]); + T1 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[5]), 0x4E)); + T1 = _mm512_mask_blend_ps(0xF0F0, kernel.packet[1], T1); + T5 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[1]), 0x4E)); + T5 = _mm512_mask_blend_ps(0xF0F0, T5, kernel.packet[5]); + T2 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[6]), 0x4E)); + T2 = _mm512_mask_blend_ps(0xF0F0, kernel.packet[2], T2); + T6 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[2]), 0x4E)); + T6 = _mm512_mask_blend_ps(0xF0F0, T6, kernel.packet[6]); + T3 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[7]), 0x4E)); + T3 = _mm512_mask_blend_ps(0xF0F0, kernel.packet[3], T3); + T7 = _mm512_castpd_ps(_mm512_permutex_pd(_mm512_castps_pd(kernel.packet[3]), 0x4E)); + T7 = _mm512_mask_blend_ps(0xF0F0, T7, kernel.packet[7]); + + kernel.packet[0] = T0; + kernel.packet[1] = T1; + kernel.packet[2] = T2; + kernel.packet[3] = T3; + kernel.packet[4] = T4; + kernel.packet[5] = T5; + kernel.packet[6] = T6; + kernel.packet[7] = T7; +} + +template <> +EIGEN_ALWAYS_INLINE void trans8x8blocks(PacketBlock &kernel) { + ptranspose(kernel); +} + +/*** + * Unrolls for transposed C stores + */ +template +class trans { + public: + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + using vecHalf = typename std::conditional::value, vecHalfFloat, vecFullDouble>::type; + static constexpr int64_t PacketSize = packet_traits::size; + + /*********************************** + * Auxiliary Functions for: + * - storeC + *********************************** + */ + + /** + * aux_storeC + * + * 1-D unroll + * for(startN = 0; startN < endN; startN++) + * + * (endN <= PacketSize) is required to handle the fp32 case, see comments in transStoreC + * + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0 && endN <= PacketSize)> aux_storeC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t remM_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + + EIGEN_IF_CONSTEXPR(startN < EIGEN_AVX_MAX_NUM_ROW) { + EIGEN_IF_CONSTEXPR(remM) { + pstoreu( + C_arr + LDC * startN, + padd(ploadu((const Scalar *)C_arr + LDC * startN, remMask(remM_)), + preinterpret(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN]), + remMask(remM_)), + remMask(remM_)); + } + else { + pstoreu(C_arr + LDC * startN, + padd(ploadu((const Scalar *)C_arr + LDC * startN), + preinterpret(zmm.packet[packetIndexOffset + (unrollN / PacketSize) * startN]))); + } + } + else { // This block is only needed for fp32 case + // Reinterpret as __m512 for _mm512_shuffle_f32x4 + vecFullFloat zmm2vecFullFloat = preinterpret( + zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)]); + // Swap lower and upper half of avx register. + zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)] = + preinterpret(_mm512_shuffle_f32x4(zmm2vecFullFloat, zmm2vecFullFloat, 0b01001110)); + + EIGEN_IF_CONSTEXPR(remM) { + pstoreu( + C_arr + LDC * startN, + padd(ploadu((const Scalar *)C_arr + LDC * startN, remMask(remM_)), + preinterpret( + zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)])), + remMask(remM_)); + } + else { + pstoreu( + C_arr + LDC * startN, + padd(ploadu((const Scalar *)C_arr + LDC * startN), + preinterpret( + zmm.packet[packetIndexOffset + (unrollN / PacketSize) * (startN - EIGEN_AVX_MAX_NUM_ROW)]))); + } + } + aux_storeC(C_arr, LDC, zmm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t 0 && endN <= PacketSize)> aux_storeC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(C_arr); + EIGEN_UNUSED_VARIABLE(LDC); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(remM_); + } + + template + static EIGEN_ALWAYS_INLINE void storeC(Scalar *C_arr, int64_t LDC, + PacketBlock &zmm, + int64_t remM_ = 0) { + aux_storeC(C_arr, LDC, zmm, remM_); + } + + /** + * Transposes LxunrollN row major block of matrices stored `EIGEN_AVX_MAX_NUM_ACC` zmm registers to + * "unrollN"xL ymm registers to be stored col-major into C. + * + * For 8x48, the 8x48 block (row-major) is stored in zmm as follows: + * + * ``` + * row0: zmm0 zmm1 zmm2 + * row1: zmm3 zmm4 zmm5 + * . + * . + * row7: zmm21 zmm22 zmm23 + * + * For 8x32, the 8x32 block (row-major) is stored in zmm as follows: + * + * row0: zmm0 zmm1 + * row1: zmm2 zmm3 + * . + * . + * row7: zmm14 zmm15 + * ``` + * + * In general we will have {1,2,3} groups of avx registers each of size + * `EIGEN_AVX_MAX_NUM_ROW`. packetIndexOffset is used to select which "block" of + * avx registers are being transposed. + */ + template + static EIGEN_ALWAYS_INLINE void transpose(PacketBlock &zmm) { + // Note: this assumes EIGEN_AVX_MAX_NUM_ROW = 8. Unrolls should be adjusted + // accordingly if EIGEN_AVX_MAX_NUM_ROW is smaller. + constexpr int64_t zmmStride = unrollN / PacketSize; + PacketBlock r; + r.packet[0] = zmm.packet[packetIndexOffset + zmmStride * 0]; + r.packet[1] = zmm.packet[packetIndexOffset + zmmStride * 1]; + r.packet[2] = zmm.packet[packetIndexOffset + zmmStride * 2]; + r.packet[3] = zmm.packet[packetIndexOffset + zmmStride * 3]; + r.packet[4] = zmm.packet[packetIndexOffset + zmmStride * 4]; + r.packet[5] = zmm.packet[packetIndexOffset + zmmStride * 5]; + r.packet[6] = zmm.packet[packetIndexOffset + zmmStride * 6]; + r.packet[7] = zmm.packet[packetIndexOffset + zmmStride * 7]; + trans8x8blocks(r); + zmm.packet[packetIndexOffset + zmmStride * 0] = r.packet[0]; + zmm.packet[packetIndexOffset + zmmStride * 1] = r.packet[1]; + zmm.packet[packetIndexOffset + zmmStride * 2] = r.packet[2]; + zmm.packet[packetIndexOffset + zmmStride * 3] = r.packet[3]; + zmm.packet[packetIndexOffset + zmmStride * 4] = r.packet[4]; + zmm.packet[packetIndexOffset + zmmStride * 5] = r.packet[5]; + zmm.packet[packetIndexOffset + zmmStride * 6] = r.packet[6]; + zmm.packet[packetIndexOffset + zmmStride * 7] = r.packet[7]; + } +}; + +/** + * Unrolls for copyBToRowMajor + * + * Idea: + * 1) Load a block of right-hand sides to registers (using loadB). + * 2) Convert the block from column-major to row-major (transposeLxL) + * 3) Store the blocks from register either to a temp array (toTemp == true), or back to B (toTemp == false). + * + * We use at most EIGEN_AVX_MAX_NUM_ACC avx registers to store the blocks of B. The remaining registers are + * used as temps for transposing. + * + * Blocks will be of size Lx{U1,U2,U3}. packetIndexOffset is used to index between these subblocks + * For fp32, PacketSize = 2*EIGEN_AVX_MAX_NUM_ROW, so we reinterpret packets as packets half the size (zmm -> ymm). + */ +template +class transB { + public: + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + using vecHalf = typename std::conditional::value, vecHalfFloat, vecFullDouble>::type; + static constexpr int64_t PacketSize = packet_traits::size; + + /*********************************** + * Auxiliary Functions for: + * - loadB + * - storeB + * - loadBBlock + * - storeBBlock + *********************************** + */ + + /** + * aux_loadB + * + * 1-D unroll + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_loadB( + Scalar *B_arr, int64_t LDB, PacketBlock &ymm, + int64_t remM_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + + EIGEN_IF_CONSTEXPR(remM) { + ymm.packet[packetIndexOffset + startN] = + ploadu((const Scalar *)&B_arr[startN * LDB], remMask(remM_)); + } + else { + EIGEN_IF_CONSTEXPR(remN_ == 0) { + ymm.packet[packetIndexOffset + startN] = ploadu((const Scalar *)&B_arr[startN * LDB]); + } + else ymm.packet[packetIndexOffset + startN] = + ploadu((const Scalar *)&B_arr[startN * LDB], remMask(remN_)); + } + + aux_loadB(B_arr, LDB, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_loadB( + Scalar *B_arr, int64_t LDB, PacketBlock &ymm, + int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(ymm); + EIGEN_UNUSED_VARIABLE(remM_); + } + + /** + * aux_storeB + * + * 1-D unroll + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_storeB( + Scalar *B_arr, int64_t LDB, PacketBlock &ymm, int64_t rem_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + + EIGEN_IF_CONSTEXPR(remK || remM) { + pstoreu(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN], + remMask(rem_)); + } + else { + pstoreu(&B_arr[startN * LDB], ymm.packet[packetIndexOffset + startN]); + } + + aux_storeB(B_arr, LDB, ymm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_storeB( + Scalar *B_arr, int64_t LDB, PacketBlock &ymm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(ymm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_loadBBlock + * + * 1-D unroll + * for(startN = 0; startN < endN; startN += EIGEN_AVX_MAX_NUM_ROW) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_loadBBlock( + Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, int64_t remM_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + transB::template loadB(&B_temp[startN], LDB_, ymm); + aux_loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_loadBBlock( + Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(B_temp); + EIGEN_UNUSED_VARIABLE(LDB_); + EIGEN_UNUSED_VARIABLE(ymm); + EIGEN_UNUSED_VARIABLE(remM_); + } + + /** + * aux_storeBBlock + * + * 1-D unroll + * for(startN = 0; startN < endN; startN += EIGEN_AVX_MAX_NUM_ROW) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_storeBBlock( + Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, int64_t remM_ = 0) { + constexpr int64_t counterReverse = endN - counter; + constexpr int64_t startN = counterReverse; + + EIGEN_IF_CONSTEXPR(toTemp) { + transB::template storeB(&B_temp[startN], LDB_, ymm, remK_); + } + else { + transB::template storeB(&B_arr[0 + startN * LDB], LDB, + ymm, remM_); + } + aux_storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_storeBBlock( + Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, int64_t remM_ = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(B_temp); + EIGEN_UNUSED_VARIABLE(LDB_); + EIGEN_UNUSED_VARIABLE(ymm); + EIGEN_UNUSED_VARIABLE(remM_); + } + + /******************************************************** + * Wrappers for aux_XXXX to hide counter parameter + ********************************************************/ + + template + static EIGEN_ALWAYS_INLINE void loadB(Scalar *B_arr, int64_t LDB, + PacketBlock &ymm, + int64_t remM_ = 0) { + aux_loadB(B_arr, LDB, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE void storeB(Scalar *B_arr, int64_t LDB, + PacketBlock &ymm, + int64_t rem_ = 0) { + aux_storeB(B_arr, LDB, ymm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE void loadBBlock(Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, + int64_t remM_ = 0) { + EIGEN_IF_CONSTEXPR(toTemp) { transB::template loadB(&B_arr[0], LDB, ymm, remM_); } + else { + aux_loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + } + + template + static EIGEN_ALWAYS_INLINE void storeBBlock(Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, + int64_t remM_ = 0) { + aux_storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + + template + static EIGEN_ALWAYS_INLINE void transposeLxL(PacketBlock &ymm) { + // Note: this assumes EIGEN_AVX_MAX_NUM_ROW = 8. Unrolls should be adjusted + // accordingly if EIGEN_AVX_MAX_NUM_ROW is smaller. + PacketBlock r; + r.packet[0] = ymm.packet[packetIndexOffset + 0]; + r.packet[1] = ymm.packet[packetIndexOffset + 1]; + r.packet[2] = ymm.packet[packetIndexOffset + 2]; + r.packet[3] = ymm.packet[packetIndexOffset + 3]; + r.packet[4] = ymm.packet[packetIndexOffset + 4]; + r.packet[5] = ymm.packet[packetIndexOffset + 5]; + r.packet[6] = ymm.packet[packetIndexOffset + 6]; + r.packet[7] = ymm.packet[packetIndexOffset + 7]; + ptranspose(r); + ymm.packet[packetIndexOffset + 0] = r.packet[0]; + ymm.packet[packetIndexOffset + 1] = r.packet[1]; + ymm.packet[packetIndexOffset + 2] = r.packet[2]; + ymm.packet[packetIndexOffset + 3] = r.packet[3]; + ymm.packet[packetIndexOffset + 4] = r.packet[4]; + ymm.packet[packetIndexOffset + 5] = r.packet[5]; + ymm.packet[packetIndexOffset + 6] = r.packet[6]; + ymm.packet[packetIndexOffset + 7] = r.packet[7]; + } + + template + static EIGEN_ALWAYS_INLINE void transB_kernel(Scalar *B_arr, int64_t LDB, Scalar *B_temp, int64_t LDB_, + PacketBlock &ymm, + int64_t remM_ = 0) { + constexpr int64_t U3 = PacketSize * 3; + constexpr int64_t U2 = PacketSize * 2; + constexpr int64_t U1 = PacketSize * 1; + /** + * Unrolls needed for each case: + * - AVX512 fp32 48 32 16 8 4 2 1 + * - AVX512 fp64 24 16 8 4 2 1 + * + * For fp32 L and U1 are 1:2 so for U3/U2 cases the loads/stores need to be split up. + */ + EIGEN_IF_CONSTEXPR(unrollN == U3) { + // load LxU3 B col major, transpose LxU3 row major + constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U3); + transB::template loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + + EIGEN_IF_CONSTEXPR(maxUBlock < U3) { + transB::template loadBBlock(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_, + ymm, remM_); + transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template storeBBlock(&B_arr[maxUBlock * LDB], LDB, &B_temp[maxUBlock], LDB_, + ymm, remM_); + } + } + else EIGEN_IF_CONSTEXPR(unrollN == U2) { + // load LxU2 B col major, transpose LxU2 row major + constexpr int64_t maxUBlock = std::min(3 * EIGEN_AVX_MAX_NUM_ROW, U2); + transB::template loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + EIGEN_IF_CONSTEXPR(maxUBlock < U2) transB::template transposeLxL<2 * EIGEN_AVX_MAX_NUM_ROW>(ymm); + transB::template storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + + EIGEN_IF_CONSTEXPR(maxUBlock < U2) { + transB::template loadBBlock(&B_arr[maxUBlock * LDB], LDB, + &B_temp[maxUBlock], LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock(&B_arr[maxUBlock * LDB], LDB, + &B_temp[maxUBlock], LDB_, ymm, remM_); + } + } + else EIGEN_IF_CONSTEXPR(unrollN == U1) { + // load LxU1 B col major, transpose LxU1 row major + transB::template loadBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + EIGEN_IF_CONSTEXPR(EIGEN_AVX_MAX_NUM_ROW < U1) { transB::template transposeLxL<1 * EIGEN_AVX_MAX_NUM_ROW>(ymm); } + transB::template storeBBlock(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + else EIGEN_IF_CONSTEXPR(unrollN == 8 && U1 > 8) { + // load Lx4 B col major, transpose Lx4 row major + transB::template loadBBlock<8, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock<8, toTemp, remM, 8>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + else EIGEN_IF_CONSTEXPR(unrollN == 4 && U1 > 4) { + // load Lx4 B col major, transpose Lx4 row major + transB::template loadBBlock<4, toTemp, remM>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock<4, toTemp, remM, 4>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + else EIGEN_IF_CONSTEXPR(unrollN == 2) { + // load Lx2 B col major, transpose Lx2 row major + transB::template loadBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock<2, toTemp, remM, 2>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + else EIGEN_IF_CONSTEXPR(unrollN == 1) { + // load Lx1 B col major, transpose Lx1 row major + transB::template loadBBlock<1, toTemp, remM, 1>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + transB::template transposeLxL<0>(ymm); + transB::template storeBBlock<1, toTemp, remM, 1>(B_arr, LDB, B_temp, LDB_, ymm, remM_); + } + } +}; + +/** + * Unrolls for triSolveKernel + * + * Idea: + * 1) Load a block of right-hand sides to registers in RHSInPacket (using loadRHS). + * 2) Do triangular solve with RHSInPacket and a small block of A (triangular matrix) + * stored in AInPacket (using triSolveMicroKernel). + * 3) Store final results (in avx registers) back into memory (using storeRHS). + * + * RHSInPacket uses at most EIGEN_AVX_MAX_NUM_ACC avx registers and AInPacket uses at most + * EIGEN_AVX_MAX_NUM_ROW registers. + */ +template +class trsm { + public: + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + static constexpr int64_t PacketSize = packet_traits::size; + + /*********************************** + * Auxiliary Functions for: + * - loadRHS + * - storeRHS + * - divRHSByDiag + * - updateRHS + * - triSolveMicroKernel + ************************************/ + /** + * aux_loadRHS + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_loadRHS( + Scalar *B_arr, int64_t LDB, PacketBlock &RHSInPacket, int64_t rem = 0) { + constexpr int64_t counterReverse = endM * endK - counter; + constexpr int64_t startM = counterReverse / (endK); + constexpr int64_t startK = counterReverse % endK; + + constexpr int64_t packetIndex = startM * endK + startK; + constexpr int64_t startM_ = isFWDSolve ? startM : -startM; + const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB; + EIGEN_IF_CONSTEXPR(krem) { + RHSInPacket.packet[packetIndex] = ploadu(&B_arr[rhsIndex], remMask(rem)); + } + else { + RHSInPacket.packet[packetIndex] = ploadu(&B_arr[rhsIndex]); + } + aux_loadRHS(B_arr, LDB, RHSInPacket, rem); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_loadRHS( + Scalar *B_arr, int64_t LDB, PacketBlock &RHSInPacket, int64_t rem = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(rem); + } + + /** + * aux_storeRHS + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_storeRHS( + Scalar *B_arr, int64_t LDB, PacketBlock &RHSInPacket, int64_t rem = 0) { + constexpr int64_t counterReverse = endM * endK - counter; + constexpr int64_t startM = counterReverse / (endK); + constexpr int64_t startK = counterReverse % endK; + + constexpr int64_t packetIndex = startM * endK + startK; + constexpr int64_t startM_ = isFWDSolve ? startM : -startM; + const int64_t rhsIndex = (startK * PacketSize) + startM_ * LDB; + EIGEN_IF_CONSTEXPR(krem) { + pstoreu(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex], remMask(rem)); + } + else { + pstoreu(&B_arr[rhsIndex], RHSInPacket.packet[packetIndex]); + } + aux_storeRHS(B_arr, LDB, RHSInPacket, rem); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_storeRHS( + Scalar *B_arr, int64_t LDB, PacketBlock &RHSInPacket, int64_t rem = 0) { + EIGEN_UNUSED_VARIABLE(B_arr); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(rem); + } + + /** + * aux_divRHSByDiag + * + * currM may be -1, (currM >=0) in enable_if checks for this + * + * 1-D unroll + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0 && currM >= 0)> aux_divRHSByDiag( + PacketBlock &RHSInPacket, PacketBlock &AInPacket) { + constexpr int64_t counterReverse = endK - counter; + constexpr int64_t startK = counterReverse; + + constexpr int64_t packetIndex = currM * endK + startK; + RHSInPacket.packet[packetIndex] = pmul(AInPacket.packet[currM], RHSInPacket.packet[packetIndex]); + aux_divRHSByDiag(RHSInPacket, AInPacket); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t 0 && currM >= 0)> aux_divRHSByDiag( + PacketBlock &RHSInPacket, PacketBlock &AInPacket) { + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(AInPacket); + } + + /** + * aux_updateRHS + * + * 2-D unroll + * for(startM = initM; startM < endM; startM++) + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_updateRHS( + Scalar *A_arr, int64_t LDA, PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + constexpr int64_t counterReverse = (endM - initM) * endK - counter; + constexpr int64_t startM = initM + counterReverse / (endK); + constexpr int64_t startK = counterReverse % endK; + + // For each row of A, first update all corresponding RHS + constexpr int64_t packetIndex = startM * endK + startK; + EIGEN_IF_CONSTEXPR(currentM > 0) { + RHSInPacket.packet[packetIndex] = + pnmadd(AInPacket.packet[startM], RHSInPacket.packet[(currentM - 1) * endK + startK], + RHSInPacket.packet[packetIndex]); + } + + EIGEN_IF_CONSTEXPR(startK == endK - 1) { + // Once all RHS for previous row of A is updated, we broadcast the next element in the column A_{i, currentM}. + EIGEN_IF_CONSTEXPR(startM == currentM && !isUnitDiag) { + // If diagonal is not unit, we broadcast reciprocals of diagonals AinPacket.packet[currentM]. + // This will be used in divRHSByDiag + EIGEN_IF_CONSTEXPR(isFWDSolve) + AInPacket.packet[currentM] = pset1(Scalar(1) / A_arr[idA(currentM, currentM, LDA)]); + else AInPacket.packet[currentM] = pset1(Scalar(1) / A_arr[idA(-currentM, -currentM, LDA)]); + } + else { + // Broadcast next off diagonal element of A + EIGEN_IF_CONSTEXPR(isFWDSolve) + AInPacket.packet[startM] = pset1(A_arr[idA(startM, currentM, LDA)]); + else AInPacket.packet[startM] = pset1(A_arr[idA(-startM, -currentM, LDA)]); + } + } + + aux_updateRHS( + A_arr, LDA, RHSInPacket, AInPacket); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_updateRHS( + Scalar *A_arr, int64_t LDA, PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + EIGEN_UNUSED_VARIABLE(A_arr); + EIGEN_UNUSED_VARIABLE(LDA); + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(AInPacket); + } + + /** + * aux_triSolverMicroKernel + * + * 1-D unroll + * for(startM = 0; startM < endM; startM++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_triSolveMicroKernel( + Scalar *A_arr, int64_t LDA, PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + constexpr int64_t counterReverse = endM - counter; + constexpr int64_t startM = counterReverse; + + constexpr int64_t currentM = startM; + // Divides the right-hand side in row startM, by digonal value of A + // broadcasted to AInPacket.packet[startM-1] in the previous iteration. + // + // Without "if constexpr" the compiler instantiates the case <-1, numK> + // this is handled with enable_if to prevent out-of-bound warnings + // from the compiler + EIGEN_IF_CONSTEXPR(!isUnitDiag && startM > 0) + trsm::template divRHSByDiag(RHSInPacket, AInPacket); + + // After division, the rhs corresponding to subsequent rows of A can be partially updated + // We also broadcast the reciprocal of the next diagonal to AInPacket.packet[currentM] (if needed) + // to be used in the next iteration. + trsm::template updateRHS(A_arr, LDA, RHSInPacket, + AInPacket); + + // Handle division for the RHS corresponding to the final row of A. + EIGEN_IF_CONSTEXPR(!isUnitDiag && startM == endM - 1) + trsm::template divRHSByDiag(RHSInPacket, AInPacket); + + aux_triSolveMicroKernel(A_arr, LDA, RHSInPacket, + AInPacket); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_triSolveMicroKernel( + Scalar *A_arr, int64_t LDA, PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + EIGEN_UNUSED_VARIABLE(A_arr); + EIGEN_UNUSED_VARIABLE(LDA); + EIGEN_UNUSED_VARIABLE(RHSInPacket); + EIGEN_UNUSED_VARIABLE(AInPacket); + } + + /******************************************************** + * Wrappers for aux_XXXX to hide counter parameter + ********************************************************/ + + /** + * Load endMxendK block of B to RHSInPacket + * Masked loads are used for cases where endK is not a multiple of PacketSize + */ + template + static EIGEN_ALWAYS_INLINE void loadRHS(Scalar *B_arr, int64_t LDB, + PacketBlock &RHSInPacket, int64_t rem = 0) { + aux_loadRHS(B_arr, LDB, RHSInPacket, rem); + } + + /** + * Load endMxendK block of B to RHSInPacket + * Masked loads are used for cases where endK is not a multiple of PacketSize + */ + template + static EIGEN_ALWAYS_INLINE void storeRHS(Scalar *B_arr, int64_t LDB, + PacketBlock &RHSInPacket, int64_t rem = 0) { + aux_storeRHS(B_arr, LDB, RHSInPacket, rem); + } + + /** + * Only used if Triangular matrix has non-unit diagonal values + */ + template + static EIGEN_ALWAYS_INLINE void divRHSByDiag(PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + aux_divRHSByDiag(RHSInPacket, AInPacket); + } + + /** + * Update right-hand sides (stored in avx registers) + * Traversing along the column A_{i,currentM}, where currentM <= i <= endM, and broadcasting each value to AInPacket. + **/ + template + static EIGEN_ALWAYS_INLINE void updateRHS(Scalar *A_arr, int64_t LDA, + PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + aux_updateRHS( + A_arr, LDA, RHSInPacket, AInPacket); + } + + /** + * endM: dimension of A. 1 <= endM <= EIGEN_AVX_MAX_NUM_ROW + * numK: number of avx registers to use for each row of B (ex fp32: 48 rhs => 3 avx reg used). 1 <= endK <= 3. + * isFWDSolve: true => forward substitution, false => backwards substitution + * isUnitDiag: true => triangular matrix has unit diagonal. + */ + template + static EIGEN_ALWAYS_INLINE void triSolveMicroKernel(Scalar *A_arr, int64_t LDA, + PacketBlock &RHSInPacket, + PacketBlock &AInPacket) { + static_assert(numK >= 1 && numK <= 3, "numK out of range"); + aux_triSolveMicroKernel(A_arr, LDA, RHSInPacket, AInPacket); + } +}; + +/** + * Unrolls for gemm kernel + * + * isAdd: true => C += A*B, false => C -= A*B + */ +template +class gemm { + public: + using vec = typename std::conditional::value, vecFullFloat, vecFullDouble>::type; + static constexpr int64_t PacketSize = packet_traits::size; + + /*********************************** + * Auxiliary Functions for: + * - setzero + * - updateC + * - storeC + * - startLoadB + * - triSolveMicroKernel + ************************************/ + + /** + * aux_setzero + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_setzero( + PacketBlock &zmm) { + constexpr int64_t counterReverse = endM * endN - counter; + constexpr int64_t startM = counterReverse / (endN); + constexpr int64_t startN = counterReverse % endN; + + zmm.packet[startN * endM + startM] = pzero(zmm.packet[startN * endM + startM]); + aux_setzero(zmm); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_setzero( + PacketBlock &zmm) { + EIGEN_UNUSED_VARIABLE(zmm); + } + + /** + * aux_updateC + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_updateC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + constexpr int64_t counterReverse = endM * endN - counter; + constexpr int64_t startM = counterReverse / (endN); + constexpr int64_t startN = counterReverse % endN; + + EIGEN_IF_CONSTEXPR(rem) + zmm.packet[startN * endM + startM] = + padd(ploadu(&C_arr[(startN)*LDC + startM * PacketSize], remMask(rem_)), + zmm.packet[startN * endM + startM], remMask(rem_)); + else zmm.packet[startN * endM + startM] = + padd(ploadu(&C_arr[(startN)*LDC + startM * PacketSize]), zmm.packet[startN * endM + startM]); + aux_updateC(C_arr, LDC, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_updateC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(C_arr); + EIGEN_UNUSED_VARIABLE(LDC); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_storeC + * + * 2-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startN = 0; startN < endN; startN++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_storeC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + constexpr int64_t counterReverse = endM * endN - counter; + constexpr int64_t startM = counterReverse / (endN); + constexpr int64_t startN = counterReverse % endN; + + EIGEN_IF_CONSTEXPR(rem) + pstoreu(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM], + remMask(rem_)); + else pstoreu(&C_arr[(startN)*LDC + startM * PacketSize], zmm.packet[startN * endM + startM]); + aux_storeC(C_arr, LDC, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_storeC( + Scalar *C_arr, int64_t LDC, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(C_arr); + EIGEN_UNUSED_VARIABLE(LDC); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_startLoadB + * + * 1-D unroll + * for(startL = 0; startL < endL; startL++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_startLoadB( + Scalar *B_t, int64_t LDB, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + constexpr int64_t counterReverse = endL - counter; + constexpr int64_t startL = counterReverse; + + EIGEN_IF_CONSTEXPR(rem) + zmm.packet[unrollM * unrollN + startL] = + ploadu(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize], remMask(rem_)); + else zmm.packet[unrollM * unrollN + startL] = + ploadu(&B_t[(startL / unrollM) * LDB + (startL % unrollM) * PacketSize]); + + aux_startLoadB(B_t, LDB, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_startLoadB( + Scalar *B_t, int64_t LDB, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(B_t); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_startBCastA + * + * 1-D unroll + * for(startB = 0; startB < endB; startB++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_startBCastA( + Scalar *A_t, int64_t LDA, PacketBlock &zmm) { + constexpr int64_t counterReverse = endB - counter; + constexpr int64_t startB = counterReverse; + + zmm.packet[unrollM * unrollN + numLoad + startB] = pload1(&A_t[idA(startB, 0, LDA)]); + + aux_startBCastA(A_t, LDA, zmm); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_startBCastA( + Scalar *A_t, int64_t LDA, PacketBlock &zmm) { + EIGEN_UNUSED_VARIABLE(A_t); + EIGEN_UNUSED_VARIABLE(LDA); + EIGEN_UNUSED_VARIABLE(zmm); + } + + /** + * aux_loadB + * currK: current K + * + * 1-D unroll + * for(startM = 0; startM < endM; startM++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_loadB( + Scalar *B_t, int64_t LDB, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + if ((numLoad / endM + currK < unrollK)) { + constexpr int64_t counterReverse = endM - counter; + constexpr int64_t startM = counterReverse; + + EIGEN_IF_CONSTEXPR(rem) { + zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] = + ploadu(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize], remMask(rem_)); + } + else { + zmm.packet[endM * unrollN + (startM + currK * endM) % numLoad] = + ploadu(&B_t[(numLoad / endM + currK) * LDB + startM * PacketSize]); + } + + aux_loadB(B_t, LDB, zmm, rem_); + } + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_loadB( + Scalar *B_t, int64_t LDB, PacketBlock &zmm, int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(B_t); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /** + * aux_microKernel + * + * 3-D unroll + * for(startM = 0; startM < endM; startM++) + * for(startN = 0; startN < endN; startN++) + * for(startK = 0; startK < endK; startK++) + **/ + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter > 0)> aux_microKernel( + Scalar *B_t, Scalar *A_t, int64_t LDB, int64_t LDA, PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + constexpr int64_t counterReverse = endM * endN * endK - counter; + constexpr int startK = counterReverse / (endM * endN); + constexpr int startN = (counterReverse / (endM)) % endN; + constexpr int startM = counterReverse % endM; + + EIGEN_IF_CONSTEXPR(startK == 0 && startM == 0 && startN == 0) { + gemm::template startLoadB(B_t, LDB, zmm, rem_); + gemm::template startBCastA(A_t, LDA, zmm); + } + + { + // Interleave FMA and Bcast + EIGEN_IF_CONSTEXPR(isAdd) { + zmm.packet[startN * endM + startM] = + pmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast], + zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]); + } + else { + zmm.packet[startN * endM + startM] = + pnmadd(zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast], + zmm.packet[endM * endN + (startM + startK * endM) % numLoad], zmm.packet[startN * endM + startM]); + } + // Bcast + EIGEN_IF_CONSTEXPR(startM == endM - 1 && (numBCast + startN + startK * endN < endK * endN)) { + zmm.packet[endM * endN + numLoad + (startN + startK * endN) % numBCast] = pload1(&A_t[idA( + (numBCast + startN + startK * endN) % endN, (numBCast + startN + startK * endN) / endN, LDA)]); + } + } + + // We have updated all accumulators, time to load next set of B's + EIGEN_IF_CONSTEXPR((startN == endN - 1) && (startM == endM - 1)) { + gemm::template loadB(B_t, LDB, zmm, rem_); + } + aux_microKernel(B_t, A_t, LDB, LDA, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE std::enable_if_t<(counter <= 0)> aux_microKernel( + Scalar *B_t, Scalar *A_t, int64_t LDB, int64_t LDA, PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(B_t); + EIGEN_UNUSED_VARIABLE(A_t); + EIGEN_UNUSED_VARIABLE(LDB); + EIGEN_UNUSED_VARIABLE(LDA); + EIGEN_UNUSED_VARIABLE(zmm); + EIGEN_UNUSED_VARIABLE(rem_); + } + + /******************************************************** + * Wrappers for aux_XXXX to hide counter parameter + ********************************************************/ + + template + static EIGEN_ALWAYS_INLINE void setzero(PacketBlock &zmm) { + aux_setzero(zmm); + } + + /** + * Ideally the compiler folds these into vaddp{s,d} with an embedded memory load. + */ + template + static EIGEN_ALWAYS_INLINE void updateC(Scalar *C_arr, int64_t LDC, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_updateC(C_arr, LDC, zmm, rem_); + } + + template + static EIGEN_ALWAYS_INLINE void storeC(Scalar *C_arr, int64_t LDC, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_storeC(C_arr, LDC, zmm, rem_); + } + + /** + * Use numLoad registers for loading B at start of microKernel + */ + template + static EIGEN_ALWAYS_INLINE void startLoadB(Scalar *B_t, int64_t LDB, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_startLoadB(B_t, LDB, zmm, rem_); + } + + /** + * Use numBCast registers for broadcasting A at start of microKernel + */ + template + static EIGEN_ALWAYS_INLINE void startBCastA(Scalar *A_t, int64_t LDA, + PacketBlock &zmm) { + aux_startBCastA(A_t, LDA, zmm); + } + + /** + * Loads next set of B into vector registers between each K unroll. + */ + template + static EIGEN_ALWAYS_INLINE void loadB(Scalar *B_t, int64_t LDB, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_loadB(B_t, LDB, zmm, rem_); + } + + /** + * Generates a microkernel for gemm (row-major) with unrolls {1,2,4,8}x{U1,U2,U3} to compute C -= A*B. + * A matrix can be row/col-major. B matrix is assumed row-major. + * + * isARowMajor: is A row major + * endM: Number registers per row + * endN: Number of rows + * endK: Loop unroll for K. + * numLoad: Number of registers for loading B. + * numBCast: Number of registers for broadcasting A. + * + * Ex: microkernel: 8x48 unroll (24 accumulators), k unrolled 4 times, + * 6 register for loading B, 2 for broadcasting A. + * + * Note: Ideally the microkernel should not have any register spilling. + * The avx instruction counts should be: + * - endK*endN vbroadcasts{s,d} + * - endK*endM vmovup{s,d} + * - endK*endN*endM FMAs + * + * From testing, there are no register spills with clang. There are register spills with GNU, which + * causes a performance hit. + */ + template + static EIGEN_ALWAYS_INLINE void microKernel(Scalar *B_t, Scalar *A_t, int64_t LDB, int64_t LDA, + PacketBlock &zmm, + int64_t rem_ = 0) { + EIGEN_UNUSED_VARIABLE(rem_); + aux_microKernel(B_t, A_t, LDB, LDA, zmm, + rem_); + } +}; +} // namespace unrolls + +#endif // EIGEN_CORE_ARCH_AVX512_TRSM_UNROLLS_H diff --git a/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h b/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h new file mode 100644 index 0000000..f06f13d --- /dev/null +++ b/Eigen/src/Core/arch/AVX512/TypeCastingFP16.h @@ -0,0 +1,130 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2025 The Eigen Authors. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TYPE_CASTING_FP16_AVX512_H +#define EIGEN_TYPE_CASTING_FP16_AVX512_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { +namespace internal { + +template <> +EIGEN_STRONG_INLINE Packet32s preinterpret(const Packet32h& a) { + return _mm512_castph_si512(a); +} +template <> +EIGEN_STRONG_INLINE Packet16s preinterpret(const Packet16h& a) { + return _mm256_castph_si256(a); +} +template <> +EIGEN_STRONG_INLINE Packet8s preinterpret(const Packet8h& a) { + return _mm_castph_si128(a); +} + +template <> +EIGEN_STRONG_INLINE Packet32h preinterpret(const Packet32s& a) { + return _mm512_castsi512_ph(a); +} +template <> +EIGEN_STRONG_INLINE Packet16h preinterpret(const Packet16s& a) { + return _mm256_castsi256_ph(a); +} +template <> +EIGEN_STRONG_INLINE Packet8h preinterpret(const Packet8s& a) { + return _mm_castsi128_ph(a); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pcast(const Packet16h& a) { + return half2float(a); +} +template <> +EIGEN_STRONG_INLINE Packet8f pcast(const Packet8h& a) { + return half2float(a); +} + +template <> +EIGEN_STRONG_INLINE Packet16h pcast(const Packet16f& a) { + return float2half(a); +} +template <> +EIGEN_STRONG_INLINE Packet8h pcast(const Packet8f& a) { + return float2half(a); +} + +template <> +EIGEN_STRONG_INLINE Packet16f pcast(const Packet32h& a) { + // Discard second-half of input. + Packet16h low = _mm256_castpd_ph(_mm512_extractf64x4_pd(_mm512_castph_pd(a), 0)); + return _mm512_cvtxph_ps(low); +} +template <> +EIGEN_STRONG_INLINE Packet8f pcast(const Packet16h& a) { + // Discard second-half of input. + Packet8h low = _mm_castps_ph(_mm256_extractf32x4_ps(_mm256_castph_ps(a), 0)); + return _mm256_cvtxph_ps(low); +} +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet8h& a) { + Packet8f full = _mm256_cvtxph_ps(a); + // Discard second-half of input. + return _mm256_extractf32x4_ps(full, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet32h pcast(const Packet16f& a, const Packet16f& b) { + __m512 result = _mm512_castsi512_ps(_mm512_castsi256_si512(_mm256_castph_si256(_mm512_cvtxps_ph(a)))); + result = _mm512_insertf32x8(result, _mm256_castph_ps(_mm512_cvtxps_ph(b)), 1); + return _mm512_castps_ph(result); +} +template <> +EIGEN_STRONG_INLINE Packet16h pcast(const Packet8f& a, const Packet8f& b) { + __m256 result = _mm256_castsi256_ps(_mm256_castsi128_si256(_mm_castph_si128(_mm256_cvtxps_ph(a)))); + result = _mm256_insertf32x4(result, _mm_castph_ps(_mm256_cvtxps_ph(b)), 1); + return _mm256_castps_ph(result); +} +template <> +EIGEN_STRONG_INLINE Packet8h pcast(const Packet4f& a, const Packet4f& b) { + __m256 result = _mm256_castsi256_ps(_mm256_castsi128_si256(_mm_castps_si128(a))); + result = _mm256_insertf128_ps(result, b, 1); + return _mm256_cvtxps_ph(result); +} + +template <> +EIGEN_STRONG_INLINE Packet32s pcast(const Packet32h& a) { + return _mm512_cvtph_epi16(a); +} +template <> +EIGEN_STRONG_INLINE Packet16s pcast(const Packet16h& a) { + return _mm256_cvtph_epi16(a); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet8h& a) { + return _mm_cvtph_epi16(a); +} + +template <> +EIGEN_STRONG_INLINE Packet32h pcast(const Packet32s& a) { + return _mm512_cvtepi16_ph(a); +} +template <> +EIGEN_STRONG_INLINE Packet16h pcast(const Packet16s& a) { + return _mm256_cvtepi16_ph(a); +} +template <> +EIGEN_STRONG_INLINE Packet8h pcast(const Packet8s& a) { + return _mm_cvtepi16_ph(a); +} + +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_TYPE_CASTING_FP16_AVX512_H diff --git a/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h b/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h new file mode 100644 index 0000000..6ecec0e --- /dev/null +++ b/Eigen/src/Core/arch/AltiVec/MatrixProductMMAbfloat16.h @@ -0,0 +1,742 @@ +#ifndef EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H +#define EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H + +#if EIGEN_COMP_LLVM +#define BFLOAT16_UNROLL _Pragma("unroll 8") +#else +#define BFLOAT16_UNROLL _Pragma("GCC unroll(8)") +#endif + +namespace Eigen { + +namespace internal { + +template +EIGEN_ALWAYS_INLINE Packet8bf loadBfloat16(const bfloat16* indexA) { + Packet8bf lhs1 = ploadu(indexA); + if (zero) { + Packet8bf lhs2 = pset1(Eigen::bfloat16(0)); + return vec_mergeh(lhs1.m_val, lhs2.m_val); + } else { + return lhs1; + } +} + +template +EIGEN_ALWAYS_INLINE Packet8bf loadRhsBfloat16(const bfloat16* blockB, Index strideB, Index i) { + return loadBfloat16(blockB + strideB * i); +} + +template +EIGEN_ALWAYS_INLINE void KLoop(const bfloat16* indexA, const bfloat16* indexB, __vector_quad (&quad_acc)[num_acc], + Index strideB, Index k, Index offsetB, Index extra_cols, Index extra_rows) { + Packet8bf lhs[num_lhs], rhs[num_rhs]; + + BFLOAT16_UNROLL + for (Index i = 0; i < (num_rhs - (rhsExtraCols ? 1 : 0)); i++) { + rhs[i] = loadRhsBfloat16(indexB + k * 4, strideB, i); + } + if (rhsExtraCols) { + rhs[num_rhs - 1] = loadRhsBfloat16(indexB + k * extra_cols - offsetB, strideB, num_rhs - 1); + } + + indexA += k * (lhsExtraRows ? extra_rows : num_packets); + if (num_lhs == 1) { + lhs[0] = loadBfloat16(indexA); + } else { + BFLOAT16_UNROLL + for (Index j = 0; j < num_lhs; j += 2) { + Packet8bf lhs1 = ploadu(indexA + (j + 0) * (zero ? 4 : 8)); + if (zero) { + Packet8bf lhs2 = pset1(Eigen::bfloat16(0)); + lhs[j + 0] = vec_mergeh(lhs1.m_val, lhs2.m_val); + lhs[j + 1] = vec_mergel(lhs1.m_val, lhs2.m_val); + } else { + lhs[j + 0] = lhs1; + lhs[j + 1] = ploadu(indexA + (j + 1) * 8); + } + } + } + + BFLOAT16_UNROLL + for (Index i = 0, x = 0; i < num_rhs; i++) { + BFLOAT16_UNROLL + for (Index j = 0; j < num_lhs; j++, x++) { + __builtin_mma_xvbf16ger2pp(&(quad_acc[x]), reinterpret_cast(rhs[i].m_val), + reinterpret_cast(lhs[j].m_val)); + } + } +} + +template +EIGEN_ALWAYS_INLINE void zeroAccumulators(__vector_quad (&quad_acc)[num_acc]) { + BFLOAT16_UNROLL + for (Index k = 0; k < num_acc; k++) __builtin_mma_xxsetaccz(&(quad_acc[k])); +} + +template +EIGEN_ALWAYS_INLINE void disassembleAccumulators(__vector_quad (&quad_acc)[num_acc], Packet4f (&acc)[num_acc][4]) { + BFLOAT16_UNROLL + for (Index k = 0; k < num_acc; k++) __builtin_mma_disassemble_acc((void*)acc[k], &(quad_acc[k])); +} + +template +EIGEN_ALWAYS_INLINE void outputResults(Packet4f (&acc)[num_acc][4], Index rows, const Packet4f pAlpha, float* result, + const Index extra_cols, Index extra_rows) { + BFLOAT16_UNROLL + for (Index i = 0, k = 0; i < num_rhs - (rhsExtraCols ? 1 : 0); i++, result += 4 * rows) { + BFLOAT16_UNROLL + for (Index j = 0; j < num_lhs; j++, k++) { + storeResults(acc[k], rows, pAlpha, result + j * 4, extra_cols, extra_rows); + } + } + if (rhsExtraCols) { + storeResults(acc[num_acc - 1], rows, pAlpha, result, extra_cols, extra_rows); + } +} + +template +EIGEN_ALWAYS_INLINE void colLoopBodyIter(Index depth, Index rows, const Packet4f pAlpha, const bfloat16* indexA, + const bfloat16* indexB, Index strideB, Index offsetB, float* result, + const Index extra_cols, const Index extra_rows) { + constexpr Index num_lhs = multiIter ? (num_packets / 4) : 1; + constexpr Index num_rhs = (num_acc + num_lhs - 1) / num_lhs; + + for (Index offset_row = 0; offset_row < num_packets; offset_row += 4, indexA += (multiIter ? 0 : 8), + indexB += (multiIter ? (num_rhs * strideB) : 0), result += (multiIter ? (4 * rows * num_rhs) : 4)) { + Packet4f acc[num_acc][4]; + __vector_quad quad_acc[num_acc]; + + zeroAccumulators(quad_acc); + + Index k; + for (k = 0; k + 2 <= depth; k += 2) { + KLoop( + indexA, indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows); + } + if (depth & 1) { + KLoop( + indexA - (multiIter ? 0 : offset_row), indexB, quad_acc, strideB, k, offsetB, extra_cols, extra_rows); + } + + disassembleAccumulators(quad_acc, acc); + + outputResults(acc, rows, pAlpha, result, extra_cols, + extra_rows); + } +} + +#define MAX_BFLOAT16_ACC 8 + +template +void colLoopBody(Index& col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA, + const bfloat16* indexB, Index strideB, Index offsetB, float* result) { + constexpr Index step = (num_acc * 4); // each accumulator has 4 elements + const Index extra_cols = (rhsExtraCols) ? (cols & 3) : 0; + const Index extra_rows = (lhsExtraRows) ? (rows & 3) : 0; + constexpr bool multiIters = !rhsExtraCols && (num_acc == MAX_BFLOAT16_ACC); + constexpr bool normIters = multiIters && ((num_acc % (num_packets / 4)) == 0); + + do { + colLoopBodyIter( + depth, rows, pAlpha, indexA, indexB, strideB, offsetB, result, extra_cols, extra_rows); + + indexB += strideB * num_acc; + result += rows * step; + } while (multiIters && (step <= cols - (col += step))); +} + +template +EIGEN_ALWAYS_INLINE void colLoopBodyExtraN(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, + const bfloat16* indexA, const bfloat16* blockB, Index strideB, Index offsetB, + float* result) { + if (MAX_BFLOAT16_ACC > num_acc) { + colLoopBody( + col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, result); + } +} + +template +void colLoopBodyExtra(Index col, Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA, + const bfloat16* blockB, Index strideB, Index offsetB, float* result) { + switch ((cols - col) >> 2) { + case 7: + colLoopBodyExtraN<7, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, + strideB, offsetB, result); + break; + case 6: + colLoopBodyExtraN<6, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, + strideB, offsetB, result); + break; + case 5: + colLoopBodyExtraN<5, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, + strideB, offsetB, result); + break; + case 4: + colLoopBodyExtraN<4, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, + strideB, offsetB, result); + break; + case 3: + colLoopBodyExtraN<3, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, + strideB, offsetB, result); + break; + case 2: + colLoopBodyExtraN<2, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, + strideB, offsetB, result); + break; + case 1: + colLoopBodyExtraN<1, num_packets, rhsExtraCols, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, + strideB, offsetB, result); + break; + default: + if (rhsExtraCols) { + colLoopBody<1, num_packets, true, lhsExtraRows>(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, + offsetB, result); + } + break; + } +} + +template +EIGEN_ALWAYS_INLINE void colLoops(Index depth, Index cols, Index rows, const Packet4f pAlpha, const bfloat16* indexA, + const bfloat16* blockB, Index strideB, Index offsetB, float* result) { + Index col = 0; + if (cols >= (MAX_BFLOAT16_ACC * 4)) { + colLoopBody(col, depth, cols, rows, pAlpha, indexA, blockB, + strideB, 0, result); + blockB += (strideB >> 2) * col; + result += rows * col; + } + if (cols & 3) { + colLoopBodyExtra(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, offsetB, + result); + } else { + colLoopBodyExtra(col, depth, cols, rows, pAlpha, indexA, blockB, strideB, 0, + result); + } +} + +EIGEN_ALWAYS_INLINE Packet8bf convertF32toBF16(const float* res) { + Packet16uc fp16[2]; + __vector_pair fp16_vp = *reinterpret_cast<__vector_pair*>(const_cast(res)); + __builtin_vsx_disassemble_pair(reinterpret_cast(fp16), &fp16_vp); + fp16[0] = __builtin_vsx_xvcvspbf16(fp16[0]); + fp16[1] = __builtin_vsx_xvcvspbf16(fp16[1]); + return vec_pack(reinterpret_cast(fp16[0]), reinterpret_cast(fp16[1])); +} + +template +EIGEN_ALWAYS_INLINE void convertArrayF32toBF16Col(float* result, Index col, Index rows, const DataMapper& res) { + const DataMapper res2 = res.getSubMapper(0, col); + Index row; + float* result2 = result + col * rows; + for (row = 0; row + 8 <= rows; row += 8, result2 += 8) { + // get and save block + PacketBlock block; + BFLOAT16_UNROLL + for (Index j = 0; j < size; j++) { + block.packet[j] = convertF32toBF16(result2 + j * rows); + } + res2.template storePacketBlock(row, 0, block); + } + // extra rows + if (row < rows) { + BFLOAT16_UNROLL + for (Index j = 0; j < size; j++) { + Packet8bf fp16 = convertF32toBF16(result2 + j * rows); + res2.template storePacketPartial(row, j, fp16, rows & 7); + } + } +} + +template +EIGEN_ALWAYS_INLINE void convertPointerF32toBF16(Index& i, float* result, Index rows, bfloat16*& dst, + Index resInc = 1) { + constexpr Index extra = ((size < 8) ? 8 : size); + while (i + size <= rows) { + PacketBlock r32; + r32.packet[0] = convertF32toBF16(result + i + 0); + if (size >= 16) { + r32.packet[1] = convertF32toBF16(result + i + 8); + } + if (size >= 32) { + r32.packet[2] = convertF32toBF16(result + i + 16); + r32.packet[3] = convertF32toBF16(result + i + 24); + } + storeBF16fromResult(dst, r32.packet[0], resInc, rows & 7); + if (size >= 16) { + storeBF16fromResult(dst, r32.packet[1], resInc); + } + if (size >= 32) { + storeBF16fromResult(dst, r32.packet[2], resInc); + storeBF16fromResult(dst, r32.packet[3], resInc); + } + i += extra; + dst += extra * resInc; + if (size != 32) break; + } +} + +template +EIGEN_ALWAYS_INLINE void convertArrayPointerF32toBF16(float* result, Index rows, bfloat16* dst, Index resInc = 1) { + Index i = 0; + convertPointerF32toBF16<32, non_unit_stride>(i, result, rows, dst, resInc); + convertPointerF32toBF16<16, non_unit_stride>(i, result, rows, dst, resInc); + convertPointerF32toBF16<8, non_unit_stride>(i, result, rows, dst, resInc); + convertPointerF32toBF16<1, non_unit_stride>(i, result, rows, dst, resInc); +} + +template +EIGEN_ALWAYS_INLINE void convertArrayF32toBF16(float* result, Index cols, Index rows, const DataMapper& res) { + Index col; + for (col = 0; col + 4 <= cols; col += 4) { + convertArrayF32toBF16Col(result, col, rows, res); + } + // extra cols + switch (cols - col) { + case 1: + convertArrayF32toBF16Col(result, col, rows, res); + break; + case 2: + convertArrayF32toBF16Col(result, col, rows, res); + break; + case 3: + convertArrayF32toBF16Col(result, col, rows, res); + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcColLoops(const bfloat16*& indexA, Index& row, Index depth, Index cols, Index rows, + const Packet4f pAlpha, const bfloat16* indexB, Index strideB, Index offsetA, + Index offsetB, Index bigSuffix, float* result) { + if ((size == 16) || (rows & size)) { + indexA += size * offsetA; + colLoops(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row); + row += size; + indexA += bigSuffix * size / 16; + } +} + +template +void gemmMMAbfloat16(const DataMapper& res, const bfloat16* indexA, const bfloat16* indexB, Index rows, Index depth, + Index cols, bfloat16 alpha, Index strideA, Index strideB, Index offsetA, Index offsetB) { + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + const Packet4f pAlpha = pset1(falpha); + ei_declare_aligned_stack_constructed_variable(float, result, cols* rows, 0); + + convertArrayBF16toF32(result, cols, rows, res); + + if (strideA == -1) strideA = depth; + if (strideB == -1) strideB = depth; + // Packing is done in blocks. + // There's 4 possible sizes of blocks + // Blocks of 8 columns with 16 elements (8x16) + // Blocks of 8 columns with 8 elements (8x8). This happens when there's 16 > rows >= 8 + // Blocks of 8 columns with 4 elements (8x4). This happens when there's 8 > rows >= 4 + // Blocks of 8 columns with < 4 elements. This happens when there's less than 4 remaining rows + + // Loop for LHS standard block (8x16) + Index bigSuffix = (2 * 8) * (strideA - offsetA); + indexB += 4 * offsetB; + strideB *= 4; + offsetB *= 3; + + Index row = 0; + while (row + 16 <= rows) { + calcColLoops<16>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result); + } + // LHS (8x8) block + calcColLoops<8>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result); + // LHS (8x4) block + calcColLoops<4>(indexA, row, depth, cols, rows, pAlpha, indexB, strideB, offsetA, offsetB, bigSuffix, result); + // extra rows + if (rows & 3) { + // This index is the beginning of remaining block. + colLoops<4, true>(depth, cols, rows, pAlpha, indexA, indexB, strideB, offsetB, result + row); + } + + // Convert back to bfloat16 + convertArrayF32toBF16(result, cols, rows, res); +} + +#undef MAX_BFLOAT16_ACC + +#if !EIGEN_ALTIVEC_DISABLE_MMA +template +EIGEN_ALWAYS_INLINE void loadVecLoop(Index k, LhsMapper& lhs, Packet8bf (&a0)[num_acc], Packet8bf b1) { + a0[k + 0] = lhs.template loadPacket(k * 4, 0); + if (!zero) { + b1 = lhs.template loadPacket(k * 4, 1); + } + if (num_acc > (k + 1)) { + a0[k + 1] = vec_mergel(a0[k + 0].m_val, b1.m_val); + } + a0[k + 0] = vec_mergeh(a0[k + 0].m_val, b1.m_val); +} + +template +EIGEN_ALWAYS_INLINE void multVec(__vector_quad (&quad_acc)[num_acc], Packet8bf (&a0)[num_acc], Packet8bf b0) { + BFLOAT16_UNROLL + for (Index k = 0; k < num_acc; k++) { + __builtin_mma_xvbf16ger2pp(&(quad_acc[k]), reinterpret_cast(b0.m_val), + reinterpret_cast(a0[k].m_val)); + } +} + +template +EIGEN_ALWAYS_INLINE void vecColLoop(Index j, LhsMapper& lhs, RhsMapper& rhs, __vector_quad (&quad_acc)[num_acc]) { + Packet8bf a0[num_acc]; + Packet8bf b1 = pset1(Eigen::bfloat16(0)); + Packet8bf b0 = loadColData(rhs, j); + + if (zero) { + b0 = vec_mergeh(b0.m_val, b1.m_val); + } + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(0, j); + BFLOAT16_UNROLL + for (Index k = 0; k < num_acc; k += 2) { + loadVecLoop(k, lhs2, a0, b1); + } + + multVec(quad_acc, a0, b0); +} + +#define MAX_BFLOAT16_VEC_ACC 8 + +template +void colVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, + float* result) { + constexpr Index step = (num_acc * 4); + const Index extra_rows = (extraRows) ? (rows & 3) : 0; + constexpr bool multiIters = !extraRows && (num_acc == MAX_BFLOAT16_VEC_ACC); + + do { + Packet4f acc[num_acc][4]; + __vector_quad quad_acc[num_acc]; + + zeroAccumulators(quad_acc); + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(row, 0); + for (Index j = 0; j + 2 <= cend; j += 2) { + vecColLoop(j, lhs2, rhs, quad_acc); + } + if (cend & 1) { + vecColLoop(cend - 1, lhs2, rhs, quad_acc); + } + + disassembleAccumulators(quad_acc, acc); + + outputVecColResults(acc, result, pAlpha, extra_rows); + + result += step; + } while (multiIters && (step <= rows - (row += step))); +} + +template +EIGEN_ALWAYS_INLINE void colVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, + const Packet4f pAlpha, float* result) { + if (MAX_BFLOAT16_VEC_ACC > num_acc) { + colVecColLoopBody(row, cend, rows, lhs, rhs, + pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void colVecColLoopBodyExtra(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, + const Packet4f pAlpha, float* result) { + switch ((rows - row) >> 2) { + case 7: + colVecColLoopBodyExtraN<7, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 6: + colVecColLoopBodyExtraN<6, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 5: + colVecColLoopBodyExtraN<5, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 4: + colVecColLoopBodyExtraN<4, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 3: + colVecColLoopBodyExtraN<3, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 2: + colVecColLoopBodyExtraN<2, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 1: + colVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + default: + if (extraRows) { + colVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + } + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcVecColLoops(Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, + float* result) { + Index row = 0; + if (rows >= (MAX_BFLOAT16_VEC_ACC * 4)) { + colVecColLoopBody(row, cend, rows, lhs, rhs, pAlpha, + result); + result += row; + } + if (rows & 3) { + colVecColLoopBodyExtra(row, cend, rows, lhs, rhs, pAlpha, result); + } else { + colVecColLoopBodyExtra(row, cend, rows, lhs, rhs, pAlpha, result); + } +} + +template +struct UseMMAStride : std::false_type { + static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha, + float* result) { + using RhsSubMapper = typename RhsMapper::SubMapper; + + RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0); + calcVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } +}; + +template +struct UseMMAStride::value>> + : std::true_type { + static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha, + float* result) { + using RhsSubMapper = typename RhsMapper::SubMapper; + + RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0); + if (rhs.stride() == 1) { + calcVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } else { + calcVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } + } +}; + +template +void gemvMMA_bfloat16_col(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, bfloat16* res, + Index resIncr, bfloat16 alpha) { + EIGEN_UNUSED_VARIABLE(resIncr); + eigen_internal_assert(resIncr == 1); + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + RhsMapper rhs2(rhs); + + const Index lhsStride = lhs.stride(); + + // TODO: improve the following heuristic: + const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(bfloat16) < 16000 ? 16 : 8); + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + Packet4f pAlpha = pset1(falpha); + + ei_declare_aligned_stack_constructed_variable(float, result, rows, 0); + + convertArrayPointerBF16toF32(result, 1, rows, res); + + for (Index j2 = 0; j2 < cols; j2 += block_cols) { + Index jend = numext::mini(j2 + block_cols, cols); + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(0, j2); + UseMMAStride::run(j2, jend, rows, lhs2, rhs2, pAlpha, result); + } + + convertArrayPointerF32toBF16(result, rows, res); +} + +static Packet16uc p16uc_ELEMENT_VEC3 = {0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f, + 0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f}; + +template +EIGEN_ALWAYS_INLINE void preduxVecResults2(Packet4f (&acc)[num_acc][4], Index k) { + if (num_acc > (k + 1)) { + acc[k][0] = vec_mergeh(acc[k][0], acc[k + 1][0]); + acc[k][1] = vec_mergeo(acc[k][1], acc[k + 1][1]); + acc[k][2] = vec_mergel(acc[k][2], acc[k + 1][2]); + acc[k][3] = vec_perm(acc[k][3], acc[k + 1][3], p16uc_ELEMENT_VEC3); + + acc[k][0] = (acc[k][0] + acc[k][2]) + (acc[k][1] + acc[k][3]); + } else { + acc[k][0] = vec_mergeh(acc[k][0], acc[k][1]); + acc[k][0] += vec_mergel(acc[k][2], acc[k][3]); +#ifdef _BIG_ENDIAN + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12); +#else + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 4); +#endif + } +} + +template +EIGEN_ALWAYS_INLINE void preduxVecResults(Packet4f (&acc)[num_acc][4]) { + BFLOAT16_UNROLL + for (Index k = 0; k < num_acc; k += 4) { + preduxVecResults2(acc, k + 0); + if (num_acc > (k + 2)) { + preduxVecResults2(acc, k + 2); + acc[k + 0][0] = reinterpret_cast( + vec_mergeh(reinterpret_cast(acc[k + 0][0]), reinterpret_cast(acc[k + 2][0]))); + } + } +} + +template +EIGEN_ALWAYS_INLINE void multVecLoop(__vector_quad (&quad_acc)[num_acc], const LhsMapper& lhs, RhsMapper& rhs, Index j, + Index extra_cols) { + Packet8bf a0[num_acc], b0; + + if (extra) { + b0 = rhs.template loadPacketPartial(j, extra_cols); + } else { + b0 = rhs.template loadPacket(j); + } + + const LhsMapper lhs2 = lhs.getSubMapper(0, j); + BFLOAT16_UNROLL + for (Index k = 0; k < num_acc; k++) { + if (extra) { + a0[k] = lhs2.template loadPacketPartial(k, 0, extra_cols); + } else { + a0[k] = lhs2.template loadPacket(k, 0); + } + } + + multVec(quad_acc, a0, b0); +} + +template +EIGEN_ALWAYS_INLINE void vecLoop(Index cols, const LhsMapper& lhs, RhsMapper& rhs, __vector_quad (&quad_acc)[num_acc], + Index extra_cols) { + Index j = 0; + for (; j + 8 <= cols; j += 8) { + multVecLoop(quad_acc, lhs, rhs, j, extra_cols); + } + + if (extra_cols) { + multVecLoop(quad_acc, lhs, rhs, j, extra_cols); + } +} + +template +void colVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, + float* result) { + constexpr bool multiIters = (num_acc == MAX_BFLOAT16_VEC_ACC); + const Index extra_cols = (cols & 7); + + do { + Packet4f acc[num_acc][4]; + __vector_quad quad_acc[num_acc]; + + zeroAccumulators(quad_acc); + + const LhsMapper lhs2 = lhs.getSubMapper(row, 0); + vecLoop(cols, lhs2, rhs, quad_acc, extra_cols); + + disassembleAccumulators(quad_acc, acc); + + preduxVecResults(acc); + + outputVecResults(acc, result, pAlpha); + + result += num_acc; + } while (multiIters && (num_acc <= rows - (row += num_acc))); +} + +template +EIGEN_ALWAYS_INLINE void colVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, + const Packet4f pAlpha, float* result) { + if (MAX_BFLOAT16_VEC_ACC > num_acc) { + colVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void colVecLoopBodyExtra(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, + const Packet4f pAlpha, float* result) { + switch (rows - row) { + case 7: + colVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 6: + colVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 5: + colVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 4: + colVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 3: + colVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 2: + colVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 1: + colVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcVecLoops(Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, + float* result) { + Index row = 0; + if (rows >= MAX_BFLOAT16_VEC_ACC) { + colVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); + result += row; + } + colVecLoopBodyExtra(row, cols, rows, lhs, rhs, pAlpha, result); +} + +template +EIGEN_STRONG_INLINE void gemvMMA_bfloat16_row(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, + bfloat16* res, Index resIncr, bfloat16 alpha) { + typedef typename RhsMapper::LinearMapper LinearMapper; + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + LinearMapper rhs2 = rhs.getLinearMapper(0, 0); + + eigen_internal_assert(rhs.stride() == 1); + + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + const Packet4f pAlpha = pset1(falpha); + + ei_declare_aligned_stack_constructed_variable(float, result, rows, 0); + if (resIncr == 1) { + convertArrayPointerBF16toF32(result, 1, rows, res); + } else { + convertArrayPointerBF16toF32(result, 1, rows, res, resIncr); + } + calcVecLoops(cols, rows, lhs, rhs2, pAlpha, result); + if (resIncr == 1) { + convertArrayPointerF32toBF16(result, rows, res); + } else { + convertArrayPointerF32toBF16(result, rows, res, resIncr); + } +} +#endif + +#undef MAX_BFLOAT16_VEC_ACC +#undef BFLOAT16_UNROLL + +} // namespace internal +} // namespace Eigen +#endif // EIGEN_MATRIX_PRODUCT_MMA_BFLOAT16_ALTIVEC_H diff --git a/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc b/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc new file mode 100644 index 0000000..90c0d39 --- /dev/null +++ b/Eigen/src/Core/arch/AltiVec/MatrixVectorProduct.inc @@ -0,0 +1,2818 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2021 Chip Kerchner (chip.kerchner@ibm.com) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H +#define EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +#if defined(__MMA__) && !EIGEN_ALTIVEC_DISABLE_MMA +#if EIGEN_COMP_LLVM || (__GNUC__ > 10 || __GNUC_MINOR__ >= 3) +#define USE_GEMV_MMA +#endif + +#if !EIGEN_COMP_LLVM && (__GNUC__ < 11) +// Only allow one vector_pair in buggy gcc - gcc 10.x has a bug +#define GCC_ONE_VECTORPAIR_BUG +#endif +#endif + +// #define USE_SLOWER_GEMV_MMA // MMA is currently not as fast as VSX in complex double GEMV (revisit when gcc is +// improved) + +// #define EIGEN_POWER_USE_GEMV_PREFETCH +#ifdef EIGEN_POWER_USE_GEMV_PREFETCH +#define EIGEN_POWER_GEMV_PREFETCH(p) prefetch(p) +#else +#define EIGEN_POWER_GEMV_PREFETCH(p) +#endif + +#ifdef __has_builtin +#if !__has_builtin(__builtin_vsx_assemble_pair) +#define __builtin_vsx_assemble_pair __builtin_mma_assemble_pair +#endif +#if !__has_builtin(__builtin_vsx_disassemble_pair) +#define __builtin_vsx_disassemble_pair __builtin_mma_disassemble_pair +#endif +#endif + +#if EIGEN_COMP_LLVM +#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \ + __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src2, (__vector unsigned char)src1) +#else +#if (__GNUC__ <= 10) +#if (__GNUC_MINOR__ > 3) +#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \ + __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src2, (__vector unsigned char)src1) +#else +#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \ + __builtin_vsx_assemble_pair(&dst, (__vector unsigned char)src1, (__vector unsigned char)src2) +#endif +#else +#define GEMV_BUILDPAIR_MMA(dst, src1, src2) \ + __builtin_vsx_build_pair(&dst, (__vector unsigned char)src1, (__vector unsigned char)src2) +#endif +#endif + +#define GEMV_IS_COMPLEX_COMPLEX ((sizeof(LhsPacket) == 16) && (sizeof(RhsPacket) == 16)) +#define GEMV_IS_FLOAT (ResPacketSize == (16 / sizeof(float))) +#define GEMV_IS_SCALAR (sizeof(ResPacket) != 16) +#define GEMV_IS_COMPLEX_FLOAT (ResPacketSize == (16 / sizeof(std::complex))) + +/** \internal multiply and add and store results */ +template +EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResPacket& palpha, ResPacket& data) { + pstoreu(res, pmadd(data, palpha, ploadu(res))); +} + +template +EIGEN_ALWAYS_INLINE void storeMaddData(ResScalar* res, ResScalar& alpha, ResScalar& data) { + *res += (alpha * data); +} + +#define GEMV_UNROLL(func, N) func(0, N) func(1, N) func(2, N) func(3, N) func(4, N) func(5, N) func(6, N) func(7, N) + +#define GEMV_UNROLL_HALF(func, N) func(0, 0, 1, N) func(1, 2, 3, N) func(2, 4, 5, N) func(3, 6, 7, N) + +#define GEMV_GETN(N) (((N) * ResPacketSize) >> 2) + +#define GEMV_LOADPACKET_COL(iter) lhs.template load(i + ((iter) * LhsPacketSize), j) + +#ifdef USE_GEMV_MMA +#define GEMV_UNROLL3(func, N, which) \ + func(0, N, which) func(1, N, which) func(2, N, which) func(3, N, which) func(4, N, which) func(5, N, which) \ + func(6, N, which) func(7, N, which) + +#define GEMV_UNUSED_VAR(iter, N, which) \ + if (GEMV_GETN(N) <= iter) { \ + EIGEN_UNUSED_VARIABLE(which##iter); \ + } + +#define GEMV_UNUSED_EXTRA_VAR(iter, N, which) \ + if (N <= iter) { \ + EIGEN_UNUSED_VARIABLE(which##iter); \ + } + +#define GEMV_UNUSED_EXTRA(N, which) GEMV_UNROLL3(GEMV_UNUSED_EXTRA_VAR, N, which) + +#define GEMV_UNUSED(N, which) GEMV_UNROLL3(GEMV_UNUSED_VAR, N, which) + +#define GEMV_INIT_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + __builtin_mma_xxsetaccz(&e##iter); \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_COL(iter2), GEMV_LOADPACKET_COL((iter2) + 1)); +#else +#define GEMV_LOADPAIR_COL_MMA(iter1, iter2) \ + const LhsScalar& src##iter1 = lhs(i + ((iter1 * 32) / sizeof(LhsScalar)), j); \ + b##iter1 = *reinterpret_cast<__vector_pair*>(const_cast(&src##iter1)); +#endif + +#define GEMV_LOAD1A_COL_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + if (GEMV_IS_FLOAT) { \ + g##iter = GEMV_LOADPACKET_COL(iter); \ + EIGEN_UNUSED_VARIABLE(b##iter); \ + } else { \ + GEMV_LOADPAIR_COL_MMA(iter, iter << 1) \ + EIGEN_UNUSED_VARIABLE(g##iter); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(b##iter); \ + EIGEN_UNUSED_VARIABLE(g##iter); \ + } + +#define GEMV_WORK1A_COL_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + if (GEMV_IS_FLOAT) { \ + pger_vecMMA_acc(&e##iter, a0, g##iter); \ + } else { \ + pger_vecMMA_acc(&e##iter, b##iter, a0); \ + } \ + } + +#define GEMV_LOAD1B_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + GEMV_LOADPAIR_COL_MMA(iter2, iter2) \ + EIGEN_UNUSED_VARIABLE(b##iter3); \ + } else { \ + GEMV_LOADPAIR_COL_MMA(iter2, iter2 << 1) \ + GEMV_LOADPAIR_COL_MMA(iter3, iter3 << 1) \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(b##iter2); \ + EIGEN_UNUSED_VARIABLE(b##iter3); \ + } \ + EIGEN_UNUSED_VARIABLE(g##iter2); \ + EIGEN_UNUSED_VARIABLE(g##iter3); + +#define GEMV_WORK1B_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + LhsPacket h[2]; \ + __builtin_vsx_disassemble_pair(reinterpret_cast(h), &b##iter2); \ + pger_vecMMA_acc(&e##iter2, a0, h[0]); \ + pger_vecMMA_acc(&e##iter3, a0, h[1]); \ + } else { \ + pger_vecMMA_acc(&e##iter2, b##iter2, a0); \ + pger_vecMMA_acc(&e##iter3, b##iter3, a0); \ + } \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_LOAD_COL_MMA(N) \ + if (GEMV_GETN(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_LOAD1B_COL_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) \ + } + +#define GEMV_WORK_COL_MMA(N) \ + if (GEMV_GETN(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_WORK1B_COL_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) \ + } +#else +#define GEMV_LOAD_COL_MMA(N) GEMV_UNROLL(GEMV_LOAD1A_COL_MMA, N) + +#define GEMV_WORK_COL_MMA(N) GEMV_UNROLL(GEMV_WORK1A_COL_MMA, N) +#endif + +#define GEMV_DISASSEMBLE_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + __builtin_mma_disassemble_acc(&result##iter.packet, &e##iter); \ + if (!GEMV_IS_FLOAT) { \ + result##iter.packet[0][1] = result##iter.packet[1][0]; \ + result##iter.packet[2][1] = result##iter.packet[3][0]; \ + } \ + } + +#define GEMV_LOADPAIR2_COL_MMA(iter1, iter2) \ + b##iter1 = *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize)); + +#define GEMV_LOAD2_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + GEMV_LOADPAIR2_COL_MMA(iter2, iter2); \ + EIGEN_UNUSED_VARIABLE(b##iter3); \ + } else { \ + GEMV_LOADPAIR2_COL_MMA(iter2, iter2 << 1); \ + GEMV_LOADPAIR2_COL_MMA(iter3, iter3 << 1); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(b##iter2); \ + EIGEN_UNUSED_VARIABLE(b##iter3); \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \ + ResPacket f##iter2[2]; \ + __builtin_vsx_disassemble_pair(reinterpret_cast(f##iter2), &b##iter2); \ + f##iter2[0] = pmadd(result##iter2.packet[0], palpha, f##iter2[0]); \ + f##iter2[1] = pmadd(result##iter3.packet[(iter2 == iter3) ? 2 : 0], palpha, f##iter2[1]); \ + GEMV_BUILDPAIR_MMA(b##iter2, f##iter2[0], f##iter2[1]); +#else +#define GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter4) \ + if (GEMV_IS_FLOAT) { \ + __asm__("xvmaddasp %0,%x1,%x3\n\txvmaddasp %L0,%x2,%x3" \ + : "+&d"(b##iter2) \ + : "wa"(result##iter3.packet[0]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \ + } else { \ + __asm__("xvmaddadp %0,%x1,%x3\n\txvmaddadp %L0,%x2,%x3" \ + : "+&d"(b##iter2) \ + : "wa"(result##iter2.packet[2]), "wa"(result##iter2.packet[0]), "wa"(palpha)); \ + } +#endif + +#define GEMV_WORK2_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + GEMV_WORKPAIR2_COL_MMA(iter2, iter3, iter2); \ + } else { \ + GEMV_WORKPAIR2_COL_MMA(iter2, iter2, iter2 << 1); \ + GEMV_WORKPAIR2_COL_MMA(iter3, iter3, iter3 << 1); \ + } \ + } + +#define GEMV_STOREPAIR2_COL_MMA(iter1, iter2) \ + *reinterpret_cast<__vector_pair*>(res + i + ((iter2) * ResPacketSize)) = b##iter1; + +#define GEMV_STORE_COL_MMA(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + if (GEMV_IS_FLOAT) { \ + storeMaddData(res + i + (iter * ResPacketSize), palpha, result##iter.packet[0]); \ + } else { \ + GEMV_LOADPAIR2_COL_MMA(iter, iter << 1) \ + GEMV_WORKPAIR2_COL_MMA(iter, iter, iter << 1) \ + GEMV_STOREPAIR2_COL_MMA(iter, iter << 1) \ + } \ + } + +#define GEMV_STORE2_COL_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN(N) > iter1) { \ + if (GEMV_IS_FLOAT) { \ + GEMV_STOREPAIR2_COL_MMA(iter2, iter2); \ + } else { \ + GEMV_STOREPAIR2_COL_MMA(iter2, iter2 << 1) \ + GEMV_STOREPAIR2_COL_MMA(iter3, iter3 << 1) \ + } \ + } + +#define GEMV_PROCESS_COL_ONE_MMA(N) \ + GEMV_UNROLL(GEMV_INIT_MMA, N) \ + Index j = j2; \ + __vector_pair b0, b1, b2, b3, b4, b5, b6, b7; \ + do { \ + LhsPacket g0, g1, g2, g3, g4, g5, g6, g7; \ + RhsPacket a0 = pset1(rhs2(j, 0)); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_LOAD_COL_MMA(N) \ + GEMV_WORK_COL_MMA(N) \ + } while (++j < jend); \ + GEMV_UNROLL(GEMV_DISASSEMBLE_MMA, N) \ + if (GEMV_GETN(N) <= 1) { \ + GEMV_UNROLL(GEMV_STORE_COL_MMA, N) \ + } else { \ + GEMV_UNROLL_HALF(GEMV_LOAD2_COL_MMA, (N >> 1)) \ + GEMV_UNROLL_HALF(GEMV_WORK2_COL_MMA, (N >> 1)) \ + GEMV_UNROLL_HALF(GEMV_STORE2_COL_MMA, (N >> 1)) \ + } \ + i += (ResPacketSize * N); +#endif + +#define GEMV_INIT(iter, N) \ + if (N > iter) { \ + c##iter = pset1(ResScalar(0)); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c##iter); \ + } + +#ifdef EIGEN_POWER_USE_GEMV_PREFETCH +#define GEMV_PREFETCH(iter, N) \ + if (GEMV_GETN(N) > ((iter >> 1) + ((N >> 1) * (iter & 1)))) { \ + lhs.prefetch(i + (iter * LhsPacketSize) + prefetch_dist, j); \ + } +#else +#define GEMV_PREFETCH(iter, N) +#endif + +#define GEMV_WORK_COL(iter, N) \ + if (N > iter) { \ + c##iter = pcj.pmadd(GEMV_LOADPACKET_COL(iter), a0, c##iter); \ + } + +#define GEMV_STORE_COL(iter, N) \ + if (N > iter) { \ + pstoreu(res + i + (iter * ResPacketSize), \ + pmadd(c##iter, palpha, ploadu(res + i + (iter * ResPacketSize)))); \ + } + +/** \internal main macro for gemv_col - initialize accumulators, multiply and add inputs, and store results */ +#define GEMV_PROCESS_COL_ONE(N) \ + GEMV_UNROLL(GEMV_INIT, N) \ + Index j = j2; \ + do { \ + RhsPacket a0 = pset1(rhs2(j, 0)); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_UNROLL(GEMV_WORK_COL, N) \ + } while (++j < jend); \ + GEMV_UNROLL(GEMV_STORE_COL, N) \ + i += (ResPacketSize * N); + +#ifdef USE_GEMV_MMA +#define GEMV_PROCESS_COL(N) GEMV_PROCESS_COL_ONE_MMA(N) +#else +#define GEMV_PROCESS_COL(N) GEMV_PROCESS_COL_ONE(N) +#endif + +/** \internal perform a matrix multiply and accumulate of packet a and packet b */ +#ifdef USE_GEMV_MMA +template +EIGEN_ALWAYS_INLINE void pger_vecMMA_acc(__vector_quad* acc, const RhsPacket& a, const LhsPacket& b) { + if (accumulate) { + __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b); + } else { + __builtin_mma_xvf32ger(acc, (__vector unsigned char)a, (__vector unsigned char)b); + } +} + +/** \internal perform a matrix multiply and accumulate of vector_pair a and packet b */ +template +EIGEN_ALWAYS_INLINE void pger_vecMMA_acc(__vector_quad* acc, __vector_pair& a, const LhsPacket& b) { + if (accumulate) { + __builtin_mma_xvf64gerpp(acc, a, (__vector unsigned char)b); + } else { + __builtin_mma_xvf64ger(acc, a, (__vector unsigned char)b); + } +} +#endif + +template +EIGEN_STRONG_INLINE void gemv_col(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, ResScalar* res, + Index resIncr, ResScalar alpha) { + typedef gemv_traits Traits; + + typedef typename Traits::LhsPacket LhsPacket; + typedef typename Traits::RhsPacket RhsPacket; + typedef typename Traits::ResPacket ResPacket; + + EIGEN_UNUSED_VARIABLE(resIncr); + eigen_internal_assert(resIncr == 1); + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + RhsMapper rhs2(rhs); + + conj_helper cj; + conj_helper pcj; + + const Index lhsStride = lhs.stride(); + // TODO: for padded aligned inputs, we could enable aligned reads + enum { + LhsAlignment = Unaligned, + ResPacketSize = Traits::ResPacketSize, + LhsPacketSize = Traits::LhsPacketSize, + RhsPacketSize = Traits::RhsPacketSize, + }; + +#ifndef GCC_ONE_VECTORPAIR_BUG + const Index n8 = rows - 8 * ResPacketSize + 1; + const Index n4 = rows - 4 * ResPacketSize + 1; + const Index n2 = rows - 2 * ResPacketSize + 1; +#endif + const Index n1 = rows - 1 * ResPacketSize + 1; +#ifdef EIGEN_POWER_USE_GEMV_PREFETCH + const Index prefetch_dist = 64 * LhsPacketSize; +#endif + + // TODO: improve the following heuristic: + const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(LhsScalar) < 16000 ? 16 : 8); + ResPacket palpha = pset1(alpha); + + for (Index j2 = 0; j2 < cols; j2 += block_cols) { + Index jend = numext::mini(j2 + block_cols, cols); + Index i = 0; + ResPacket c0, c1, c2, c3, c4, c5, c6, c7; +#ifdef USE_GEMV_MMA + __vector_quad e0, e1, e2, e3, e4, e5, e6, e7; + PacketBlock result0, result1, result2, result3, result4, result5, result6, result7; + GEMV_UNUSED(8, e) + GEMV_UNUSED(8, result) + GEMV_UNUSED_EXTRA(1, c) +#endif +#ifndef GCC_ONE_VECTORPAIR_BUG + while (i < n8) { + GEMV_PROCESS_COL(8) + } + if (i < n4) { + GEMV_PROCESS_COL(4) + } + if (i < n2) { + GEMV_PROCESS_COL(2) + } + if (i < n1) +#else + while (i < n1) +#endif + { + GEMV_PROCESS_COL_ONE(1) + } + for (; i < rows; ++i) { + ResScalar d0(0); + Index j = j2; + do { + d0 += cj.pmul(lhs(i, j), rhs2(j, 0)); + } while (++j < jend); + res[i] += alpha * d0; + } + } +} + +template +EIGEN_ALWAYS_INLINE void outputVecCol(Packet4f acc, float* result, Packet4f pAlpha, Index extra_rows) { + Packet4f d0 = ploadu(result); + d0 = pmadd(acc, pAlpha, d0); + if (extraRows) { + pstoreu_partial(result, d0, extra_rows); + } else { + pstoreu(result, d0); + } +} + +template +EIGEN_ALWAYS_INLINE void outputVecColResults(Packet4f (&acc)[num_acc][size], float* result, Packet4f pAlpha, + Index extra_rows) { + constexpr Index real_acc = (num_acc - (extraRows ? 1 : 0)); + for (Index k = 0; k < real_acc; k++) { + outputVecCol(acc[k][0], result + k * 4, pAlpha, extra_rows); + } + if (extraRows) { + outputVecCol(acc[real_acc][0], result + real_acc * 4, pAlpha, extra_rows); + } +} + +static Packet16uc p16uc_MERGE16_32_V1 = {0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17, 0, 1, 16, 17}; +static Packet16uc p16uc_MERGE16_32_V2 = {2, 3, 18, 19, 2, 3, 18, 19, 2, 3, 18, 19, 2, 3, 18, 19}; + +template +EIGEN_ALWAYS_INLINE void loadVecLoopVSX(Index k, LhsMapper& lhs, Packet4f (&a0)[num_acc][2]) { + Packet8bf c0 = lhs.template loadPacket(k * 4, 0); + Packet8bf b1; + if (!zero) { + b1 = lhs.template loadPacket(k * 4, 1); + + a0[k + 0][1] = oneConvertBF16Hi(b1.m_val); + } + a0[k + 0][0] = oneConvertBF16Hi(c0.m_val); + + if (num_acc > (k + 1)) { + a0[k + 1][0] = oneConvertBF16Lo(c0.m_val); + if (!zero) { + a0[k + 1][1] = oneConvertBF16Lo(b1.m_val); + } + } +} + +template +EIGEN_ALWAYS_INLINE void multVecVSX(Packet4f (&acc)[num_acc][2], Packet4f (&a0)[num_acc][2], Packet4f (&b0)[2]) { + for (Index k = 0; k < num_acc; k++) { + for (Index i = 0; i < (zero ? 1 : 2); i++) { + acc[k][i] = pmadd(b0[i], a0[k][i], acc[k][i]); + } + } +} + +template +struct loadColData_impl { + // linear == false + static EIGEN_ALWAYS_INLINE Packet8bf run(RhsMapper& rhs, Index j) { + const Index n = unpacket_traits::size; + EIGEN_ALIGN16 bfloat16 to[n]; + LOAD_STORE_UNROLL_16 + for (Index i = 0; i < n; i++) { + to[i] = rhs(j + i, 0); + } + return pload(to); + } +}; + +template +struct loadColData_impl { + // linear == true + static EIGEN_ALWAYS_INLINE Packet8bf run(RhsMapper& rhs, Index j) { + return rhs.template loadPacket(j + 0, 0); + } +}; + +template +EIGEN_ALWAYS_INLINE Packet8bf loadColData(RhsMapper& rhs, Index j) { + return loadColData_impl::run(rhs, j); +} + +template +EIGEN_ALWAYS_INLINE void vecColLoopVSX(Index j, LhsMapper& lhs, RhsMapper& rhs, Packet4f (&acc)[num_acc][2]) { + Packet4f a0[num_acc][2], b0[2]; + Packet8bf b2 = loadColData(rhs, j); + + b0[0] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V1); + if (!zero) { + b0[1] = oneConvertBF16Perm(b2.m_val, p16uc_MERGE16_32_V2); + } + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(0, j); + for (Index k = 0; k < num_acc; k += 2) { + loadVecLoopVSX(k, lhs2, a0); + } + + multVecVSX(acc, a0, b0); +} + +template +EIGEN_ALWAYS_INLINE void addResultsVSX(Packet4f (&acc)[num_acc][2]) { + for (Index i = 0; i < num_acc; i++) { + acc[i][0] = acc[i][0] + acc[i][1]; + } +} + +// Uses 2X the accumulators or 4X the number of VSX registers +#define MAX_BFLOAT16_VEC_ACC_VSX 8 + +template +void colVSXVecColLoopBody(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, + float* result) { + constexpr Index step = (num_acc * 4); + const Index extra_rows = (extraRows) ? (rows & 3) : 0; + constexpr bool multiIters = !extraRows && (num_acc == MAX_BFLOAT16_VEC_ACC_VSX); + + do { + Packet4f acc[num_acc][2]; + + zeroAccumulators(acc); + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(row, 0); + for (Index j = 0; j + 2 <= cend; j += 2) { + vecColLoopVSX(j, lhs2, rhs, acc); + } + if (cend & 1) { + vecColLoopVSX(cend - 1, lhs2, rhs, acc); + } + + addResultsVSX(acc); + + outputVecColResults(acc, result, pAlpha, extra_rows); + + result += step; + } while (multiIters && (step <= rows - (row += step))); +} + +template +EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtraN(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, + const Packet4f pAlpha, float* result) { + if (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { + colVSXVecColLoopBody(row, cend, rows, lhs, + rhs, pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void colVSXVecColLoopBodyExtra(Index& row, Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, + const Packet4f pAlpha, float* result) { + switch ((rows - row) >> 2) { + case 7: + colVSXVecColLoopBodyExtraN<7, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 6: + colVSXVecColLoopBodyExtraN<6, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 5: + colVSXVecColLoopBodyExtraN<5, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 4: + colVSXVecColLoopBodyExtraN<4, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 3: + colVSXVecColLoopBodyExtraN<3, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 2: + colVSXVecColLoopBodyExtraN<2, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + case 1: + colVSXVecColLoopBodyExtraN<1, LhsMapper, RhsMapper, extraRows, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + break; + default: + if (extraRows) { + colVSXVecColLoopBody<1, LhsMapper, RhsMapper, true, linear>(row, cend, rows, lhs, rhs, pAlpha, result); + } + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcVSXVecColLoops(Index cend, Index rows, LhsMapper& lhs, RhsMapper& rhs, + const Packet4f pAlpha, float* result) { + Index row = 0; + if (rows >= (MAX_BFLOAT16_VEC_ACC_VSX * 4)) { + colVSXVecColLoopBody(row, cend, rows, lhs, rhs, + pAlpha, result); + result += row; + } + if (rows & 3) { + colVSXVecColLoopBodyExtra(row, cend, rows, lhs, rhs, pAlpha, result); + } else { + colVSXVecColLoopBodyExtra(row, cend, rows, lhs, rhs, pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void storeBF16fromResult(bfloat16* dst, Packet8bf data, Index resInc, Index extra) { + if (inc) { + if (size < 8) { + pscatter_partial(dst + delta * resInc, data, resInc, extra); + } else { + pscatter(dst + delta * resInc, data, resInc); + } + } else { + if (size < 8) { + pstoreu_partial(dst + delta, data, extra); + } else { + pstoreu(dst + delta, data); + } + } +} + +template +EIGEN_ALWAYS_INLINE void convertPointerF32toBF16VSX(Index& i, float* result, Index rows, bfloat16*& dst, + Index resInc = 1) { + constexpr Index extra = ((size < 8) ? 8 : size); + while (i + size <= rows) { + PacketBlock r32; + r32.packet[0] = convertF32toBF16VSX(result + i + 0); + if (size >= 16) { + r32.packet[1] = convertF32toBF16VSX(result + i + 8); + } + if (size >= 32) { + r32.packet[2] = convertF32toBF16VSX(result + i + 16); + r32.packet[3] = convertF32toBF16VSX(result + i + 24); + } + storeBF16fromResult(dst, r32.packet[0], resInc, rows & 7); + if (size >= 16) { + storeBF16fromResult(dst, r32.packet[1], resInc); + } + if (size >= 32) { + storeBF16fromResult(dst, r32.packet[2], resInc); + storeBF16fromResult(dst, r32.packet[3], resInc); + } + i += extra; + dst += extra * resInc; + if (size != 32) break; + } +} + +template +EIGEN_ALWAYS_INLINE void convertArrayPointerF32toBF16VSX(float* result, Index rows, bfloat16* dst, Index resInc = 1) { + Index i = 0; + convertPointerF32toBF16VSX<32, inc>(i, result, rows, dst, resInc); + convertPointerF32toBF16VSX<16, inc>(i, result, rows, dst, resInc); + convertPointerF32toBF16VSX<8, inc>(i, result, rows, dst, resInc); + convertPointerF32toBF16VSX<1, inc>(i, result, rows, dst, resInc); +} + +template +struct UseStride : std::false_type { + static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha, + float* result) { + using RhsSubMapper = typename RhsMapper::SubMapper; + + RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0); + calcVSXVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } +}; + +template +struct UseStride::value>> + : std::true_type { + static EIGEN_ALWAYS_INLINE void run(Index j2, Index jend, Index rows, LhsMapper& lhs, RhsMapper& rhs, Packet4f pAlpha, + float* result) { + using RhsSubMapper = typename RhsMapper::SubMapper; + + RhsSubMapper rhs2 = rhs.getSubMapper(j2, 0); + if (rhs.stride() == 1) { + calcVSXVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } else { + calcVSXVecColLoops(jend - j2, rows, lhs, rhs2, pAlpha, result); + } + } +}; + +template +void gemv_bfloat16_col(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, bfloat16* res, + Index resIncr, bfloat16 alpha) { + EIGEN_UNUSED_VARIABLE(resIncr); + eigen_internal_assert(resIncr == 1); + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + RhsMapper rhs2(rhs); + + const Index lhsStride = lhs.stride(); + + // TODO: improve the following heuristic: + const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(bfloat16) < 16000 ? 16 : 8); + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + Packet4f pAlpha = pset1(falpha); + + ei_declare_aligned_stack_constructed_variable(float, result, rows, 0); + + convertArrayPointerBF16toF32(result, 1, rows, res); + + for (Index j2 = 0; j2 < cols; j2 += block_cols) { + Index jend = numext::mini(j2 + block_cols, cols); + + using LhsSubMapper = typename LhsMapper::SubMapper; + + LhsSubMapper lhs2 = lhs.getSubMapper(0, j2); + UseStride::run(j2, jend, rows, lhs2, rhs2, pAlpha, result); + } + + convertArrayPointerF32toBF16VSX(result, rows, res); +} + +template +EIGEN_ALWAYS_INLINE void outputVecResults(Packet4f (&acc)[num_acc][size], float* result, Packet4f pAlpha) { + constexpr Index extra = num_acc & 3; + + for (Index k = 0; k < num_acc; k += 4) { + Packet4f d0 = ploadu(result + k); + d0 = pmadd(acc[k + 0][0], pAlpha, d0); + + if (num_acc > (k + 3)) { + pstoreu(result + k, d0); + } else { + if (extra == 3) { + pstoreu_partial(result + k, d0, extra); + } else { + memcpy((void*)(result + k), (void*)(&d0), sizeof(float) * extra); + } + } + } +} + +template +EIGEN_ALWAYS_INLINE void preduxVecResults2VSX(Packet4f (&acc)[num_acc][2], Index k) { + if (num_acc > (k + 1)) { + acc[k][1] = vec_mergel(acc[k + 0][0], acc[k + 1][0]); + acc[k][0] = vec_mergeh(acc[k + 0][0], acc[k + 1][0]); + acc[k][0] = acc[k][0] + acc[k][1]; + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8); + } else { + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 8); +#ifdef _BIG_ENDIAN + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 12); +#else + acc[k][0] += vec_sld(acc[k][0], acc[k][0], 4); +#endif + } +} + +template +EIGEN_ALWAYS_INLINE void preduxVecResultsVSX(Packet4f (&acc)[num_acc][2]) { + for (Index k = 0; k < num_acc; k += 4) { + preduxVecResults2VSX(acc, k + 0); + if (num_acc > (k + 2)) { + preduxVecResults2VSX(acc, k + 2); +#ifdef EIGEN_VECTORIZE_VSX + acc[k + 0][0] = reinterpret_cast( + vec_mergeh(reinterpret_cast(acc[k + 0][0]), reinterpret_cast(acc[k + 2][0]))); +#else + acc[k + 0][0] = reinterpret_cast(vec_perm(acc[k + 0][0], acc[k + 2][0], p16uc_TRANSPOSE64_HI)); +#endif + } + } +} + +#ifndef _ARCH_PWR9 +EIGEN_ALWAYS_INLINE Packet8us loadPacketPartialZero(Packet8us data, Index extra_cols) { + Packet16uc shift = pset1(8 * 2 * (8 - extra_cols)); +#ifdef _BIG_ENDIAN + return reinterpret_cast(vec_slo(vec_sro(reinterpret_cast(data), shift), shift)); +#else + return reinterpret_cast(vec_sro(vec_slo(reinterpret_cast(data), shift), shift)); +#endif +} +#endif + +template +EIGEN_ALWAYS_INLINE void multVSXVecLoop(Packet4f (&acc)[num_acc][2], const LhsMapper& lhs, RhsMapper& rhs, Index j, + Index extra_cols) { + Packet4f a0[num_acc][2], b0[2]; + Packet8bf a1, b1; + + if (extra) { + b1 = rhs.template loadPacketPartial(j, extra_cols); +#ifndef _ARCH_PWR9 + b1 = loadPacketPartialZero(b1.m_val, extra_cols); +#endif + } else { + b1 = rhs.template loadPacket(j); + } + b0[0] = oneConvertBF16Hi(b1.m_val); + b0[1] = oneConvertBF16Lo(b1.m_val); + + const LhsMapper lhs2 = lhs.getSubMapper(0, j); + for (Index k = 0; k < num_acc; k++) { + if (extra) { + a1 = lhs2.template loadPacketPartial(k, 0, extra_cols); +#ifndef _ARCH_PWR9 + a1 = loadPacketPartialZero(a1.m_val, extra_cols); +#endif + } else { + a1 = lhs2.template loadPacket(k, 0); + } + a0[k][0] = oneConvertBF16Hi(a1.m_val); + a0[k][1] = oneConvertBF16Lo(a1.m_val); + } + + multVecVSX(acc, a0, b0); +} + +template +EIGEN_ALWAYS_INLINE void vecVSXLoop(Index cols, const LhsMapper& lhs, RhsMapper& rhs, Packet4f (&acc)[num_acc][2], + Index extra_cols) { + Index j = 0; + for (; j + 8 <= cols; j += 8) { + multVSXVecLoop(acc, lhs, rhs, j, extra_cols); + } + + if (extra_cols) { + multVSXVecLoop(acc, lhs, rhs, j, extra_cols); + } +} + +template +void colVSXVecLoopBody(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, + float* result) { + constexpr bool multiIters = (num_acc == MAX_BFLOAT16_VEC_ACC_VSX); + const Index extra_cols = (cols & 7); + + do { + Packet4f acc[num_acc][2]; + + zeroAccumulators(acc); + + const LhsMapper lhs2 = lhs.getSubMapper(row, 0); + vecVSXLoop(cols, lhs2, rhs, acc, extra_cols); + + addResultsVSX(acc); + + preduxVecResultsVSX(acc); + + outputVecResults(acc, result, pAlpha); + + result += num_acc; + } while (multiIters && (num_acc <= rows - (row += num_acc))); +} + +template +EIGEN_ALWAYS_INLINE void colVSXVecLoopBodyExtraN(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, + const Packet4f pAlpha, float* result) { + if (MAX_BFLOAT16_VEC_ACC_VSX > num_acc) { + colVSXVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); + } +} + +template +EIGEN_ALWAYS_INLINE void colVSXVecLoopBodyExtra(Index& row, Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, + const Packet4f pAlpha, float* result) { + switch (rows - row) { + case 7: + colVSXVecLoopBodyExtraN<7, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 6: + colVSXVecLoopBodyExtraN<6, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 5: + colVSXVecLoopBodyExtraN<5, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 4: + colVSXVecLoopBodyExtraN<4, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 3: + colVSXVecLoopBodyExtraN<3, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 2: + colVSXVecLoopBodyExtraN<2, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + case 1: + colVSXVecLoopBodyExtraN<1, LhsMapper, RhsMapper>(row, cols, rows, lhs, rhs, pAlpha, result); + break; + } +} + +template +EIGEN_ALWAYS_INLINE void calcVSXVecLoops(Index cols, Index rows, LhsMapper& lhs, RhsMapper& rhs, const Packet4f pAlpha, + float* result) { + Index row = 0; + if (rows >= MAX_BFLOAT16_VEC_ACC_VSX) { + colVSXVecLoopBody(row, cols, rows, lhs, rhs, pAlpha, result); + result += row; + } + colVSXVecLoopBodyExtra(row, cols, rows, lhs, rhs, pAlpha, result); +} + +template +EIGEN_STRONG_INLINE void gemv_bfloat16_row(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, + bfloat16* res, Index resIncr, bfloat16 alpha) { + typedef typename RhsMapper::LinearMapper LinearMapper; + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + LinearMapper rhs2 = rhs.getLinearMapper(0, 0); + + eigen_internal_assert(rhs.stride() == 1); + + float falpha = Eigen::bfloat16_impl::bfloat16_to_float(alpha); + const Packet4f pAlpha = pset1(falpha); + + ei_declare_aligned_stack_constructed_variable(float, result, rows, 0); + if (resIncr == 1) { + convertArrayPointerBF16toF32(result, 1, rows, res); + } else { + convertArrayPointerBF16toF32(result, 1, rows, res, resIncr); + } + calcVSXVecLoops(cols, rows, lhs, rhs2, pAlpha, result); + if (resIncr == 1) { + convertArrayPointerF32toBF16VSX(result, rows, res); + } else { + convertArrayPointerF32toBF16VSX(result, rows, res, resIncr); + } +} + +#undef MAX_BFLOAT16_VEC_ACC_VSX + +const Packet16uc p16uc_COMPLEX32_XORFLIP = {0x44, 0x55, 0x66, 0x77, 0x00, 0x11, 0x22, 0x33, + 0xcc, 0xdd, 0xee, 0xff, 0x88, 0x99, 0xaa, 0xbb}; +const Packet16uc p16uc_COMPLEX64_XORFLIP = {0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff, + 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77}; + +#ifdef _BIG_ENDIAN +const Packet16uc p16uc_COMPLEX32_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00}; +const Packet16uc p16uc_COMPLEX64_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const Packet16uc p16uc_COMPLEX32_CONJ_XOR2 = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const Packet16uc p16uc_COMPLEX64_CONJ_XOR2 = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const Packet16uc p16uc_COMPLEX32_NEGATE = {0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00}; +const Packet16uc p16uc_COMPLEX64_NEGATE = {0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +#else +const Packet16uc p16uc_COMPLEX32_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}; +const Packet16uc p16uc_COMPLEX64_CONJ_XOR = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}; +const Packet16uc p16uc_COMPLEX32_CONJ_XOR2 = {0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00}; +const Packet16uc p16uc_COMPLEX64_CONJ_XOR2 = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const Packet16uc p16uc_COMPLEX32_NEGATE = {0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x80, 0x00, 0x00, 0x00, 0x80}; +const Packet16uc p16uc_COMPLEX64_NEGATE = {0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x80}; +#endif + +#ifdef _BIG_ENDIAN +#define COMPLEX_DELTA 0 +#else +#define COMPLEX_DELTA 2 +#endif + +/** \internal packet conjugate (same as pconj but uses the constants in pcplxflipconj for better code generation) */ +EIGEN_ALWAYS_INLINE Packet2cf pconj2(const Packet2cf& a) { + return Packet2cf(pxor(a.v, reinterpret_cast(p16uc_COMPLEX32_CONJ_XOR))); +} + +EIGEN_ALWAYS_INLINE Packet1cd pconj2(const Packet1cd& a) { + return Packet1cd(pxor(a.v, reinterpret_cast(p16uc_COMPLEX64_CONJ_XOR))); +} + +/** \internal packet conjugate with real & imaginary operation inverted */ +EIGEN_ALWAYS_INLINE Packet2cf pconjinv(const Packet2cf& a) { +#ifdef __POWER8_VECTOR__ + return Packet2cf(Packet4f(vec_neg(Packet2d(a.v)))); +#else + return Packet2cf(pxor(a.v, reinterpret_cast(p16uc_COMPLEX32_CONJ_XOR2))); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pconjinv(const Packet1cd& a) { + return Packet1cd(pxor(a.v, reinterpret_cast(p16uc_COMPLEX64_CONJ_XOR2))); +} + +#if defined(_ARCH_PWR8) && (!EIGEN_COMP_LLVM || __clang_major__ >= 12) +#define PERMXOR_GOOD // Clang had a bug with vec_permxor and endianness prior to version 12 +#endif + +/** \internal flip the real & imaginary results and packet conjugate */ +EIGEN_ALWAYS_INLINE Packet2cf pcplxflipconj(Packet2cf a) { +#ifdef PERMXOR_GOOD + return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_CONJ_XOR, p16uc_COMPLEX32_XORFLIP))); +#else + return pcplxflip(pconj2(a)); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pcplxflipconj(Packet1cd a) { +#ifdef PERMXOR_GOOD + return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_CONJ_XOR, p16uc_COMPLEX64_XORFLIP))); +#else + return pcplxflip(pconj2(a)); +#endif +} + +/** \internal packet conjugate and flip the real & imaginary results */ +EIGEN_ALWAYS_INLINE Packet2cf pcplxconjflip(Packet2cf a) { +#ifdef PERMXOR_GOOD + return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_CONJ_XOR2, p16uc_COMPLEX32_XORFLIP))); +#else + return pconj2(pcplxflip(a)); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pcplxconjflip(Packet1cd a) { +#ifdef PERMXOR_GOOD + return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_CONJ_XOR2, p16uc_COMPLEX64_XORFLIP))); +#else + return pconj2(pcplxflip(a)); +#endif +} + +/** \internal packet negate */ +EIGEN_ALWAYS_INLINE Packet2cf pnegate2(Packet2cf a) { +#ifdef __POWER8_VECTOR__ + return Packet2cf(vec_neg(a.v)); +#else + return Packet2cf(pxor(a.v, reinterpret_cast(p16uc_COMPLEX32_NEGATE))); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pnegate2(Packet1cd a) { +#ifdef __POWER8_VECTOR__ + return Packet1cd(vec_neg(a.v)); +#else + return Packet1cd(pxor(a.v, reinterpret_cast(p16uc_COMPLEX64_NEGATE))); +#endif +} + +/** \internal flip the real & imaginary results and negate */ +EIGEN_ALWAYS_INLINE Packet2cf pcplxflipnegate(Packet2cf a) { +#ifdef PERMXOR_GOOD + return Packet2cf(Packet4f(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX32_NEGATE, p16uc_COMPLEX32_XORFLIP))); +#else + return pcplxflip(pnegate2(a)); +#endif +} + +EIGEN_ALWAYS_INLINE Packet1cd pcplxflipnegate(Packet1cd a) { +#ifdef PERMXOR_GOOD + return Packet1cd(Packet2d(vec_permxor(Packet16uc(a.v), p16uc_COMPLEX64_NEGATE, p16uc_COMPLEX64_XORFLIP))); +#else + return pcplxflip(pnegate2(a)); +#endif +} + +/** \internal flip the real & imaginary results */ +EIGEN_ALWAYS_INLINE Packet2cf pcplxflip2(Packet2cf a) { + return Packet2cf(Packet4f(vec_perm(Packet16uc(a.v), Packet16uc(a.v), p16uc_COMPLEX32_XORFLIP))); +} + +EIGEN_ALWAYS_INLINE Packet1cd pcplxflip2(Packet1cd a) { +#ifdef EIGEN_VECTORIZE_VSX + return Packet1cd(__builtin_vsx_xxpermdi(a.v, a.v, 2)); +#else + return Packet1cd(Packet2d(vec_perm(Packet16uc(a.v), Packet16uc(a.v), p16uc_COMPLEX64_XORFLIP))); +#endif +} + +/** \internal load half a vector with one complex value */ +EIGEN_ALWAYS_INLINE Packet4f pload_complex_half(std::complex* src) { + Packet4f t; +#ifdef EIGEN_VECTORIZE_VSX + // Load float64/two float32 (doubleword alignment) + __asm__("lxsdx %x0,%y1" : "=wa"(t) : "Z"(*src)); +#else + *reinterpret_cast*>(reinterpret_cast(&t) + COMPLEX_DELTA) = *src; +#endif + return t; +} + +/** \internal load two vectors from the real and imaginary portions of a complex value */ +template +EIGEN_ALWAYS_INLINE void pload_realimag(RhsScalar* src, Packet4f& r, Packet4f& i) { +#ifdef _ARCH_PWR9 + __asm__("lxvwsx %x0,%y1" : "=wa"(r) : "Z"(*(reinterpret_cast(src) + 0))); + __asm__("lxvwsx %x0,%y1" : "=wa"(i) : "Z"(*(reinterpret_cast(src) + 1))); +#else + Packet4f t = pload_complex_half(src); + r = vec_splat(t, COMPLEX_DELTA + 0); + i = vec_splat(t, COMPLEX_DELTA + 1); +#endif +} + +template +EIGEN_ALWAYS_INLINE void pload_realimag(RhsScalar* src, Packet2d& r, Packet2d& i) { +#ifdef EIGEN_VECTORIZE_VSX + __asm__("lxvdsx %x0,%y1" : "=wa"(r) : "Z"(*(reinterpret_cast(src) + 0))); + __asm__("lxvdsx %x0,%y1" : "=wa"(i) : "Z"(*(reinterpret_cast(src) + 1))); +#else + Packet2d t = ploadu(reinterpret_cast(src)); + r = vec_splat(t, 0); + i = vec_splat(t, 1); +#endif +} + +#ifndef __POWER8_VECTOR__ +const Packet16uc p16uc_MERGEE = {0x00, 0x01, 0x02, 0x03, 0x10, 0x11, 0x12, 0x13, + 0x08, 0x09, 0x0A, 0x0B, 0x18, 0x19, 0x1A, 0x1B}; + +const Packet16uc p16uc_MERGEO = {0x04, 0x05, 0x06, 0x07, 0x14, 0x15, 0x16, 0x17, + 0x0C, 0x0D, 0x0E, 0x0F, 0x1C, 0x1D, 0x1E, 0x1F}; +#endif + +/** \internal load two vectors from the interleaved real & imaginary values of src */ +template +EIGEN_ALWAYS_INLINE void pload_realimag_row(RhsScalar* src, Packet4f& r, Packet4f& i) { + Packet4f t = ploadu(reinterpret_cast(src)); +#ifdef __POWER8_VECTOR__ + r = vec_mergee(t, t); + i = vec_mergeo(t, t); +#else + r = vec_perm(t, t, p16uc_MERGEE); + i = vec_perm(t, t, p16uc_MERGEO); +#endif +} + +template +EIGEN_ALWAYS_INLINE void pload_realimag_row(RhsScalar* src, Packet2d& r, Packet2d& i) { + return pload_realimag(src, r, i); +} + +/** \internal load and splat a complex value into a vector - column-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_realimag_combine(std::complex* src) { +#ifdef EIGEN_VECTORIZE_VSX + Packet4f ret; + __asm__("lxvdsx %x0,%y1" : "=wa"(ret) : "Z"(*(reinterpret_cast(src) + 0))); + return ret; +#else + return Packet4f(ploaddup(reinterpret_cast(src))); +#endif +} + +EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine(std::complex* src) { return ploadu(src).v; } + +/** \internal load a complex value into a vector - row-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_realimag_combine_row(std::complex* src) { return ploadu(src).v; } + +EIGEN_ALWAYS_INLINE Packet2d pload_realimag_combine_row(std::complex* src) { return ploadu(src).v; } + +/** \internal load a scalar or a vector from complex location */ +template +EIGEN_ALWAYS_INLINE Packet4f pload_complex(std::complex* src) { + if (GEMV_IS_SCALAR) { + return pload_complex_half(src); + } else { + return ploadu(reinterpret_cast(src)); + } +} + +template +EIGEN_ALWAYS_INLINE Packet2d pload_complex(std::complex* src) { + return ploadu(reinterpret_cast(src)); +} + +/** \internal load from a complex vector and convert to a real vector */ +template +EIGEN_ALWAYS_INLINE Packet4f pload_complex(Packet2cf* src) { + return src->v; +} + +template +EIGEN_ALWAYS_INLINE Packet2d pload_complex(Packet1cd* src) { + return src->v; +} + +/** \internal load a full vector from complex location - column-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_complex_full(std::complex* src) { + return Packet4f(ploaddup(reinterpret_cast(src))); +} + +EIGEN_ALWAYS_INLINE Packet2d pload_complex_full(std::complex* src) { return ploadu(src).v; } + +/** \internal load a full vector from complex location - row-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_complex_full_row(std::complex* src) { return ploadu(src).v; } + +EIGEN_ALWAYS_INLINE Packet2d pload_complex_full_row(std::complex* src) { return pload_complex_full(src); } + +/** \internal load a vector from a real-only scalar location - column-wise */ +EIGEN_ALWAYS_INLINE Packet4f pload_real(float* src) { return pset1(*src); } + +EIGEN_ALWAYS_INLINE Packet2d pload_real(double* src) { return pset1(*src); } + +EIGEN_ALWAYS_INLINE Packet4f pload_real(Packet4f& src) { return src; } + +EIGEN_ALWAYS_INLINE Packet2d pload_real(Packet2d& src) { return src; } + +/** \internal load a vector from a real-only vector location */ +EIGEN_ALWAYS_INLINE Packet4f pload_real_full(float* src) { + Packet4f ret = ploadu(src); + return vec_mergeh(ret, ret); +} + +EIGEN_ALWAYS_INLINE Packet2d pload_real_full(double* src) { return pload_real(src); } + +EIGEN_ALWAYS_INLINE Packet4f pload_real_full(std::complex* src) { + return pload_complex_full(src); // Just for compilation +} + +EIGEN_ALWAYS_INLINE Packet2d pload_real_full(std::complex* src) { + return pload_complex_full(src); // Just for compilation +} + +/** \internal load a vector from a real-only scalar location - row-wise */ +template +EIGEN_ALWAYS_INLINE Packet4f pload_real_row(float* src) { + if (GEMV_IS_SCALAR) { + return pload_real_full(src); + } else { + return ploadu(src); + } +} + +template +EIGEN_ALWAYS_INLINE Packet2d pload_real_row(double* src) { + return pload_real(src); +} + +EIGEN_ALWAYS_INLINE Packet2cf padd(Packet2cf& a, std::complex& b) { + EIGEN_UNUSED_VARIABLE(b); + return a; // Just for compilation +} + +EIGEN_ALWAYS_INLINE Packet1cd padd(Packet1cd& a, std::complex& b) { + EIGEN_UNUSED_VARIABLE(b); + return a; // Just for compilation +} + +/** \internal set a scalar from complex location */ +template +EIGEN_ALWAYS_INLINE Scalar pset1_realimag(ResScalar& alpha, int which, int conj) { + return (which) ? ((conj) ? -alpha.real() : alpha.real()) : ((conj) ? -alpha.imag() : alpha.imag()); +} + +/** \internal set a vector from complex location */ +template +EIGEN_ALWAYS_INLINE Packet2cf pset1_complex(std::complex& alpha) { + Packet2cf ret; + ret.v[COMPLEX_DELTA + 0] = pset1_realimag(alpha, (which & 0x01), (which & 0x04)); + ret.v[COMPLEX_DELTA + 1] = pset1_realimag(alpha, (which & 0x02), (which & 0x08)); + ret.v[2 - COMPLEX_DELTA] = ret.v[COMPLEX_DELTA + 0]; + ret.v[3 - COMPLEX_DELTA] = ret.v[COMPLEX_DELTA + 1]; + return ret; +} + +template +EIGEN_ALWAYS_INLINE Packet1cd pset1_complex(std::complex& alpha) { + Packet1cd ret; + ret.v[0] = pset1_realimag(alpha, (which & 0x01), (which & 0x04)); + ret.v[1] = pset1_realimag(alpha, (which & 0x02), (which & 0x08)); + return ret; +} + +/** \internal zero out a vector for real or complex forms */ +template +EIGEN_ALWAYS_INLINE Packet pset_zero() { + return pset1(__UNPACK_TYPE__(Packet)(0)); +} + +template <> +EIGEN_ALWAYS_INLINE Packet2cf pset_zero() { + return Packet2cf(pset1(float(0))); +} + +template <> +EIGEN_ALWAYS_INLINE Packet1cd pset_zero() { + return Packet1cd(pset1(double(0))); +} + +/** \internal initialize a vector from another vector */ +template +EIGEN_ALWAYS_INLINE Packet pset_init(Packet& c1) { + if (GEMV_IS_COMPLEX_COMPLEX) { + EIGEN_UNUSED_VARIABLE(c1); + return pset_zero(); + } else { + return c1; // Intentionally left uninitialized + } +} + +template +struct alpha_store { + alpha_store(ResScalar& alpha) { + separate.r = pset1_complex(alpha); + separate.i = pset1_complex(alpha); + } + struct ri { + PResPacket r; + PResPacket i; + } separate; +}; + +/** \internal multiply and add for complex math */ +template +EIGEN_ALWAYS_INLINE ScalarPacket pmadd_complex(ScalarPacket& c0, ScalarPacket& c2, ScalarPacket& c4, AlphaData& b0) { + return pmadd(c2, b0.separate.i.v, pmadd(c0, b0.separate.r.v, c4)); +} + +/** \internal store and madd for complex math */ +template +EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, AlphaData& b0, ResScalar* res) { + PResPacket c2 = pcplxflipconj(c0); + if (GEMV_IS_SCALAR) { + ScalarPacket c4 = ploadu(reinterpret_cast(res)); + ScalarPacket c3 = pmadd_complex(c0.v, c2.v, c4, b0); + pstoreu(reinterpret_cast(res), c3); + } else { + ScalarPacket c4 = pload_complex(res); + PResPacket c3 = PResPacket(pmadd_complex(c0.v, c2.v, c4, b0)); + pstoreu(res, c3); + } +} + +template +EIGEN_ALWAYS_INLINE void pstoreu_pmadd_complex(PResPacket& c0, PResPacket& c1, AlphaData& b0, ResScalar* res) { + PResPacket c2 = pcplxflipconj(c0); + PResPacket c3 = pcplxflipconj(c1); +#if !defined(_ARCH_PWR10) + ScalarPacket c4 = pload_complex(res + (iter2 * ResPacketSize)); + ScalarPacket c5 = pload_complex(res + ((iter2 + 1) * ResPacketSize)); + PResPacket c6 = PResPacket(pmadd_complex(c0.v, c2.v, c4, b0)); + PResPacket c7 = PResPacket(pmadd_complex(c1.v, c3.v, c5, b0)); + pstoreu(res + (iter2 * ResPacketSize), c6); + pstoreu(res + ((iter2 + 1) * ResPacketSize), c7); +#else + __vector_pair a = *reinterpret_cast<__vector_pair*>(res + (iter2 * ResPacketSize)); +#if EIGEN_COMP_LLVM + PResPacket c6[2]; + __builtin_vsx_disassemble_pair(reinterpret_cast(c6), &a); + c6[0] = PResPacket(pmadd_complex(c0.v, c2.v, c6[0].v, b0)); + c6[1] = PResPacket(pmadd_complex(c1.v, c3.v, c6[1].v, b0)); + GEMV_BUILDPAIR_MMA(a, c6[0].v, c6[1].v); +#else + if (GEMV_IS_COMPLEX_FLOAT) { + __asm__("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.r.v), "wa"(c0.v), "wa"(c1.v)); + __asm__("xvmaddasp %L0,%x1,%x2\n\txvmaddasp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.i.v), "wa"(c2.v), "wa"(c3.v)); + } else { + __asm__("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.r.v), "wa"(c0.v), "wa"(c1.v)); + __asm__("xvmaddadp %L0,%x1,%x2\n\txvmaddadp %0,%x1,%x3" : "+&d"(a) : "wa"(b0.separate.i.v), "wa"(c2.v), "wa"(c3.v)); + } +#endif + *reinterpret_cast<__vector_pair*>(res + (iter2 * ResPacketSize)) = a; +#endif +} + +/** \internal load lhs packet */ +template +EIGEN_ALWAYS_INLINE LhsPacket loadLhsPacket(LhsMapper& lhs, Index i, Index j) { + if (sizeof(Scalar) == sizeof(LhsScalar)) { + const LhsScalar& src = lhs(i + 0, j); + return LhsPacket(pload_real_full(const_cast(&src))); + } + return lhs.template load(i + 0, j); +} + +/** \internal madd for complex times complex */ +template +EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_complex(RealPacket& a, RealPacket& b, RealPacket& c) { + if (ConjugateLhs && ConjugateRhs) { + return vec_madd(a, pconj2(ComplexPacket(b)).v, c); + } else if (Negate && !ConjugateLhs && ConjugateRhs) { + return vec_nmsub(a, b, c); + } else { + return vec_madd(a, b, c); + } +} + +/** \internal madd for complex times real */ +template +EIGEN_ALWAYS_INLINE RealPacket pmadd_complex_real(RealPacket& a, RealPacket& b, RealPacket& c) { + if (Conjugate) { + return vec_madd(a, pconj2(ComplexPacket(b)).v, c); + } else { + return vec_madd(a, b, c); + } +} + +template +EIGEN_ALWAYS_INLINE void gemv_mult_generic(LhsPacket& a0, RhsScalar* b, PResPacket& c0) { + conj_helper pcj; + RhsPacket b0; + if (StorageOrder == ColMajor) { + b0 = pset1(*b); + } else { + b0 = ploadu(b); + } + c0 = pcj.pmadd(a0, b0, c0); +} + +/** \internal core multiply operation for vectors - complex times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0, ResPacket& c1) { + ScalarPacket br, bi; + if (StorageOrder == ColMajor) { + pload_realimag(b, br, bi); + } else { + pload_realimag_row(b, br, bi); + } + if (ConjugateLhs && !ConjugateRhs) a0 = pconj2(a0); + LhsPacket a1 = pcplxflipconj(a0); + ScalarPacket cr = pmadd_complex_complex(a0.v, br, c0.v); + ScalarPacket ci = pmadd_complex_complex(a1.v, bi, c1.v); + c1 = ResPacket(ci); + c0 = PResPacket(cr); +} + +/** \internal core multiply operation for vectors - real times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_real_complex(LhsPacket& a0, RhsScalar* b, PResPacket& c0) { + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_complex_full(b); + } else { + b0 = pload_complex_full_row(b); + } + ScalarPacket cri = pmadd_complex_real(a0, b0, c0.v); + c0 = PResPacket(cri); +} + +/** \internal core multiply operation for vectors - complex times real */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_real(LhsPacket& a0, RhsScalar* b, PResPacket& c0) { + ScalarPacket a1 = pload_complex(&a0); + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_real(b); + } else { + b0 = pload_real_row(b); + } + ScalarPacket cri = pmadd_complex_real(a1, b0, c0.v); + c0 = PResPacket(cri); +} + +#define GEMV_MULT_COMPLEX_COMPLEX(LhsType, RhsType, ResType) \ + template \ + EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType& c0, ResType& c1) { \ + gemv_mult_complex_complex(a0, b, c0, c1); \ + } + +GEMV_MULT_COMPLEX_COMPLEX(Packet2cf, std::complex, Packet2cf) +GEMV_MULT_COMPLEX_COMPLEX(Packet1cd, std::complex, Packet1cd) + +#define GEMV_MULT_REAL_COMPLEX(LhsType, RhsType, ResType) \ + template \ + EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType& c0, RhsType&) { \ + gemv_mult_real_complex(a0, b, c0); \ + } + +GEMV_MULT_REAL_COMPLEX(float, std::complex, Packet2cf) +GEMV_MULT_REAL_COMPLEX(double, std::complex, Packet1cd) +GEMV_MULT_REAL_COMPLEX(Packet4f, std::complex, Packet2cf) +GEMV_MULT_REAL_COMPLEX(Packet2d, std::complex, Packet1cd) + +#define GEMV_MULT_COMPLEX_REAL(LhsType, RhsType, ResType1, ResType2) \ + template \ + EIGEN_ALWAYS_INLINE void gemv_mult_complex(LhsType& a0, RhsType* b, ResType1& c0, ResType2&) { \ + gemv_mult_complex_real(a0, b, c0); \ + } + +GEMV_MULT_COMPLEX_REAL(Packet2cf, float, Packet2cf, std::complex) +GEMV_MULT_COMPLEX_REAL(Packet1cd, double, Packet1cd, std::complex) +GEMV_MULT_COMPLEX_REAL(std::complex, float, Packet2cf, std::complex) +GEMV_MULT_COMPLEX_REAL(std::complex, double, Packet1cd, std::complex) + +#ifdef USE_GEMV_MMA +/** \internal convert packet to real form */ +template +EIGEN_ALWAYS_INLINE T convertReal(T a) { + return a; +} + +EIGEN_ALWAYS_INLINE Packet4f convertReal(Packet2cf a) { return a.v; } + +EIGEN_ALWAYS_INLINE Packet2d convertReal(Packet1cd a) { return a.v; } + +/** \internal convert packet to complex form */ +template +EIGEN_ALWAYS_INLINE T convertComplex(T a) { + return a; +} + +EIGEN_ALWAYS_INLINE Packet2cf convertComplex(Packet4f a) { return Packet2cf(a); } + +EIGEN_ALWAYS_INLINE Packet1cd convertComplex(Packet2d a) { return Packet1cd(a); } + +/** \internal load a vector from a complex location (for MMA version) */ +template +EIGEN_ALWAYS_INLINE void pload_complex_MMA(SLhsPacket& a) { + a = SLhsPacket(pload_complex(&a)); +} + +template +EIGEN_ALWAYS_INLINE void pload_complex_MMA(__vector_pair&) { + // Pass thru +} + +/** \internal perform a matrix multiply and accumulate (positive and negative) of packet a and packet b */ +template +EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, RhsPacket& a, LhsPacket& b) { + if (NegativeAccumulate) { + __builtin_mma_xvf32gernp(acc, (__vector unsigned char)a, (__vector unsigned char)b); + } else { + __builtin_mma_xvf32gerpp(acc, (__vector unsigned char)a, (__vector unsigned char)b); + } +} + +/** \internal perform a matrix multiply and accumulate (positive and negative) of vector_pair a and packet b */ +template +EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad* acc, __vector_pair& a, Packet2d& b) { + if (NegativeAccumulate) { + __builtin_mma_xvf64gernp(acc, (__vector_pair)a, (__vector unsigned char)b); + } else { + __builtin_mma_xvf64gerpp(acc, (__vector_pair)a, (__vector unsigned char)b); + } +} + +template +EIGEN_ALWAYS_INLINE void pger_vecMMA(__vector_quad*, __vector_pair&, Packet4f&) { + // Just for compilation +} + +/** \internal madd for complex times complex (MMA version) */ +template +EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(LhsPacket& a, RealPacket& b, __vector_quad* c) { + if (ConjugateLhs && ConjugateRhs) { + RealPacket b2 = pconj2(convertComplex(b)).v; + return pger_vecMMA(c, b2, a.v); + } else if (Negate && !ConjugateLhs && ConjugateRhs) { + return pger_vecMMA(c, b, a.v); + } else { + return pger_vecMMA(c, b, a.v); + } +} + +template +EIGEN_ALWAYS_INLINE void pmadd_complex_complex_MMA(__vector_pair& a, RealPacket& b, __vector_quad* c) { + if (ConjugateLhs && ConjugateRhs) { + RealPacket b2 = pconj2(convertComplex(b)).v; + return pger_vecMMA(c, a, b2); + } else if (Negate && !ConjugateLhs && ConjugateRhs) { + return pger_vecMMA(c, a, b); + } else { + return pger_vecMMA(c, a, b); + } +} + +/** \internal madd for complex times real (MMA version) */ +template +EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(LhsPacket& a, RealPacket& b, __vector_quad* c) { + RealPacket a2 = convertReal(a); + if (Conjugate) { + RealPacket b2 = pconj2(convertComplex(b)).v; + if (StorageOrder == ColMajor) { + return pger_vecMMA(c, b2, a2); + } else { + return pger_vecMMA(c, a2, b2); + } + } else { + if (StorageOrder == ColMajor) { + return pger_vecMMA(c, b, a2); + } else { + return pger_vecMMA(c, a2, b); + } + } +} + +/** \internal madd for real times complex (MMA version) */ +template +EIGEN_ALWAYS_INLINE void pmadd_complex_real_MMA(__vector_pair& a, RealPacket& b, __vector_quad* c) { + if (Conjugate) { + RealPacket b2 = pconj2(convertComplex(b)).v; + return pger_vecMMA(c, a, b2); + } else { + return pger_vecMMA(c, a, b); + } +} + +/** \internal core multiply operation for vectors (MMA version) - complex times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) { + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_realimag_combine(b); + } else { + b0 = pload_realimag_combine_row(b); + } + pmadd_complex_complex_MMA(a0, b0, c0); +} + +/** \internal core multiply operation for vectors (MMA version) - complex times real */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_real_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) { + pload_complex_MMA(a0); + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_real(b); + } else { + b0 = pload_real_row(b); + } + pmadd_complex_real_MMA(a0, b0, c0); +} + +/** \internal core multiply operation for vectors (MMA version) - real times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_real_complex_MMA(SLhsPacket& a0, RhsScalar* b, __vector_quad* c0) { + ScalarPacket b0; + if (StorageOrder == ColMajor) { + b0 = pload_complex_full(b); + } else { + b0 = pload_complex_full_row(b); + } + pmadd_complex_real_MMA)) ? StorageOrder : ColMajor>(a0, b0, c0); +} + +#define GEMV_MULT_COMPLEX_COMPLEX_MMA(LhsType, RhsType) \ + template \ + EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) { \ + gemv_mult_complex_complex_MMA(a0, b, c0); \ + } + +GEMV_MULT_COMPLEX_COMPLEX_MMA(Packet2cf, std::complex) +GEMV_MULT_COMPLEX_COMPLEX_MMA(__vector_pair, std::complex) +GEMV_MULT_COMPLEX_COMPLEX_MMA(Packet1cd, std::complex) + +/** \internal core multiply operation for vectors (MMA version) - complex times complex */ +template +EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(__vector_pair& a0, std::complex* b, __vector_quad* c0) { + if (sizeof(LhsScalar) == 16) { + gemv_mult_complex_complex_MMA(a0, b, c0); + } else { + gemv_mult_real_complex_MMA(a0, b, c0); + } +} + +#define GEMV_MULT_REAL_COMPLEX_MMA(LhsType, RhsType) \ + template \ + EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) { \ + gemv_mult_real_complex_MMA(a0, b, c0); \ + } + +GEMV_MULT_REAL_COMPLEX_MMA(Packet4f, std::complex) +GEMV_MULT_REAL_COMPLEX_MMA(Packet2d, std::complex) + +#define GEMV_MULT_COMPLEX_REAL_MMA(LhsType, RhsType) \ + template \ + EIGEN_ALWAYS_INLINE void gemv_mult_complex_MMA(LhsType& a0, RhsType* b, __vector_quad* c0) { \ + gemv_mult_complex_real_MMA(a0, b, c0); \ + } + +GEMV_MULT_COMPLEX_REAL_MMA(Packet2cf, float) +GEMV_MULT_COMPLEX_REAL_MMA(Packet1cd, double) +GEMV_MULT_COMPLEX_REAL_MMA(__vector_pair, float) +GEMV_MULT_COMPLEX_REAL_MMA(__vector_pair, double) + +/** \internal disassemble MMA accumulator results into packets */ +template +EIGEN_ALWAYS_INLINE void disassembleResults2(__vector_quad* c0, PacketBlock& result0) { + __builtin_mma_disassemble_acc(&result0.packet, c0); + if (sizeof(LhsPacket) == 16) { + if (sizeof(RhsPacket) == 16) { + ScalarPacket tmp0, tmp2; + tmp2 = vec_mergeh(result0.packet[2], result0.packet[3]); + tmp0 = vec_mergeh(result0.packet[0], result0.packet[1]); + result0.packet[3] = vec_mergel(result0.packet[3], result0.packet[2]); + result0.packet[1] = vec_mergel(result0.packet[1], result0.packet[0]); + result0.packet[2] = tmp2; + result0.packet[0] = tmp0; + + if (ConjugateLhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v; + } else if (ConjugateRhs) { + result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v; + result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v; + } else { + result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v; + result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v; + } + result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); + result0.packet[2] = vec_add(result0.packet[2], result0.packet[3]); + } else { + result0.packet[0][1] = result0.packet[1][1]; + result0.packet[2][1] = result0.packet[3][1]; + } + } +} + +template +EIGEN_ALWAYS_INLINE void disassembleResults4(__vector_quad* c0, PacketBlock& result0) { + __builtin_mma_disassemble_acc(&result0.packet, c0); + if (GEMV_IS_COMPLEX_COMPLEX) { + if (ConjugateLhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v; + } else { + if (ConjugateRhs) { + result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; + } else { + result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v; + } + } + result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); + } else if (sizeof(LhsPacket) == sizeof(std::complex)) { + if (ConjugateLhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + } + } else { + result0.packet[0] = vec_mergee(result0.packet[0], result0.packet[1]); + } +} + +template +EIGEN_ALWAYS_INLINE void disassembleResults(__vector_quad* c0, PacketBlock& result0) { + if (!GEMV_IS_COMPLEX_FLOAT) { + disassembleResults2(c0, result0); + } else { + disassembleResults4(c0, result0); + } +} +#endif + +#define GEMV_GETN_COMPLEX(N) (((N) * ResPacketSize) >> 1) + +#define GEMV_LOADPACKET_COL_COMPLEX(iter) \ + loadLhsPacket(lhs, i + ((iter) * ResPacketSize), j) + +#define GEMV_LOADPACKET_COL_COMPLEX_DATA(iter) convertReal(GEMV_LOADPACKET_COL_COMPLEX(iter)) + +#ifdef USE_GEMV_MMA +#define GEMV_INIT_COL_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + __builtin_mma_xxsetaccz(&e0##iter); \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_COL_COMPLEX_DATA(iter2), \ + GEMV_LOADPACKET_COL_COMPLEX_DATA((iter2) + 1)); \ + EIGEN_UNUSED_VARIABLE(f##iter1); +#else +#define GEMV_LOADPAIR_COL_COMPLEX_MMA(iter1, iter2) \ + if (sizeof(LhsPacket) == 16) { \ + const LhsScalar& src = lhs(i + ((32 * iter1) / sizeof(LhsScalar)), j); \ + a##iter1 = *reinterpret_cast<__vector_pair*>(const_cast(&src)); \ + EIGEN_UNUSED_VARIABLE(f##iter1); \ + } else { \ + f##iter1 = lhs.template load(i + ((iter2) * ResPacketSize), j); \ + GEMV_BUILDPAIR_MMA(a##iter1, vec_splat(convertReal(f##iter1), 0), vec_splat(convertReal(f##iter1), 1)); \ + } +#endif + +#define GEMV_LOAD1_COL_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \ + EIGEN_UNUSED_VARIABLE(a##iter); \ + } else { \ + GEMV_LOADPAIR_COL_COMPLEX_MMA(iter, iter << 1) \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(a##iter); \ + EIGEN_UNUSED_VARIABLE(f##iter); \ + } + +#define GEMV_WORK1_COL_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + gemv_mult_complex_MMA(f##iter, b, &e0##iter); \ + } else { \ + gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ + } \ + } + +#define GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_COL_COMPLEX_DATA(iter2), GEMV_LOADPACKET_COL_COMPLEX_DATA((iter2) + 1)); + +#define GEMV_LOAD2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN_COMPLEX(N) > iter1) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2); \ + EIGEN_UNUSED_VARIABLE(a##iter3) \ + } else { \ + GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter2, iter2 << 1); \ + GEMV_LOADPAIR2_COL_COMPLEX_MMA(iter3, iter3 << 1); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(a##iter2); \ + EIGEN_UNUSED_VARIABLE(a##iter3); \ + } \ + EIGEN_UNUSED_VARIABLE(f##iter2); \ + EIGEN_UNUSED_VARIABLE(f##iter3); + +#define GEMV_WORK2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN_COMPLEX(N) > iter1) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + PLhsPacket g[2]; \ + __builtin_vsx_disassemble_pair(reinterpret_cast(g), &a##iter2); \ + gemv_mult_complex_MMA(g[0], b, &e0##iter2); \ + gemv_mult_complex_MMA(g[1], b, &e0##iter3); \ + } else { \ + gemv_mult_complex_MMA(a##iter2, b, &e0##iter2); \ + gemv_mult_complex_MMA(a##iter3, b, &e0##iter3); \ + } \ + } + +#if EIGEN_COMP_LLVM +#define GEMV_LOAD_COL_COMPLEX_MMA(N) \ + if (GEMV_GETN_COMPLEX(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_LOAD2_COL_COMPLEX_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) \ + } + +#define GEMV_WORK_COL_COMPLEX_MMA(N) \ + if (GEMV_GETN_COMPLEX(N) > 1) { \ + GEMV_UNROLL_HALF(GEMV_WORK2_COL_COMPLEX_MMA, (N >> 1)) \ + } else { \ + GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) \ + } +#else +#define GEMV_LOAD_COL_COMPLEX_MMA(N) GEMV_UNROLL(GEMV_LOAD1_COL_COMPLEX_MMA, N) + +#define GEMV_WORK_COL_COMPLEX_MMA(N) GEMV_UNROLL(GEMV_WORK1_COL_COMPLEX_MMA, N) +#endif + +#define GEMV_DISASSEMBLE_COMPLEX_MMA(iter) \ + disassembleResults( \ + &e0##iter, result0##iter); + +#define GEMV_STORE_COL_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + GEMV_DISASSEMBLE_COMPLEX_MMA(iter); \ + c0##iter = PResPacket(result0##iter.packet[0]); \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + pstoreu_pmadd_complex( \ + c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \ + } else { \ + pstoreu_pmadd_complex( \ + c0##iter, alpha_data, res + i + ((iter << 1) * ResPacketSize)); \ + c0##iter = PResPacket(result0##iter.packet[2]); \ + pstoreu_pmadd_complex( \ + c0##iter, alpha_data, res + i + (((iter << 1) + 1) * ResPacketSize)); \ + } \ + } + +#define GEMV_STORE2_COL_COMPLEX_MMA(iter1, iter2, iter3, N) \ + if (GEMV_GETN_COMPLEX(N) > iter1) { \ + GEMV_DISASSEMBLE_COMPLEX_MMA(iter2); \ + GEMV_DISASSEMBLE_COMPLEX_MMA(iter3); \ + c0##iter2 = PResPacket(result0##iter2.packet[0]); \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + c0##iter3 = PResPacket(result0##iter3.packet[0]); \ + pstoreu_pmadd_complex( \ + c0##iter2, c0##iter3, alpha_data, res + i); \ + } else { \ + c0##iter3 = PResPacket(result0##iter2.packet[2]); \ + pstoreu_pmadd_complex( \ + c0##iter2, c0##iter3, alpha_data, res + i); \ + c0##iter2 = PResPacket(result0##iter3.packet[0]); \ + c0##iter3 = PResPacket(result0##iter3.packet[2]); \ + pstoreu_pmadd_complex( \ + c0##iter2, c0##iter3, alpha_data, res + i); \ + } \ + } + +#define GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \ + GEMV_UNROLL(GEMV_INIT_COL_COMPLEX_MMA, N) \ + Index j = j2; \ + do { \ + const RhsScalar& b1 = rhs2(j, 0); \ + RhsScalar* b = const_cast(&b1); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_LOAD_COL_COMPLEX_MMA(N) \ + GEMV_WORK_COL_COMPLEX_MMA(N) \ + } while (++j < jend); \ + if (GEMV_GETN(N) <= 2) { \ + GEMV_UNROLL(GEMV_STORE_COL_COMPLEX_MMA, N) \ + } else { \ + GEMV_UNROLL_HALF(GEMV_STORE2_COL_COMPLEX_MMA, (N >> 1)) \ + } \ + i += (ResPacketSize * N); +#endif + +#define GEMV_INIT_COMPLEX(iter, N) \ + if (N > iter) { \ + c0##iter = pset_zero(); \ + c1##iter = pset_init(c1##iter); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c0##iter); \ + EIGEN_UNUSED_VARIABLE(c1##iter); \ + } + +#define GEMV_WORK_COL_COMPLEX(iter, N) \ + if (N > iter) { \ + f##iter = GEMV_LOADPACKET_COL_COMPLEX(iter); \ + gemv_mult_complex(f##iter, b, c0##iter, c1##iter); \ + } else { \ + EIGEN_UNUSED_VARIABLE(f##iter); \ + } + +#define GEMV_STORE_COL_COMPLEX(iter, N) \ + if (N > iter) { \ + if (GEMV_IS_COMPLEX_COMPLEX) { \ + c0##iter = padd(c0##iter, c1##iter); \ + } \ + pstoreu_pmadd_complex( \ + c0##iter, alpha_data, res + i + (iter * ResPacketSize)); \ + } + +/** \internal main macro for gemv_complex_col - initialize accumulators, multiply and add inputs, and store results */ +#define GEMV_PROCESS_COL_COMPLEX_ONE(N) \ + GEMV_UNROLL(GEMV_INIT_COMPLEX, N) \ + Index j = j2; \ + do { \ + const RhsScalar& b1 = rhs2(j, 0); \ + RhsScalar* b = const_cast(&b1); \ + GEMV_UNROLL(GEMV_PREFETCH, N) \ + GEMV_UNROLL(GEMV_WORK_COL_COMPLEX, N) \ + } while (++j < jend); \ + GEMV_UNROLL(GEMV_STORE_COL_COMPLEX, N) \ + i += (ResPacketSize * N); + +#if defined(USE_GEMV_MMA) && (EIGEN_COMP_LLVM || defined(USE_SLOWER_GEMV_MMA)) +#define USE_GEMV_COL_COMPLEX_MMA +#endif + +#ifdef USE_GEMV_COL_COMPLEX_MMA +#define GEMV_PROCESS_COL_COMPLEX(N) GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) +#else +#if defined(USE_GEMV_MMA) && (__GNUC__ > 10) +#define GEMV_PROCESS_COL_COMPLEX(N) \ + if (sizeof(Scalar) != sizeof(LhsPacket)) { \ + GEMV_PROCESS_COL_COMPLEX_ONE_MMA(N) \ + } else { \ + GEMV_PROCESS_COL_COMPLEX_ONE(N) \ + } +#else +#define GEMV_PROCESS_COL_COMPLEX(N) GEMV_PROCESS_COL_COMPLEX_ONE(N) +#endif +#endif + +template +EIGEN_STRONG_INLINE void gemv_complex_col(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, + ResScalar* res, Index resIncr, ResScalar alpha) { + typedef gemv_traits Traits; + + typedef typename Traits::LhsPacket LhsPacket; + typedef typename Traits::RhsPacket RhsPacket; + typedef typename Traits::ResPacket ResPacket; + + typedef typename packet_traits::type ScalarPacket; + typedef typename packet_traits::type PLhsPacket; + typedef typename packet_traits::type PResPacket; + typedef gemv_traits PTraits; + + EIGEN_UNUSED_VARIABLE(resIncr); + eigen_internal_assert(resIncr == 1); + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + RhsMapper rhs2(rhs); + + conj_helper cj; + + const Index lhsStride = lhs.stride(); + // TODO: for padded aligned inputs, we could enable aligned reads + enum { + LhsAlignment = Unaligned, + ResPacketSize = PTraits::ResPacketSize, + LhsPacketSize = PTraits::LhsPacketSize, + RhsPacketSize = PTraits::RhsPacketSize, + }; +#ifdef EIGEN_POWER_USE_GEMV_PREFETCH + const Index prefetch_dist = 64 * LhsPacketSize; +#endif + +#ifndef GCC_ONE_VECTORPAIR_BUG + const Index n8 = rows - 8 * ResPacketSize + 1; + const Index n4 = rows - 4 * ResPacketSize + 1; + const Index n2 = rows - 2 * ResPacketSize + 1; +#endif + const Index n1 = rows - 1 * ResPacketSize + 1; + + // TODO: improve the following heuristic: + const Index block_cols = cols < 128 ? cols : (lhsStride * sizeof(LhsScalar) < 16000 ? 16 : 8); + + typedef alpha_store AlphaData; + AlphaData alpha_data(alpha); + + for (Index j2 = 0; j2 < cols; j2 += block_cols) { + Index jend = numext::mini(j2 + block_cols, cols); + Index i = 0; + PResPacket c00, c01, c02, c03, c04, c05, c06, c07; + ResPacket c10, c11, c12, c13, c14, c15, c16, c17; + PLhsPacket f0, f1, f2, f3, f4, f5, f6, f7; +#ifdef USE_GEMV_MMA + __vector_quad e00, e01, e02, e03, e04, e05, e06, e07; + __vector_pair a0, a1, a2, a3, a4, a5, a6, a7; + PacketBlock result00, result01, result02, result03, result04, result05, result06, result07; + GEMV_UNUSED(8, e0) + GEMV_UNUSED(8, result0) + GEMV_UNUSED(8, a) + GEMV_UNUSED(8, f) +#if !defined(GCC_ONE_VECTORPAIR_BUG) && defined(USE_GEMV_COL_COMPLEX_MMA) + if (GEMV_IS_COMPLEX_COMPLEX || !GEMV_IS_COMPLEX_FLOAT) +#endif +#endif +#ifndef GCC_ONE_VECTORPAIR_BUG + { + while (i < n8) { + GEMV_PROCESS_COL_COMPLEX(8) + } + } + while (i < n4) { + GEMV_PROCESS_COL_COMPLEX(4) + } + if (i < n2) { + GEMV_PROCESS_COL_COMPLEX(2) + } + if (i < n1) +#else + while (i < n1) +#endif + { + GEMV_PROCESS_COL_COMPLEX_ONE(1) + } + for (; i < rows; ++i) { + ResScalar d0(0); + Index j = j2; + do { + d0 += cj.pmul(lhs(i, j), rhs2(j, 0)); + } while (++j < jend); + res[i] += alpha * d0; + } + } +} + +template +struct ScalarBlock { + Scalar scalar[N]; +}; + +#ifdef USE_GEMV_MMA +static Packet16uc p16uc_ELEMENT_3 = {0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f, + 0x0c, 0x0d, 0x0e, 0x0f, 0x1c, 0x1d, 0x1e, 0x1f}; + +/** \internal predux (add elements of a vector) from a MMA accumulator - real results */ +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_real(__vector_quad* acc0, __vector_quad* acc1) { + PacketBlock result0, result1; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + __builtin_mma_disassemble_acc(&result1.packet, acc1); + result0.packet[0] = vec_mergeh(result0.packet[0], result1.packet[0]); + result0.packet[1] = vec_mergeo(result0.packet[1], result1.packet[1]); + result0.packet[2] = vec_mergel(result0.packet[2], result1.packet[2]); + result0.packet[3] = vec_perm(result0.packet[3], result1.packet[3], p16uc_ELEMENT_3); + result0.packet[0] = + vec_add(vec_add(result0.packet[0], result0.packet[2]), vec_add(result0.packet[1], result0.packet[3])); + return *reinterpret_cast*>(&result0.packet[0]); +} + +template <> +EIGEN_ALWAYS_INLINE ScalarBlock predux_real(__vector_quad* acc0, __vector_quad* acc1) { + PacketBlock result0, result1; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + __builtin_mma_disassemble_acc(&result1.packet, acc1); + result0.packet[0] = + vec_add(vec_mergeh(result0.packet[0], result1.packet[0]), vec_mergel(result0.packet[1], result1.packet[1])); + return *reinterpret_cast*>(&result0.packet[0]); +} + +/** \internal add complex results together */ +template +EIGEN_ALWAYS_INLINE ScalarBlock, 2> addComplexResults(PacketBlock& result0, + PacketBlock& result1) { + ScalarBlock, 2> cc0; + result0.packet[0] = reinterpret_cast( + vec_mergeh(reinterpret_cast(result0.packet[0]), reinterpret_cast(result1.packet[0]))); + result0.packet[2] = reinterpret_cast( + vec_mergel(reinterpret_cast(result0.packet[2]), reinterpret_cast(result1.packet[2]))); + result0.packet[0] = vec_add(result0.packet[0], result0.packet[2]); + if (GEMV_IS_COMPLEX_COMPLEX) { + result0.packet[1] = reinterpret_cast( + vec_mergeh(reinterpret_cast(result0.packet[1]), reinterpret_cast(result1.packet[1]))); + result0.packet[3] = reinterpret_cast( + vec_mergel(reinterpret_cast(result0.packet[3]), reinterpret_cast(result1.packet[3]))); + result0.packet[1] = vec_add(result0.packet[1], result0.packet[3]); + if (ConjugateLhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + result0.packet[1] = pcplxflip2(convertComplex(result0.packet[1])).v; + } else if (ConjugateRhs) { + result0.packet[1] = pcplxconjflip(convertComplex(result0.packet[1])).v; + } else { + result0.packet[1] = pcplxflipconj(convertComplex(result0.packet[1])).v; + } + result0.packet[0] = vec_add(result0.packet[0], result0.packet[1]); + } else { + if (ConjugateLhs && (sizeof(LhsPacket) == sizeof(std::complex))) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + } + } + cc0.scalar[0].real(result0.packet[0][0]); + cc0.scalar[0].imag(result0.packet[0][1]); + cc0.scalar[1].real(result0.packet[0][2]); + cc0.scalar[1].imag(result0.packet[0][3]); + return cc0; +} + +template +EIGEN_ALWAYS_INLINE ScalarBlock, 2> addComplexResults(PacketBlock&, + PacketBlock&) { + ScalarBlock, 2> cc0; + EIGEN_UNUSED_VARIABLE(cc0); + return cc0; // Just for compilation +} + +/** \internal predux (add elements of a vector) from a MMA accumulator - complex results */ +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(__vector_quad* acc0, __vector_quad* acc1) { + PacketBlock result0, result1; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + __builtin_mma_disassemble_acc(&result1.packet, acc1); + return addComplexResults(result0, result1); +} + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_real(__vector_quad* acc0) { + PacketBlock result0; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + result0.packet[0] = + vec_add(vec_mergeh(result0.packet[0], result0.packet[2]), vec_mergel(result0.packet[1], result0.packet[3])); + return *reinterpret_cast*>(&result0.packet[0]); +} + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(__vector_quad* acc0) { + ScalarBlock cc0; + PacketBlock result0; + __builtin_mma_disassemble_acc(&result0.packet, acc0); + if (GEMV_IS_COMPLEX_COMPLEX) { + if (ConjugateLhs) { + result0.packet[1] = pconjinv(convertComplex(result0.packet[1])).v; + result0.packet[3] = pconjinv(convertComplex(result0.packet[3])).v; + } else if (ConjugateRhs) { + result0.packet[0] = pconj2(convertComplex(result0.packet[0])).v; + result0.packet[2] = pconj2(convertComplex(result0.packet[2])).v; + } else { + result0.packet[1] = pconj2(convertComplex(result0.packet[1])).v; + result0.packet[3] = pconj2(convertComplex(result0.packet[3])).v; + } + result0.packet[0] = vec_add(result0.packet[0], __builtin_vsx_xxpermdi(result0.packet[1], result0.packet[1], 2)); + result0.packet[2] = vec_add(result0.packet[2], __builtin_vsx_xxpermdi(result0.packet[3], result0.packet[3], 2)); + } else { + result0.packet[0] = __builtin_vsx_xxpermdi(result0.packet[0], result0.packet[1], 1); + result0.packet[2] = __builtin_vsx_xxpermdi(result0.packet[2], result0.packet[3], 1); + } + cc0.scalar[0].real(result0.packet[0][0]); + cc0.scalar[0].imag(result0.packet[0][1]); + cc0.scalar[1].real(result0.packet[2][0]); + cc0.scalar[1].imag(result0.packet[2][1]); + return cc0; +} +#endif + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_real(ResPacket& a, ResPacket& b) { + ScalarBlock cc0; + cc0.scalar[0] = predux(a); + cc0.scalar[1] = predux(b); + return cc0; +} + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(ResPacket& a, ResPacket& b) { + return predux_real(a, b); +} + +#define GEMV_UNROLL_ROW(func, N) func(0, N) func(1, N) func(2, N) func(3, N) func(4, N) func(5, N) func(6, N) func(7, N) + +#define GEMV_UNROLL_ROW_HALF(func, N) func(0, 0, 1, N) func(1, 2, 3, N) func(2, 4, 5, N) func(3, 6, 7, N) + +#define GEMV_LOADPACKET_ROW(iter) lhs.template load(i + (iter), j) + +#ifdef USE_GEMV_MMA +#define GEMV_UNROLL3_ROW(func, N, which) \ + func(0, N, which) func(1, N, which) func(2, N, which) func(3, N, which) func(4, N, which) func(5, N, which) \ + func(6, N, which) func(7, N, which) + +#define GEMV_UNUSED_ROW(N, which) GEMV_UNROLL3_ROW(GEMV_UNUSED_VAR, N, which) + +#define GEMV_INIT_ROW(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + __builtin_mma_xxsetaccz(&c##iter); \ + } + +#define GEMV_LOADPAIR_ROW(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(b##iter1, GEMV_LOADPACKET_ROW(iter2), GEMV_LOADPACKET_ROW((iter2) + 1)); + +#define GEMV_WORK_ROW(iter, N) \ + if (GEMV_GETN(N) > iter) { \ + if (GEMV_IS_FLOAT) { \ + pger_vecMMA_acc(&c##iter, a0, GEMV_LOADPACKET_ROW(iter)); \ + } else { \ + __vector_pair b##iter; \ + GEMV_LOADPAIR_ROW(iter, iter << 1) \ + pger_vecMMA_acc(&c##iter, b##iter, a0); \ + } \ + } + +#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + if (GEMV_IS_FLOAT) { \ + cc##iter1 = predux_real(&c##iter2, &c##iter3); \ + } else { \ + cc##iter1 = predux_real(&c##iter1); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } +#else +#define GEMV_INIT_ROW(iter, N) \ + if (N > iter) { \ + c##iter = pset1(ResScalar(0)); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c##iter); \ + } + +#define GEMV_WORK_ROW(iter, N) \ + if (N > iter) { \ + c##iter = pcj.pmadd(GEMV_LOADPACKET_ROW(iter), a0, c##iter); \ + } + +#define GEMV_PREDUX2(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1 = predux_real(c##iter2, c##iter3); \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } +#endif + +#define GEMV_MULT(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), a0); \ + cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), a0); \ + } + +#define GEMV_STORE_ROW(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + storeMaddData(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \ + storeMaddData(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \ + } + +/** \internal main macro for gemv_row - initialize accumulators, multiply and add inputs, predux and store results */ +#define GEMV_PROCESS_ROW(N) \ + for (; i < n##N; i += N) { \ + GEMV_UNROLL_ROW(GEMV_INIT_ROW, N) \ + Index j = 0; \ + for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \ + RhsPacket a0 = rhs2.template load(j); \ + GEMV_UNROLL_ROW(GEMV_WORK_ROW, N) \ + } \ + GEMV_UNROLL_ROW_HALF(GEMV_PREDUX2, (N >> 1)) \ + for (; j < cols; ++j) { \ + RhsScalar a0 = rhs2(j); \ + GEMV_UNROLL_ROW_HALF(GEMV_MULT, (N >> 1)) \ + } \ + GEMV_UNROLL_ROW_HALF(GEMV_STORE_ROW, (N >> 1)) \ + } + +template +EIGEN_STRONG_INLINE void gemv_row(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, ResScalar* res, + Index resIncr, ResScalar alpha) { + typedef gemv_traits Traits; + + typedef typename Traits::LhsPacket LhsPacket; + typedef typename Traits::RhsPacket RhsPacket; + typedef typename Traits::ResPacket ResPacket; + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + typename RhsMapper::LinearMapper rhs2 = rhs.getLinearMapper(0, 0); + + eigen_internal_assert(rhs.stride() == 1); + conj_helper cj; + conj_helper pcj; + + // TODO: fine tune the following heuristic. The rationale is that if the matrix is very large, + // processing 8 rows at once might be counter productive wrt cache. +#ifndef GCC_ONE_VECTORPAIR_BUG + const Index n8 = lhs.stride() * sizeof(LhsScalar) > 32000 ? (rows - 7) : (rows - 7); + const Index n4 = rows - 3; + const Index n2 = rows - 1; +#endif + + // TODO: for padded aligned inputs, we could enable aligned reads + enum { + LhsAlignment = Unaligned, + ResPacketSize = Traits::ResPacketSize, + LhsPacketSize = Traits::LhsPacketSize, + RhsPacketSize = Traits::RhsPacketSize, + }; + + Index i = 0; +#ifdef USE_GEMV_MMA + __vector_quad c0, c1, c2, c3, c4, c5, c6, c7; + GEMV_UNUSED_ROW(8, c) +#else + ResPacket c0, c1, c2, c3, c4, c5, c6, c7; +#endif +#ifndef GCC_ONE_VECTORPAIR_BUG + ScalarBlock cc0, cc1, cc2, cc3; + GEMV_PROCESS_ROW(8) + GEMV_PROCESS_ROW(4) + GEMV_PROCESS_ROW(2) +#endif + for (; i < rows; ++i) { + ResPacket d0 = pset1(ResScalar(0)); + Index j = 0; + for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { + RhsPacket b0 = rhs2.template load(j); + + d0 = pcj.pmadd(lhs.template load(i + 0, j), b0, d0); + } + ResScalar dd0 = predux(d0); + for (; j < cols; ++j) { + dd0 += cj.pmul(lhs(i, j), rhs2(j)); + } + res[i * resIncr] += alpha * dd0; + } +} + +#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(Scalar) \ + template \ + struct general_matrix_vector_product { \ + typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; \ + \ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \ + const RhsMapper& rhs, ResScalar* res, Index resIncr, \ + ResScalar alpha) { \ + gemv_col(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ + }; + +#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(Scalar) \ + template \ + struct general_matrix_vector_product { \ + typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; \ + \ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \ + const RhsMapper& rhs, ResScalar* res, Index resIncr, \ + ResScalar alpha) { \ + gemv_row(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ + }; + +EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(float) +EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL(double) +EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(float) +EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW(double) + +#ifdef USE_GEMV_MMA +#define gemv_bf16_col gemvMMA_bfloat16_col +#define gemv_bf16_row gemvMMA_bfloat16_row +#else +#define gemv_bf16_col gemv_bfloat16_col +#define gemv_bf16_row gemv_bfloat16_row +#endif + +#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL_BFLOAT16() \ + template \ + struct general_matrix_vector_product { \ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \ + const RhsMapper& rhs, bfloat16* res, Index resIncr, \ + bfloat16 alpha) { \ + gemv_bf16_col(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ + }; + +#define EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16() \ + template \ + struct general_matrix_vector_product { \ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \ + const RhsMapper& rhs, bfloat16* res, Index resIncr, \ + bfloat16 alpha) { \ + gemv_bf16_row(rows, cols, lhs, rhs, res, resIncr, alpha); \ + } \ + }; + +EIGEN_POWER_GEMV_REAL_SPECIALIZE_COL_BFLOAT16() +EIGEN_POWER_GEMV_REAL_SPECIALIZE_ROW_BFLOAT16() + +template +EIGEN_ALWAYS_INLINE ScalarBlock predux_complex(PResPacket& a0, PResPacket& b0, ResPacket& a1, + ResPacket& b1) { + if (GEMV_IS_COMPLEX_COMPLEX) { + a0 = padd(a0, a1); + b0 = padd(b0, b1); + } + return predux_complex(a0, b0); +} + +#define GEMV_LOADPACKET_ROW_COMPLEX(iter) loadLhsPacket(lhs, i + (iter), j) + +#define GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter) convertReal(GEMV_LOADPACKET_ROW_COMPLEX(iter)) + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(which, N) \ + j = 0; \ + for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \ + const RhsScalar& b1 = rhs2(j); \ + RhsScalar* b = const_cast(&b1); \ + GEMV_UNROLL_ROW(which, N) \ + } + +#define GEMV_PROCESS_END_ROW_COMPLEX(N) \ + for (; j < cols; ++j) { \ + RhsScalar b0 = rhs2(j); \ + GEMV_UNROLL_ROW_HALF(GEMV_MULT_COMPLEX, (N >> 1)) \ + } \ + GEMV_UNROLL_ROW_HALF(GEMV_STORE_ROW_COMPLEX, (N >> 1)) + +#ifdef USE_GEMV_MMA +#define GEMV_INIT_ROW_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + __builtin_mma_xxsetaccz(&e0##iter); \ + } + +#define GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter1, iter2) \ + GEMV_BUILDPAIR_MMA(a##iter1, GEMV_LOADPACKET_ROW_COMPLEX_DATA(iter2), GEMV_LOADPACKET_ROW_COMPLEX_DATA((iter2) + 1)); + +#define GEMV_WORK_ROW_COMPLEX_MMA(iter, N) \ + if (GEMV_GETN_COMPLEX(N) > iter) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \ + gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ + } else { \ + __vector_pair a##iter; \ + GEMV_LOADPAIR_ROW_COMPLEX_MMA(iter, iter << 1) \ + gemv_mult_complex_MMA(a##iter, b, &e0##iter); \ + } \ + } + +#define GEMV_PREDUX4_COMPLEX_MMA(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + if (GEMV_IS_COMPLEX_FLOAT) { \ + cc##iter1 = predux_complex( \ + &e0##iter2, &e0##iter3); \ + } else { \ + cc##iter1 = \ + predux_complex(&e0##iter1); \ + } \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_MMA(N) \ + GEMV_UNROLL_ROW(GEMV_INIT_ROW_COMPLEX_MMA, N) \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(GEMV_WORK_ROW_COMPLEX_MMA, N) + +#define GEMV_PROCESS_ROW_COMPLEX_ONE_MMA(N) \ + for (; i < n##N; i += N) { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_MMA(N) \ + GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX_MMA, (N >> 1)) \ + GEMV_PROCESS_END_ROW_COMPLEX(N); \ + } +#endif + +#define GEMV_WORK_ROW_COMPLEX(iter, N) \ + if (N > iter) { \ + PLhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX(iter); \ + gemv_mult_complex(a##iter, b, c0##iter, c1##iter); \ + } + +#define GEMV_PREDUX4_COMPLEX(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1 = predux_complex(c0##iter2, c0##iter3, \ + c1##iter2, c1##iter3); \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } + +#define GEMV_MULT_COMPLEX(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1.scalar[0] += cj.pmul(lhs(i + iter2, j), b0); \ + cc##iter1.scalar[1] += cj.pmul(lhs(i + iter3, j), b0); \ + } + +#define GEMV_STORE_ROW_COMPLEX(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + storeMaddData(res + ((i + iter2) * resIncr), alpha, cc##iter1.scalar[0]); \ + storeMaddData(res + ((i + iter3) * resIncr), alpha, cc##iter1.scalar[1]); \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \ + GEMV_UNROLL_ROW(GEMV_INIT_COMPLEX, N) \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_WORK(GEMV_WORK_ROW_COMPLEX, N) + +/** \internal main macro for gemv_complex_row - initialize accumulators, multiply and add inputs, predux and store + * results */ +#define GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) \ + for (; i < n##N; i += N) { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \ + GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX, (N >> 1)) \ + GEMV_PROCESS_END_ROW_COMPLEX(N); \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \ + if (GEMV_IS_COMPLEX_COMPLEX) { \ + c0##iter = padd(c0##iter, c1##iter); \ + } \ + dd0 = predux(c0##iter); + +#if EIGEN_COMP_LLVM +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE(N) GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) + +#define GEMV_PROCESS_ROW_COMPLEX_ONE(N) GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) + +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX(iter) GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) +#else +// gcc seems to be reading and writing registers unnecessarily to memory. +// Use the old way for complex double until it is fixed. + +#define GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter) lhs.template load(i + (iter), j) + +#define GEMV_INIT_COMPLEX_OLD(iter, N) \ + EIGEN_UNUSED_VARIABLE(c0##iter); \ + if (N > iter) { \ + c1##iter = pset_zero(); \ + } else { \ + EIGEN_UNUSED_VARIABLE(c1##iter); \ + } + +#define GEMV_WORK_ROW_COMPLEX_OLD(iter, N) \ + if (N > iter) { \ + LhsPacket a##iter = GEMV_LOADPACKET_ROW_COMPLEX_OLD(iter); \ + c1##iter = pcj.pmadd(a##iter, b0, c1##iter); \ + } + +#define GEMV_PREDUX4_COMPLEX_OLD(iter1, iter2, iter3, N) \ + if (N > iter1) { \ + cc##iter1.scalar[0] = predux(c1##iter2); \ + cc##iter1.scalar[1] = predux(c1##iter3); \ + } else { \ + EIGEN_UNUSED_VARIABLE(cc##iter1); \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \ + GEMV_UNROLL_ROW(GEMV_INIT_COMPLEX_OLD, N) \ + j = 0; \ + for (; j + LhsPacketSize <= cols; j += LhsPacketSize) { \ + RhsPacket b0 = rhs2.template load(j); \ + GEMV_UNROLL_ROW(GEMV_WORK_ROW_COMPLEX_OLD, N) \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_ONE_OLD(N) \ + for (; i < n##N; i += N) { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \ + GEMV_UNROLL_ROW_HALF(GEMV_PREDUX4_COMPLEX_OLD, (N >> 1)) \ + GEMV_PROCESS_END_ROW_COMPLEX(N) \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX_OLD(iter) dd0 = predux(c1##iter); + +#if (__GNUC__ > 10) +#define GEMV_PROCESS_ROW_COMPLEX_IS_NEW 1 +#else +#define GEMV_PROCESS_ROW_COMPLEX_IS_NEW (sizeof(Scalar) == sizeof(float)) || GEMV_IS_COMPLEX_COMPLEX +#endif + +#define GEMV_PROCESS_ROW_COMPLEX_SINGLE(N) \ + if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_NEW(N) \ + } else { \ + GEMV_PROCESS_ROW_COMPLEX_SINGLE_OLD(N) \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_ONE(N) \ + if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \ + GEMV_PROCESS_ROW_COMPLEX_ONE_NEW(N) \ + } else { \ + GEMV_PROCESS_ROW_COMPLEX_ONE_OLD(N) \ + } + +#define GEMV_PROCESS_ROW_COMPLEX_PREDUX(iter) \ + if (GEMV_PROCESS_ROW_COMPLEX_IS_NEW) { \ + GEMV_PROCESS_ROW_COMPLEX_PREDUX_NEW(iter) \ + } else { \ + GEMV_PROCESS_ROW_COMPLEX_PREDUX_OLD(iter) \ + } +#endif + +#ifdef USE_GEMV_MMA +#define GEMV_PROCESS_ROW_COMPLEX(N) GEMV_PROCESS_ROW_COMPLEX_ONE_MMA(N) +#else +#define GEMV_PROCESS_ROW_COMPLEX(N) GEMV_PROCESS_ROW_COMPLEX_ONE(N) +#endif + +template +EIGEN_STRONG_INLINE void gemv_complex_row(Index rows, Index cols, const LhsMapper& alhs, const RhsMapper& rhs, + ResScalar* res, Index resIncr, ResScalar alpha) { + typedef gemv_traits Traits; + + typedef typename Traits::LhsPacket LhsPacket; + typedef typename Traits::RhsPacket RhsPacket; + typedef typename Traits::ResPacket ResPacket; + + typedef typename packet_traits::type ScalarPacket; + typedef typename packet_traits::type PLhsPacket; + typedef typename packet_traits::type PResPacket; + typedef gemv_traits PTraits; + + // The following copy tells the compiler that lhs's attributes are not modified outside this function + // This helps GCC to generate proper code. + LhsMapper lhs(alhs); + typename RhsMapper::LinearMapper rhs2 = rhs.getLinearMapper(0, 0); + + eigen_internal_assert(rhs.stride() == 1); + conj_helper cj; +#if !EIGEN_COMP_LLVM + conj_helper pcj; +#endif + + // TODO: fine tune the following heuristic. The rationale is that if the matrix is very large, + // processing 8 rows at once might be counter productive wrt cache. +#ifndef GCC_ONE_VECTORPAIR_BUG + const Index n8 = lhs.stride() * sizeof(LhsScalar) > 32000 ? (rows - 7) : (rows - 7); + const Index n4 = rows - 3; + const Index n2 = rows - 1; +#endif + + // TODO: for padded aligned inputs, we could enable aligned reads + enum { + LhsAlignment = Unaligned, + ResPacketSize = PTraits::ResPacketSize, + LhsPacketSize = PTraits::LhsPacketSize, + RhsPacketSize = PTraits::RhsPacketSize, + }; + + Index i = 0, j; + PResPacket c00, c01, c02, c03, c04, c05, c06, c07; + ResPacket c10, c11, c12, c13, c14, c15, c16, c17; +#ifdef USE_GEMV_MMA + __vector_quad e00, e01, e02, e03, e04, e05, e06, e07; + GEMV_UNUSED_ROW(8, e0) + GEMV_UNUSED_EXTRA(1, c0) + GEMV_UNUSED_EXTRA(1, c1) +#endif + ResScalar dd0; +#ifndef GCC_ONE_VECTORPAIR_BUG + ScalarBlock cc0, cc1, cc2, cc3; +#ifdef USE_GEMV_MMA + if (!GEMV_IS_COMPLEX_COMPLEX) +#endif + { + GEMV_PROCESS_ROW_COMPLEX(8) + } + GEMV_PROCESS_ROW_COMPLEX(4) + GEMV_PROCESS_ROW_COMPLEX(2) +#endif + for (; i < rows; ++i) { + GEMV_PROCESS_ROW_COMPLEX_SINGLE(1) + GEMV_PROCESS_ROW_COMPLEX_PREDUX(0) + for (; j < cols; ++j) { + dd0 += cj.pmul(lhs(i, j), rhs2(j)); + } + res[i * resIncr] += alpha * dd0; + } +} + +#define EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(Scalar, LhsScalar, RhsScalar) \ + template \ + struct general_matrix_vector_product { \ + typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; \ + \ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \ + const RhsMapper& rhs, ResScalar* res, Index resIncr, \ + ResScalar alpha) { \ + gemv_complex_col(rows, cols, lhs, rhs, \ + res, resIncr, alpha); \ + } \ + }; + +#define EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(Scalar, LhsScalar, RhsScalar) \ + template \ + struct general_matrix_vector_product { \ + typedef typename ScalarBinaryOpTraits::ReturnType ResScalar; \ + \ + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static void run(Index rows, Index cols, const LhsMapper& lhs, \ + const RhsMapper& rhs, ResScalar* res, Index resIncr, \ + ResScalar alpha) { \ + gemv_complex_row(rows, cols, lhs, rhs, \ + res, resIncr, alpha); \ + } \ + }; + +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(float, float, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(float, std::complex, float) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(float, std::complex, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(double, double, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(double, std::complex, double) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_COL(double, std::complex, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(float, float, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(float, std::complex, float) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(float, std::complex, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(double, double, std::complex) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(double, std::complex, double) +EIGEN_POWER_GEMV_COMPLEX_SPECIALIZE_ROW(double, std::complex, std::complex) + +#endif // EIGEN_MATRIX_VECTOR_PRODUCT_ALTIVEC_H diff --git a/Eigen/src/Core/arch/AltiVec/TypeCasting.h b/Eigen/src/Core/arch/AltiVec/TypeCasting.h new file mode 100644 index 0000000..439339e --- /dev/null +++ b/Eigen/src/Core/arch/AltiVec/TypeCasting.h @@ -0,0 +1,153 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2019 Rasmus Munk Larsen +// Copyright (C) 2023 Chip Kerchner (chip.kerchner@ibm.com) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TYPE_CASTING_ALTIVEC_H +#define EIGEN_TYPE_CASTING_ALTIVEC_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { +template <> +struct type_casting_traits { + enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; +}; + +template <> +struct type_casting_traits { + enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; +}; + +template <> +struct type_casting_traits { + enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; +}; + +template <> +struct type_casting_traits { + enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 1 }; +}; + +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet4f& a) { + return vec_cts(a, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4f& a) { + return vec_ctu(a, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet4i& a) { + return vec_ctf(a, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet4ui& a) { + return vec_ctf(a, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet8us pcast(const Packet8bf& a) { + Packet4f float_even = Bf16ToF32Even(a); + Packet4f float_odd = Bf16ToF32Odd(a); + Packet4ui int_even = pcast(float_even); + Packet4ui int_odd = pcast(float_odd); + const EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF); + Packet4ui low_even = pand(int_even, p4ui_low_mask); + Packet4ui low_odd = pand(int_odd, p4ui_low_mask); + + // Check values that are bigger than USHRT_MAX (0xFFFF) + Packet4bi overflow_selector; + if (vec_any_gt(int_even, p4ui_low_mask)) { + overflow_selector = vec_cmpgt(int_even, p4ui_low_mask); + low_even = vec_sel(low_even, p4ui_low_mask, overflow_selector); + } + if (vec_any_gt(int_odd, p4ui_low_mask)) { + overflow_selector = vec_cmpgt(int_odd, p4ui_low_mask); + low_odd = vec_sel(low_even, p4ui_low_mask, overflow_selector); + } + + return pmerge(low_even, low_odd); +} + +template <> +EIGEN_STRONG_INLINE Packet8bf pcast(const Packet8us& a) { + // short -> int -> float -> bfloat16 + const EIGEN_DECLARE_CONST_FAST_Packet4ui(low_mask, 0x0000FFFF); + Packet4ui int_cast = reinterpret_cast(a); + Packet4ui int_even = pand(int_cast, p4ui_low_mask); + Packet4ui int_odd = plogical_shift_right<16>(int_cast); + Packet4f float_even = pcast(int_even); + Packet4f float_odd = pcast(int_odd); + return F32ToBf16(float_even, float_odd); +} + +template <> +struct type_casting_traits { + enum { VectorizedCast = 1, SrcCoeffRatio = 1, TgtCoeffRatio = 2 }; +}; + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet8bf& a) { + Packet8us z = pset1(0); +#ifdef _BIG_ENDIAN + return reinterpret_cast(vec_mergeh(a.m_val, z)); +#else + return reinterpret_cast(vec_mergeh(z, a.m_val)); +#endif +} + +template <> +struct type_casting_traits { + enum { VectorizedCast = 1, SrcCoeffRatio = 2, TgtCoeffRatio = 1 }; +}; + +template <> +EIGEN_STRONG_INLINE Packet8bf pcast(const Packet4f& a, const Packet4f& b) { + return F32ToBf16Both(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { + return reinterpret_cast(a); +} + +template <> +EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) { + return reinterpret_cast(a); +} + +#ifdef EIGEN_VECTORIZE_VSX +template <> +inline Packet2l pcast(const Packet2d& x) { + EIGEN_ALIGN_MAX double dtmp[2]; + pstore(dtmp, x); + EIGEN_ALIGN_MAX long long itmp[2] = {static_cast(dtmp[0]), static_cast(dtmp[1])}; + return vec_xl(0, itmp); +} + +template <> +inline Packet2d pcast(const Packet2l& x) { + EIGEN_ALIGN_MAX long long itmp[2]; + vec_xst(x, 0, itmp); + EIGEN_ALIGN_MAX double dtmp[2] = {static_cast(itmp[0]), static_cast(itmp[1])}; + return pload(dtmp); +} +#endif + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_TYPE_CASTING_ALTIVEC_H diff --git a/Eigen/src/Core/arch/GPU/Complex.h b/Eigen/src/Core/arch/GPU/Complex.h new file mode 100644 index 0000000..fa46aec --- /dev/null +++ b/Eigen/src/Core/arch/GPU/Complex.h @@ -0,0 +1,244 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// Copyright (C) 2021 C. Antonio Sanchez +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_COMPLEX_GPU_H +#define EIGEN_COMPLEX_GPU_H + +// Many std::complex methods such as operator+, operator-, operator* and +// operator/ are not constexpr. Due to this, GCC and older versions of clang do +// not treat them as device functions and thus Eigen functors making use of +// these operators fail to compile. Here, we manually specialize these +// operators and functors for complex types when building for CUDA to enable +// their use on-device. +// +// NOTES: +// - Compound assignment operators +=,-=,*=,/=(Scalar) will not work on device, +// since they are already specialized in the standard. Using them will result +// in silent kernel failures. +// - Compiling with MSVC and using +=,-=,*=,/=(std::complex) will lead +// to duplicate definition errors, since these are already specialized in +// Visual Studio's header (contrary to the standard). This is +// preferable to removing such definitions, which will lead to silent kernel +// failures. +// - Compiling with ICC requires defining _USE_COMPLEX_SPECIALIZATION_ prior +// to the first inclusion of . + +#if defined(EIGEN_GPUCC) && defined(EIGEN_GPU_COMPILE_PHASE) + +// ICC already specializes std::complex and std::complex +// operators, preventing us from making them device functions here. +// This will lead to silent runtime errors if the operators are used on device. +// +// To allow std::complex operator use on device, define _OVERRIDE_COMPLEX_SPECIALIZATION_ +// prior to first inclusion of . This prevents ICC from adding +// its own specializations, so our custom ones below can be used instead. +#if !(EIGEN_COMP_ICC && defined(_USE_COMPLEX_SPECIALIZATION_)) + +// Import Eigen's internal operator specializations. +#define EIGEN_USING_STD_COMPLEX_OPERATORS \ + using Eigen::complex_operator_detail::operator+; \ + using Eigen::complex_operator_detail::operator-; \ + using Eigen::complex_operator_detail::operator*; \ + using Eigen::complex_operator_detail::operator/; \ + using Eigen::complex_operator_detail::operator+=; \ + using Eigen::complex_operator_detail::operator-=; \ + using Eigen::complex_operator_detail::operator*=; \ + using Eigen::complex_operator_detail::operator/=; \ + using Eigen::complex_operator_detail::operator==; \ + using Eigen::complex_operator_detail::operator!=; + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +// Specialized std::complex overloads. +namespace complex_operator_detail { + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex complex_multiply(const std::complex& a, + const std::complex& b) { + const T a_real = numext::real(a); + const T a_imag = numext::imag(a); + const T b_real = numext::real(b); + const T b_imag = numext::imag(b); + return std::complex(a_real * b_real - a_imag * b_imag, a_imag * b_real + a_real * b_imag); +} + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex complex_divide_fast(const std::complex& a, + const std::complex& b) { + const T a_real = numext::real(a); + const T a_imag = numext::imag(a); + const T b_real = numext::real(b); + const T b_imag = numext::imag(b); + const T norm = (b_real * b_real + b_imag * b_imag); + return std::complex((a_real * b_real + a_imag * b_imag) / norm, (a_imag * b_real - a_real * b_imag) / norm); +} + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex complex_divide_stable(const std::complex& a, + const std::complex& b) { + const T a_real = numext::real(a); + const T a_imag = numext::imag(a); + const T b_real = numext::real(b); + const T b_imag = numext::imag(b); + // Smith's complex division (https://arxiv.org/pdf/1210.4539.pdf), + // guards against over/under-flow. + const bool scale_imag = numext::abs(b_imag) <= numext::abs(b_real); + const T rscale = scale_imag ? T(1) : b_real / b_imag; + const T iscale = scale_imag ? b_imag / b_real : T(1); + const T denominator = b_real * rscale + b_imag * iscale; + return std::complex((a_real * rscale + a_imag * iscale) / denominator, + (a_imag * rscale - a_real * iscale) / denominator); +} + +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex complex_divide(const std::complex& a, + const std::complex& b) { +#if EIGEN_FAST_MATH + return complex_divide_fast(a, b); +#else + return complex_divide_stable(a, b); +#endif +} + +// NOTE: We cannot specialize compound assignment operators with Scalar T, +// (i.e. operator@=(const T&), for @=+,-,*,/) +// since they are already specialized for float/double/long double within +// the standard header. We also do not specialize the stream +// operators. +#define EIGEN_CREATE_STD_COMPLEX_OPERATOR_SPECIALIZATIONS(T) \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator+(const std::complex& a) { return a; } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator-(const std::complex& a) { \ + return std::complex(-numext::real(a), -numext::imag(a)); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator+(const std::complex& a, \ + const std::complex& b) { \ + return std::complex(numext::real(a) + numext::real(b), numext::imag(a) + numext::imag(b)); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator+(const std::complex& a, const T& b) { \ + return std::complex(numext::real(a) + b, numext::imag(a)); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator+(const T& a, const std::complex& b) { \ + return std::complex(a + numext::real(b), numext::imag(b)); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator-(const std::complex& a, \ + const std::complex& b) { \ + return std::complex(numext::real(a) - numext::real(b), numext::imag(a) - numext::imag(b)); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator-(const std::complex& a, const T& b) { \ + return std::complex(numext::real(a) - b, numext::imag(a)); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator-(const T& a, const std::complex& b) { \ + return std::complex(a - numext::real(b), -numext::imag(b)); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator*(const std::complex& a, \ + const std::complex& b) { \ + return complex_multiply(a, b); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator*(const std::complex& a, const T& b) { \ + return std::complex(numext::real(a) * b, numext::imag(a) * b); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator*(const T& a, const std::complex& b) { \ + return std::complex(a * numext::real(b), a * numext::imag(b)); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator/(const std::complex& a, \ + const std::complex& b) { \ + return complex_divide(a, b); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator/(const std::complex& a, const T& b) { \ + return std::complex(numext::real(a) / b, numext::imag(a) / b); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex operator/(const T& a, const std::complex& b) { \ + return complex_divide(std::complex(a, 0), b); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex& operator+=(std::complex& a, const std::complex& b) { \ + numext::real_ref(a) += numext::real(b); \ + numext::imag_ref(a) += numext::imag(b); \ + return a; \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex& operator-=(std::complex& a, const std::complex& b) { \ + numext::real_ref(a) -= numext::real(b); \ + numext::imag_ref(a) -= numext::imag(b); \ + return a; \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex& operator*=(std::complex& a, const std::complex& b) { \ + a = complex_multiply(a, b); \ + return a; \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE std::complex& operator/=(std::complex& a, const std::complex& b) { \ + a = complex_divide(a, b); \ + return a; \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator==(const std::complex& a, const std::complex& b) { \ + return numext::real(a) == numext::real(b) && numext::imag(a) == numext::imag(b); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator==(const std::complex& a, const T& b) { \ + return numext::real(a) == b && numext::imag(a) == 0; \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator==(const T& a, const std::complex& b) { \ + return a == numext::real(b) && 0 == numext::imag(b); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator!=(const std::complex& a, const std::complex& b) { \ + return !(a == b); \ + } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator!=(const std::complex& a, const T& b) { return !(a == b); } \ + \ + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool operator!=(const T& a, const std::complex& b) { return !(a == b); } + +// Do not specialize for long double, since that reduces to double on device. +EIGEN_CREATE_STD_COMPLEX_OPERATOR_SPECIALIZATIONS(float) +EIGEN_CREATE_STD_COMPLEX_OPERATOR_SPECIALIZATIONS(double) + +#undef EIGEN_CREATE_STD_COMPLEX_OPERATOR_SPECIALIZATIONS + +} // namespace complex_operator_detail + +EIGEN_USING_STD_COMPLEX_OPERATORS + +namespace numext { +EIGEN_USING_STD_COMPLEX_OPERATORS +} // namespace numext + +namespace internal { +EIGEN_USING_STD_COMPLEX_OPERATORS + +} // namespace internal +} // namespace Eigen + +#endif // !(EIGEN_COMP_ICC && _USE_COMPLEX_SPECIALIZATION_) + +#endif // EIGEN_GPUCC && EIGEN_GPU_COMPILE_PHASE + +#endif // EIGEN_COMPLEX_GPU_H diff --git a/Eigen/src/Core/arch/GPU/Tuple.h b/Eigen/src/Core/arch/GPU/Tuple.h new file mode 100644 index 0000000..402d92f --- /dev/null +++ b/Eigen/src/Core/arch/GPU/Tuple.h @@ -0,0 +1,268 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2021 The Eigen Team +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TUPLE_GPU +#define EIGEN_TUPLE_GPU + +#include +#include + +// This is a replacement of std::tuple that can be used in device code. + +namespace Eigen { +namespace internal { +namespace tuple_impl { + +// Internal tuple implementation. +template +class TupleImpl; + +// Generic recursive tuple. +template +class TupleImpl { + public: + // Tuple may contain Eigen types. + EIGEN_MAKE_ALIGNED_OPERATOR_NEW + + // Default constructor, enable if all types are default-constructible. + template ::value && + reduce_all::value...>::value>> + constexpr EIGEN_DEVICE_FUNC TupleImpl() : head_{}, tail_{} {} + + // Element constructor. + template 1 || std::is_convertible::value)>> + constexpr EIGEN_DEVICE_FUNC TupleImpl(U1&& arg1, Us&&... args) + : head_(std::forward(arg1)), tail_(std::forward(args)...) {} + + // The first stored value. + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T1& head() { return head_; } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE const T1& head() const { return head_; } + + // The tail values. + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE TupleImpl& tail() { return tail_; } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE const TupleImpl& tail() const { return tail_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void swap(TupleImpl& other) { + using numext::swap; + swap(head_, other.head_); + swap(tail_, other.tail_); + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TupleImpl& operator=(const TupleImpl& other) { + head_ = other.head_; + tail_ = other.tail_; + return *this; + } + + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TupleImpl& operator=(TupleImpl&& other) { + head_ = std::move(other.head_); + tail_ = std::move(other.tail_); + return *this; + } + + private: + // Allow related tuples to reference head_/tail_. + template + friend class TupleImpl; + + T1 head_; + TupleImpl tail_; +}; + +// Empty tuple specialization. +template <> +class TupleImpl {}; + +template +struct is_tuple : std::false_type {}; + +template +struct is_tuple> : std::true_type {}; + +// Gets an element from a tuple. +template +struct tuple_get_impl { + using TupleType = TupleImpl; + using ReturnType = typename tuple_get_impl::ReturnType; + + static constexpr EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE ReturnType& run(TupleType& tuple) { + return tuple_get_impl::run(tuple.tail()); + } + + static constexpr EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE const ReturnType& run(const TupleType& tuple) { + return tuple_get_impl::run(tuple.tail()); + } +}; + +// Base case, getting the head element. +template +struct tuple_get_impl<0, T1, Ts...> { + using TupleType = TupleImpl; + using ReturnType = T1; + + static constexpr EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE T1& run(TupleType& tuple) { return tuple.head(); } + + static constexpr EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE const T1& run(const TupleType& tuple) { return tuple.head(); } +}; + +// Concatenates N Tuples. +template +struct tuple_cat_impl; + +template +struct tuple_cat_impl, TupleImpl, Tuples...> { + using TupleType1 = TupleImpl; + using TupleType2 = TupleImpl; + using MergedTupleType = TupleImpl; + + using ReturnType = typename tuple_cat_impl::ReturnType; + + // Uses the index sequences to extract and merge elements from tuple1 and tuple2, + // then recursively calls again. + template + static constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ReturnType run(Tuple1&& tuple1, std::index_sequence, + Tuple2&& tuple2, std::index_sequence, + MoreTuples&&... tuples) { + return tuple_cat_impl::run( + MergedTupleType(tuple_get_impl::run(std::forward(tuple1))..., + tuple_get_impl::run(std::forward(tuple2))...), + std::forward(tuples)...); + } + + // Concatenates the first two tuples. + template + static constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ReturnType run(Tuple1&& tuple1, Tuple2&& tuple2, + MoreTuples&&... tuples) { + return run(std::forward(tuple1), std::make_index_sequence{}, std::forward(tuple2), + std::make_index_sequence{}, std::forward(tuples)...); + } +}; + +// Base case with a single tuple. +template +struct tuple_cat_impl<1, TupleImpl> { + using ReturnType = TupleImpl; + + template + static constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ReturnType run(Tuple1&& tuple1) { + return tuple1; + } +}; + +// Special case of no tuples. +template <> +struct tuple_cat_impl<0> { + using ReturnType = TupleImpl<0>; + static constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ReturnType run() { return ReturnType{}; } +}; + +// For use in make_tuple, unwraps a reference_wrapper. +template +struct unwrap_reference_wrapper { + using type = T; +}; + +template +struct unwrap_reference_wrapper> { + using type = T&; +}; + +// For use in make_tuple, decays a type and unwraps a reference_wrapper. +template +struct unwrap_decay { + using type = typename unwrap_reference_wrapper::type>::type; +}; + +/** + * Utility for determining a tuple's size. + */ +template +struct tuple_size; + +template +struct tuple_size> : std::integral_constant {}; + +/** + * Gets an element of a tuple. + * \tparam Idx index of the element. + * \tparam Types ... tuple element types. + * \param tuple the tuple. + * \return a reference to the desired element. + */ +template +constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const typename tuple_get_impl::ReturnType& get( + const TupleImpl& tuple) { + return tuple_get_impl::run(tuple); +} + +template +constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE typename tuple_get_impl::ReturnType& get( + TupleImpl& tuple) { + return tuple_get_impl::run(tuple); +} + +/** + * Concatenate multiple tuples. + * \param tuples ... list of tuples. + * \return concatenated tuple. + */ +template ::type>::value...>::value>> +constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE + typename tuple_cat_impl::type...>::ReturnType + tuple_cat(Tuples&&... tuples) { + return tuple_cat_impl::type...>::run(std::forward(tuples)...); +} + +/** + * Tie arguments together into a tuple. + */ +template > +constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ReturnType tie(Args&... args) noexcept { + return ReturnType{args...}; +} + +/** + * Create a tuple of l-values with the supplied arguments. + */ +template ::type...>> +constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ReturnType make_tuple(Args&&... args) { + return ReturnType{std::forward(args)...}; +} + +/** + * Forward a set of arguments as a tuple. + */ +template +constexpr EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE TupleImpl forward_as_tuple(Args&&... args) { + return TupleImpl(std::forward(args)...); +} + +/** + * Alternative to std::tuple that can be used on device. + */ +template +using tuple = TupleImpl; + +} // namespace tuple_impl +} // namespace internal +} // namespace Eigen + +#endif // EIGEN_TUPLE_GPU diff --git a/Eigen/src/Core/arch/HVX/PacketMath.h b/Eigen/src/Core/arch/HVX/PacketMath.h new file mode 100644 index 0000000..9b6ceb3 --- /dev/null +++ b/Eigen/src/Core/arch/HVX/PacketMath.h @@ -0,0 +1,1088 @@ + +#ifndef EIGEN_HVX_PACKET_MATH_H +#define EIGEN_HVX_PACKET_MATH_H + +// Only support 128B HVX now. +// Floating-point operations are supported only since V68. +#if defined __HVX__ && (__HVX_LENGTH__ == 128) && __HVX_ARCH__ >= 68 + +// All the floating-point operations do not support IEEE standard. +// From HVX document: +// There is no concept of infinity or NaN. QFloat saturates to maximum +// exponent with maximum positive or minimum negative significand. + +#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS +#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 +#endif + +namespace Eigen { +namespace internal { + +// HVX utilities. + +template +EIGEN_STRONG_INLINE HVX_Vector HVX_vmem(const void* m) { + HVX_Vector v; +#if EIGEN_COMP_CLANG + // Use inlined assembly for aligned vmem load on unaligned memory. + // Use type cast to HVX_Vector* may mess up with compiler data alignment. + __asm__("%0 = vmem(%1+#%2)" : "=v"(v) : "r"(m), "i"(D) : "memory"); +#else + void* aligned_mem = + reinterpret_cast((reinterpret_cast(m) & ~(__HVX_LENGTH__ - 1)) + D * __HVX_LENGTH__); + memcpy(&v, aligned_mem, __HVX_LENGTH__); +#endif + return v; +} + +template +EIGEN_STRONG_INLINE HVX_Vector HVX_load(const T* mem) { + HVX_Vector v; + memcpy(&v, reinterpret_cast(mem), __HVX_LENGTH__); + return v; +} + +template +EIGEN_STRONG_INLINE HVX_Vector HVX_loadu(const T* mem) { + HVX_Vector v; + memcpy(&v, mem, __HVX_LENGTH__); + return v; +} + +template +EIGEN_STRONG_INLINE HVX_Vector HVX_load_partial(const T* mem) { +#if defined(EIGEN_HVX_FAST_PARTIAL_VECTOR_LOAD) + // Fast partial vector load through aligned vmem load. + // The load may past end of array but is aligned to prevent memory fault. + HVX_Vector v0 = HVX_vmem<0>(mem); + HVX_Vector v1 = v0; + uintptr_t mem_addr = reinterpret_cast(mem); + EIGEN_IF_CONSTEXPR(Size * sizeof(T) <= Alignment) { + // Data size less than alignment will never cross multiple aligned vectors. + v1 = v0; + } + else { + uintptr_t left_off = mem_addr & (__HVX_LENGTH__ - 1); + if (left_off + Size * sizeof(T) > __HVX_LENGTH__) { + v1 = HVX_vmem<1>(mem); + } else { + v1 = v0; + } + } + return Q6_V_valign_VVR(v1, v0, mem_addr); +#else + HVX_Vector v; + memcpy(&v, mem, Size * sizeof(T)); + return v; +#endif +} + +template +EIGEN_STRONG_INLINE void HVX_store(T* mem, HVX_Vector v) { + memcpy(reinterpret_cast(mem), &v, __HVX_LENGTH__); +} + +template +EIGEN_STRONG_INLINE void HVX_storeu(T* mem, HVX_Vector v) { + memcpy(mem, &v, __HVX_LENGTH__); +} + +template +EIGEN_STRONG_INLINE void HVX_store_partial(T* mem, HVX_Vector v) { + uintptr_t mem_addr = reinterpret_cast(mem); + HVX_Vector value = Q6_V_vlalign_VVR(v, v, mem_addr); + uintptr_t left_off = mem_addr & (__HVX_LENGTH__ - 1); + uintptr_t right_off = left_off + Size * sizeof(T); + + HVX_VectorPred ql_not = Q6_Q_vsetq_R(mem_addr); + HVX_VectorPred qr = Q6_Q_vsetq2_R(right_off); + + EIGEN_IF_CONSTEXPR(Size * sizeof(T) > Alignment) { + if (right_off > __HVX_LENGTH__) { + Q6_vmem_QRIV(qr, mem + __HVX_LENGTH__ / sizeof(T), value); + qr = Q6_Q_vcmp_eq_VbVb(value, value); + } + } + + ql_not = Q6_Q_or_QQn(ql_not, qr); + Q6_vmem_QnRIV(ql_not, mem, value); +} + +// Packet definitions. +enum class HVXPacketSize { + Full, + Half, + Quarter, +}; + +// Hexagon compiler uses same HVX_Vector to represent all HVX vector types. +// Wrap different vector type (float32, int32, etc) to different class with +// explicit constructor and casting back-and-force to HVX_Vector. +template +class HVXPacket { + public: + HVXPacket() = default; + static HVXPacket Create(HVX_Vector v) { return HVXPacket(v); } + HVX_Vector Get() const { return m_val; } + + private: + explicit HVXPacket(HVX_Vector v) : m_val(v) {} + HVX_Vector m_val = Q6_V_vzero(); +}; + +typedef HVXPacket Packet32f; +typedef HVXPacket Packet16f; +typedef HVXPacket Packet8f; + +// Packet traits. +template <> +struct packet_traits : default_packet_traits { + typedef Packet32f type; + typedef Packet16f half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 32, + + HasCmp = 1, + HasAdd = 1, + HasSub = 1, + HasShift = 0, + HasMul = 1, + HasNegate = 1, + HasAbs = 1, + HasArg = 0, + HasAbs2 = 0, + HasAbsDiff = 0, + HasMin = 1, + HasMax = 1, + HasConj = 0, + HasSetLinear = 0, + HasBlend = 0, + + HasDiv = 0, + + HasSin = 0, + HasCos = 0, + HasACos = 0, + HasASin = 0, + HasATan = 0, + HasATanh = 0, + HasLog = 0, + HasExp = 0, + HasSqrt = 0, + HasRsqrt = 0, + HasTanh = 0, + HasErf = 0, + HasBessel = 0, + HasNdtri = 0 + }; +}; + +template <> +struct unpacket_traits { + typedef float type; + typedef Packet16f half; + enum { + size = 32, + alignment = Aligned128, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +struct unpacket_traits { + typedef float type; + typedef Packet8f half; + enum { + size = 16, + // Many code assume alignment on packet size instead of following trait + // So we do not use Aligned128 to optimize aligned load/store, + alignment = Aligned64, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +struct unpacket_traits { + typedef float type; + typedef Packet8f half; + enum { + size = 8, + // Many code assume alignment on packet size instead of following trait + // So we do not use Aligned128 to optimize aligned load/store, + alignment = Aligned32, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +// float32 operations. +template +EIGEN_STRONG_INLINE HVXPacket pzero_hvx(const HVXPacket&) { + return HVXPacket::Create(Q6_V_vzero()); +} +template <> +EIGEN_STRONG_INLINE Packet32f pzero(const Packet32f&) { + return pzero_hvx(Packet32f()); +} +template <> +EIGEN_STRONG_INLINE Packet16f pzero(const Packet16f&) { + return pzero_hvx(Packet16f()); +} +template <> +EIGEN_STRONG_INLINE Packet8f pzero(const Packet8f&) { + return pzero_hvx(Packet8f()); +} + +template +EIGEN_STRONG_INLINE typename unpacket_traits>::half predux_half_dowto4_hvx(const HVXPacket& a) { + const Index packet_size = unpacket_traits>::size; + return unpacket_traits>::half::Create( + Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(Q6_V_vror_VR(a.Get(), sizeof(float) * packet_size / 2), a.Get()))); +} +template <> +EIGEN_STRONG_INLINE Packet16f predux_half_dowto4(const Packet32f& a) { + return predux_half_dowto4_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet8f predux_half_dowto4(const Packet16f& a) { + return predux_half_dowto4_hvx(a); +} + +template +EIGEN_STRONG_INLINE HVXPacket pset1_hvx(const float& from) { + union { + float f; + int32_t i; + } u; + u.f = from; + return HVXPacket::Create(Q6_V_vsplat_R(u.i)); +} +template <> +EIGEN_STRONG_INLINE Packet32f pset1(const float& from) { + return pset1_hvx(from); +} +template <> +EIGEN_STRONG_INLINE Packet16f pset1(const float& from) { + return pset1_hvx(from); +} +template <> +EIGEN_STRONG_INLINE Packet8f pset1(const float& from) { + return pset1_hvx(from); +} + +template <> +EIGEN_STRONG_INLINE Packet32f pload(const float* from) { + return Packet32f::Create(HVX_load(from)); +} +template <> +EIGEN_STRONG_INLINE Packet16f pload(const float* from) { + return Packet16f::Create( + HVX_load_partial::size, unpacket_traits::alignment>(from)); +} +template <> +EIGEN_STRONG_INLINE Packet8f pload(const float* from) { + return Packet8f::Create( + HVX_load_partial::size, unpacket_traits::alignment>(from)); +} + +template <> +EIGEN_STRONG_INLINE Packet32f ploadu(const float* from) { + return Packet32f::Create(HVX_loadu(from)); +} +template <> +EIGEN_STRONG_INLINE Packet16f ploadu(const float* from) { + return Packet16f::Create(HVX_load_partial::size, 0>(from)); +} +template <> +EIGEN_STRONG_INLINE Packet8f ploadu(const float* from) { + return Packet8f::Create(HVX_load_partial::size, 0>(from)); +} + +template <> +EIGEN_STRONG_INLINE void pstore(float* to, const Packet32f& from) { + HVX_store(to, from.Get()); +} +template <> +EIGEN_STRONG_INLINE void pstore(float* to, const Packet16f& from) { + HVX_store_partial::size, unpacket_traits::alignment>(to, from.Get()); +} +template <> +EIGEN_STRONG_INLINE void pstore(float* to, const Packet8f& from) { + HVX_store_partial::size, unpacket_traits::alignment>(to, from.Get()); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet32f& from) { + HVX_storeu(to, from.Get()); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet16f& from) { + HVX_store_partial::size, 0>(to, from.Get()); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet8f& from) { + HVX_store_partial::size, 0>(to, from.Get()); +} + +template +EIGEN_STRONG_INLINE HVXPacket pmul_hvx(const HVXPacket& a, const HVXPacket& b) { + return HVXPacket::Create(Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(a.Get(), b.Get()))); +} +template <> +EIGEN_STRONG_INLINE Packet32f pmul(const Packet32f& a, const Packet32f& b) { + return pmul_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pmul(const Packet16f& a, const Packet16f& b) { + return pmul_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pmul(const Packet8f& a, const Packet8f& b) { + return pmul_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket padd_hvx(const HVXPacket& a, const HVXPacket& b) { + return HVXPacket::Create(Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(a.Get(), b.Get()))); +} +template <> +EIGEN_STRONG_INLINE Packet32f padd(const Packet32f& a, const Packet32f& b) { + return padd_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f padd(const Packet16f& a, const Packet16f& b) { + return padd_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f padd(const Packet8f& a, const Packet8f& b) { + return padd_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket psub_hvx(const HVXPacket& a, const HVXPacket& b) { + return HVXPacket::Create(Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(a.Get(), b.Get()))); +} +template <> +EIGEN_STRONG_INLINE Packet32f psub(const Packet32f& a, const Packet32f& b) { + return psub_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f psub(const Packet16f& a, const Packet16f& b) { + return psub_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f psub(const Packet8f& a, const Packet8f& b) { + return psub_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket pnegate_hvx(const HVXPacket& a) { + return HVXPacket::Create(a.Get() ^ Q6_V_vsplat_R(0x80000000)); +} +template <> +EIGEN_STRONG_INLINE Packet32f pnegate(const Packet32f& a) { + return pnegate_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet16f pnegate(const Packet16f& a) { + return pnegate_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet8f pnegate(const Packet8f& a) { + return pnegate_hvx(a); +} + +template +EIGEN_STRONG_INLINE HVXPacket ptrue_hvx(const HVXPacket& a) { + return HVXPacket::Create(Q6_V_vsplat_R(0x3f800000)); +} +template <> +EIGEN_STRONG_INLINE Packet32f ptrue(const Packet32f& a) { + return ptrue_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet16f ptrue(const Packet16f& a) { + return ptrue_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet8f ptrue(const Packet8f& a) { + return ptrue_hvx(a); +} + +template +EIGEN_STRONG_INLINE HVXPacket pcmp_le_hvx(const HVXPacket& a, const HVXPacket& b) { + HVX_Vector v_true = ptrue(a).Get(); + HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(a.Get(), b.Get()); + return HVXPacket::Create(Q6_V_vmux_QVV(pred, Q6_V_vzero(), v_true)); +} +template <> +EIGEN_STRONG_INLINE Packet32f pcmp_le(const Packet32f& a, const Packet32f& b) { + return pcmp_le_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pcmp_le(const Packet16f& a, const Packet16f& b) { + return pcmp_le_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pcmp_le(const Packet8f& a, const Packet8f& b) { + return pcmp_le_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket pcmp_eq_hvx(const HVXPacket& a, const HVXPacket& b) { + HVX_Vector v_true = ptrue(a).Get(); + HVX_VectorPred pred = Q6_Q_vcmp_eq_VwVw(a.Get(), b.Get()); + return HVXPacket::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero())); +} +template <> +EIGEN_STRONG_INLINE Packet32f pcmp_eq(const Packet32f& a, const Packet32f& b) { + return pcmp_eq_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pcmp_eq(const Packet16f& a, const Packet16f& b) { + return pcmp_eq_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pcmp_eq(const Packet8f& a, const Packet8f& b) { + return pcmp_eq_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket pcmp_lt_hvx(const HVXPacket& a, const HVXPacket& b) { + HVX_Vector v_true = ptrue(a).Get(); + HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(b.Get(), a.Get()); + return HVXPacket::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero())); +} +template <> +EIGEN_STRONG_INLINE Packet32f pcmp_lt(const Packet32f& a, const Packet32f& b) { + return pcmp_lt_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pcmp_lt(const Packet16f& a, const Packet16f& b) { + return pcmp_lt_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pcmp_lt(const Packet8f& a, const Packet8f& b) { + return pcmp_lt_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket pcmp_lt_or_nan_hvx(const HVXPacket& a, const HVXPacket& b) { + HVX_Vector v_true = ptrue(a).Get(); + HVX_VectorPred pred = Q6_Q_vcmp_gt_VsfVsf(b.Get(), a.Get()); + return HVXPacket::Create(Q6_V_vmux_QVV(pred, v_true, Q6_V_vzero())); +} +template <> +EIGEN_STRONG_INLINE Packet32f pcmp_lt_or_nan(const Packet32f& a, const Packet32f& b) { + return pcmp_lt_or_nan_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pcmp_lt_or_nan(const Packet16f& a, const Packet16f& b) { + return pcmp_lt_or_nan_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pcmp_lt_or_nan(const Packet8f& a, const Packet8f& b) { + return pcmp_lt_or_nan_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket pabs_hvx(const HVXPacket& a) { + return HVXPacket::Create(a.Get() & Q6_V_vsplat_R(0x7FFFFFFF)); +} +template <> +EIGEN_STRONG_INLINE Packet32f pabs(const Packet32f& a) { + return pabs_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet16f pabs(const Packet16f& a) { + return pabs_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet8f pabs(const Packet8f& a) { + return pabs_hvx(a); +} + +template +EIGEN_STRONG_INLINE float pfirst_hvx(const HVXPacket& a) { + union { + float array[1]; + HVX_Vector vector; + } HVX_and_array; + HVX_and_array.vector = a.Get(); + return HVX_and_array.array[0]; +} +template <> +EIGEN_STRONG_INLINE float pfirst(const Packet32f& a) { + return pfirst_hvx(a); +} +template <> +EIGEN_STRONG_INLINE float pfirst(const Packet16f& a) { + return pfirst_hvx(a); +} +template <> +EIGEN_STRONG_INLINE float pfirst(const Packet8f& a) { + return pfirst_hvx(a); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + // Shuffle the 32-bit lanes. + HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4); + HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4); + + // Shuffle the 64-bit lanes. + HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8); + HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_3_2), HEXAGON_HVX_GET_V1(v_0_1_0), -8); + kernel.packet[0] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_1_1_0)); + kernel.packet[1] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_1_1_0)); + kernel.packet[2] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_1_3_2)); + kernel.packet[3] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_1_3_2)); +} +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + // Shuffle the 32-bit lanes. + HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4); + HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4); + + // Shuffle the 64-bit lanes. + HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8); + + kernel.packet[0] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_1_0)); + kernel.packet[1] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 64)); + kernel.packet[2] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_1_0)); + kernel.packet[3] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_1_0), HEXAGON_HVX_GET_V1(v_1_1_0), 64)); +} +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + // Shuffle the 32-bit lanes. + HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4); + HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4); + + // Shuffle the 64-bit lanes. + HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8); + + kernel.packet[0] = Packet8f::Create(HEXAGON_HVX_GET_V0(v_1_1_0)); + kernel.packet[1] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 32)); + kernel.packet[2] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 64)); + kernel.packet[3] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 96)); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + // Shuffle the 32-bit lanes. + HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4); + HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4); + HVX_VectorPair v_0_5_4 = Q6_W_vshuff_VVR(kernel.packet[5].Get(), kernel.packet[4].Get(), -4); + HVX_VectorPair v_0_7_6 = Q6_W_vshuff_VVR(kernel.packet[7].Get(), kernel.packet[6].Get(), -4); + + // Shuffle the 64-bit lanes. + HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8); + HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_7_6), HEXAGON_HVX_GET_V0(v_0_5_4), -8); + + // Shuffle the 128-bit lanes. + v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_3_2), HEXAGON_HVX_GET_V0(v_1_1_0), -16); + + kernel.packet[0] = Packet8f::Create(HEXAGON_HVX_GET_V0(v_0_1_0)); + kernel.packet[1] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_0_1_0), HEXAGON_HVX_GET_V0(v_0_1_0), 32)); + kernel.packet[2] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_0_1_0), HEXAGON_HVX_GET_V0(v_0_1_0), 64)); + kernel.packet[3] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_0_1_0), HEXAGON_HVX_GET_V0(v_0_1_0), 96)); + kernel.packet[4] = Packet8f::Create(HEXAGON_HVX_GET_V1(v_0_1_0)); + kernel.packet[5] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_0_1_0), HEXAGON_HVX_GET_V1(v_0_1_0), 32)); + kernel.packet[6] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_0_1_0), HEXAGON_HVX_GET_V1(v_0_1_0), 64)); + kernel.packet[7] = Packet8f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_0_1_0), HEXAGON_HVX_GET_V1(v_0_1_0), 96)); +} +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + // Shuffle the 32-bit lanes. + HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4); + HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4); + HVX_VectorPair v_0_5_4 = Q6_W_vshuff_VVR(kernel.packet[5].Get(), kernel.packet[4].Get(), -4); + HVX_VectorPair v_0_7_6 = Q6_W_vshuff_VVR(kernel.packet[7].Get(), kernel.packet[6].Get(), -4); + HVX_VectorPair v_0_9_8 = Q6_W_vshuff_VVR(kernel.packet[9].Get(), kernel.packet[8].Get(), -4); + HVX_VectorPair v_0_11_10 = Q6_W_vshuff_VVR(kernel.packet[11].Get(), kernel.packet[10].Get(), -4); + HVX_VectorPair v_0_13_12 = Q6_W_vshuff_VVR(kernel.packet[13].Get(), kernel.packet[12].Get(), -4); + HVX_VectorPair v_0_15_14 = Q6_W_vshuff_VVR(kernel.packet[15].Get(), kernel.packet[14].Get(), -4); + + // Shuffle the 64-bit lanes. + HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8); + HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_7_6), HEXAGON_HVX_GET_V0(v_0_5_4), -8); + HVX_VectorPair v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), HEXAGON_HVX_GET_V0(v_0_9_8), -8); + HVX_VectorPair v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_15_14), HEXAGON_HVX_GET_V0(v_0_13_12), -8); + + // Shuffle the 128-bit lanes. + v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_3_2), HEXAGON_HVX_GET_V0(v_1_1_0), -16); + v_0_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_3_2), HEXAGON_HVX_GET_V1(v_1_1_0), -16); + v_0_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_7_6), HEXAGON_HVX_GET_V0(v_1_5_4), -16); + v_0_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_7_6), HEXAGON_HVX_GET_V1(v_1_5_4), -16); + + // Shuffle the 256-bit lanes. + v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_9_8), HEXAGON_HVX_GET_V0(v_0_1_0), -32); + v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_9_8), HEXAGON_HVX_GET_V1(v_0_1_0), -32); + v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), HEXAGON_HVX_GET_V0(v_0_3_2), -32); + v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_11_10), HEXAGON_HVX_GET_V1(v_0_3_2), -32); + + kernel.packet[0] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_1_0)); + kernel.packet[1] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_1_0), HEXAGON_HVX_GET_V0(v_1_1_0), 64)); + kernel.packet[2] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_1_0)); + kernel.packet[3] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_1_0), HEXAGON_HVX_GET_V1(v_1_1_0), 64)); + kernel.packet[4] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_3_2)); + kernel.packet[5] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_3_2), HEXAGON_HVX_GET_V0(v_1_3_2), 64)); + kernel.packet[6] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_3_2)); + kernel.packet[7] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_3_2), HEXAGON_HVX_GET_V1(v_1_3_2), 64)); + kernel.packet[8] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_5_4)); + kernel.packet[9] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_5_4), HEXAGON_HVX_GET_V0(v_1_5_4), 64)); + kernel.packet[10] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_5_4)); + kernel.packet[11] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_5_4), HEXAGON_HVX_GET_V1(v_1_5_4), 64)); + kernel.packet[12] = Packet16f::Create(HEXAGON_HVX_GET_V0(v_1_7_6)); + kernel.packet[13] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V0(v_1_7_6), HEXAGON_HVX_GET_V0(v_1_7_6), 64)); + kernel.packet[14] = Packet16f::Create(HEXAGON_HVX_GET_V1(v_1_7_6)); + kernel.packet[15] = Packet16f::Create(Q6_V_valign_VVR(HEXAGON_HVX_GET_V1(v_1_7_6), HEXAGON_HVX_GET_V1(v_1_7_6), 64)); +} +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + // Shuffle the 32-bit lanes. + HVX_VectorPair v_0_1_0 = Q6_W_vshuff_VVR(kernel.packet[1].Get(), kernel.packet[0].Get(), -4); + HVX_VectorPair v_0_3_2 = Q6_W_vshuff_VVR(kernel.packet[3].Get(), kernel.packet[2].Get(), -4); + HVX_VectorPair v_0_5_4 = Q6_W_vshuff_VVR(kernel.packet[5].Get(), kernel.packet[4].Get(), -4); + HVX_VectorPair v_0_7_6 = Q6_W_vshuff_VVR(kernel.packet[7].Get(), kernel.packet[6].Get(), -4); + HVX_VectorPair v_0_9_8 = Q6_W_vshuff_VVR(kernel.packet[9].Get(), kernel.packet[8].Get(), -4); + HVX_VectorPair v_0_11_10 = Q6_W_vshuff_VVR(kernel.packet[11].Get(), kernel.packet[10].Get(), -4); + HVX_VectorPair v_0_13_12 = Q6_W_vshuff_VVR(kernel.packet[13].Get(), kernel.packet[12].Get(), -4); + HVX_VectorPair v_0_15_14 = Q6_W_vshuff_VVR(kernel.packet[15].Get(), kernel.packet[14].Get(), -4); + HVX_VectorPair v_0_17_16 = Q6_W_vshuff_VVR(kernel.packet[17].Get(), kernel.packet[16].Get(), -4); + HVX_VectorPair v_0_19_18 = Q6_W_vshuff_VVR(kernel.packet[19].Get(), kernel.packet[18].Get(), -4); + HVX_VectorPair v_0_21_20 = Q6_W_vshuff_VVR(kernel.packet[21].Get(), kernel.packet[20].Get(), -4); + HVX_VectorPair v_0_23_22 = Q6_W_vshuff_VVR(kernel.packet[23].Get(), kernel.packet[22].Get(), -4); + HVX_VectorPair v_0_25_24 = Q6_W_vshuff_VVR(kernel.packet[25].Get(), kernel.packet[24].Get(), -4); + HVX_VectorPair v_0_27_26 = Q6_W_vshuff_VVR(kernel.packet[27].Get(), kernel.packet[26].Get(), -4); + HVX_VectorPair v_0_29_28 = Q6_W_vshuff_VVR(kernel.packet[29].Get(), kernel.packet[28].Get(), -4); + HVX_VectorPair v_0_31_30 = Q6_W_vshuff_VVR(kernel.packet[31].Get(), kernel.packet[30].Get(), -4); + + // Shuffle the 64-bit lanes. + HVX_VectorPair v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_3_2), HEXAGON_HVX_GET_V0(v_0_1_0), -8); + HVX_VectorPair v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_3_2), HEXAGON_HVX_GET_V1(v_0_1_0), -8); + HVX_VectorPair v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_7_6), HEXAGON_HVX_GET_V0(v_0_5_4), -8); + HVX_VectorPair v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_7_6), HEXAGON_HVX_GET_V1(v_0_5_4), -8); + HVX_VectorPair v_1_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), HEXAGON_HVX_GET_V0(v_0_9_8), -8); + HVX_VectorPair v_1_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_11_10), HEXAGON_HVX_GET_V1(v_0_9_8), -8); + HVX_VectorPair v_1_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_15_14), HEXAGON_HVX_GET_V0(v_0_13_12), -8); + HVX_VectorPair v_1_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_15_14), HEXAGON_HVX_GET_V1(v_0_13_12), -8); + HVX_VectorPair v_1_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_19_18), HEXAGON_HVX_GET_V0(v_0_17_16), -8); + HVX_VectorPair v_1_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_19_18), HEXAGON_HVX_GET_V1(v_0_17_16), -8); + HVX_VectorPair v_1_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_23_22), HEXAGON_HVX_GET_V0(v_0_21_20), -8); + HVX_VectorPair v_1_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_23_22), HEXAGON_HVX_GET_V1(v_0_21_20), -8); + HVX_VectorPair v_1_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_27_26), HEXAGON_HVX_GET_V0(v_0_25_24), -8); + HVX_VectorPair v_1_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_27_26), HEXAGON_HVX_GET_V1(v_0_25_24), -8); + HVX_VectorPair v_1_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_31_30), HEXAGON_HVX_GET_V0(v_0_29_28), -8); + HVX_VectorPair v_1_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_31_30), HEXAGON_HVX_GET_V1(v_0_29_28), -8); + + // Shuffle the 128-bit lanes. + v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_5_4), HEXAGON_HVX_GET_V0(v_1_1_0), -16); + v_0_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_5_4), HEXAGON_HVX_GET_V1(v_1_1_0), -16); + v_0_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_7_6), HEXAGON_HVX_GET_V0(v_1_3_2), -16); + v_0_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_7_6), HEXAGON_HVX_GET_V1(v_1_3_2), -16); + v_0_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_13_12), HEXAGON_HVX_GET_V0(v_1_9_8), -16); + v_0_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_13_12), HEXAGON_HVX_GET_V1(v_1_9_8), -16); + v_0_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_15_14), HEXAGON_HVX_GET_V0(v_1_11_10), -16); + v_0_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_15_14), HEXAGON_HVX_GET_V1(v_1_11_10), -16); + v_0_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_21_20), HEXAGON_HVX_GET_V0(v_1_17_16), -16); + v_0_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_21_20), HEXAGON_HVX_GET_V1(v_1_17_16), -16); + v_0_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_23_22), HEXAGON_HVX_GET_V0(v_1_19_18), -16); + v_0_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_23_22), HEXAGON_HVX_GET_V1(v_1_19_18), -16); + v_0_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_29_28), HEXAGON_HVX_GET_V0(v_1_25_24), -16); + v_0_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_29_28), HEXAGON_HVX_GET_V1(v_1_25_24), -16); + v_0_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_31_30), HEXAGON_HVX_GET_V0(v_1_27_26), -16); + v_0_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_31_30), HEXAGON_HVX_GET_V1(v_1_27_26), -16); + + // Shuffle the 256-bit lanes. + v_1_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_9_8), HEXAGON_HVX_GET_V0(v_0_1_0), -32); + v_1_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_9_8), HEXAGON_HVX_GET_V1(v_0_1_0), -32); + v_1_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_11_10), HEXAGON_HVX_GET_V0(v_0_3_2), -32); + v_1_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_11_10), HEXAGON_HVX_GET_V1(v_0_3_2), -32); + v_1_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_13_12), HEXAGON_HVX_GET_V0(v_0_5_4), -32); + v_1_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_13_12), HEXAGON_HVX_GET_V1(v_0_5_4), -32); + v_1_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_15_14), HEXAGON_HVX_GET_V0(v_0_7_6), -32); + v_1_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_15_14), HEXAGON_HVX_GET_V1(v_0_7_6), -32); + v_1_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_25_24), HEXAGON_HVX_GET_V0(v_0_17_16), -32); + v_1_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_25_24), HEXAGON_HVX_GET_V1(v_0_17_16), -32); + v_1_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_27_26), HEXAGON_HVX_GET_V0(v_0_19_18), -32); + v_1_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_27_26), HEXAGON_HVX_GET_V1(v_0_19_18), -32); + v_1_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_29_28), HEXAGON_HVX_GET_V0(v_0_21_20), -32); + v_1_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_29_28), HEXAGON_HVX_GET_V1(v_0_21_20), -32); + v_1_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_0_31_30), HEXAGON_HVX_GET_V0(v_0_23_22), -32); + v_1_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_0_31_30), HEXAGON_HVX_GET_V1(v_0_23_22), -32); + + // Shuffle the 512-bit lanes. + v_0_1_0 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_17_16), HEXAGON_HVX_GET_V0(v_1_1_0), -64); + v_0_3_2 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_17_16), HEXAGON_HVX_GET_V1(v_1_1_0), -64); + v_0_5_4 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_19_18), HEXAGON_HVX_GET_V0(v_1_3_2), -64); + v_0_7_6 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_19_18), HEXAGON_HVX_GET_V1(v_1_3_2), -64); + v_0_9_8 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_21_20), HEXAGON_HVX_GET_V0(v_1_5_4), -64); + v_0_11_10 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_21_20), HEXAGON_HVX_GET_V1(v_1_5_4), -64); + v_0_13_12 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_23_22), HEXAGON_HVX_GET_V0(v_1_7_6), -64); + v_0_15_14 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_23_22), HEXAGON_HVX_GET_V1(v_1_7_6), -64); + v_0_17_16 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_25_24), HEXAGON_HVX_GET_V0(v_1_9_8), -64); + v_0_19_18 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_25_24), HEXAGON_HVX_GET_V1(v_1_9_8), -64); + v_0_21_20 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_27_26), HEXAGON_HVX_GET_V0(v_1_11_10), -64); + v_0_23_22 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_27_26), HEXAGON_HVX_GET_V1(v_1_11_10), -64); + v_0_25_24 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_29_28), HEXAGON_HVX_GET_V0(v_1_13_12), -64); + v_0_27_26 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_29_28), HEXAGON_HVX_GET_V1(v_1_13_12), -64); + v_0_29_28 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(v_1_31_30), HEXAGON_HVX_GET_V0(v_1_15_14), -64); + v_0_31_30 = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V1(v_1_31_30), HEXAGON_HVX_GET_V1(v_1_15_14), -64); + + kernel.packet[0] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_1_0)); + kernel.packet[1] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_1_0)); + kernel.packet[2] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_3_2)); + kernel.packet[3] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_3_2)); + kernel.packet[4] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_5_4)); + kernel.packet[5] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_5_4)); + kernel.packet[6] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_7_6)); + kernel.packet[7] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_7_6)); + kernel.packet[8] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_9_8)); + kernel.packet[9] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_9_8)); + kernel.packet[10] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_11_10)); + kernel.packet[11] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_11_10)); + kernel.packet[12] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_13_12)); + kernel.packet[13] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_13_12)); + kernel.packet[14] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_15_14)); + kernel.packet[15] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_15_14)); + kernel.packet[16] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_17_16)); + kernel.packet[17] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_17_16)); + kernel.packet[18] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_19_18)); + kernel.packet[19] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_19_18)); + kernel.packet[20] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_21_20)); + kernel.packet[21] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_21_20)); + kernel.packet[22] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_23_22)); + kernel.packet[23] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_23_22)); + kernel.packet[24] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_25_24)); + kernel.packet[25] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_25_24)); + kernel.packet[26] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_27_26)); + kernel.packet[27] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_27_26)); + kernel.packet[28] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_29_28)); + kernel.packet[29] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_29_28)); + kernel.packet[30] = Packet32f::Create(HEXAGON_HVX_GET_V0(v_0_31_30)); + kernel.packet[31] = Packet32f::Create(HEXAGON_HVX_GET_V1(v_0_31_30)); +} + +template +EIGEN_STRONG_INLINE float predux_hvx(const HVXPacket& a) { + const Index packet_size = unpacket_traits>::size; + HVX_Vector vsum = Q6_Vqf32_vadd_VsfVsf(a.Get(), Q6_V_vror_VR(a.Get(), sizeof(float))); + for (int i = 2; i < packet_size; i <<= 1) { + vsum = Q6_Vqf32_vadd_Vqf32Vqf32(vsum, Q6_V_vror_VR(vsum, i * sizeof(float))); + } + return pfirst(HVXPacket::Create(Q6_Vsf_equals_Vqf32(vsum))); +} +template <> +EIGEN_STRONG_INLINE float predux(const Packet32f& a) { + return predux_hvx(a); +} +template <> +EIGEN_STRONG_INLINE float predux(const Packet16f& a) { + return predux_hvx(a); +} +template <> +EIGEN_STRONG_INLINE float predux(const Packet8f& a) { + return predux_hvx(a); +} + +template +EIGEN_STRONG_INLINE HVXPacket ploaddup_hvx(const float* from) { + constexpr Index size = unpacket_traits>::size / 2; + HVX_Vector load = HVX_load_partial(from); + HVX_VectorPair dup = Q6_W_vshuff_VVR(load, load, -4); + return HVXPacket::Create(HEXAGON_HVX_GET_V0(dup)); +} +template <> +EIGEN_STRONG_INLINE Packet32f ploaddup(const float* from) { + return ploaddup_hvx(from); +} +template <> +EIGEN_STRONG_INLINE Packet16f ploaddup(const float* from) { + return ploaddup_hvx(from); +} +template <> +EIGEN_STRONG_INLINE Packet8f ploaddup(const float* from) { + return ploaddup_hvx(from); +} + +template +EIGEN_STRONG_INLINE HVXPacket ploadquad_hvx(const float* from) { + constexpr Index size = unpacket_traits>::size / 4; + HVX_Vector load = HVX_load_partial(from); + HVX_VectorPair dup = Q6_W_vshuff_VVR(load, load, -4); + HVX_VectorPair quad = Q6_W_vshuff_VVR(HEXAGON_HVX_GET_V0(dup), HEXAGON_HVX_GET_V0(dup), -8); + return HVXPacket::Create(HEXAGON_HVX_GET_V0(quad)); +} +template <> +EIGEN_STRONG_INLINE Packet32f ploadquad(const float* from) { + return ploadquad_hvx(from); +} +template <> +EIGEN_STRONG_INLINE Packet16f ploadquad(const float* from) { + return ploadquad_hvx(from); +} +template <> +EIGEN_STRONG_INLINE Packet8f ploadquad(const float* from) { + return ploadquad_hvx(from); +} + +template <> +EIGEN_STRONG_INLINE Packet32f preverse(const Packet32f& a) { + HVX_Vector delta = Q6_Vb_vsplat_R(0x7c); + return Packet32f::Create(Q6_V_vdelta_VV(a.Get(), delta)); +} + +template <> +EIGEN_STRONG_INLINE Packet16f preverse(const Packet16f& a) { + HVX_Vector delta = Q6_Vb_vsplat_R(0x3c); + return Packet16f::Create(Q6_V_vdelta_VV(a.Get(), delta)); +} + +template <> +EIGEN_STRONG_INLINE Packet8f preverse(const Packet8f& a) { + HVX_Vector delta = Q6_Vb_vsplat_R(0x1c); + return Packet8f::Create(Q6_V_vdelta_VV(a.Get(), delta)); +} + +template +EIGEN_STRONG_INLINE HVXPacket pmin_hvx(const HVXPacket& a, const HVXPacket& b) { + return HVXPacket::Create(Q6_Vsf_vmin_VsfVsf(a.Get(), b.Get())); +} +template <> +EIGEN_STRONG_INLINE Packet32f pmin(const Packet32f& a, const Packet32f& b) { + return pmin_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pmin(const Packet16f& a, const Packet16f& b) { + return pmin_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pmin(const Packet8f& a, const Packet8f& b) { + return pmin_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket pmax_hvx(const HVXPacket& a, const HVXPacket& b) { + return HVXPacket::Create(Q6_Vsf_vmax_VsfVsf(a.Get(), b.Get())); +} +template <> +EIGEN_STRONG_INLINE Packet32f pmax(const Packet32f& a, const Packet32f& b) { + return pmax_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pmax(const Packet16f& a, const Packet16f& b) { + return pmax_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pmax(const Packet8f& a, const Packet8f& b) { + return pmax_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket pand_hvx(const HVXPacket& a, const HVXPacket& b) { + return HVXPacket::Create(a.Get() & b.Get()); +} +template <> +EIGEN_STRONG_INLINE Packet32f pand(const Packet32f& a, const Packet32f& b) { + return pand_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pand(const Packet16f& a, const Packet16f& b) { + return pand_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pand(const Packet8f& a, const Packet8f& b) { + return pand_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket por_hvx(const HVXPacket& a, const HVXPacket& b) { + return HVXPacket::Create(a.Get() | b.Get()); +} +template <> +EIGEN_STRONG_INLINE Packet32f por(const Packet32f& a, const Packet32f& b) { + return por_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f por(const Packet16f& a, const Packet16f& b) { + return por_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f por(const Packet8f& a, const Packet8f& b) { + return por_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket pxor_hvx(const HVXPacket& a, const HVXPacket& b) { + return HVXPacket::Create(a.Get() ^ b.Get()); +} +template <> +EIGEN_STRONG_INLINE Packet32f pxor(const Packet32f& a, const Packet32f& b) { + return pxor_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pxor(const Packet16f& a, const Packet16f& b) { + return pxor_hvx(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pxor(const Packet8f& a, const Packet8f& b) { + return pxor_hvx(a, b); +} + +template +EIGEN_STRONG_INLINE HVXPacket pnot_hvx(const HVXPacket& a) { + return HVXPacket::Create(~a.Get()); +} +template <> +EIGEN_STRONG_INLINE Packet32f pnot(const Packet32f& a) { + return pnot_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet16f pnot(const Packet16f& a) { + return pnot_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet8f pnot(const Packet8f& a) { + return pnot_hvx(a); +} + +template +EIGEN_STRONG_INLINE HVXPacket pselect_hvx(const HVXPacket& mask, const HVXPacket& a, const HVXPacket& b) { + HVX_VectorPred pred = Q6_Q_vcmp_eq_VwVw(mask.Get(), Q6_V_vzero()); + return HVXPacket::Create(Q6_V_vmux_QVV(pred, b.Get(), a.Get())); +} +template <> +EIGEN_STRONG_INLINE Packet32f pselect(const Packet32f& mask, const Packet32f& a, const Packet32f& b) { + return pselect_hvx(mask, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16f pselect(const Packet16f& mask, const Packet16f& a, const Packet16f& b) { + return pselect_hvx(mask, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8f pselect(const Packet8f& mask, const Packet8f& a, const Packet8f& b) { + return pselect_hvx(mask, a, b); +} + +template +EIGEN_STRONG_INLINE float predux_generic(const HVXPacket& a, Op op) { + const Index packet_size = unpacket_traits>::size; + HVXPacket vredux = a; + for (int i = 1; i < packet_size; i <<= 1) { + vredux = op(vredux, HVXPacket::Create(Q6_V_vror_VR(vredux.Get(), i * sizeof(float)))); + } + return pfirst(vredux); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet32f& a) { + return predux_generic(a, pmax); +} +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet16f& a) { + return predux_generic(a, pmax); +} +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet8f& a) { + return predux_generic(a, pmax); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet32f& a) { + return predux_generic(a, pmin); +} +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet16f& a) { + return predux_generic(a, pmin); +} +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet8f& a) { + return predux_generic(a, pmin); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet32f& a) { + return predux_generic(a, por) != 0.0f; +} +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet16f& a) { + return predux_generic(a, por) != 0.0f; +} +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet8f& a) { + return predux_generic(a, por) != 0.0f; +} + +static const float index_vsf[32] + __attribute__((aligned(__HVX_LENGTH__))) = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, + 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31}; + +template +EIGEN_STRONG_INLINE HVXPacket plset_hvx(const float& a) { + return padd(pload>(index_vsf), pset1>(a)); +} +template <> +EIGEN_STRONG_INLINE Packet32f plset(const float& a) { + return plset_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet16f plset(const float& a) { + return plset_hvx(a); +} +template <> +EIGEN_STRONG_INLINE Packet8f plset(const float& a) { + return plset_hvx(a); +} + +template +EIGEN_STRONG_INLINE void pscatter_hvx(float* to, const HVXPacket& from, Index stride) { + const Index packet_size = unpacket_traits>::size; + float elements[packet_size] __attribute__((aligned(__HVX_LENGTH__))); + pstore(elements, from); + for (Index i = 0; i < packet_size; ++i) { + to[i * stride] = elements[i]; + } +} +template <> +EIGEN_STRONG_INLINE void pscatter(float* to, const Packet32f& from, Index stride) { + pscatter_hvx(to, from, stride); +} +template <> +EIGEN_STRONG_INLINE void pscatter(float* to, const Packet16f& from, Index stride) { + pscatter_hvx(to, from, stride); +} +template <> +EIGEN_STRONG_INLINE void pscatter(float* to, const Packet8f& from, Index stride) { + pscatter_hvx(to, from, stride); +} + +template +EIGEN_STRONG_INLINE HVXPacket pgather_hvx(const float* from, Index stride) { + const Index packet_size = unpacket_traits>::size; + float elements[packet_size] __attribute__((aligned(__HVX_LENGTH__))); + for (Index i = 0; i < packet_size; i++) { + elements[i] = from[i * stride]; + } + return pload>(elements); +} +template <> +EIGEN_STRONG_INLINE Packet32f pgather(const float* from, Index stride) { + return pgather_hvx(from, stride); +} +template <> +EIGEN_STRONG_INLINE Packet16f pgather(const float* from, Index stride) { + return pgather_hvx(from, stride); +} +template <> +EIGEN_STRONG_INLINE Packet8f pgather(const float* from, Index stride) { + return pgather_hvx(from, stride); +} + +} // end namespace internal +} // end namespace Eigen + +#endif // __HVX__ && (__HVX_LENGTH__ == 128) && __HVX_ARCH__ >= 68 + +#endif // EIGEN_HVX_PACKET_MATH_H diff --git a/Eigen/src/Core/arch/LSX/Complex.h b/Eigen/src/Core/arch/LSX/Complex.h new file mode 100644 index 0000000..0b60a83 --- /dev/null +++ b/Eigen/src/Core/arch/LSX/Complex.h @@ -0,0 +1,520 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// copyright (c) 2023 zang ruochen +// copyright (c) 2024 XiWei Gu +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_COMPLEX_LSX_H +#define EIGEN_COMPLEX_LSX_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +//---------- float ---------- +struct Packet2cf { + EIGEN_STRONG_INLINE Packet2cf() {} + EIGEN_STRONG_INLINE explicit Packet2cf(const __m128& a) : v(a) {} + Packet4f v; +}; + +template <> +struct packet_traits > : default_packet_traits { + typedef Packet2cf type; + typedef Packet2cf half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 2, + + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasSqrt = 1, + HasExp = 1, + HasAbs = 0, + HasLog = 1, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasSetLinear = 0 + }; +}; + +template <> +struct unpacket_traits { + typedef std::complex type; + typedef Packet2cf half; + typedef Packet4f as_real; + enum { + size = 2, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +EIGEN_STRONG_INLINE Packet2cf padd(const Packet2cf& a, const Packet2cf& b) { + return Packet2cf(__lsx_vfadd_s(a.v, b.v)); +} +template <> +EIGEN_STRONG_INLINE Packet2cf psub(const Packet2cf& a, const Packet2cf& b) { + return Packet2cf(__lsx_vfsub_s(a.v, b.v)); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pnegate(const Packet2cf& a) { + const uint32_t b[4] = {0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u}; + Packet4i mask = (Packet4i)__lsx_vld(b, 0); + Packet2cf res; + res.v = (Packet4f)__lsx_vxor_v((__m128i)a.v, mask); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pconj(const Packet2cf& a) { + const uint32_t b[4] = {0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u}; + Packet4i mask = (__m128i)__lsx_vld(b, 0); + Packet2cf res; + res.v = (Packet4f)__lsx_vxor_v((__m128i)a.v, mask); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pmul(const Packet2cf& a, const Packet2cf& b) { + Packet4f part0_tmp = (Packet4f)__lsx_vfmul_s(a.v, b.v); + Packet4f part0 = __lsx_vfsub_s(part0_tmp, (__m128)__lsx_vshuf4i_w(part0_tmp, 0x31)); + Packet4f part1_tmp = __lsx_vfmul_s((__m128)__lsx_vshuf4i_w(a.v, 0xb1), b.v); + Packet4f part1 = __lsx_vfadd_s(part1_tmp, (__m128)__lsx_vshuf4i_w(part1_tmp, 0x31)); + Packet2cf res; + res.v = (Packet4f)__lsx_vpackev_w((__m128i)part1, (__m128i)part0); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf ptrue(const Packet2cf& a) { + return Packet2cf(ptrue(Packet4f(a.v))); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pand(const Packet2cf& a, const Packet2cf& b) { + Packet2cf res; + res.v = (Packet4f)__lsx_vand_v((__m128i)a.v, (__m128i)b.v); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf por(const Packet2cf& a, const Packet2cf& b) { + Packet2cf res; + res.v = (Packet4f)__lsx_vor_v((__m128i)a.v, (__m128i)b.v); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pxor(const Packet2cf& a, const Packet2cf& b) { + Packet2cf res; + res.v = (Packet4f)__lsx_vxor_v((__m128i)a.v, (__m128i)b.v); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pandnot(const Packet2cf& a, const Packet2cf& b) { + Packet2cf res; + res.v = (Packet4f)__lsx_vandn_v((__m128i)b.v, (__m128i)a.v); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pload(const std::complex* from) { + EIGEN_DEBUG_ALIGNED_LOAD return Packet2cf(pload(&numext::real_ref(*from))); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf ploadu(const std::complex* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return Packet2cf(ploadu(&numext::real_ref(*from))); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pset1(const std::complex& from) { + float f0 = from.real(), f1 = from.imag(); + Packet4f re = {f0, f0, f0, f0}; + Packet4f im = {f1, f1, f1, f1}; + return Packet2cf((Packet4f)__lsx_vilvl_w((__m128i)im, (__m128i)re)); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf ploaddup(const std::complex* from) { + return pset1(*from); +} + +template <> +EIGEN_STRONG_INLINE void pstore >(std::complex* to, const Packet2cf& from) { + EIGEN_DEBUG_ALIGNED_STORE pstore(&numext::real_ref(*to), Packet4f(from.v)); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu >(std::complex* to, const Packet2cf& from) { + EIGEN_DEBUG_UNALIGNED_STORE pstoreu(&numext::real_ref(*to), Packet4f(from.v)); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet2cf pgather, Packet2cf>(const std::complex* from, + Index stride) { + Packet2cf res; + __m128i tmp = __lsx_vldrepl_d(from, 0); + __m128i tmp1 = __lsx_vldrepl_d(from + stride, 0); + tmp = __lsx_vilvl_d(tmp1, tmp); + res.v = (__m128)tmp; + return res; +} + +template <> +EIGEN_DEVICE_FUNC inline void pscatter, Packet2cf>(std::complex* to, const Packet2cf& from, + Index stride) { + __lsx_vstelm_d((__m128i)from.v, to, 0, 0); + __lsx_vstelm_d((__m128i)from.v, to + stride, 0, 1); +} + +template <> +EIGEN_STRONG_INLINE void prefetch >(const std::complex* addr) { + __builtin_prefetch(addr); +} + +template <> +EIGEN_STRONG_INLINE std::complex pfirst(const Packet2cf& a) { + EIGEN_ALIGN16 std::complex res[2]; + __lsx_vst(a.v, res, 0); + return res[0]; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf preverse(const Packet2cf& a) { + Packet2cf res; + res.v = (Packet4f)__lsx_vshuf4i_w(a.v, 0x4e); + return res; +} + +template <> +EIGEN_STRONG_INLINE std::complex predux(const Packet2cf& a) { + return pfirst(Packet2cf(__lsx_vfadd_s(a.v, vec4f_movehl(a.v, a.v)))); +} + +template <> +EIGEN_STRONG_INLINE std::complex predux_mul(const Packet2cf& a) { + return pfirst(pmul(a, Packet2cf(vec4f_movehl(a.v, a.v)))); +} + +EIGEN_STRONG_INLINE Packet2cf pcplxflip /* */ (const Packet2cf& x) { + return Packet2cf(vec4f_swizzle1(x.v, 1, 0, 3, 2)); +} + +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet2cf, Packet4f) + +template <> +EIGEN_STRONG_INLINE Packet2cf pdiv(const Packet2cf& a, const Packet2cf& b) { + return pdiv_complex(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf plog(const Packet2cf& a) { + return plog_complex(a); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pzero(const Packet2cf& /* a */) { + __m128 v = {0.0f, 0.0f, 0.0f, 0.0f}; + return (Packet2cf)v; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pmadd(const Packet2cf& a, const Packet2cf& b, const Packet2cf& c) { + Packet2cf result, t0, t1, t2; + t1 = pzero(t1); + t0.v = (__m128)__lsx_vpackev_w((__m128i)a.v, (__m128i)a.v); + t2.v = __lsx_vfmadd_s(t0.v, b.v, c.v); + result.v = __lsx_vfadd_s(t2.v, t1.v); + t1.v = __lsx_vfsub_s(t1.v, a.v); + t1.v = (__m128)__lsx_vpackod_w((__m128i)a.v, (__m128i)t1.v); + t2.v = (__m128)__lsx_vshuf4i_w((__m128i)b.v, 0xb1); + result.v = __lsx_vfmadd_s(t1.v, t2.v, result.v); + return result; +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pexp(const Packet2cf& a) { + return pexp_complex(a); +} + +//---------- double ---------- +struct Packet1cd { + EIGEN_STRONG_INLINE Packet1cd() {} + EIGEN_STRONG_INLINE explicit Packet1cd(const __m128d& a) : v(a) {} + Packet2d v; +}; + +template <> +struct packet_traits > : default_packet_traits { + typedef Packet1cd type; + typedef Packet1cd half; + enum { + Vectorizable = 1, + AlignedOnScalar = 0, + size = 1, + + HasAdd = 1, + HasSub = 1, + HasMul = 1, + HasDiv = 1, + HasNegate = 1, + HasSqrt = 1, + HasAbs = 0, + HasLog = 1, + HasAbs2 = 0, + HasMin = 0, + HasMax = 0, + HasSetLinear = 0 + }; +}; + +template <> +struct unpacket_traits { + typedef std::complex type; + typedef Packet1cd half; + typedef Packet2d as_real; + enum { + size = 1, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +EIGEN_STRONG_INLINE Packet1cd padd(const Packet1cd& a, const Packet1cd& b) { + return Packet1cd(__lsx_vfadd_d(a.v, b.v)); +} +template <> +EIGEN_STRONG_INLINE Packet1cd psub(const Packet1cd& a, const Packet1cd& b) { + return Packet1cd(__lsx_vfsub_d(a.v, b.v)); +} +template <> +EIGEN_STRONG_INLINE Packet1cd pnegate(const Packet1cd& a) { + return Packet1cd(pnegate(Packet2d(a.v))); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pconj(const Packet1cd& a) { + const uint64_t tmp[2] = {0x0000000000000000u, 0x8000000000000000u}; + __m128i mask = __lsx_vld(tmp, 0); + Packet1cd res; + res.v = (Packet2d)__lsx_vxor_v((__m128i)a.v, mask); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pmul(const Packet1cd& a, const Packet1cd& b) { + Packet2d tmp_real = __lsx_vfmul_d(a.v, b.v); + Packet2d real = __lsx_vfsub_d(tmp_real, preverse(tmp_real)); + + Packet2d tmp_imag = __lsx_vfmul_d(preverse(a.v), b.v); + Packet2d imag = (__m128d)__lsx_vfadd_d((__m128d)tmp_imag, preverse(tmp_imag)); + Packet1cd res; + res.v = (__m128d)__lsx_vilvl_d((__m128i)imag, (__m128i)real); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet1cd ptrue(const Packet1cd& a) { + return Packet1cd(ptrue(Packet2d(a.v))); +} +template <> +EIGEN_STRONG_INLINE Packet1cd pand(const Packet1cd& a, const Packet1cd& b) { + Packet1cd res; + res.v = (Packet2d)__lsx_vand_v((__m128i)a.v, (__m128i)b.v); + return res; +} +template <> +EIGEN_STRONG_INLINE Packet1cd por(const Packet1cd& a, const Packet1cd& b) { + Packet1cd res; + res.v = (Packet2d)__lsx_vor_v((__m128i)a.v, (__m128i)b.v); + return res; +} +template <> +EIGEN_STRONG_INLINE Packet1cd pxor(const Packet1cd& a, const Packet1cd& b) { + Packet1cd res; + res.v = (Packet2d)__lsx_vxor_v((__m128i)a.v, (__m128i)b.v); + return res; +} +template <> +EIGEN_STRONG_INLINE Packet1cd pandnot(const Packet1cd& a, const Packet1cd& b) { + Packet1cd res; + res.v = (Packet2d)__lsx_vandn_v((__m128i)b.v, (__m128i)a.v); + return res; +} + +// FIXME force unaligned load, this is a temporary fix +template <> +EIGEN_STRONG_INLINE Packet1cd pload(const std::complex* from) { + EIGEN_DEBUG_ALIGNED_LOAD return Packet1cd(pload((const double*)from)); +} +template <> +EIGEN_STRONG_INLINE Packet1cd ploadu(const std::complex* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return Packet1cd(ploadu((const double*)from)); +} +template <> +EIGEN_STRONG_INLINE Packet1cd +pset1(const std::complex& from) { /* here we really have to use unaligned loads :( */ + return ploadu(&from); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd ploaddup(const std::complex* from) { + return pset1(*from); +} + +// FIXME force unaligned store, this is a temporary fix +template <> +EIGEN_STRONG_INLINE void pstore >(std::complex* to, const Packet1cd& from) { + EIGEN_DEBUG_ALIGNED_STORE pstore((double*)to, Packet2d(from.v)); +} +template <> +EIGEN_STRONG_INLINE void pstoreu >(std::complex* to, const Packet1cd& from) { + EIGEN_DEBUG_UNALIGNED_STORE pstoreu((double*)to, Packet2d(from.v)); +} + +template <> +EIGEN_STRONG_INLINE void prefetch >(const std::complex* addr) { + __builtin_prefetch(addr); +} + +template <> +EIGEN_STRONG_INLINE std::complex pfirst(const Packet1cd& a) { + EIGEN_ALIGN16 double res[2]; + __lsx_vst(a.v, res, 0); + return std::complex(res[0], res[1]); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd preverse(const Packet1cd& a) { + return a; +} + +template <> +EIGEN_STRONG_INLINE std::complex predux(const Packet1cd& a) { + return pfirst(a); +} + +template <> +EIGEN_STRONG_INLINE std::complex predux_mul(const Packet1cd& a) { + return pfirst(a); +} + +EIGEN_MAKE_CONJ_HELPER_CPLX_REAL(Packet1cd, Packet2d) + +template <> +EIGEN_STRONG_INLINE Packet1cd pdiv(const Packet1cd& a, const Packet1cd& b) { + return pdiv_complex(a, b); +} + +EIGEN_STRONG_INLINE Packet1cd pcplxflip /* */ (const Packet1cd& x) { + return Packet1cd(preverse(Packet2d(x.v))); +} + +EIGEN_DEVICE_FUNC inline void ptranspose(PacketBlock& kernel) { + Packet4f tmp1 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v); + Packet4f tmp2 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v); + kernel.packet[0].v = (Packet4f)__lsx_vshuf4i_w(tmp1, 0xd8); + kernel.packet[1].v = (Packet4f)__lsx_vshuf4i_w(tmp2, 0xd8); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf pcmp_eq(const Packet2cf& a, const Packet2cf& b) { + Packet4f eq = (Packet4f)__lsx_vfcmp_ceq_s(a.v, b.v); + return Packet2cf(pand(eq, vec4f_swizzle1(eq, 1, 0, 3, 2))); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pcmp_eq(const Packet1cd& a, const Packet1cd& b) { + Packet2d eq = (Packet2d)__lsx_vfcmp_ceq_d(a.v, b.v); + return Packet1cd(pand(eq, preverse(eq))); +} + +template <> +EIGEN_DEVICE_FUNC inline Packet2cf pselect(const Packet2cf& mask, const Packet2cf& a, const Packet2cf& b) { + Packet2cf res; + res.v = (Packet4f)__lsx_vbitsel_v((__m128i)b.v, (__m128i)a.v, (__m128i)mask.v); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet1cd psqrt(const Packet1cd& a) { + return psqrt_complex(a); +} + +template <> +EIGEN_STRONG_INLINE Packet2cf psqrt(const Packet2cf& a) { + return psqrt_complex(a); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd plog(const Packet1cd& a) { + return plog_complex(a); +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pzero(const Packet1cd& /* a */) { + __m128d v = {0.0, 0.0}; + return (Packet1cd)v; +} + +template <> +EIGEN_STRONG_INLINE Packet1cd pmadd(const Packet1cd& a, const Packet1cd& b, const Packet1cd& c) { + Packet1cd result, t0, t1, t2; + t1 = pzero(t1); + t0.v = (__m128d)__lsx_vpackev_d((__m128i)a.v, (__m128i)a.v); + t2.v = __lsx_vfmadd_d(t0.v, b.v, c.v); + result.v = __lsx_vfadd_d(t2.v, t1.v); + t1.v = __lsx_vfsub_d(t1.v, a.v); + t1.v = (__m128d)__lsx_vpackod_d((__m128i)a.v, (__m128i)t1.v); + t2.v = (__m128d)__lsx_vshuf4i_d((__m128i)t2.v, (__m128i)b.v, 0xb); + result.v = __lsx_vfmadd_d(t1.v, t2.v, result.v); + return result; +} + +template <> +EIGEN_DEVICE_FUNC inline Packet1cd pgather, Packet1cd>(const std::complex* from, + Index /* stride */) { + Packet1cd res; + __m128i tmp = __lsx_vld((void*)from, 0); + res.v = (__m128d)tmp; + return res; +} + +template <> +EIGEN_DEVICE_FUNC inline void pscatter, Packet1cd>(std::complex* to, const Packet1cd& from, + Index /* stride */) { + __lsx_vst((__m128i)from.v, (void*)to, 0); +} + +EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + Packet2d tmp = (__m128d)__lsx_vilvl_d((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v); + kernel.packet[1].v = (__m128d)__lsx_vilvh_d((__m128i)kernel.packet[1].v, (__m128i)kernel.packet[0].v); + kernel.packet[0].v = tmp; +} + +} // end namespace internal +} // end namespace Eigen + +#endif // EIGEN_COMPLEX_LSX_H diff --git a/Eigen/src/Core/arch/LSX/GeneralBlockPanelKernel.h b/Eigen/src/Core/arch/LSX/GeneralBlockPanelKernel.h new file mode 100644 index 0000000..4b07062 --- /dev/null +++ b/Eigen/src/Core/arch/LSX/GeneralBlockPanelKernel.h @@ -0,0 +1,23 @@ +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { +namespace internal { + +#ifndef EIGEN_LSX_GEBP_NR +#define EIGEN_LSX_GEBP_NR 8 +#endif + +template <> +struct gebp_traits + : gebp_traits { + enum { nr = EIGEN_LSX_GEBP_NR }; +}; + +template <> +struct gebp_traits + : gebp_traits { + enum { nr = EIGEN_LSX_GEBP_NR }; +}; +} // namespace internal +} // namespace Eigen diff --git a/Eigen/src/Core/arch/LSX/MathFunctions.h b/Eigen/src/Core/arch/LSX/MathFunctions.h new file mode 100644 index 0000000..cead463 --- /dev/null +++ b/Eigen/src/Core/arch/LSX/MathFunctions.h @@ -0,0 +1,43 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2024 XiWei Gu (guxiwei-hf@loongson.cn) +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_MATH_FUNCTIONS_LSX_H +#define EIGEN_MATH_FUNCTIONS_LSX_H + +/* The sin and cos functions of this file are loosely derived from + * Julien Pommier's sse math library: http://gruntthepeon.free.fr/ssemath/ + */ + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +EIGEN_DOUBLE_PACKET_FUNCTION(atanh, Packet2d) +EIGEN_DOUBLE_PACKET_FUNCTION(log, Packet2d) +EIGEN_DOUBLE_PACKET_FUNCTION(log2, Packet2d) +EIGEN_DOUBLE_PACKET_FUNCTION(tanh, Packet2d) + +EIGEN_FLOAT_PACKET_FUNCTION(atanh, Packet4f) +EIGEN_FLOAT_PACKET_FUNCTION(log, Packet4f) +EIGEN_FLOAT_PACKET_FUNCTION(log2, Packet4f) +EIGEN_FLOAT_PACKET_FUNCTION(tanh, Packet4f) + +EIGEN_GENERIC_PACKET_FUNCTION(atan, Packet2d) +EIGEN_GENERIC_PACKET_FUNCTION(atan, Packet4f) +EIGEN_GENERIC_PACKET_FUNCTION(exp2, Packet2d) +EIGEN_GENERIC_PACKET_FUNCTION(exp2, Packet4f) + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_MATH_FUNCTIONS_LSX_H diff --git a/Eigen/src/Core/arch/LSX/PacketMath.h b/Eigen/src/Core/arch/LSX/PacketMath.h new file mode 100644 index 0000000..87232aa --- /dev/null +++ b/Eigen/src/Core/arch/LSX/PacketMath.h @@ -0,0 +1,2866 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2023 Zang Ruochen +// Copyright (C) 2024 XiWei Gu +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_PACKET_MATH_LSX_H +#define EIGEN_PACKET_MATH_LSX_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +#ifndef EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD +#define EIGEN_CACHEFRIENDLY_PRODUCT_THRESHOLD 8 +#endif + +#ifndef EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS +#if EIGEN_ARCH_LOONGARCH64 +#define EIGEN_ARCH_DEFAULT_NUMBER_OF_REGISTERS 32 +#endif +#endif + +#ifndef EIGEN_HAS_SINGLE_INSTRUCTION_MADD +#define EIGEN_HAS_SINGLE_INSTRUCTION_MADD +#endif + +typedef __m128 Packet4f; +typedef __m128d Packet2d; + +typedef eigen_packet_wrapper<__m128i, 0> Packet16c; +typedef eigen_packet_wrapper<__m128i, 1> Packet8s; +typedef eigen_packet_wrapper<__m128i, 2> Packet4i; +typedef eigen_packet_wrapper<__m128i, 3> Packet2l; +typedef eigen_packet_wrapper<__m128i, 4> Packet16uc; +typedef eigen_packet_wrapper<__m128i, 5> Packet8us; +typedef eigen_packet_wrapper<__m128i, 6> Packet4ui; +typedef eigen_packet_wrapper<__m128i, 7> Packet2ul; + +template <> +struct is_arithmetic<__m128> { + enum { value = true }; +}; +template <> +struct is_arithmetic<__m128i> { + enum { value = true }; +}; +template <> +struct is_arithmetic<__m128d> { + enum { value = true }; +}; +template <> +struct is_arithmetic { + enum { value = true }; +}; +template <> +struct is_arithmetic { + enum { value = true }; +}; +template <> +struct is_arithmetic { + enum { value = true }; +}; +template <> +struct is_arithmetic { + enum { value = true }; +}; +template <> +struct is_arithmetic { + enum { value = false }; +}; +template <> +struct is_arithmetic { + enum { value = false }; +}; +template <> +struct is_arithmetic { + enum { value = false }; +}; +template <> +struct is_arithmetic { + enum { value = false }; +}; + +EIGEN_ALWAYS_INLINE Packet4f make_packet4f(float a, float b, float c, float d) { + float from[4] = {a, b, c, d}; + return (Packet4f)__lsx_vld(from, 0); +} + +EIGEN_STRONG_INLINE Packet4f shuffle1(const Packet4f& m, int mask) { + const float* a = reinterpret_cast(&m); + Packet4f res = + make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(a + ((mask >> 6) & 3))); + return res; +} + +template +EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f& m, const Packet4f& n, int mask) { + const float* a = reinterpret_cast(&m); + const float* b = reinterpret_cast(&n); + Packet4f res = + make_packet4f(*(a + (mask & 3)), *(a + ((mask >> 2) & 3)), *(b + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))); + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet4f shuffle2(const Packet4f& m, const Packet4f& n, int mask) { + const float* a = reinterpret_cast(&m); + const float* b = reinterpret_cast(&n); + Packet4f res = + make_packet4f(*(a + (mask & 3)), *(b + ((mask >> 2) & 3)), *(a + ((mask >> 4) & 3)), *(b + ((mask >> 6) & 3))); + return res; +} + +EIGEN_STRONG_INLINE static int eigen_lsx_shuffle_mask(int p, int q, int r, int s) { + return ((s) << 6 | (r) << 4 | (q) << 2 | (p)); +} + +EIGEN_STRONG_INLINE Packet4f vec4f_swizzle1(const Packet4f& a, int p, int q, int r, int s) { + return shuffle1(a, eigen_lsx_shuffle_mask(p, q, r, s)); +} +EIGEN_STRONG_INLINE Packet4f vec4f_swizzle2(const Packet4f& a, const Packet4f& b, int p, int q, int r, int s) { + return shuffle2(a, b, eigen_lsx_shuffle_mask(p, q, r, s)); +} +EIGEN_STRONG_INLINE Packet4f vec4f_movelh(const Packet4f& a, const Packet4f& b) { + return shuffle2(a, b, eigen_lsx_shuffle_mask(0, 1, 0, 1)); +} +EIGEN_STRONG_INLINE Packet4f vec4f_movehl(const Packet4f& a, const Packet4f& b) { + return shuffle2(b, a, eigen_lsx_shuffle_mask(2, 3, 2, 3)); +} +EIGEN_STRONG_INLINE Packet4f vec4f_unpacklo(const Packet4f& a, const Packet4f& b) { + return shuffle2(a, b, eigen_lsx_shuffle_mask(0, 0, 1, 1)); +} +EIGEN_STRONG_INLINE Packet4f vec4f_unpackhi(const Packet4f& a, const Packet4f& b) { + return shuffle2(a, b, eigen_lsx_shuffle_mask(2, 2, 3, 3)); +} + +EIGEN_ALWAYS_INLINE Packet2d make_packet2d(double a, double b) { + double from[2] = {a, b}; + return (Packet2d)__lsx_vld(from, 0); +} + +EIGEN_STRONG_INLINE Packet2d shuffle(const Packet2d& m, const Packet2d& n, int mask) { + const double* a = reinterpret_cast(&m); + const double* b = reinterpret_cast(&n); + Packet2d res = make_packet2d(*(a + (mask & 1)), *(b + ((mask >> 1) & 1))); + return res; +} + +EIGEN_STRONG_INLINE Packet2d vec2d_swizzle2(const Packet2d& a, const Packet2d& b, int mask) { + return shuffle(a, b, mask); +} +EIGEN_STRONG_INLINE Packet2d vec2d_unpacklo(const Packet2d& a, const Packet2d& b) { return shuffle(a, b, 0); } +EIGEN_STRONG_INLINE Packet2d vec2d_unpackhi(const Packet2d& a, const Packet2d& b) { return shuffle(a, b, 3); } + +template <> +struct packet_traits : default_packet_traits { + typedef Packet16c type; + typedef Packet16c half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 16, + + HasAbs2 = 0, + HasSetLinear = 0, + HasCmp = 1, + HasBlend = 0 + }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet8s type; + typedef Packet8s half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 8, + + HasAbs2 = 0, + HasSetLinear = 0, + HasCmp = 1, + HasDiv = 1, + HasBlend = 0 + }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet4i type; + typedef Packet4i half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 4, + + HasAbs2 = 0, + HasSetLinear = 0, + HasCmp = 1, + HasDiv = 1, + HasBlend = 0 + }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet2l type; + typedef Packet2l half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 2, + + HasAbs2 = 0, + HasSetLinear = 0, + HasCmp = 1, + HasDiv = 1, + HasBlend = 0 + }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet16uc type; + typedef Packet16uc half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 16, + + HasAbs2 = 0, + HasSetLinear = 0, + HasNegate = 0, + HasCmp = 1, + HasBlend = 0 + }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet8us type; + typedef Packet8us half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 8, + + HasAbs2 = 0, + HasSetLinear = 0, + HasNegate = 0, + HasCmp = 1, + HasDiv = 1, + HasBlend = 0 + }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet4ui type; + typedef Packet4ui half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 4, + + HasAbs2 = 0, + HasSetLinear = 0, + HasNegate = 0, + HasCmp = 1, + HasDiv = 1, + HasBlend = 0 + }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet2ul type; + typedef Packet2ul half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 2, + + HasAbs2 = 0, + HasSetLinear = 0, + HasNegate = 0, + HasCmp = 1, + HasDiv = 1, + HasBlend = 0 + }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet4f type; + typedef Packet4f half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 4, + + HasAbs2 = 0, + HasSetLinear = 0, + HasBlend = 0, + HasSign = 0, + HasDiv = 1, + HasExp = 1, + HasSqrt = 1, + HasLog = 1, + HasRsqrt = 1 + }; +}; + +template <> +struct packet_traits : default_packet_traits { + typedef Packet2d type; + typedef Packet2d half; + enum { + Vectorizable = 1, + AlignedOnScalar = 1, + size = 2, + + HasAbs2 = 0, + HasSetLinear = 0, + HasBlend = 0, + HasSign = 0, + HasDiv = 1, + HasSqrt = 1, + HasLog = 1, + HasRsqrt = 1 + }; +}; + +template <> +struct unpacket_traits { + typedef int8_t type; + typedef Packet16c half; + enum { + size = 16, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; +template <> +struct unpacket_traits { + typedef int16_t type; + typedef Packet8s half; + enum { + size = 8, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; +template <> +struct unpacket_traits { + typedef int32_t type; + typedef Packet4i half; + enum { + size = 4, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; +template <> +struct unpacket_traits { + typedef int64_t type; + typedef Packet2l half; + enum { + size = 2, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; +template <> +struct unpacket_traits { + typedef uint8_t type; + typedef Packet16uc half; + enum { + size = 16, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; +template <> +struct unpacket_traits { + typedef uint16_t type; + typedef Packet8us half; + enum { + size = 8, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; +template <> +struct unpacket_traits { + typedef uint32_t type; + typedef Packet4ui half; + enum { + size = 4, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; +template <> +struct unpacket_traits { + typedef uint64_t type; + typedef Packet2ul half; + enum { + size = 2, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; +template <> +struct unpacket_traits { + typedef float type; + typedef Packet4f half; + typedef Packet4i integer_packet; + enum { + size = 4, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; +template <> +struct unpacket_traits { + typedef double type; + typedef Packet2d half; + typedef Packet2l integer_packet; + enum { + size = 2, + alignment = Aligned16, + vectorizable = true, + masked_load_available = false, + masked_store_available = false + }; +}; + +template <> +EIGEN_STRONG_INLINE Packet16c pset1(const int8_t& from) { + return __lsx_vreplgr2vr_b(from); +} +template <> +EIGEN_STRONG_INLINE Packet8s pset1(const int16_t& from) { + return __lsx_vreplgr2vr_h(from); +} +template <> +EIGEN_STRONG_INLINE Packet4i pset1(const int32_t& from) { + return __lsx_vreplgr2vr_w(from); +} +template <> +EIGEN_STRONG_INLINE Packet2l pset1(const int64_t& from) { + return __lsx_vreplgr2vr_d(from); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pset1(const uint8_t& from) { + return __lsx_vreplgr2vr_b(from); +} +template <> +EIGEN_STRONG_INLINE Packet8us pset1(const uint16_t& from) { + return __lsx_vreplgr2vr_h(from); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pset1(const uint32_t& from) { + return __lsx_vreplgr2vr_w(from); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pset1(const uint64_t& from) { + return __lsx_vreplgr2vr_d(from); +} +template <> +EIGEN_STRONG_INLINE Packet4f pset1(const float& from) { + Packet4f v = {from, from, from, from}; + return v; +} +template <> +EIGEN_STRONG_INLINE Packet2d pset1(const double& from) { + Packet2d v = {from, from}; + return v; +} + +template <> +EIGEN_STRONG_INLINE Packet4f pset1frombits(uint32_t from) { + return reinterpret_cast<__m128>((__m128i)pset1(from)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pset1frombits(uint64_t from) { + return reinterpret_cast<__m128d>((__m128i)pset1(from)); +} + +template <> +EIGEN_STRONG_INLINE Packet16c plset(const int8_t& a) { + const int8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + return __lsx_vadd_b(pset1(a), __lsx_vld(countdown, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet8s plset(const int16_t& a) { + const int16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7}; + return __lsx_vadd_h(pset1(a), __lsx_vld(countdown, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet4i plset(const int32_t& a) { + const int32_t countdown[] = {0, 1, 2, 3}; + return __lsx_vadd_w(pset1(a), __lsx_vld(countdown, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet2l plset(const int64_t& a) { + const int64_t countdown[] = {0, 1}; + return __lsx_vadd_d(pset1(a), __lsx_vld(countdown, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet16uc plset(const uint8_t& a) { + const uint8_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}; + return __lsx_vadd_b(pset1(a), __lsx_vld(countdown, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet8us plset(const uint16_t& a) { + const uint16_t countdown[] = {0, 1, 2, 3, 4, 5, 6, 7}; + return __lsx_vadd_h(pset1(a), __lsx_vld(countdown, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet4ui plset(const uint32_t& a) { + const uint32_t countdown[] = {0, 1, 2, 3}; + return __lsx_vadd_w(pset1(a), __lsx_vld(countdown, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet2ul plset(const uint64_t& a) { + const uint64_t countdown[] = {0, 1}; + return __lsx_vadd_d(pset1(a), __lsx_vld(countdown, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet4f plset(const float& a) { + static const Packet4f countdown = {0.0f, 1.0f, 2.0f, 3.0f}; + return __lsx_vfadd_s(pset1(a), countdown); +} +template <> +EIGEN_STRONG_INLINE Packet2d plset(const double& a) { + static const Packet2d countdown = {0.0f, 1.0f}; + return __lsx_vfadd_d(pset1(a), countdown); +} + +template <> +EIGEN_STRONG_INLINE Packet16c padd(const Packet16c& a, const Packet16c& b) { + return __lsx_vadd_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s padd(const Packet8s& a, const Packet8s& b) { + return __lsx_vadd_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i padd(const Packet4i& a, const Packet4i& b) { + return __lsx_vadd_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l padd(const Packet2l& a, const Packet2l& b) { + return __lsx_vadd_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc padd(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vadd_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us padd(const Packet8us& a, const Packet8us& b) { + return __lsx_vadd_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui padd(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vadd_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul padd(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vadd_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4f padd(const Packet4f& a, const Packet4f& b) { + return __lsx_vfadd_s(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d padd(const Packet2d& a, const Packet2d& b) { + return __lsx_vfadd_d(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16c psub(const Packet16c& a, const Packet16c& b) { + return __lsx_vsub_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s psub(const Packet8s& a, const Packet8s& b) { + return __lsx_vsub_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i psub(const Packet4i& a, const Packet4i& b) { + return __lsx_vsub_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l psub(const Packet2l& a, const Packet2l& b) { + return __lsx_vsub_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc psub(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vsub_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us psub(const Packet8us& a, const Packet8us& b) { + return __lsx_vsub_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui psub(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vsub_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul psub(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vsub_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4f psub(const Packet4f& a, const Packet4f& b) { + return __lsx_vfsub_s(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d psub(const Packet2d& a, const Packet2d& b) { + return __lsx_vfsub_d(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b); +template <> +EIGEN_STRONG_INLINE Packet4f paddsub(const Packet4f& a, const Packet4f& b) { + const Packet4f mask = + make_packet4f(numext::bit_cast(0x80000000u), 0.0f, numext::bit_cast(0x80000000u), 0.0f); + return padd(a, pxor(mask, b)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& a, const Packet2d& b); +template <> +EIGEN_STRONG_INLINE Packet2d paddsub(const Packet2d& a, const Packet2d& b) { + const Packet2d mask = make_packet2d(numext::bit_cast(0x8000000000000000ull), 0.0); + return padd(a, pxor(mask, b)); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pnegate(const Packet4f& a) { + Packet4f mask = make_packet4f(numext::bit_cast(0x80000000), numext::bit_cast(0x80000000), + numext::bit_cast(0x80000000), numext::bit_cast(0x80000000)); + return (Packet4f)__lsx_vxor_v(numext::bit_cast<__m128i>(mask), numext::bit_cast<__m128i>(a)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pnegate(const Packet2d& a) { + Packet2d mask = + make_packet2d(numext::bit_cast(0x8000000000000000), numext::bit_cast(0x8000000000000000)); + return (Packet2d)__lsx_vxor_v(numext::bit_cast<__m128i>(mask), numext::bit_cast<__m128i>(a)); +} +template <> +EIGEN_STRONG_INLINE Packet16c pnegate(const Packet16c& a) { + return __lsx_vneg_b(a); +} +template <> +EIGEN_STRONG_INLINE Packet8s pnegate(const Packet8s& a) { + return __lsx_vneg_h(a); +} +template <> +EIGEN_STRONG_INLINE Packet4i pnegate(const Packet4i& a) { + return __lsx_vneg_w(a); +} +template <> +EIGEN_STRONG_INLINE Packet2l pnegate(const Packet2l& a) { + return __lsx_vneg_d(a); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pconj(const Packet4f& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet2d pconj(const Packet2d& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet16c pconj(const Packet16c& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet8s pconj(const Packet8s& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet4i pconj(const Packet4i& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet2l pconj(const Packet2l& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet16uc pconj(const Packet16uc& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet8us pconj(const Packet8us& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet4ui pconj(const Packet4ui& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet2ul pconj(const Packet2ul& a) { + return a; +} + +template <> +EIGEN_STRONG_INLINE Packet4f pmul(const Packet4f& a, const Packet4f& b) { + return __lsx_vfmul_s(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d pmul(const Packet2d& a, const Packet2d& b) { + return __lsx_vfmul_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16c pmul(const Packet16c& a, const Packet16c& b) { + return __lsx_vmul_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pmul(const Packet8s& a, const Packet8s& b) { + return __lsx_vmul_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pmul(const Packet4i& a, const Packet4i& b) { + return __lsx_vmul_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pmul(const Packet2l& a, const Packet2l& b) { + return __lsx_vmul_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pmul(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vmul_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pmul(const Packet8us& a, const Packet8us& b) { + return __lsx_vmul_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pmul(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vmul_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pmul(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vmul_d(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pdiv(const Packet4f& a, const Packet4f& b) { + return __lsx_vfdiv_s(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d pdiv(const Packet2d& a, const Packet2d& b) { + return __lsx_vfdiv_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pdiv(const Packet8s& a, const Packet8s& b) { + return __lsx_vdiv_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pdiv(const Packet4i& a, const Packet4i& b) { + return __lsx_vdiv_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pdiv(const Packet2l& a, const Packet2l& b) { + return __lsx_vdiv_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pdiv(const Packet8us& a, const Packet8us& b) { + return __lsx_vdiv_hu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pdiv(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vdiv_wu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pdiv(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vdiv_du(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { + return __lsx_vfmadd_s(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet2d pmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { + return __lsx_vfmadd_d(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet4f pmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) { + return __lsx_vfmsub_s(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet2d pmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) { + return __lsx_vfmsub_d(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet4f pnmadd(const Packet4f& a, const Packet4f& b, const Packet4f& c) { + return __lsx_vfnmsub_s(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet2d pnmadd(const Packet2d& a, const Packet2d& b, const Packet2d& c) { + return __lsx_vfnmsub_d(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet4f pnmsub(const Packet4f& a, const Packet4f& b, const Packet4f& c) { + return __lsx_vfnmadd_s(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet2d pnmsub(const Packet2d& a, const Packet2d& b, const Packet2d& c) { + return __lsx_vfnmadd_d(a, b, c); +} +template <> +EIGEN_STRONG_INLINE Packet16c pmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c) { + return __lsx_vmadd_b(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) { + return __lsx_vmadd_h(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { + return __lsx_vmadd_w(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pmadd(const Packet2l& a, const Packet2l& b, const Packet2l& c) { + return __lsx_vmadd_d(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pmadd(const Packet16uc& a, const Packet16uc& b, const Packet16uc& c) { + return __lsx_vmadd_b(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pmadd(const Packet8us& a, const Packet8us& b, const Packet8us& c) { + return __lsx_vmadd_h(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pmadd(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c) { + return __lsx_vmadd_w(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pmadd(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c) { + return __lsx_vmadd_d(c, a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pand(const Packet4f& a, const Packet4f& b) { + return (Packet4f)__lsx_vand_v((__m128i)a, (__m128i)b); +} +template <> +EIGEN_STRONG_INLINE Packet2d pand(const Packet2d& a, const Packet2d& b) { + return (Packet2d)__lsx_vand_v((__m128i)a, (__m128i)b); +} +template <> +EIGEN_STRONG_INLINE Packet16c pand(const Packet16c& a, const Packet16c& b) { + return __lsx_vand_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pand(const Packet8s& a, const Packet8s& b) { + return __lsx_vand_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pand(const Packet4i& a, const Packet4i& b) { + return __lsx_vand_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pand(const Packet2l& a, const Packet2l& b) { + return __lsx_vand_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pand(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vand_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pand(const Packet8us& a, const Packet8us& b) { + return __lsx_vand_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pand(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vand_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pand(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vand_v(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f por(const Packet4f& a, const Packet4f& b) { + return (Packet4f)__lsx_vor_v((__m128i)a, (__m128i)b); +} +template <> +EIGEN_STRONG_INLINE Packet2d por(const Packet2d& a, const Packet2d& b) { + return (Packet2d)__lsx_vor_v((__m128i)a, (__m128i)b); +} +template <> +EIGEN_STRONG_INLINE Packet16c por(const Packet16c& a, const Packet16c& b) { + return __lsx_vor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s por(const Packet8s& a, const Packet8s& b) { + return __lsx_vor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i por(const Packet4i& a, const Packet4i& b) { + return __lsx_vor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l por(const Packet2l& a, const Packet2l& b) { + return __lsx_vor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc por(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us por(const Packet8us& a, const Packet8us& b) { + return __lsx_vor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui por(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul por(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vor_v(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pxor(const Packet4f& a, const Packet4f& b) { + return (Packet4f)__lsx_vxor_v((__m128i)a, (__m128i)b); +} +template <> +EIGEN_STRONG_INLINE Packet2d pxor(const Packet2d& a, const Packet2d& b) { + return (Packet2d)__lsx_vxor_v((__m128i)a, (__m128i)b); +} +template <> +EIGEN_STRONG_INLINE Packet16c pxor(const Packet16c& a, const Packet16c& b) { + return __lsx_vxor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pxor(const Packet8s& a, const Packet8s& b) { + return __lsx_vxor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pxor(const Packet4i& a, const Packet4i& b) { + return __lsx_vxor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pxor(const Packet2l& a, const Packet2l& b) { + return __lsx_vxor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pxor(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vxor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pxor(const Packet8us& a, const Packet8us& b) { + return __lsx_vxor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pxor(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vxor_v(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pxor(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vxor_v(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pandnot(const Packet4f& a, const Packet4f& b) { + return (Packet4f)__lsx_vandn_v((__m128i)b, (__m128i)a); +} +template <> +EIGEN_STRONG_INLINE Packet2d pandnot(const Packet2d& a, const Packet2d& b) { + return (Packet2d)__lsx_vandn_v((__m128i)b, (__m128i)a); +} +template <> +EIGEN_STRONG_INLINE Packet16c pandnot(const Packet16c& a, const Packet16c& b) { + return __lsx_vandn_v(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet8s pandnot(const Packet8s& a, const Packet8s& b) { + return __lsx_vandn_v(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet4i pandnot(const Packet4i& a, const Packet4i& b) { + return __lsx_vandn_v(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet2l pandnot(const Packet2l& a, const Packet2l& b) { + return __lsx_vandn_v(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pandnot(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vandn_v(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet8us pandnot(const Packet8us& a, const Packet8us& b) { + return __lsx_vandn_v(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pandnot(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vandn_v(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pandnot(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vandn_v(b, a); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcmp_le(const Packet4f& a, const Packet4f& b) { + return (Packet4f)__lsx_vfcmp_cle_s(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcmp_le(const Packet2d& a, const Packet2d& b) { + return (Packet2d)__lsx_vfcmp_cle_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcmp_le(const Packet16c& a, const Packet16c& b) { + return __lsx_vsle_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcmp_le(const Packet8s& a, const Packet8s& b) { + return __lsx_vsle_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcmp_le(const Packet4i& a, const Packet4i& b) { + return __lsx_vsle_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcmp_le(const Packet2l& a, const Packet2l& b) { + return __lsx_vsle_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcmp_le(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vsle_bu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcmp_le(const Packet8us& a, const Packet8us& b) { + return __lsx_vsle_hu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcmp_le(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vsle_wu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcmp_le(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vsle_du(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcmp_lt(const Packet4f& a, const Packet4f& b) { + return (Packet4f)__lsx_vfcmp_clt_s(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcmp_lt(const Packet2d& a, const Packet2d& b) { + return (Packet2d)__lsx_vfcmp_clt_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcmp_lt(const Packet16c& a, const Packet16c& b) { + return __lsx_vslt_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcmp_lt(const Packet8s& a, const Packet8s& b) { + return __lsx_vslt_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcmp_lt(const Packet4i& a, const Packet4i& b) { + return __lsx_vslt_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcmp_lt(const Packet2l& a, const Packet2l& b) { + return __lsx_vslt_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcmp_lt(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vslt_bu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcmp_lt(const Packet8us& a, const Packet8us& b) { + return __lsx_vslt_hu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcmp_lt(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vslt_wu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcmp_lt(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vslt_du(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcmp_lt_or_nan(const Packet4f& a, const Packet4f& b) { + return (Packet4f)__lsx_vfcmp_sult_s(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcmp_lt_or_nan(const Packet2d& a, const Packet2d& b) { + return (Packet2d)__lsx_vfcmp_sult_d(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcmp_eq(const Packet4f& a, const Packet4f& b) { + return (Packet4f)__lsx_vfcmp_seq_s(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcmp_eq(const Packet2d& a, const Packet2d& b) { + return (Packet2d)__lsx_vfcmp_seq_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcmp_eq(const Packet16c& a, const Packet16c& b) { + return __lsx_vseq_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcmp_eq(const Packet8s& a, const Packet8s& b) { + return __lsx_vseq_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcmp_eq(const Packet4i& a, const Packet4i& b) { + return __lsx_vseq_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcmp_eq(const Packet2l& a, const Packet2l& b) { + return __lsx_vseq_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcmp_eq(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vseq_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcmp_eq(const Packet8us& a, const Packet8us& b) { + return __lsx_vseq_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcmp_eq(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vseq_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcmp_eq(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vseq_d(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16c pmin(const Packet16c& a, const Packet16c& b) { + return __lsx_vmin_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pmin(const Packet8s& a, const Packet8s& b) { + return __lsx_vmin_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pmin(const Packet4i& a, const Packet4i& b) { + return __lsx_vmin_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pmin(const Packet2l& a, const Packet2l& b) { + return __lsx_vmin_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pmin(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vmin_bu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pmin(const Packet8us& a, const Packet8us& b) { + return __lsx_vmin_hu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pmin(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vmin_wu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pmin(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vmin_du(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16c pmax(const Packet16c& a, const Packet16c& b) { + return __lsx_vmax_b(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pmax(const Packet8s& a, const Packet8s& b) { + return __lsx_vmax_h(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pmax(const Packet4i& a, const Packet4i& b) { + return __lsx_vmax_w(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pmax(const Packet2l& a, const Packet2l& b) { + return __lsx_vmax_d(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pmax(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vmax_bu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8us pmax(const Packet8us& a, const Packet8us& b) { + return __lsx_vmax_hu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pmax(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vmax_wu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pmax(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vmax_du(a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { + Packet4i aNaN = __lsx_vfcmp_cun_s(a, a); + Packet4i aMinOrNaN = por(__lsx_vfcmp_clt_s(a, b), aNaN); + return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMinOrNaN); +} +template <> +EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { + Packet2l aNaN = __lsx_vfcmp_cun_d(a, a); + Packet2l aMinOrNaN = por(__lsx_vfcmp_clt_d(a, b), aNaN); + return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMinOrNaN); +} +template <> +EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { + Packet4i aNaN = __lsx_vfcmp_cun_s(a, a); + Packet4i aMaxOrNaN = por(__lsx_vfcmp_clt_s(b, a), aNaN); + return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMaxOrNaN); +} +template <> +EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { + Packet2l aNaN = __lsx_vfcmp_cun_d(a, a); + Packet2l aMaxOrNaN = por(__lsx_vfcmp_clt_d(b, a), aNaN); + return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, aMaxOrNaN); +} + +template +EIGEN_STRONG_INLINE Packet16c parithmetic_shift_right(const Packet16c& a) { + return __lsx_vsrai_b((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet8s parithmetic_shift_right(const Packet8s& a) { + return __lsx_vsrai_h((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet4i parithmetic_shift_right(const Packet4i& a) { + return __lsx_vsrai_w((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet2l parithmetic_shift_right(const Packet2l& a) { + return __lsx_vsrai_d((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet16uc parithmetic_shift_right(const Packet16uc& a) { + return __lsx_vsrli_b((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet8us parithmetic_shift_right(const Packet8us& a) { + return __lsx_vsrli_h((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet4ui parithmetic_shift_right(const Packet4ui& a) { + return __lsx_vsrli_w((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet2ul parithmetic_shift_right(const Packet2ul& a) { + return __lsx_vsrli_d((__m128i)a, N); +} + +template +EIGEN_STRONG_INLINE Packet16c plogical_shift_right(const Packet16c& a) { + return __lsx_vsrli_b((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet8s plogical_shift_right(const Packet8s& a) { + return __lsx_vsrli_h((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet4i plogical_shift_right(const Packet4i& a) { + return __lsx_vsrli_w((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet2l plogical_shift_right(const Packet2l& a) { + return __lsx_vsrli_d((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet16uc plogical_shift_right(const Packet16uc& a) { + return __lsx_vsrli_b((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet8us plogical_shift_right(const Packet8us& a) { + return __lsx_vsrli_h((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet4ui plogical_shift_right(const Packet4ui& a) { + return __lsx_vsrli_w((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet2ul plogical_shift_right(const Packet2ul& a) { + return __lsx_vsrli_d((__m128i)a, N); +} + +template +EIGEN_STRONG_INLINE Packet16c plogical_shift_left(const Packet16c& a) { + return __lsx_vslli_b((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet8s plogical_shift_left(const Packet8s& a) { + return __lsx_vslli_h((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet4i plogical_shift_left(const Packet4i& a) { + return __lsx_vslli_w((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet2l plogical_shift_left(const Packet2l& a) { + return __lsx_vslli_d((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet16uc plogical_shift_left(const Packet16uc& a) { + return __lsx_vslli_b((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet8us plogical_shift_left(const Packet8us& a) { + return __lsx_vslli_h((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet4ui plogical_shift_left(const Packet4ui& a) { + return __lsx_vslli_w((__m128i)a, N); +} +template +EIGEN_STRONG_INLINE Packet2ul plogical_shift_left(const Packet2ul& a) { + return __lsx_vslli_d((__m128i)a, N); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pabs(const Packet4f& a) { + return (Packet4f)__lsx_vbitclri_w((__m128i)a, 31); +} +template <> +EIGEN_STRONG_INLINE Packet2d pabs(const Packet2d& a) { + return (Packet2d)__lsx_vbitclri_d((__m128i)a, 63); +} +template <> +EIGEN_STRONG_INLINE Packet16c pabs(const Packet16c& a) { + return __lsx_vabsd_b(a, pzero(a)); +} +template <> +EIGEN_STRONG_INLINE Packet8s pabs(const Packet8s& a) { + return __lsx_vabsd_h(a, pzero(a)); +} +template <> +EIGEN_STRONG_INLINE Packet4i pabs(const Packet4i& a) { + return __lsx_vabsd_w(a, pzero(a)); +} +template <> +EIGEN_STRONG_INLINE Packet2l pabs(const Packet2l& a) { + return __lsx_vabsd_d(a, pzero(a)); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pabs(const Packet16uc& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet8us pabs(const Packet8us& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet4ui pabs(const Packet4ui& a) { + return a; +} +template <> +EIGEN_STRONG_INLINE Packet2ul pabs(const Packet2ul& a) { + return a; +} + +template <> +EIGEN_STRONG_INLINE Packet4f pload(const float* from) { + EIGEN_DEBUG_ALIGNED_LOAD return (Packet4f)__lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2d pload(const double* from) { + EIGEN_DEBUG_ALIGNED_LOAD return (Packet2d)__lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c pload(const int8_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s pload(const int16_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4i pload(const int32_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2l pload(const int64_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pload(const uint8_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us pload(const uint16_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pload(const uint32_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pload(const uint64_t* from) { + EIGEN_DEBUG_ALIGNED_LOAD return __lsx_vld(from, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f ploadu(const float* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return (Packet4f)__lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2d ploadu(const double* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return (Packet2d)__lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c ploadu(const int8_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s ploadu(const int16_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4i ploadu(const int32_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2l ploadu(const int64_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc ploadu(const uint8_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us ploadu(const uint16_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4ui ploadu(const uint32_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2ul ploadu(const uint64_t* from) { + EIGEN_DEBUG_UNALIGNED_LOAD return __lsx_vld(from, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f ploaddup(const float* from) { + float f0 = from[0], f1 = from[1]; + return make_packet4f(f0, f0, f1, f1); +} +template <> +EIGEN_STRONG_INLINE Packet2d ploaddup(const double* from) { + return pset1(from[0]); +} +template <> +EIGEN_STRONG_INLINE Packet16c ploaddup(const int8_t* from) { + Packet16c tmp = pload(from); + return __lsx_vilvl_b(tmp, tmp); +} +template <> +EIGEN_STRONG_INLINE Packet8s ploaddup(const int16_t* from) { + Packet8s tmp = pload(from); + return __lsx_vilvl_h(tmp, tmp); +} +template <> +EIGEN_STRONG_INLINE Packet4i ploaddup(const int32_t* from) { + Packet4i tmp = pload(from); + return __lsx_vilvl_w(tmp, tmp); +} +template <> +EIGEN_STRONG_INLINE Packet2l ploaddup(const int64_t* from) { + return pset1(from[0]); +} +template <> +EIGEN_STRONG_INLINE Packet16uc ploaddup(const uint8_t* from) { + Packet16uc tmp = pload(from); + return __lsx_vilvl_b(tmp, tmp); +} +template <> +EIGEN_STRONG_INLINE Packet8us ploaddup(const uint16_t* from) { + Packet8us tmp = pload(from); + return __lsx_vilvl_h(tmp, tmp); +} +template <> +EIGEN_STRONG_INLINE Packet4ui ploaddup(const uint32_t* from) { + Packet4ui tmp = pload(from); + return __lsx_vilvl_w(tmp, tmp); +} +template <> +EIGEN_STRONG_INLINE Packet2ul ploaddup(const uint64_t* from) { + return pset1(from[0]); +} + +template <> +EIGEN_STRONG_INLINE void pstore(float* to, const Packet4f& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst(from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstore(double* to, const Packet2d& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst(from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstore(int8_t* to, const Packet16c& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstore(int16_t* to, const Packet8s& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstore(int32_t* to, const Packet4i& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstore(int64_t* to, const Packet2l& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstore(uint8_t* to, const Packet16uc& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstore(uint16_t* to, const Packet8us& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstore(uint32_t* to, const Packet4ui& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstore(uint64_t* to, const Packet2ul& from) { + EIGEN_DEBUG_ALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu(float* to, const Packet4f& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst(from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(double* to, const Packet2d& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst(from, to, 0); +} + +template <> +EIGEN_STRONG_INLINE void pstoreu(int8_t* to, const Packet16c& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(int16_t* to, const Packet8s& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(int32_t* to, const Packet4i& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(int64_t* to, const Packet2l& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(uint8_t* to, const Packet16uc& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(uint16_t* to, const Packet8us& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(uint32_t* to, const Packet4ui& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} +template <> +EIGEN_STRONG_INLINE void pstoreu(uint64_t* to, const Packet2ul& from) { + EIGEN_DEBUG_UNALIGNED_STORE __lsx_vst((__m128i)from, to, 0); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pgather(const float* from, Index stride) { + Packet4f v = {from[0], from[stride], from[2 * stride], from[3 * stride]}; + return v; +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pgather(const double* from, Index stride) { + Packet2d v = {from[0], from[stride]}; + return v; +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pgather(const int8_t* from, Index stride) { + int8_t v[16] __attribute__((aligned(16))); + v[0] = from[0]; + v[1] = from[stride]; + v[2] = from[2 * stride]; + v[3] = from[3 * stride]; + v[4] = from[4 * stride]; + v[5] = from[5 * stride]; + v[6] = from[6 * stride]; + v[7] = from[7 * stride]; + v[8] = from[8 * stride]; + v[9] = from[9 * stride]; + v[10] = from[10 * stride]; + v[11] = from[11 * stride]; + v[12] = from[12 * stride]; + v[13] = from[13 * stride]; + v[14] = from[14 * stride]; + v[15] = from[15 * stride]; + return __lsx_vld(v, 0); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pgather(const int16_t* from, Index stride) { + int16_t v[8] __attribute__((aligned(16))); + v[0] = from[0]; + v[1] = from[stride]; + v[2] = from[2 * stride]; + v[3] = from[3 * stride]; + v[4] = from[4 * stride]; + v[5] = from[5 * stride]; + v[6] = from[6 * stride]; + v[7] = from[7 * stride]; + return __lsx_vld(v, 0); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pgather(const int32_t* from, Index stride) { + int32_t v[4] __attribute__((aligned(16))); + v[0] = from[0]; + v[1] = from[stride]; + v[2] = from[2 * stride]; + v[3] = from[3 * stride]; + return __lsx_vld(v, 0); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pgather(const int64_t* from, Index stride) { + int64_t v[2] __attribute__((aligned(16))); + v[0] = from[0]; + v[1] = from[stride]; + return __lsx_vld(v, 0); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pgather(const uint8_t* from, Index stride) { + uint8_t v[16] __attribute__((aligned(16))); + v[0] = from[0]; + v[1] = from[stride]; + v[2] = from[2 * stride]; + v[3] = from[3 * stride]; + v[4] = from[4 * stride]; + v[5] = from[5 * stride]; + v[6] = from[6 * stride]; + v[7] = from[7 * stride]; + v[8] = from[8 * stride]; + v[9] = from[9 * stride]; + v[10] = from[10 * stride]; + v[11] = from[11 * stride]; + v[12] = from[12 * stride]; + v[13] = from[13 * stride]; + v[14] = from[14 * stride]; + v[15] = from[15 * stride]; + return __lsx_vld(v, 0); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pgather(const uint16_t* from, Index stride) { + uint16_t v[8] __attribute__((aligned(16))); + v[0] = from[0]; + v[1] = from[stride]; + v[2] = from[2 * stride]; + v[3] = from[3 * stride]; + v[4] = from[4 * stride]; + v[5] = from[5 * stride]; + v[6] = from[6 * stride]; + v[7] = from[7 * stride]; + return __lsx_vld(v, 0); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pgather(const uint32_t* from, Index stride) { + uint32_t v[4] __attribute__((aligned(16))); + v[0] = from[0]; + v[1] = from[stride]; + v[2] = from[2 * stride]; + v[3] = from[3 * stride]; + return __lsx_vld(v, 0); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pgather(const uint64_t* from, Index stride) { + uint64_t v[2] __attribute__((aligned(16))); + v[0] = from[0]; + v[1] = from[stride]; + return __lsx_vld(v, 0); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(float* to, const Packet4f& from, Index stride) { + __lsx_vstelm_w(from, to, 0, 0); + __lsx_vstelm_w(from, to + stride * 1, 0, 1); + __lsx_vstelm_w(from, to + stride * 2, 0, 2); + __lsx_vstelm_w(from, to + stride * 3, 0, 3); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(double* to, const Packet2d& from, Index stride) { + __lsx_vstelm_d(from, to, 0, 0); + __lsx_vstelm_d(from, to + stride, 0, 1); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int8_t* to, const Packet16c& from, + Index stride) { + __lsx_vstelm_b((__m128i)from, to, 0, 0); + __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1); + __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2); + __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3); + __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4); + __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5); + __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6); + __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7); + __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8); + __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9); + __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10); + __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11); + __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12); + __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13); + __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14); + __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int16_t* to, const Packet8s& from, + Index stride) { + __lsx_vstelm_h((__m128i)from, to, 0, 0); + __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1); + __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2); + __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3); + __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4); + __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5); + __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6); + __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int32_t* to, const Packet4i& from, + Index stride) { + __lsx_vstelm_w((__m128i)from, to, 0, 0); + __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1); + __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2); + __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(int64_t* to, const Packet2l& from, + Index stride) { + __lsx_vstelm_d((__m128i)from, to, 0, 0); + __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint8_t* to, const Packet16uc& from, + Index stride) { + __lsx_vstelm_b((__m128i)from, to, 0, 0); + __lsx_vstelm_b((__m128i)from, to + stride * 1, 0, 1); + __lsx_vstelm_b((__m128i)from, to + stride * 2, 0, 2); + __lsx_vstelm_b((__m128i)from, to + stride * 3, 0, 3); + __lsx_vstelm_b((__m128i)from, to + stride * 4, 0, 4); + __lsx_vstelm_b((__m128i)from, to + stride * 5, 0, 5); + __lsx_vstelm_b((__m128i)from, to + stride * 6, 0, 6); + __lsx_vstelm_b((__m128i)from, to + stride * 7, 0, 7); + __lsx_vstelm_b((__m128i)from, to + stride * 8, 0, 8); + __lsx_vstelm_b((__m128i)from, to + stride * 9, 0, 9); + __lsx_vstelm_b((__m128i)from, to + stride * 10, 0, 10); + __lsx_vstelm_b((__m128i)from, to + stride * 11, 0, 11); + __lsx_vstelm_b((__m128i)from, to + stride * 12, 0, 12); + __lsx_vstelm_b((__m128i)from, to + stride * 13, 0, 13); + __lsx_vstelm_b((__m128i)from, to + stride * 14, 0, 14); + __lsx_vstelm_b((__m128i)from, to + stride * 15, 0, 15); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint16_t* to, const Packet8us& from, + Index stride) { + __lsx_vstelm_h((__m128i)from, to, 0, 0); + __lsx_vstelm_h((__m128i)from, to + stride * 1, 0, 1); + __lsx_vstelm_h((__m128i)from, to + stride * 2, 0, 2); + __lsx_vstelm_h((__m128i)from, to + stride * 3, 0, 3); + __lsx_vstelm_h((__m128i)from, to + stride * 4, 0, 4); + __lsx_vstelm_h((__m128i)from, to + stride * 5, 0, 5); + __lsx_vstelm_h((__m128i)from, to + stride * 6, 0, 6); + __lsx_vstelm_h((__m128i)from, to + stride * 7, 0, 7); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint32_t* to, const Packet4ui& from, + Index stride) { + __lsx_vstelm_w((__m128i)from, to, 0, 0); + __lsx_vstelm_w((__m128i)from, to + stride * 1, 0, 1); + __lsx_vstelm_w((__m128i)from, to + stride * 2, 0, 2); + __lsx_vstelm_w((__m128i)from, to + stride * 3, 0, 3); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pscatter(uint64_t* to, const Packet2ul& from, + Index stride) { + __lsx_vstelm_d((__m128i)from, to, 0, 0); + __lsx_vstelm_d((__m128i)from, to + stride * 1, 0, 1); +} + +template <> +EIGEN_STRONG_INLINE void prefetch(const float* addr) { + __builtin_prefetch(addr); +} +template <> +EIGEN_STRONG_INLINE void prefetch(const double* addr) { + __builtin_prefetch(addr); +} +template <> +EIGEN_STRONG_INLINE void prefetch(const int8_t* addr) { + __builtin_prefetch(addr); +} +template <> +EIGEN_STRONG_INLINE void prefetch(const int16_t* addr) { + __builtin_prefetch(addr); +} +template <> +EIGEN_STRONG_INLINE void prefetch(const int32_t* addr) { + __builtin_prefetch(addr); +} +template <> +EIGEN_STRONG_INLINE void prefetch(const int64_t* addr) { + __builtin_prefetch(addr); +} +template <> +EIGEN_STRONG_INLINE void prefetch(const uint8_t* addr) { + __builtin_prefetch(addr); +} +template <> +EIGEN_STRONG_INLINE void prefetch(const uint16_t* addr) { + __builtin_prefetch(addr); +} +template <> +EIGEN_STRONG_INLINE void prefetch(const uint32_t* addr) { + __builtin_prefetch(addr); +} +template <> +EIGEN_STRONG_INLINE void prefetch(const uint64_t* addr) { + __builtin_prefetch(addr); +} + +template <> +EIGEN_STRONG_INLINE float pfirst(const Packet4f& a) { + float v; + __lsx_vstelm_w(a, &v, 0, 0); + return v; +} +template <> +EIGEN_STRONG_INLINE double pfirst(const Packet2d& a) { + double v; + __lsx_vstelm_d(a, &v, 0, 0); + return v; +} + +template <> +EIGEN_STRONG_INLINE int8_t pfirst(const Packet16c& a) { + return (int8_t)__lsx_vpickve2gr_b((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE int16_t pfirst(const Packet8s& a) { + return (int16_t)__lsx_vpickve2gr_h((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE int32_t pfirst(const Packet4i& a) { + return __lsx_vpickve2gr_w((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE int64_t pfirst(const Packet2l& a) { + return __lsx_vpickve2gr_d((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE uint8_t pfirst(const Packet16uc& a) { + return (uint8_t)__lsx_vpickve2gr_bu((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE uint16_t pfirst(const Packet8us& a) { + return (uint16_t)__lsx_vpickve2gr_hu((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE uint32_t pfirst(const Packet4ui& a) { + return __lsx_vpickve2gr_wu((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE uint64_t pfirst(const Packet2ul& a) { + return __lsx_vpickve2gr_du((__m128i)a, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f preverse(const Packet4f& a) { + return (Packet4f)__lsx_vshuf4i_w(a, 0x1B); +} +template <> +EIGEN_STRONG_INLINE Packet2d preverse(const Packet2d& a) { + return (Packet2d)__lsx_vshuf4i_d(a, a, 0x1); +} +template <> +EIGEN_STRONG_INLINE Packet16c preverse(const Packet16c& a) { + return __lsx_vshuf4i_b(__lsx_vshuf4i_w((__m128i)a, 0x1B), 0x1B); +} +template <> +EIGEN_STRONG_INLINE Packet8s preverse(const Packet8s& a) { + return __lsx_vshuf4i_h(__lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1), 0x1B); +} +template <> +EIGEN_STRONG_INLINE Packet4i preverse(const Packet4i& a) { + return __lsx_vshuf4i_w((__m128i)a, 0x1B); +} +template <> +EIGEN_STRONG_INLINE Packet2l preverse(const Packet2l& a) { + return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1); +} +template <> +EIGEN_STRONG_INLINE Packet16uc preverse(const Packet16uc& a) { + return __lsx_vshuf4i_b(__lsx_vshuf4i_w((__m128i)a, 0x1B), 0x1B); +} +template <> +EIGEN_STRONG_INLINE Packet8us preverse(const Packet8us& a) { + return __lsx_vshuf4i_h(__lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1), 0x1B); +} +template <> +EIGEN_STRONG_INLINE Packet4ui preverse(const Packet4ui& a) { + return __lsx_vshuf4i_w((__m128i)a, 0x1B); +} +template <> +EIGEN_STRONG_INLINE Packet2ul preverse(const Packet2ul& a) { + return __lsx_vshuf4i_d((__m128i)a, (__m128i)a, 0x1); +} + +template <> +EIGEN_STRONG_INLINE float predux(const Packet4f& a) { + Packet4f tmp = __lsx_vfadd_s(a, vec4f_swizzle1(a, 2, 3, 2, 3)); + return pfirst(__lsx_vfadd_s(tmp, vec4f_swizzle1(tmp, 1, 1, 1, 1))); +} +template <> +EIGEN_STRONG_INLINE double predux(const Packet2d& a) { + return pfirst(__lsx_vfadd_d(a, preverse(a))); +} +template <> +EIGEN_STRONG_INLINE int8_t predux(const Packet16c& a) { + Packet8s tmp1 = __lsx_vhaddw_h_b(a, a); + Packet4i tmp2 = __lsx_vhaddw_w_h(tmp1, tmp1); + Packet2l tmp3 = __lsx_vhaddw_d_w(tmp2, tmp2); + return (int8_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp3, tmp3), 0); +} +template <> +EIGEN_STRONG_INLINE int16_t predux(const Packet8s& a) { + Packet4i tmp1 = __lsx_vhaddw_w_h(a, a); + Packet2l tmp2 = __lsx_vhaddw_d_w(tmp1, tmp1); + return (int16_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp2, tmp2), 0); +} +template <> +EIGEN_STRONG_INLINE int32_t predux(const Packet4i& a) { + Packet2l tmp = __lsx_vhaddw_d_w(a, a); + return (int32_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(tmp, tmp), 0); +} +template <> +EIGEN_STRONG_INLINE int64_t predux(const Packet2l& a) { + return (int64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_q_d(a, a), 0); +} +template <> +EIGEN_STRONG_INLINE uint8_t predux(const Packet16uc& a) { + Packet8us tmp1 = __lsx_vhaddw_hu_bu(a, a); + Packet4ui tmp2 = __lsx_vhaddw_wu_hu(tmp1, tmp1); + Packet2ul tmp3 = __lsx_vhaddw_du_wu(tmp2, tmp2); + return (uint8_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp3, tmp3), 0); +} +template <> +EIGEN_STRONG_INLINE uint16_t predux(const Packet8us& a) { + Packet4ui tmp1 = __lsx_vhaddw_wu_hu(a, a); + Packet2ul tmp2 = __lsx_vhaddw_du_wu(tmp1, tmp1); + return (uint16_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp2, tmp2), 0); +} +template <> +EIGEN_STRONG_INLINE uint32_t predux(const Packet4ui& a) { + Packet2ul tmp = __lsx_vhaddw_du_wu(a, a); + return (uint32_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(tmp, tmp), 0); +} +template <> +EIGEN_STRONG_INLINE uint64_t predux(const Packet2ul& a) { + return (uint64_t)__lsx_vpickve2gr_d(__lsx_vhaddw_qu_du(a, a), 0); +} + +template <> +EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) { + Packet4f tmp = __lsx_vfmul_s(a, vec4f_swizzle1(a, 2, 3, 2, 3)); + return pfirst(__lsx_vfmul_s(tmp, vec4f_swizzle1(tmp, 1, 1, 1, 1))); +} +template <> +EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) { + return pfirst(__lsx_vfmul_d(a, preverse(a))); +} +template <> +EIGEN_STRONG_INLINE int8_t predux_mul(const Packet16c& a) { + Packet8s tmp1 = __lsx_vmulwev_h_b(a, preverse(a)); + Packet4i tmp2 = __lsx_vmulwev_w_h(tmp1, preverse(tmp1)); + Packet2l tmp3 = __lsx_vmulwev_d_w(tmp2, preverse(tmp2)); + return (int8_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp3, preverse(tmp3)), 0); +} +template <> +EIGEN_STRONG_INLINE int16_t predux_mul(const Packet8s& a) { + Packet4i tmp1 = __lsx_vmulwev_w_h(a, preverse(a)); + Packet2l tmp2 = __lsx_vmulwev_d_w(tmp1, preverse(tmp1)); + return (int16_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp2, preverse(tmp2)), 0); +} +template <> +EIGEN_STRONG_INLINE int32_t predux_mul(const Packet4i& a) { + Packet2l tmp = __lsx_vmulwev_d_w(a, preverse(a)); + return (int32_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp, preverse(tmp)), 0); +} +template <> +EIGEN_STRONG_INLINE int64_t predux_mul(const Packet2l& a) { + return (int64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(a, preverse(a)), 0); +} +template <> +EIGEN_STRONG_INLINE uint8_t predux_mul(const Packet16uc& a) { + Packet8us tmp1 = __lsx_vmulwev_h_bu(a, preverse(a)); + Packet4ui tmp2 = __lsx_vmulwev_w_h(tmp1, preverse(tmp1)); + Packet2ul tmp3 = __lsx_vmulwev_d_w(tmp2, preverse(tmp2)); + return (uint8_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp3, preverse(tmp3)), 0); +} +template <> +EIGEN_STRONG_INLINE uint16_t predux_mul(const Packet8us& a) { + Packet4ui tmp1 = __lsx_vmulwev_w_hu(a, preverse(a)); + Packet2ul tmp2 = __lsx_vmulwev_d_w(tmp1, preverse(tmp1)); + return (uint16_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp2, preverse(tmp2)), 0); +} +template <> +EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet4ui& a) { + Packet2ul tmp = __lsx_vmulwev_d_wu(a, preverse(a)); + return (uint32_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_d(tmp, preverse(tmp)), 0); +} +template <> +EIGEN_STRONG_INLINE uint64_t predux_mul(const Packet2ul& a) { + return (uint64_t)__lsx_vpickve2gr_d(__lsx_vmulwev_q_du(a, preverse(a)), 0); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { + Packet4f tmp = __lsx_vfmin_s(a, (Packet4f)__lsx_vshuf4i_w(a, 0x4E)); + return pfirst(__lsx_vfmin_s(tmp, (Packet4f)__lsx_vshuf4i_w(tmp, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) { + return pfirst(__lsx_vfmin_d(a, preverse(a))); +} +template <> +EIGEN_STRONG_INLINE int8_t predux_min(const Packet16c& a) { + Packet16c tmp1 = __lsx_vmin_b(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + Packet16c tmp2 = __lsx_vmin_b(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E)); + Packet16c tmp3 = __lsx_vmin_b(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E)); + return pfirst((Packet16c)__lsx_vmin_b(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE int16_t predux_min(const Packet8s& a) { + Packet8s tmp1 = __lsx_vmin_h(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + Packet8s tmp2 = __lsx_vmin_h(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E)); + return pfirst((Packet8s)__lsx_vmin_h(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE int32_t predux_min(const Packet4i& a) { + Packet4i tmp = __lsx_vmin_w(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + return pfirst((Packet4i)__lsx_vmin_w(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE int64_t predux_min(const Packet2l& a) { + return pfirst((Packet2l)__lsx_vmin_d(a, preverse(a))); +} +template <> +EIGEN_STRONG_INLINE uint8_t predux_min(const Packet16uc& a) { + Packet16uc tmp1 = __lsx_vmin_bu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + Packet16uc tmp2 = __lsx_vmin_bu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E)); + Packet16uc tmp3 = __lsx_vmin_bu(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E)); + return pfirst((Packet16uc)__lsx_vmin_bu(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE uint16_t predux_min(const Packet8us& a) { + Packet8us tmp1 = __lsx_vmin_hu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + Packet8us tmp2 = __lsx_vmin_hu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E)); + return pfirst((Packet8us)__lsx_vmin_hu(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE uint32_t predux_min(const Packet4ui& a) { + Packet4ui tmp = __lsx_vmin_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + return pfirst((Packet4ui)__lsx_vmin_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE uint64_t predux_min(const Packet2ul& a) { + return pfirst((Packet2ul)__lsx_vmin_du(a, preverse(a))); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { + Packet4f tmp = __lsx_vfmax_s(a, (Packet4f)__lsx_vshuf4i_w(a, 0x4E)); + return pfirst(__lsx_vfmax_s(tmp, (Packet4f)__lsx_vshuf4i_w(tmp, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { + return pfirst(__lsx_vfmax_d(a, preverse(a))); +} +template <> +EIGEN_STRONG_INLINE int8_t predux_max(const Packet16c& a) { + Packet16c tmp1 = __lsx_vmax_b(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + Packet16c tmp2 = __lsx_vmax_b(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E)); + Packet16c tmp3 = __lsx_vmax_b(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E)); + return pfirst((Packet16c)__lsx_vmax_b(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE int16_t predux_max(const Packet8s& a) { + Packet8s tmp1 = __lsx_vmax_h(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + Packet8s tmp2 = __lsx_vmax_h(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E)); + return pfirst((Packet8s)__lsx_vmax_h(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE int32_t predux_max(const Packet4i& a) { + Packet4i tmp = __lsx_vmax_w(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + return pfirst((Packet4i)__lsx_vmax_w(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE int64_t predux_max(const Packet2l& a) { + return pfirst((Packet2l)__lsx_vmax_d(a, preverse(a))); +} +template <> +EIGEN_STRONG_INLINE uint8_t predux_max(const Packet16uc& a) { + Packet16uc tmp1 = __lsx_vmax_bu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + Packet16uc tmp2 = __lsx_vmax_bu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E)); + Packet16uc tmp3 = __lsx_vmax_bu(tmp2, __lsx_vshuf4i_b((__m128i)tmp2, 0x4E)); + return pfirst((Packet16uc)__lsx_vmax_bu(tmp3, __lsx_vshuf4i_b((__m128i)tmp3, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE uint16_t predux_max(const Packet8us& a) { + Packet8us tmp1 = __lsx_vmax_hu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + Packet8us tmp2 = __lsx_vmax_hu(tmp1, __lsx_vshuf4i_h((__m128i)tmp1, 0x4E)); + return pfirst((Packet8us)__lsx_vmax_hu(tmp2, __lsx_vshuf4i_h((__m128i)tmp2, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE uint32_t predux_max(const Packet4ui& a) { + Packet4ui tmp = __lsx_vmax_wu(a, __lsx_vshuf4i_w((__m128i)a, 0x4E)); + return pfirst((Packet4ui)__lsx_vmax_wu(tmp, __lsx_vshuf4i_w((__m128i)tmp, 0xB1))); +} +template <> +EIGEN_STRONG_INLINE uint64_t predux_max(const Packet2ul& a) { + return pfirst((Packet2ul)__lsx_vmax_du(a, preverse(a))); +} + +template <> +EIGEN_STRONG_INLINE Packet4f psqrt(const Packet4f& a) { + return __lsx_vfsqrt_s(a); +} +template <> +EIGEN_STRONG_INLINE Packet2d psqrt(const Packet2d& a) { + return __lsx_vfsqrt_d(a); +} + +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + Packet4f T0 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]); + Packet4f T1 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]); + Packet4f T2 = (Packet4f)__lsx_vilvl_w((__m128i)kernel.packet[3], (__m128i)kernel.packet[2]); + Packet4f T3 = (Packet4f)__lsx_vilvh_w((__m128i)kernel.packet[3], (__m128i)kernel.packet[2]); + + kernel.packet[0] = (Packet4f)__lsx_vilvl_d((__m128i)T2, (__m128i)T0); + kernel.packet[1] = (Packet4f)__lsx_vilvh_d((__m128i)T2, (__m128i)T0); + kernel.packet[2] = (Packet4f)__lsx_vilvl_d((__m128i)T3, (__m128i)T1); + kernel.packet[3] = (Packet4f)__lsx_vilvh_d((__m128i)T3, (__m128i)T1); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + Packet2d tmp = (Packet2d)__lsx_vilvh_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]); + kernel.packet[0] = (Packet2d)__lsx_vilvl_d((__m128i)kernel.packet[1], (__m128i)kernel.packet[0]); + kernel.packet[1] = tmp; +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]); + __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]); + __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]); + __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]); + __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]); + __m128i t8 = __lsx_vilvl_b(kernel.packet[9], kernel.packet[8]); + __m128i t9 = __lsx_vilvh_b(kernel.packet[9], kernel.packet[8]); + __m128i ta = __lsx_vilvl_b(kernel.packet[11], kernel.packet[10]); + __m128i tb = __lsx_vilvh_b(kernel.packet[11], kernel.packet[10]); + __m128i tc = __lsx_vilvl_b(kernel.packet[13], kernel.packet[12]); + __m128i td = __lsx_vilvh_b(kernel.packet[13], kernel.packet[12]); + __m128i te = __lsx_vilvl_b(kernel.packet[15], kernel.packet[14]); + __m128i tf = __lsx_vilvh_b(kernel.packet[15], kernel.packet[14]); + + __m128i s0 = __lsx_vilvl_h(t2, t0); + __m128i s1 = __lsx_vilvh_h(t2, t0); + __m128i s2 = __lsx_vilvl_h(t3, t1); + __m128i s3 = __lsx_vilvh_h(t3, t1); + __m128i s4 = __lsx_vilvl_h(t6, t4); + __m128i s5 = __lsx_vilvh_h(t6, t4); + __m128i s6 = __lsx_vilvl_h(t7, t5); + __m128i s7 = __lsx_vilvh_h(t7, t5); + __m128i s8 = __lsx_vilvl_h(ta, t8); + __m128i s9 = __lsx_vilvh_h(ta, t8); + __m128i sa = __lsx_vilvl_h(tb, t9); + __m128i sb = __lsx_vilvh_h(tb, t9); + __m128i sc = __lsx_vilvl_h(te, tc); + __m128i sd = __lsx_vilvh_h(te, tc); + __m128i se = __lsx_vilvl_h(tf, td); + __m128i sf = __lsx_vilvh_h(tf, td); + + __m128i u0 = __lsx_vilvl_w(s4, s0); + __m128i u1 = __lsx_vilvh_w(s4, s0); + __m128i u2 = __lsx_vilvl_w(s5, s1); + __m128i u3 = __lsx_vilvh_w(s5, s1); + __m128i u4 = __lsx_vilvl_w(s6, s2); + __m128i u5 = __lsx_vilvh_w(s6, s2); + __m128i u6 = __lsx_vilvl_w(s7, s3); + __m128i u7 = __lsx_vilvh_w(s7, s3); + __m128i u8 = __lsx_vilvl_w(sc, s8); + __m128i u9 = __lsx_vilvh_w(sc, s8); + __m128i ua = __lsx_vilvl_w(sd, s9); + __m128i ub = __lsx_vilvh_w(sd, s9); + __m128i uc = __lsx_vilvl_w(se, sa); + __m128i ud = __lsx_vilvh_w(se, sa); + __m128i ue = __lsx_vilvl_w(sf, sb); + __m128i uf = __lsx_vilvh_w(sf, sb); + + kernel.packet[0] = __lsx_vilvl_d(u8, u0); + kernel.packet[1] = __lsx_vilvh_d(u8, u0); + kernel.packet[2] = __lsx_vilvl_d(u9, u1); + kernel.packet[3] = __lsx_vilvh_d(u9, u1); + kernel.packet[4] = __lsx_vilvl_d(ua, u2); + kernel.packet[5] = __lsx_vilvh_d(ua, u2); + kernel.packet[6] = __lsx_vilvl_d(ub, u3); + kernel.packet[7] = __lsx_vilvh_d(ub, u3); + kernel.packet[8] = __lsx_vilvl_d(uc, u4); + kernel.packet[9] = __lsx_vilvh_d(uc, u4); + kernel.packet[10] = __lsx_vilvl_d(ud, u5); + kernel.packet[11] = __lsx_vilvh_d(ud, u5); + kernel.packet[12] = __lsx_vilvl_d(ue, u6); + kernel.packet[13] = __lsx_vilvh_d(ue, u6); + kernel.packet[14] = __lsx_vilvl_d(uf, u7); + kernel.packet[15] = __lsx_vilvh_d(uf, u7); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]); + __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]); + __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]); + __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]); + __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]); + + __m128i s0 = __lsx_vilvl_h(t2, t0); + __m128i s1 = __lsx_vilvh_h(t2, t0); + __m128i s2 = __lsx_vilvl_h(t3, t1); + __m128i s3 = __lsx_vilvh_h(t3, t1); + __m128i s4 = __lsx_vilvl_h(t6, t4); + __m128i s5 = __lsx_vilvh_h(t6, t4); + __m128i s6 = __lsx_vilvl_h(t7, t5); + __m128i s7 = __lsx_vilvh_h(t7, t5); + + kernel.packet[0] = __lsx_vilvl_w(s4, s0); + kernel.packet[1] = __lsx_vilvh_w(s4, s0); + kernel.packet[2] = __lsx_vilvl_w(s5, s1); + kernel.packet[3] = __lsx_vilvh_w(s5, s1); + kernel.packet[4] = __lsx_vilvl_w(s6, s2); + kernel.packet[5] = __lsx_vilvh_w(s6, s2); + kernel.packet[6] = __lsx_vilvl_w(s7, s3); + kernel.packet[7] = __lsx_vilvh_w(s7, s3); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]); + + kernel.packet[0] = __lsx_vilvl_h(t2, t0); + kernel.packet[1] = __lsx_vilvh_h(t2, t0); + kernel.packet[2] = __lsx_vilvl_h(t3, t1); + kernel.packet[3] = __lsx_vilvh_h(t3, t1); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]); + __m128i t4 = __lsx_vilvl_h(kernel.packet[5], kernel.packet[4]); + __m128i t5 = __lsx_vilvh_h(kernel.packet[5], kernel.packet[4]); + __m128i t6 = __lsx_vilvl_h(kernel.packet[7], kernel.packet[6]); + __m128i t7 = __lsx_vilvh_h(kernel.packet[7], kernel.packet[6]); + + __m128i s0 = __lsx_vilvl_w(t2, t0); + __m128i s1 = __lsx_vilvh_w(t2, t0); + __m128i s2 = __lsx_vilvl_w(t3, t1); + __m128i s3 = __lsx_vilvh_w(t3, t1); + __m128i s4 = __lsx_vilvl_w(t6, t4); + __m128i s5 = __lsx_vilvh_w(t6, t4); + __m128i s6 = __lsx_vilvl_w(t7, t5); + __m128i s7 = __lsx_vilvh_w(t7, t5); + + kernel.packet[0] = __lsx_vilvl_d(s4, s0); + kernel.packet[1] = __lsx_vilvh_d(s4, s0); + kernel.packet[2] = __lsx_vilvl_d(s5, s1); + kernel.packet[3] = __lsx_vilvh_d(s5, s1); + kernel.packet[4] = __lsx_vilvl_d(s6, s2); + kernel.packet[5] = __lsx_vilvh_d(s6, s2); + kernel.packet[6] = __lsx_vilvl_d(s7, s3); + kernel.packet[7] = __lsx_vilvh_d(s7, s3); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]); + + kernel.packet[0] = __lsx_vilvl_w(t2, t0); + kernel.packet[1] = __lsx_vilvh_w(t2, t0); + kernel.packet[2] = __lsx_vilvl_w(t3, t1); + kernel.packet[3] = __lsx_vilvh_w(t3, t1); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i T0 = __lsx_vilvl_w(kernel.packet[1], kernel.packet[0]); + __m128i T1 = __lsx_vilvh_w(kernel.packet[1], kernel.packet[0]); + __m128i T2 = __lsx_vilvl_w(kernel.packet[3], kernel.packet[2]); + __m128i T3 = __lsx_vilvh_w(kernel.packet[3], kernel.packet[2]); + + kernel.packet[0] = __lsx_vilvl_d(T2, T0); + kernel.packet[1] = __lsx_vilvh_d(T2, T0); + kernel.packet[2] = __lsx_vilvl_d(T3, T1); + kernel.packet[3] = __lsx_vilvh_d(T3, T1); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]); + kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]); + kernel.packet[1] = tmp; +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]); + __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]); + __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]); + __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]); + __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]); + __m128i t8 = __lsx_vilvl_b(kernel.packet[9], kernel.packet[8]); + __m128i t9 = __lsx_vilvh_b(kernel.packet[9], kernel.packet[8]); + __m128i ta = __lsx_vilvl_b(kernel.packet[11], kernel.packet[10]); + __m128i tb = __lsx_vilvh_b(kernel.packet[11], kernel.packet[10]); + __m128i tc = __lsx_vilvl_b(kernel.packet[13], kernel.packet[12]); + __m128i td = __lsx_vilvh_b(kernel.packet[13], kernel.packet[12]); + __m128i te = __lsx_vilvl_b(kernel.packet[15], kernel.packet[14]); + __m128i tf = __lsx_vilvh_b(kernel.packet[15], kernel.packet[14]); + + __m128i s0 = __lsx_vilvl_h(t2, t0); + __m128i s1 = __lsx_vilvh_h(t2, t0); + __m128i s2 = __lsx_vilvl_h(t3, t1); + __m128i s3 = __lsx_vilvh_h(t3, t1); + __m128i s4 = __lsx_vilvl_h(t6, t4); + __m128i s5 = __lsx_vilvh_h(t6, t4); + __m128i s6 = __lsx_vilvl_h(t7, t5); + __m128i s7 = __lsx_vilvh_h(t7, t5); + __m128i s8 = __lsx_vilvl_h(ta, t8); + __m128i s9 = __lsx_vilvh_h(ta, t8); + __m128i sa = __lsx_vilvl_h(tb, t9); + __m128i sb = __lsx_vilvh_h(tb, t9); + __m128i sc = __lsx_vilvl_h(te, tc); + __m128i sd = __lsx_vilvh_h(te, tc); + __m128i se = __lsx_vilvl_h(tf, td); + __m128i sf = __lsx_vilvh_h(tf, td); + + __m128i u0 = __lsx_vilvl_w(s4, s0); + __m128i u1 = __lsx_vilvh_w(s4, s0); + __m128i u2 = __lsx_vilvl_w(s5, s1); + __m128i u3 = __lsx_vilvh_w(s5, s1); + __m128i u4 = __lsx_vilvl_w(s6, s2); + __m128i u5 = __lsx_vilvh_w(s6, s2); + __m128i u6 = __lsx_vilvl_w(s7, s3); + __m128i u7 = __lsx_vilvh_w(s7, s3); + __m128i u8 = __lsx_vilvl_w(sc, s8); + __m128i u9 = __lsx_vilvh_w(sc, s8); + __m128i ua = __lsx_vilvl_w(sd, s9); + __m128i ub = __lsx_vilvh_w(sd, s9); + __m128i uc = __lsx_vilvl_w(se, sa); + __m128i ud = __lsx_vilvh_w(se, sa); + __m128i ue = __lsx_vilvl_w(sf, sb); + __m128i uf = __lsx_vilvh_w(sf, sb); + + kernel.packet[0] = __lsx_vilvl_d(u8, u0); + kernel.packet[1] = __lsx_vilvh_d(u8, u0); + kernel.packet[2] = __lsx_vilvl_d(u9, u1); + kernel.packet[3] = __lsx_vilvh_d(u9, u1); + kernel.packet[4] = __lsx_vilvl_d(ua, u2); + kernel.packet[5] = __lsx_vilvh_d(ua, u2); + kernel.packet[6] = __lsx_vilvl_d(ub, u3); + kernel.packet[7] = __lsx_vilvh_d(ub, u3); + kernel.packet[8] = __lsx_vilvl_d(uc, u4); + kernel.packet[9] = __lsx_vilvh_d(uc, u4); + kernel.packet[10] = __lsx_vilvl_d(ud, u5); + kernel.packet[11] = __lsx_vilvh_d(ud, u5); + kernel.packet[12] = __lsx_vilvl_d(ue, u6); + kernel.packet[13] = __lsx_vilvh_d(ue, u6); + kernel.packet[14] = __lsx_vilvl_d(uf, u7); + kernel.packet[15] = __lsx_vilvh_d(uf, u7); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]); + __m128i t4 = __lsx_vilvl_b(kernel.packet[5], kernel.packet[4]); + __m128i t5 = __lsx_vilvh_b(kernel.packet[5], kernel.packet[4]); + __m128i t6 = __lsx_vilvl_b(kernel.packet[7], kernel.packet[6]); + __m128i t7 = __lsx_vilvh_b(kernel.packet[7], kernel.packet[6]); + + __m128i s0 = __lsx_vilvl_h(t2, t0); + __m128i s1 = __lsx_vilvh_h(t2, t0); + __m128i s2 = __lsx_vilvl_h(t3, t1); + __m128i s3 = __lsx_vilvh_h(t3, t1); + __m128i s4 = __lsx_vilvl_h(t6, t4); + __m128i s5 = __lsx_vilvh_h(t6, t4); + __m128i s6 = __lsx_vilvl_h(t7, t5); + __m128i s7 = __lsx_vilvh_h(t7, t5); + + kernel.packet[0] = __lsx_vilvl_w(s4, s0); + kernel.packet[1] = __lsx_vilvh_w(s4, s0); + kernel.packet[2] = __lsx_vilvl_w(s5, s1); + kernel.packet[3] = __lsx_vilvh_w(s5, s1); + kernel.packet[4] = __lsx_vilvl_w(s6, s2); + kernel.packet[5] = __lsx_vilvh_w(s6, s2); + kernel.packet[6] = __lsx_vilvl_w(s7, s3); + kernel.packet[7] = __lsx_vilvh_w(s7, s3); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_b(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_b(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_b(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_b(kernel.packet[3], kernel.packet[2]); + + kernel.packet[0] = __lsx_vilvl_h(t2, t0); + kernel.packet[1] = __lsx_vilvh_h(t2, t0); + kernel.packet[2] = __lsx_vilvl_h(t3, t1); + kernel.packet[3] = __lsx_vilvh_h(t3, t1); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]); + __m128i t4 = __lsx_vilvl_h(kernel.packet[5], kernel.packet[4]); + __m128i t5 = __lsx_vilvh_h(kernel.packet[5], kernel.packet[4]); + __m128i t6 = __lsx_vilvl_h(kernel.packet[7], kernel.packet[6]); + __m128i t7 = __lsx_vilvh_h(kernel.packet[7], kernel.packet[6]); + + __m128i s0 = __lsx_vilvl_w(t2, t0); + __m128i s1 = __lsx_vilvh_w(t2, t0); + __m128i s2 = __lsx_vilvl_w(t3, t1); + __m128i s3 = __lsx_vilvh_w(t3, t1); + __m128i s4 = __lsx_vilvl_w(t6, t4); + __m128i s5 = __lsx_vilvh_w(t6, t4); + __m128i s6 = __lsx_vilvl_w(t7, t5); + __m128i s7 = __lsx_vilvh_w(t7, t5); + + kernel.packet[0] = __lsx_vilvl_d(s4, s0); + kernel.packet[1] = __lsx_vilvh_d(s4, s0); + kernel.packet[2] = __lsx_vilvl_d(s5, s1); + kernel.packet[3] = __lsx_vilvh_d(s5, s1); + kernel.packet[4] = __lsx_vilvl_d(s6, s2); + kernel.packet[5] = __lsx_vilvh_d(s6, s2); + kernel.packet[6] = __lsx_vilvl_d(s7, s3); + kernel.packet[7] = __lsx_vilvh_d(s7, s3); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i t0 = __lsx_vilvl_h(kernel.packet[1], kernel.packet[0]); + __m128i t1 = __lsx_vilvh_h(kernel.packet[1], kernel.packet[0]); + __m128i t2 = __lsx_vilvl_h(kernel.packet[3], kernel.packet[2]); + __m128i t3 = __lsx_vilvh_h(kernel.packet[3], kernel.packet[2]); + + kernel.packet[0] = __lsx_vilvl_w(t2, t0); + kernel.packet[1] = __lsx_vilvh_w(t2, t0); + kernel.packet[2] = __lsx_vilvl_w(t3, t1); + kernel.packet[3] = __lsx_vilvh_w(t3, t1); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i T0 = __lsx_vilvl_w(kernel.packet[1], kernel.packet[0]); + __m128i T1 = __lsx_vilvh_w(kernel.packet[1], kernel.packet[0]); + __m128i T2 = __lsx_vilvl_w(kernel.packet[3], kernel.packet[2]); + __m128i T3 = __lsx_vilvh_w(kernel.packet[3], kernel.packet[2]); + + kernel.packet[0] = __lsx_vilvl_d(T2, T0); + kernel.packet[1] = __lsx_vilvh_d(T2, T0); + kernel.packet[2] = __lsx_vilvl_d(T3, T1); + kernel.packet[3] = __lsx_vilvh_d(T3, T1); +} +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void ptranspose(PacketBlock& kernel) { + __m128i tmp = __lsx_vilvh_d(kernel.packet[1], kernel.packet[0]); + kernel.packet[0] = __lsx_vilvl_d(kernel.packet[1], kernel.packet[0]); + kernel.packet[1] = tmp; +} + +template <> +EIGEN_STRONG_INLINE Packet4f prsqrt(const Packet4f& a) { + return __lsx_vfrsqrt_s(a); +} +template <> +EIGEN_STRONG_INLINE Packet2d prsqrt(const Packet2d& a) { + return __lsx_vfrsqrt_d(a); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pfloor(const Packet4f& a) { + return __lsx_vfrintrm_s(a); +} +template <> +EIGEN_STRONG_INLINE Packet2d pfloor(const Packet2d& a) { + return __lsx_vfrintrm_d(a); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pceil(const Packet4f& a) { + return __lsx_vfrintrp_s(a); +} +template <> +EIGEN_STRONG_INLINE Packet2d pceil(const Packet2d& a) { + return __lsx_vfrintrp_d(a); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pround(const Packet4f& a) { + const Packet4f mask = pset1frombits(static_cast(0x80000000u)); + const Packet4f prev0dot5 = pset1frombits(static_cast(0x3EFFFFFFu)); + return __lsx_vfrintrz_s(padd(pxor(pand(a, mask), prev0dot5), a)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pround(const Packet2d& a) { + const Packet2d mask = pset1frombits(static_cast(0x8000000000000000ull)); + const Packet2d prev0dot5 = pset1frombits(static_cast(0x3FDFFFFFFFFFFFFFull)); + return __lsx_vfrintrz_d(padd(por(pand(a, mask), prev0dot5), a)); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4f pselect(const Packet4f& mask, const Packet4f& a, const Packet4f& b) { + return (Packet4f)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16c pselect(const Packet16c& mask, const Packet16c& a, const Packet16c& b) { + return (Packet16c)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask); +} + +template <> +EIGEN_STRONG_INLINE Packet16c ploadquad(const int8_t* from) { + int8_t tmp[16] = {*from, *from, *from, *from, *(from + 1), *(from + 1), + *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2), + *(from + 3), *(from + 3), *(from + 3), *(from + 3)}; + return __lsx_vld(tmp, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc ploadquad(const uint8_t* from) { + uint8_t tmp[16] = {*from, *from, *from, *from, *(from + 1), *(from + 1), + *(from + 1), *(from + 1), *(from + 2), *(from + 2), *(from + 2), *(from + 2), + *(from + 3), *(from + 3), *(from + 3), *(from + 3)}; + return __lsx_vld(tmp, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s ploadquad(const int16_t* from) { + int16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)}; + return __lsx_vld(tmp, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us ploadquad(const uint16_t* from) { + uint16_t tmp[8] = {*from, *from, *from, *from, *(from + 1), *(from + 1), *(from + 1), *(from + 1)}; + return __lsx_vld(tmp, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4i ploadquad(const int32_t* from) { + int32_t tmp[4] = {*from, *from, *from, *from}; + return __lsx_vld(tmp, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4ui ploadquad(const uint32_t* from) { + uint32_t tmp[4] = {*from, *from, *from, *from}; + return __lsx_vld(tmp, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet16c pnmsub(const Packet16c& a, const Packet16c& b, const Packet16c& c) { + return __lsx_vmsub_b(pnegate(c), a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pnmsub(const Packet8s& a, const Packet8s& b, const Packet8s& c) { + return __lsx_vmsub_h(pnegate(c), a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pnmsub(const Packet4i& a, const Packet4i& b, const Packet4i& c) { + return __lsx_vmsub_w(pnegate(c), a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pnmsub(const Packet2l& a, const Packet2l& b, const Packet2l& c) { + return __lsx_vmsub_d(pnegate(c), a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16c pmsub(const Packet16c& a, const Packet16c& b, const Packet16c& c) { + return __lsx_vmadd_b(pnegate(c), a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pmsub(const Packet8s& a, const Packet8s& b, const Packet8s& c) { + return __lsx_vmadd_h(pnegate(c), a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pmsub(const Packet4i& a, const Packet4i& b, const Packet4i& c) { + return __lsx_vmadd_w(pnegate(c), a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pmsub(const Packet2l& a, const Packet2l& b, const Packet2l& c) { + return __lsx_vmadd_d(pnegate(c), a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet16c pnmadd(const Packet16c& a, const Packet16c& b, const Packet16c& c) { + return __lsx_vmsub_b(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet8s pnmadd(const Packet8s& a, const Packet8s& b, const Packet8s& c) { + return __lsx_vmsub_h(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4i pnmadd(const Packet4i& a, const Packet4i& b, const Packet4i& c) { + return __lsx_vmsub_w(c, a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2l pnmadd(const Packet2l& a, const Packet2l& b, const Packet2l& c) { + return __lsx_vmsub_d(c, a, b); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pexp(const Packet4f& _x) { + return pexp_float(_x); +} +template <> +EIGEN_STRONG_INLINE Packet2d pexp(const Packet2d& _x) { + return pexp_double(_x); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pldexp(const Packet4f& a, const Packet4f& exponent) { + return pldexp_generic(a, exponent); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pfrexp(const Packet2d& a, Packet2d& exponent) { + return pfrexp_generic(a, exponent); +} +template <> +EIGEN_STRONG_INLINE Packet4f pfrexp(const Packet4f& a, Packet4f& exponent) { + return pfrexp_generic(a, exponent); +} +template <> +EIGEN_STRONG_INLINE Packet4f pzero(const Packet4f& /* a */) { + Packet4f v = {0.0f, 0.0f, 0.0f, 0.0f}; + return v; +} +template <> +EIGEN_STRONG_INLINE Packet4f pabsdiff(const Packet4f& a, const Packet4f& b) { + Packet4f v = psub(a, b); + return pabs(v); +} +template <> +EIGEN_STRONG_INLINE Packet4f pmin(const Packet4f& a, const Packet4f& b) { + return pmin(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4f pmax(const Packet4f& a, const Packet4f& b) { + return pmax(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet4f ploadquad(const float* from) { + return (__m128)__lsx_vldrepl_w(from, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4f psignbit(const Packet4f& a) { + return (__m128)__lsx_vsrai_w((__m128i)a, 31); +} +template <> +EIGEN_STRONG_INLINE Packet4f print(const Packet4f& a) { + return __lsx_vfrintrne_s(a); +} +template <> +EIGEN_STRONG_INLINE Packet4f ptrunc(const Packet4f& a) { + return __lsx_vfrintrz_s(a); +} +template <> +EIGEN_STRONG_INLINE Packet4f preciprocal(const Packet4f& a) { + return __lsx_vfrecip_s(a); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pzero(const Packet2d& /* a */) { + Packet2d v = {0.0, 0.0}; + return v; +} +template <> +EIGEN_STRONG_INLINE Packet2d pmin(const Packet2d& a, const Packet2d& b) { + return pmin(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d pmax(const Packet2d& a, const Packet2d& b) { + return pmax(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet2d psignbit(const Packet2d& a) { + return (__m128d)(__lsx_vsrai_d((__m128i)a, 63)); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2d pselect(const Packet2d& mask, const Packet2d& a, const Packet2d& b) { + return (Packet2d)__lsx_vbitsel_v((__m128i)b, (__m128i)a, (__m128i)mask); +} +template <> +EIGEN_STRONG_INLINE Packet2d print(const Packet2d& a) { + return __lsx_vfrintrne_d(a); +} +template <> +EIGEN_STRONG_INLINE Packet2d ptrunc(const Packet2d& a) { + return __lsx_vfrintrz_d(a); +} +template <> +EIGEN_STRONG_INLINE Packet2d pldexp(const Packet2d& a, const Packet2d& exponent) { + return pldexp_generic(a, exponent); +} + +template <> +EIGEN_STRONG_INLINE Packet16c pabsdiff(const Packet16c& a, const Packet16c& b) { + Packet16c v = psub(a, b); + return pabs(v); +} + +template <> +EIGEN_STRONG_INLINE Packet8s pabsdiff(const Packet8s& a, const Packet8s& b) { + Packet8s v = psub(a, b); + return pabs(v); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8s pselect(const Packet8s& mask, const Packet8s& a, const Packet8s& b) { + return __lsx_vbitsel_v(b, a, mask); +} + +template <> +EIGEN_STRONG_INLINE Packet4i pabsdiff(const Packet4i& a, const Packet4i& b) { + Packet4i v = psub(a, b); + return pabs(v); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4i pselect(const Packet4i& mask, const Packet4i& a, const Packet4i& b) { + return __lsx_vbitsel_v(b, a, mask); +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2l pselect(const Packet2l& mask, const Packet2l& a, const Packet2l& b) { + return __lsx_vbitsel_v(b, a, mask); +} + +template <> +EIGEN_STRONG_INLINE Packet16uc pdiv(const Packet16uc& a, const Packet16uc& b) { + return __lsx_vdiv_bu(a, b); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pabsdiff(const Packet16uc& a, const Packet16uc& b) { + Packet16uc v = psub(a, b); + return pabs(v); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet16uc pselect(const Packet16uc& mask, const Packet16uc& a, + const Packet16uc& b) { + return __lsx_vbitsel_v(b, a, mask); +} +template <> +EIGEN_STRONG_INLINE Packet16uc psqrt(const Packet16uc& a) { + __m128i res = {0, 0}; + __m128i add = {0x0808080808080808, 0x0808080808080808}; + for (int i = 0; i < 4; i++) { + const __m128i temp = __lsx_vor_v(res, add); + const __m128i tmul = __lsx_vpackev_b(__lsx_vmulwod_h_bu(temp, temp), __lsx_vmulwev_h_bu(temp, temp)); + res = __lsx_vbitsel_v(res, temp, __lsx_vsle_bu(tmul, a)); + add = __lsx_vsrli_b(add, 1); + } + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet8us pabsdiff(const Packet8us& a, const Packet8us& b) { + Packet8us v = psub(a, b); + return pabs(v); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8us pselect(const Packet8us& mask, const Packet8us& a, const Packet8us& b) { + return __lsx_vbitsel_v(b, a, mask); +} +template <> +EIGEN_STRONG_INLINE Packet8us psqrt(const Packet8us& a) { + __m128i res = {0, 0}; + __m128i add = {0x0080008000800080, 0x0080008000800080}; + for (int i = 0; i < 4; i++) { + const __m128i temp = __lsx_vor_v(res, add); + const __m128i tmul = __lsx_vpackev_h(__lsx_vmulwod_w_hu(temp, temp), __lsx_vmulwev_w_hu(temp, temp)); + res = __lsx_vbitsel_v(res, temp, __lsx_vsle_hu(tmul, a)); + add = __lsx_vsrli_h(add, 1); + } + return res; +} + +template <> +EIGEN_STRONG_INLINE Packet4ui pabsdiff(const Packet4ui& a, const Packet4ui& b) { + Packet4ui v = psub(a, b); + return pabs(v); +} +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4ui pselect(const Packet4ui& mask, const Packet4ui& a, const Packet4ui& b) { + return __lsx_vbitsel_v(b, a, mask); +} +template <> +EIGEN_STRONG_INLINE Packet4ui psqrt(const Packet4ui& a) { + __m128i res = {0, 0}; + __m128i add = {0x0000800000008000, 0x0000800000008000}; + for (int i = 0; i < 4; i++) { + const __m128i temp = __lsx_vor_v(res, add); + const __m128i tmul = __lsx_vpackev_w(__lsx_vmulwod_d_wu(temp, temp), __lsx_vmulwev_d_wu(temp, temp)); + res = __lsx_vbitsel_v(res, temp, __lsx_vsle_wu(tmul, a)); + add = __lsx_vsrli_w(add, 1); + } + return res; +} + +template <> +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet2ul pselect(const Packet2ul& mask, const Packet2ul& a, const Packet2ul& b) { + return __lsx_vbitsel_v(b, a, mask); +} + +} // namespace internal +} // namespace Eigen +#endif diff --git a/Eigen/src/Core/arch/LSX/TypeCasting.h b/Eigen/src/Core/arch/LSX/TypeCasting.h new file mode 100644 index 0000000..cda8680 --- /dev/null +++ b/Eigen/src/Core/arch/LSX/TypeCasting.h @@ -0,0 +1,526 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2023 Zang Ruochen +// Copyright (C) 2024 XiWei Gu +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_TYPE_CASTING_LSX_H +#define EIGEN_TYPE_CASTING_LSX_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +//============================================================================== +// preinterpret +//============================================================================== +template <> +EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4i& a) { + return (__m128)((__m128i)a); +} +template <> +EIGEN_STRONG_INLINE Packet4f preinterpret(const Packet4ui& a) { + return (__m128)((__m128i)a); +} +template <> +EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet2l& a) { + return (__m128d)((__m128i)a); +} +template <> +EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet2ul& a) { + return (__m128d)((__m128i)a); +} +template <> +EIGEN_STRONG_INLINE Packet2d preinterpret(const Packet4i& a) { + return (__m128d)((__m128i)a); +} +template <> +EIGEN_STRONG_INLINE Packet16c preinterpret(const Packet16uc& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet8s preinterpret(const Packet8us& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4f& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet4ui& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet4i preinterpret(const Packet2d& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet2l preinterpret(const Packet2d& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet16uc preinterpret(const Packet16c& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet8us preinterpret(const Packet8s& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet4f& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet4ui preinterpret(const Packet4i& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet2ul preinterpret(const Packet2d& a) { + return (__m128i)a; +} +template <> +EIGEN_STRONG_INLINE Packet2ul preinterpret(const Packet2l& a) { + return (__m128i)a; +} + +template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet4f& a) { + Packet2d tmp = __lsx_vfcvtl_d_s(a); + return __lsx_vftint_l_d(tmp); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4f& a) { + Packet2d tmp = __lsx_vfcvtl_d_s(a); + return __lsx_vftint_lu_d(tmp); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet4f& a) { + return __lsx_vftint_w_s(a); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet4f& a) { + return __lsx_vftint_wu_s(a); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet4f& a, const Packet4f& b) { + return __lsx_vssrlni_h_w(__lsx_vftint_w_s(a), __lsx_vftint_w_s(b), 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcast(const Packet4f& a, const Packet4f& b) { + return __lsx_vssrlni_hu_w(__lsx_vftint_wu_s(a), __lsx_vftint_wu_s(b), 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcast(const Packet4f& a, const Packet4f& b, const Packet4f& c, + const Packet4f& d) { + Packet8s tmp1 = __lsx_vssrlni_h_w(__lsx_vftint_w_s(a), __lsx_vftint_w_s(b), 0); + Packet8s tmp2 = __lsx_vssrlni_h_w(__lsx_vftint_w_s(c), __lsx_vftint_w_s(d), 0); + return __lsx_vssrlni_b_h((__m128i)tmp1, (__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcast(const Packet4f& a, const Packet4f& b, const Packet4f& c, + const Packet4f& d) { + Packet8us tmp1 = __lsx_vssrlni_hu_w(__lsx_vftint_wu_s(a), __lsx_vftint_wu_s(b), 0); + Packet8us tmp2 = __lsx_vssrlni_hu_w(__lsx_vftint_wu_s(c), __lsx_vftint_wu_s(d), 0); + return __lsx_vssrlni_bu_h((__m128i)tmp1, (__m128i)tmp2, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet16c& a) { + Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0); + Packet4i tmp2 = __lsx_vsllwil_w_h((__m128i)tmp1, 0); + return __lsx_vffint_s_w(tmp2); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet16c& a) { + Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0); + Packet4i tmp2 = __lsx_vsllwil_w_h((__m128i)tmp1, 0); + return __lsx_vsllwil_d_w((__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet16c& a) { + Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0); + Packet4i tmp2 = __lsx_vsllwil_w_h((__m128i)tmp1, 0); + return (Packet2ul)__lsx_vsllwil_d_w((__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet16c& a) { + Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0); + return __lsx_vsllwil_w_h((__m128i)tmp1, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet16c& a) { + Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0); + return (Packet4ui)__lsx_vsllwil_w_h((__m128i)tmp1, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet16c& a) { + return __lsx_vsllwil_h_b((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcast(const Packet16c& a) { + return (Packet8us)__lsx_vsllwil_h_b((__m128i)a, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet16uc& a) { + Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0); + Packet4ui tmp2 = __lsx_vsllwil_wu_hu((__m128i)tmp1, 0); + return __lsx_vffint_s_wu(tmp2); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet16uc& a) { + Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0); + Packet4ui tmp2 = __lsx_vsllwil_wu_hu((__m128i)tmp1, 0); + return __lsx_vsllwil_du_wu((__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet16uc& a) { + Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0); + Packet4ui tmp2 = __lsx_vsllwil_wu_hu((__m128i)tmp1, 0); + return (Packet2l)__lsx_vsllwil_du_wu((__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet16uc& a) { + Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0); + return __lsx_vsllwil_wu_hu((__m128i)tmp1, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet16uc& a) { + Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0); + return (Packet4i)__lsx_vsllwil_wu_hu((__m128i)tmp1, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcast(const Packet16uc& a) { + return __lsx_vsllwil_hu_bu((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet16uc& a) { + return (Packet8s)__lsx_vsllwil_hu_bu((__m128i)a, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet8s& a) { + Packet4i tmp1 = __lsx_vsllwil_w_h((__m128i)a, 0); + return __lsx_vffint_s_w(tmp1); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet8s& a) { + Packet4i tmp1 = __lsx_vsllwil_w_h((__m128i)a, 0); + return __lsx_vsllwil_d_w((__m128i)tmp1, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet8s& a) { + Packet4i tmp1 = __lsx_vsllwil_w_h((__m128i)a, 0); + return (Packet2ul)__lsx_vsllwil_d_w((__m128i)tmp1, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet8s& a) { + return __lsx_vsllwil_w_h((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet8s& a) { + return (Packet4ui)__lsx_vsllwil_w_h((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcast(const Packet8s& a, const Packet8s& b) { + return __lsx_vssrlni_b_h((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcast(const Packet8s& a, const Packet8s& b) { + return (Packet16uc)__lsx_vssrlni_b_h((__m128i)a, (__m128i)b, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet8us& a) { + Packet4ui tmp1 = __lsx_vsllwil_wu_hu((__m128i)a, 0); + return __lsx_vffint_s_wu(tmp1); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet8us& a) { + Packet4ui tmp1 = __lsx_vsllwil_wu_hu((__m128i)a, 0); + return __lsx_vsllwil_du_wu((__m128i)tmp1, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet8us& a) { + Packet4ui tmp1 = __lsx_vsllwil_wu_hu((__m128i)a, 0); + return (Packet2l)__lsx_vsllwil_du_wu((__m128i)tmp1, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet8us& a) { + return __lsx_vsllwil_wu_hu((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet8us& a) { + return (Packet4i)__lsx_vsllwil_wu_hu((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcast(const Packet8us& a, const Packet8us& b) { + return __lsx_vssrlni_bu_h((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcast(const Packet8us& a, const Packet8us& b) { + return (Packet16c)__lsx_vssrlni_bu_h((__m128i)a, (__m128i)b, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet4i& a) { + return __lsx_vffint_s_w(a); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet4i& a) { + return __lsx_vsllwil_d_w((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4i& a) { + return (Packet2ul)__lsx_vsllwil_d_w((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet4i& a, const Packet4i& b) { + return __lsx_vssrlni_h_w((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcast(const Packet4i& a, const Packet4i& b) { + return (Packet8us)__lsx_vssrlni_h_w((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcast(const Packet4i& a, const Packet4i& b, const Packet4i& c, + const Packet4i& d) { + Packet8s tmp1 = __lsx_vssrlni_h_w((__m128i)a, (__m128i)b, 0); + Packet8s tmp2 = __lsx_vssrlni_h_w((__m128i)c, (__m128i)d, 0); + return __lsx_vssrlni_b_h((__m128i)tmp1, (__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcast(const Packet4i& a, const Packet4i& b, const Packet4i& c, + const Packet4i& d) { + Packet8s tmp1 = __lsx_vssrlni_h_w((__m128i)a, (__m128i)b, 0); + Packet8s tmp2 = __lsx_vssrlni_h_w((__m128i)c, (__m128i)d, 0); + return (Packet16uc)__lsx_vssrlni_b_h((__m128i)tmp1, (__m128i)tmp2, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet4ui& a) { + return __lsx_vffint_s_wu(a); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet4ui& a) { + return __lsx_vsllwil_du_wu((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet4ui& a) { + return (Packet2l)__lsx_vsllwil_du_wu((__m128i)a, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcast(const Packet4ui& a, const Packet4ui& b) { + return __lsx_vssrlni_hu_w((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet4ui& a, const Packet4ui& b) { + return (Packet8s)__lsx_vssrlni_hu_w((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcast(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c, + const Packet4ui& d) { + Packet8us tmp1 = __lsx_vssrlni_hu_w((__m128i)a, (__m128i)b, 0); + Packet8us tmp2 = __lsx_vssrlni_hu_w((__m128i)c, (__m128i)d, 0); + return __lsx_vssrlni_bu_h((__m128i)tmp1, (__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcast(const Packet4ui& a, const Packet4ui& b, const Packet4ui& c, + const Packet4ui& d) { + Packet8us tmp1 = __lsx_vssrlni_hu_w((__m128i)a, (__m128i)b, 0); + Packet8us tmp2 = __lsx_vssrlni_hu_w((__m128i)c, (__m128i)d, 0); + return (Packet16c)__lsx_vssrlni_bu_h((__m128i)tmp1, (__m128i)tmp2, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet2l& a, const Packet2l& b) { + return __lsx_vffint_s_w(__lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet2l& a, const Packet2l& b) { + return __lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet2l& a, const Packet2l& b) { + return (Packet4ui)__lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, + const Packet2l& d) { + Packet4i tmp1 = __lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0); + Packet4i tmp2 = __lsx_vssrlni_w_d((__m128i)c, (__m128i)d, 0); + return __lsx_vssrlni_h_w((__m128i)tmp1, (__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, + const Packet2l& d) { + Packet4i tmp1 = __lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0); + Packet4i tmp2 = __lsx_vssrlni_w_d((__m128i)c, (__m128i)d, 0); + return (Packet8us)__lsx_vssrlni_h_w((__m128i)tmp1, (__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, + const Packet2l& d, const Packet2l& e, const Packet2l& f, + const Packet2l& g, const Packet2l& h) { + const Packet8s abcd = pcast(a, b, c, d); + const Packet8s efgh = pcast(e, f, g, h); + return __lsx_vssrlni_b_h((__m128i)abcd, (__m128i)efgh, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcast(const Packet2l& a, const Packet2l& b, const Packet2l& c, + const Packet2l& d, const Packet2l& e, const Packet2l& f, + const Packet2l& g, const Packet2l& h) { + const Packet8us abcd = pcast(a, b, c, d); + const Packet8us efgh = pcast(e, f, g, h); + return __lsx_vssrlni_bu_h((__m128i)abcd, (__m128i)efgh, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vffint_s_wu(__lsx_vssrlni_w_d((__m128i)a, (__m128i)b, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet2ul& a, const Packet2ul& b) { + return __lsx_vssrlni_wu_d((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet2ul& a, const Packet2ul& b) { + return (Packet4i)__lsx_vssrlni_wu_d((__m128i)a, (__m128i)b, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, + const Packet2ul& d) { + Packet4ui tmp1 = __lsx_vssrlni_wu_d((__m128i)a, (__m128i)b, 0); + Packet4ui tmp2 = __lsx_vssrlni_wu_d((__m128i)c, (__m128i)d, 0); + return __lsx_vssrlni_hu_w((__m128i)tmp1, (__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, + const Packet2ul& d) { + Packet4ui tmp1 = __lsx_vssrlni_wu_d((__m128i)a, (__m128i)b, 0); + Packet4ui tmp2 = __lsx_vssrlni_wu_d((__m128i)c, (__m128i)d, 0); + return (Packet8s)__lsx_vssrlni_hu_w((__m128i)tmp1, (__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, + const Packet2ul& d, const Packet2ul& e, const Packet2ul& f, + const Packet2ul& g, const Packet2ul& h) { + const Packet8s abcd = pcast(a, b, c, d); + const Packet8s efgh = pcast(e, f, g, h); + return __lsx_vssrlni_b_h((__m128i)abcd, (__m128i)efgh, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcast(const Packet2ul& a, const Packet2ul& b, const Packet2ul& c, + const Packet2ul& d, const Packet2ul& e, const Packet2ul& f, + const Packet2ul& g, const Packet2ul& h) { + const Packet8us abcd = pcast(a, b, c, d); + const Packet8us efgh = pcast(e, f, g, h); + return __lsx_vssrlni_bu_h((__m128i)abcd, (__m128i)efgh, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet4f pcast(const Packet2d& a, const Packet2d& b) { + return __lsx_vfcvt_s_d(b, a); +} +template <> +EIGEN_STRONG_INLINE Packet2l pcast(const Packet2d& a) { + return __lsx_vftint_l_d(a); +} +template <> +EIGEN_STRONG_INLINE Packet2ul pcast(const Packet2d& a) { + return __lsx_vftint_lu_d(a); +} +template <> +EIGEN_STRONG_INLINE Packet4i pcast(const Packet2d& a, const Packet2d& b) { + return __lsx_vssrlni_w_d(__lsx_vftint_l_d(a), __lsx_vftint_l_d(b), 0); +} +template <> +EIGEN_STRONG_INLINE Packet4ui pcast(const Packet2d& a, const Packet2d& b) { + return __lsx_vssrlni_wu_d(__lsx_vftint_lu_d(a), __lsx_vftint_lu_d(b), 0); +} +template <> +EIGEN_STRONG_INLINE Packet8s pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, + const Packet2d& d) { + Packet4i tmp1 = __lsx_vssrlni_w_d(__lsx_vftint_l_d(a), __lsx_vftint_l_d(b), 0); + Packet4i tmp2 = __lsx_vssrlni_w_d(__lsx_vftint_l_d(c), __lsx_vftint_l_d(d), 0); + return __lsx_vssrlni_h_w((__m128i)tmp1, (__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet8us pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, + const Packet2d& d) { + Packet4ui tmp1 = __lsx_vssrlni_wu_d(__lsx_vftint_lu_d(a), __lsx_vftint_lu_d(b), 0); + Packet4ui tmp2 = __lsx_vssrlni_wu_d(__lsx_vftint_lu_d(c), __lsx_vftint_lu_d(d), 0); + return __lsx_vssrlni_hu_w((__m128i)tmp1, (__m128i)tmp2, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16c pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, + const Packet2d& d, const Packet2d& e, const Packet2d& f, + const Packet2d& g, const Packet2d& h) { + const Packet8s abcd = pcast(a, b, c, d); + const Packet8s efgh = pcast(e, f, g, h); + return __lsx_vssrlni_b_h((__m128i)abcd, (__m128i)efgh, 0); +} +template <> +EIGEN_STRONG_INLINE Packet16uc pcast(const Packet2d& a, const Packet2d& b, const Packet2d& c, + const Packet2d& d, const Packet2d& e, const Packet2d& f, + const Packet2d& g, const Packet2d& h) { + const Packet8us abcd = pcast(a, b, c, d); + const Packet8us efgh = pcast(e, f, g, h); + return __lsx_vssrlni_bu_h((__m128i)abcd, (__m128i)efgh, 0); +} + +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet4f& a) { + return __lsx_vfcvtl_d_s(a); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet16c& a) { + Packet8s tmp1 = __lsx_vsllwil_h_b((__m128i)a, 0); + Packet4i tmp2 = __lsx_vsllwil_w_h((__m128i)tmp1, 0); + return __lsx_vffint_d_l(__lsx_vsllwil_d_w((__m128i)tmp2, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet16uc& a) { + Packet8us tmp1 = __lsx_vsllwil_hu_bu((__m128i)a, 0); + Packet4ui tmp2 = __lsx_vsllwil_wu_hu((__m128i)tmp1, 0); + return __lsx_vffint_d_lu(__lsx_vsllwil_du_wu((__m128i)tmp2, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet8s& a) { + Packet4i tmp = __lsx_vsllwil_w_h((__m128i)a, 0); + return __lsx_vffint_d_l(__lsx_vsllwil_d_w((__m128i)tmp, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet8us& a) { + Packet4ui tmp = __lsx_vsllwil_wu_hu((__m128i)a, 0); + return __lsx_vffint_d_lu(__lsx_vsllwil_du_wu((__m128i)tmp, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet4i& a) { + return __lsx_vffint_d_l(__lsx_vsllwil_d_w((__m128i)a, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet4ui& a) { + return __lsx_vffint_d_lu(__lsx_vsllwil_du_wu((__m128i)a, 0)); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet2l& a) { + return __lsx_vffint_d_l(a); +} +template <> +EIGEN_STRONG_INLINE Packet2d pcast(const Packet2ul& a) { + return __lsx_vffint_d_lu(a); +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_TYPE_CASTING_LSX_H diff --git a/Eigen/src/Core/arch/NEON/UnaryFunctors.h b/Eigen/src/Core/arch/NEON/UnaryFunctors.h new file mode 100644 index 0000000..8be5bb0 --- /dev/null +++ b/Eigen/src/Core/arch/NEON/UnaryFunctors.h @@ -0,0 +1,57 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_NEON_UNARY_FUNCTORS_H +#define EIGEN_NEON_UNARY_FUNCTORS_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +#if EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC +/** \internal + * \brief Template specialization of the logistic function for Eigen::half. + */ +template <> +struct scalar_logistic_op { + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half operator()(const Eigen::half& x) const { + // Convert to float and call scalar_logistic_op. + const scalar_logistic_op float_op; + return Eigen::half(float_op(float(x))); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Eigen::half packetOp(const Eigen::half& x) const { return this->operator()(x); } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet4hf packetOp(const Packet4hf& x) const { + const scalar_logistic_op float_op; + return vcvt_f16_f32(float_op.packetOp(vcvt_f32_f16(x))); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE Packet8hf packetOp(const Packet8hf& x) const { + const scalar_logistic_op float_op; + return vcombine_f16(vcvt_f16_f32(float_op.packetOp(vcvt_f32_f16(vget_low_f16(x)))), + vcvt_f16_f32(float_op.packetOp(vcvt_high_f32_f16(x)))); + } +}; + +template <> +struct functor_traits> { + enum { + Cost = functor_traits>::Cost, + PacketAccess = functor_traits>::PacketAccess, + }; +}; +#endif // EIGEN_HAS_ARM64_FP16_VECTOR_ARITHMETIC + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_NEON_UNARY_FUNCTORS_H diff --git a/Eigen/src/Core/arch/SSE/Reductions.h b/Eigen/src/Core/arch/SSE/Reductions.h new file mode 100644 index 0000000..f38df4e --- /dev/null +++ b/Eigen/src/Core/arch/SSE/Reductions.h @@ -0,0 +1,324 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2025 Charlie Schlosser +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_REDUCTIONS_SSE_H +#define EIGEN_REDUCTIONS_SSE_H + +// IWYU pragma: private +#include "../../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +template +struct sse_add_wrapper { + static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return padd(a, b); } +}; + +template +struct sse_mul_wrapper { + static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return pmul(a, b); } +}; + +template +struct sse_min_wrapper { + static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return pmin(a, b); } +}; + +template +struct sse_min_prop_wrapper { + static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { + return pmin(a, b); + } +}; + +template +struct sse_max_wrapper { + static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { return pmax(a, b); } +}; + +template +struct sse_max_prop_wrapper { + static EIGEN_STRONG_INLINE Packet packetOp(const Packet& a, const Packet& b) { + return pmax(a, b); + } +}; + +template +struct sse_predux_common; + +template +struct sse_predux_impl : sse_predux_common> {}; + +template +struct sse_predux_mul_impl : sse_predux_common> {}; + +template +struct sse_predux_min_impl : sse_predux_common> {}; + +template +struct sse_predux_min_prop_impl : sse_predux_common> {}; + +template +struct sse_predux_max_impl : sse_predux_common> {}; + +template +struct sse_predux_max_prop_impl : sse_predux_common> {}; + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet16b -- -- -- -- -- -- -- -- -- -- -- -- */ + +template <> +EIGEN_STRONG_INLINE bool predux(const Packet16b& a) { + Packet4i tmp = _mm_or_si128(a, _mm_unpackhi_epi64(a, a)); + return (pfirst(tmp) != 0) || (pfirst(_mm_shuffle_epi32(tmp, 1)) != 0); +} + +template <> +EIGEN_STRONG_INLINE bool predux_mul(const Packet16b& a) { + Packet4i tmp = _mm_and_si128(a, _mm_unpackhi_epi64(a, a)); + return ((pfirst(tmp) == 0x01010101) && (pfirst(_mm_shuffle_epi32(tmp, 1)) == 0x01010101)); +} + +template <> +EIGEN_STRONG_INLINE bool predux_min(const Packet16b& a) { + return predux_mul(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_max(const Packet16b& a) { + return predux(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet16b& a) { + return predux(a); +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4i -- -- -- -- -- -- -- -- -- -- -- -- */ + +template +struct sse_predux_common { + static EIGEN_STRONG_INLINE int run(const Packet4i& a) { + Packet4i tmp; + tmp = Op::packetOp(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 1, 2, 3))); + tmp = Op::packetOp(tmp, _mm_unpackhi_epi32(tmp, tmp)); + return _mm_cvtsi128_si32(tmp); + } +}; + +template <> +EIGEN_STRONG_INLINE int predux(const Packet4i& a) { + return sse_predux_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE int predux_mul(const Packet4i& a) { + return sse_predux_mul_impl::run(a); +} + +#ifdef EIGEN_VECTORIZE_SSE4_1 +template <> +EIGEN_STRONG_INLINE int predux_min(const Packet4i& a) { + return sse_predux_min_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE int predux_max(const Packet4i& a) { + return sse_predux_max_impl::run(a); +} +#endif + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4i& a) { + return _mm_movemask_ps(_mm_castsi128_ps(a)) != 0x0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4ui -- -- -- -- -- -- -- -- -- -- -- -- */ + +template +struct sse_predux_common { + static EIGEN_STRONG_INLINE uint32_t run(const Packet4ui& a) { + Packet4ui tmp; + tmp = Op::packetOp(a, _mm_shuffle_epi32(a, _MM_SHUFFLE(0, 1, 2, 3))); + tmp = Op::packetOp(tmp, _mm_unpackhi_epi32(tmp, tmp)); + return static_cast(_mm_cvtsi128_si32(tmp)); + } +}; + +template <> +EIGEN_STRONG_INLINE uint32_t predux(const Packet4ui& a) { + return sse_predux_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE uint32_t predux_mul(const Packet4ui& a) { + return sse_predux_mul_impl::run(a); +} + +#ifdef EIGEN_VECTORIZE_SSE4_1 +template <> +EIGEN_STRONG_INLINE uint32_t predux_min(const Packet4ui& a) { + return sse_predux_min_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE uint32_t predux_max(const Packet4ui& a) { + return sse_predux_max_impl::run(a); +} +#endif + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4ui& a) { + return _mm_movemask_ps(_mm_castsi128_ps(a)) != 0x0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet2l -- -- -- -- -- -- -- -- -- -- -- -- */ + +template +struct sse_predux_common { + static EIGEN_STRONG_INLINE int64_t run(const Packet2l& a) { + Packet2l tmp; + tmp = Op::packetOp(a, _mm_unpackhi_epi64(a, a)); + return pfirst(tmp); + } +}; + +template <> +EIGEN_STRONG_INLINE int64_t predux(const Packet2l& a) { + return sse_predux_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2l& a) { + return _mm_movemask_pd(_mm_castsi128_pd(a)) != 0x0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet4f -- -- -- -- -- -- -- -- -- -- -- -- */ + +template +struct sse_predux_common { + static EIGEN_STRONG_INLINE float run(const Packet4f& a) { + Packet4f tmp; + tmp = Op::packetOp(a, _mm_movehl_ps(a, a)); +#ifdef EIGEN_VECTORIZE_SSE3 + tmp = Op::packetOp(tmp, _mm_movehdup_ps(tmp)); +#else + tmp = Op::packetOp(tmp, _mm_shuffle_ps(tmp, tmp, 1)); +#endif + return _mm_cvtss_f32(tmp); + } +}; + +template <> +EIGEN_STRONG_INLINE float predux(const Packet4f& a) { + return sse_predux_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_mul(const Packet4f& a) { + return sse_predux_mul_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { + return sse_predux_min_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { + return sse_predux_min_prop_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_min(const Packet4f& a) { + return sse_predux_min_prop_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { + return sse_predux_max_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { + return sse_predux_max_prop_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE float predux_max(const Packet4f& a) { + return sse_predux_max_prop_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet4f& a) { + return _mm_movemask_ps(a) != 0x0; +} + +/* -- -- -- -- -- -- -- -- -- -- -- -- Packet2d -- -- -- -- -- -- -- -- -- -- -- -- */ + +template +struct sse_predux_common { + static EIGEN_STRONG_INLINE double run(const Packet2d& a) { + Packet2d tmp; + tmp = Op::packetOp(a, _mm_unpackhi_pd(a, a)); + return _mm_cvtsd_f64(tmp); + } +}; + +template <> +EIGEN_STRONG_INLINE double predux(const Packet2d& a) { + return sse_predux_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_mul(const Packet2d& a) { + return sse_predux_mul_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) { + return sse_predux_min_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) { + return sse_predux_min_prop_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_min(const Packet2d& a) { + return sse_predux_min_prop_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { + return sse_predux_max_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { + return sse_predux_max_prop_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE double predux_max(const Packet2d& a) { + return sse_predux_max_prop_impl::run(a); +} + +template <> +EIGEN_STRONG_INLINE bool predux_any(const Packet2d& a) { + return _mm_movemask_pd(a) != 0x0; +} + +} // end namespace internal + +} // end namespace Eigen + +#endif // EIGEN_REDUCTIONS_SSE_H diff --git a/Eigen/src/Core/util/Assert.h b/Eigen/src/Core/util/Assert.h new file mode 100644 index 0000000..09a411a --- /dev/null +++ b/Eigen/src/Core/util/Assert.h @@ -0,0 +1,158 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2022, The Eigen authors. +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_CORE_UTIL_ASSERT_H +#define EIGEN_CORE_UTIL_ASSERT_H + +// Eigen custom assert function. +// +// The combination of Eigen's relative includes and cassert's `assert` function +// (or any usage of the __FILE__ macro) can lead to ODR issues: +// a header included using different relative paths in two different TUs will +// have two different token-for-token definitions, since __FILE__ is expanded +// as an in-line string with different values. Normally this would be +// harmless - the linker would just choose one definition. However, it breaks +// with C++20 modules when functions in different modules have different +// definitions. +// +// To get around this, we need to use __builtin_FILE() when available, which is +// considered a single token, and thus satisfies the ODR. + +// Only define eigen_plain_assert if we are debugging, and either +// - we are not compiling for GPU, or +// - gpu debugging is enabled. +#if !defined(EIGEN_NO_DEBUG) && (!defined(EIGEN_GPU_COMPILE_PHASE) || !defined(EIGEN_NO_DEBUG_GPU)) + +#include + +#ifndef EIGEN_USE_CUSTOM_PLAIN_ASSERT +// Disable new custom asserts by default for now. +#define EIGEN_USE_CUSTOM_PLAIN_ASSERT 0 +#endif + +#if EIGEN_USE_CUSTOM_PLAIN_ASSERT + +#ifndef EIGEN_HAS_BUILTIN_FILE +// Clang can check if __builtin_FILE() is supported. +// GCC > 5, MSVC 2019 14.26 (1926) all have __builtin_FILE(). +// +// For NVCC, it's more complicated. Through trial-and-error: +// - nvcc+gcc supports __builtin_FILE() on host, and on device after CUDA 11. +// - nvcc+msvc supports __builtin_FILE() only after CUDA 11. +#if (EIGEN_HAS_BUILTIN(__builtin_FILE) && (EIGEN_COMP_CLANG || !defined(EIGEN_CUDA_ARCH))) || \ + (EIGEN_GNUC_STRICT_AT_LEAST(5, 0, 0) && (EIGEN_COMP_NVCC >= 110000 || !defined(EIGEN_CUDA_ARCH))) || \ + (EIGEN_COMP_MSVC >= 1926 && (!EIGEN_COMP_NVCC || EIGEN_COMP_NVCC >= 110000)) +#define EIGEN_HAS_BUILTIN_FILE 1 +#else +#define EIGEN_HAS_BUILTIN_FILE 0 +#endif +#endif // EIGEN_HAS_BUILTIN_FILE + +#if EIGEN_HAS_BUILTIN_FILE +#define EIGEN_BUILTIN_FILE __builtin_FILE() +#define EIGEN_BUILTIN_LINE __builtin_LINE() +#else +// Default (potentially unsafe) values. +#define EIGEN_BUILTIN_FILE __FILE__ +#define EIGEN_BUILTIN_LINE __LINE__ +#endif + +// Use __PRETTY_FUNCTION__ when available, since it is more descriptive, as +// __builtin_FUNCTION() only returns the undecorated function name. +// This should still be okay ODR-wise since it is a compiler-specific fixed +// value. Mixing compilers will likely lead to ODR violations anyways. +#if EIGEN_COMP_MSVC +#define EIGEN_BUILTIN_FUNCTION __FUNCSIG__ +#elif EIGEN_COMP_GNUC +#define EIGEN_BUILTIN_FUNCTION __PRETTY_FUNCTION__ +#else +#define EIGEN_BUILTIN_FUNCTION __func__ +#endif + +namespace Eigen { +namespace internal { + +// Generic default assert handler. +template +struct assert_handler_impl { + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static inline void run(const char* expression, const char* file, unsigned line, + const char* function) { +#ifdef EIGEN_GPU_COMPILE_PHASE + // GPU device code doesn't allow stderr or abort, so use printf and raise an + // illegal instruction exception to trigger a kernel failure. +#ifndef EIGEN_NO_IO + printf("Assertion failed at %s:%u in %s: %s\n", file == nullptr ? "" : file, line, + function == nullptr ? "" : function, expression); +#endif + __trap(); + +#else // EIGEN_GPU_COMPILE_PHASE + + // Print to stderr and abort, as specified in . +#ifndef EIGEN_NO_IO + fprintf(stderr, "Assertion failed at %s:%u in %s: %s\n", file == nullptr ? "" : file, line, + function == nullptr ? "" : function, expression); +#endif + std::abort(); + +#endif // EIGEN_GPU_COMPILE_PHASE + } +}; + +// Use POSIX __assert_fail handler when available. +// +// This allows us to integrate with systems that have custom handlers. +// +// NOTE: this handler is not always available on all POSIX systems (otherwise +// we could simply test for __unix__ or similar). The handler function name +// seems to depend on the specific toolchain implementation, and differs between +// compilers, platforms, OSes, etc. Hence, we detect support via SFINAE. +template +struct assert_handler_impl()... // Empty substitution required + // for SFINAE. + ))>, + EmptyArgs...> { + EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE static inline void run(const char* expression, const char* file, unsigned line, + const char* function) { + // GCC requires this call to be dependent on the template parameters. + __assert_fail(expression, file, line, function, std::declval()...); + } +}; + +EIGEN_DEVICE_FUNC EIGEN_DONT_INLINE inline void __assert_handler(const char* expression, const char* file, + unsigned line, const char* function) { + assert_handler_impl<>::run(expression, file, line, function); +} + +} // namespace internal +} // namespace Eigen + +#define eigen_plain_assert(expression) \ + (EIGEN_PREDICT_FALSE(!(expression)) ? Eigen::internal::__assert_handler(#expression, EIGEN_BUILTIN_FILE, \ + EIGEN_BUILTIN_LINE, EIGEN_BUILTIN_FUNCTION) \ + : (void)0) + +#else // EIGEN_USE_CUSTOM_PLAIN_ASSERT + +// Use regular assert. +#define eigen_plain_assert(condition) assert(condition) + +#endif // EIGEN_USE_CUSTOM_PLAIN_ASSERT + +#else // EIGEN_NO_DEBUG + +#define eigen_plain_assert(condition) ((void)0) + +#endif // EIGEN_NO_DEBUG + +#endif // EIGEN_CORE_UTIL_ASSERT_H diff --git a/Eigen/src/Core/util/EmulateArray.h b/Eigen/src/Core/util/EmulateArray.h new file mode 100644 index 0000000..6c4c22d --- /dev/null +++ b/Eigen/src/Core/util/EmulateArray.h @@ -0,0 +1,270 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_EMULATE_ARRAY_H +#define EIGEN_EMULATE_ARRAY_H + +// CUDA doesn't support the STL containers, so we use our own instead. +#if defined(EIGEN_GPUCC) || defined(EIGEN_AVOID_STL_ARRAY) + +namespace Eigen { +template +class array { + public: + typedef T value_type; + typedef T* iterator; + typedef const T* const_iterator; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE iterator begin() { return values; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const_iterator begin() const { return values; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE iterator end() { return values + n; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const_iterator end() const { return values + n; } + + typedef std::reverse_iterator reverse_iterator; + typedef std::reverse_iterator const_reverse_iterator; + + EIGEN_STRONG_INLINE reverse_iterator rbegin() { return reverse_iterator(end()); } + EIGEN_STRONG_INLINE const_reverse_iterator rbegin() const { return const_reverse_iterator(end()); } + + EIGEN_STRONG_INLINE reverse_iterator rend() { return reverse_iterator(begin()); } + EIGEN_STRONG_INLINE const_reverse_iterator rend() const { return const_reverse_iterator(begin()); } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& operator[](size_t index) { + eigen_internal_assert(index < size()); + return values[index]; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& operator[](size_t index) const { + eigen_internal_assert(index < size()); + return values[index]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& at(size_t index) { + eigen_assert(index < size()); + return values[index]; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& at(size_t index) const { + eigen_assert(index < size()); + return values[index]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& front() { return values[0]; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& front() const { return values[0]; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& back() { return values[n - 1]; } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& back() const { return values[n - 1]; } + + EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE static std::size_t size() { return n; } + + T values[n]; + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array() {} + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(const T& v) { + EIGEN_STATIC_ASSERT(n == 1, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(const T& v1, const T& v2) { + EIGEN_STATIC_ASSERT(n == 2, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3) { + EIGEN_STATIC_ASSERT(n == 3, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3, const T& v4) { + EIGEN_STATIC_ASSERT(n == 4, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3, const T& v4, const T& v5) { + EIGEN_STATIC_ASSERT(n == 5, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + values[4] = v5; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3, const T& v4, const T& v5, + const T& v6) { + EIGEN_STATIC_ASSERT(n == 6, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + values[4] = v5; + values[5] = v6; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3, const T& v4, const T& v5, + const T& v6, const T& v7) { + EIGEN_STATIC_ASSERT(n == 7, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + values[4] = v5; + values[5] = v6; + values[6] = v7; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(const T& v1, const T& v2, const T& v3, const T& v4, const T& v5, + const T& v6, const T& v7, const T& v8) { + EIGEN_STATIC_ASSERT(n == 8, YOU_MADE_A_PROGRAMMING_MISTAKE) + values[0] = v1; + values[1] = v2; + values[2] = v3; + values[3] = v4; + values[4] = v5; + values[5] = v6; + values[6] = v7; + values[7] = v8; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array(std::initializer_list l) { + eigen_assert(l.size() == n); + internal::smart_copy(l.begin(), l.end(), values); + } +}; + +// Specialize array for zero size +template +class array { + public: + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& operator[](size_t) { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& operator[](size_t) const { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& front() { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& front() const { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& back() { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& back() const { + eigen_assert(false && "Can't index a zero size array"); + return dummy; + } + + static EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::size_t size() { return 0; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE array() : dummy() {} + + EIGEN_DEVICE_FUNC array(std::initializer_list l) : dummy() { + EIGEN_UNUSED_VARIABLE(l); + eigen_assert(l.size() == 0); + } + + private: + T dummy; +}; + +// Comparison operator +// Todo: implement !=, <, <=, >, and >= +template +EIGEN_DEVICE_FUNC bool operator==(const array& lhs, const array& rhs) { + for (std::size_t i = 0; i < N; ++i) { + if (lhs[i] != rhs[i]) { + return false; + } + } + return true; +} + +namespace internal { +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& array_get(array& a) { + return a[I_]; +} +template +EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& array_get(const array& a) { + return a[I_]; +} + +template +struct array_size > { + static constexpr Index value = N; +}; +template +struct array_size&> { + static constexpr Index value = N; +}; +template +struct array_size > { + static constexpr Index value = N; +}; +template +struct array_size&> { + static constexpr Index value = N; +}; + +} // end namespace internal +} // end namespace Eigen + +#else + +// The compiler supports c++11, and we're not targeting cuda: use std::array as Eigen::array +#include + +namespace Eigen { + +template +using array = std::array; + +namespace internal { +/* std::get is only constexpr in C++14, not yet in C++11 + * - libstdc++ from version 4.7 onwards has it nevertheless, + * so use that + * - libstdc++ older versions: use _M_instance directly + * - libc++ all versions so far: use __elems_ directly + * - all other libs: use std::get to be portable, but + * this may not be constexpr + */ +#if defined(__GLIBCXX__) && __GLIBCXX__ < 20120322 +#define STD_GET_ARR_HACK a._M_instance[I_] +#elif defined(_LIBCPP_VERSION) +#define STD_GET_ARR_HACK a.__elems_[I_] +#else +#define STD_GET_ARR_HACK std::template get(a) +#endif + +template +constexpr T& array_get(std::array& a) { + return (T&)STD_GET_ARR_HACK; +} +template +constexpr T&& array_get(std::array&& a) { + return (T&&)STD_GET_ARR_HACK; +} +template +constexpr T const& array_get(std::array const& a) { + return (T const&)STD_GET_ARR_HACK; +} + +#undef STD_GET_ARR_HACK + +} // end namespace internal +} // end namespace Eigen + +#endif + +#endif // EIGEN_EMULATE_ARRAY_H diff --git a/Eigen/src/Core/util/GpuHipCudaDefines.inc b/Eigen/src/Core/util/GpuHipCudaDefines.inc new file mode 100644 index 0000000..4e10500 --- /dev/null +++ b/Eigen/src/Core/util/GpuHipCudaDefines.inc @@ -0,0 +1,101 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// Copyright (C) 2018 Deven Desai +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#if defined(EIGEN_USE_GPU) && !defined(EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H) +#define EIGEN_CXX11_TENSOR_GPU_HIP_CUDA_DEFINES_H + +// Note that we are using EIGEN_USE_HIP here instead of EIGEN_HIPCC...this is by design +// There is code in the Tensorflow codebase that will define EIGEN_USE_GPU, but +// for some reason gets sent to the gcc/host compiler instead of the gpu/nvcc/hipcc compiler +// When compiling such files, gcc will end up trying to pick up the CUDA headers by +// default (see the code within "unsupported/Eigen/CXX11/Tensor" that is guarded by EIGEN_USE_GPU) +// This will obviously not work when trying to compile tensorflow on a system with no CUDA +// To work around this issue for HIP systems (and leave the default behaviour intact), the +// HIP tensorflow build defines EIGEN_USE_HIP when compiling all source files, and +// "unsupported/Eigen/CXX11/Tensor" has been updated to use HIP header when EIGEN_USE_HIP is +// defined. In continuation of that requirement, the guard here needs to be EIGEN_USE_HIP as well + +#if defined(EIGEN_USE_HIP) + +#define gpuStream_t hipStream_t +#define gpuDeviceProp_t hipDeviceProp_t +#define gpuError_t hipError_t +#define gpuSuccess hipSuccess +#define gpuErrorNotReady hipErrorNotReady +#define gpuGetDeviceCount hipGetDeviceCount +#define gpuGetLastError hipGetLastError +#define gpuPeekAtLastError hipPeekAtLastError +#define gpuGetErrorName hipGetErrorName +#define gpuGetErrorString hipGetErrorString +#define gpuGetDeviceProperties hipGetDeviceProperties +#define gpuStreamDefault hipStreamDefault +#define gpuGetDevice hipGetDevice +#define gpuSetDevice hipSetDevice +#define gpuMalloc hipMalloc +#define gpuFree hipFree +#define gpuMemsetAsync hipMemsetAsync +#define gpuMemset2DAsync hipMemset2DAsync +#define gpuMemcpyAsync hipMemcpyAsync +#define gpuMemcpyDeviceToDevice hipMemcpyDeviceToDevice +#define gpuMemcpyDeviceToHost hipMemcpyDeviceToHost +#define gpuMemcpyHostToDevice hipMemcpyHostToDevice +#define gpuStreamQuery hipStreamQuery +#define gpuSharedMemConfig hipSharedMemConfig +#define gpuDeviceSetSharedMemConfig hipDeviceSetSharedMemConfig +#define gpuStreamSynchronize hipStreamSynchronize +#define gpuDeviceSynchronize hipDeviceSynchronize +#define gpuMemcpy hipMemcpy + +#else + +#define gpuStream_t cudaStream_t +#define gpuDeviceProp_t cudaDeviceProp +#define gpuError_t cudaError_t +#define gpuSuccess cudaSuccess +#define gpuErrorNotReady cudaErrorNotReady +#define gpuGetDeviceCount cudaGetDeviceCount +#define gpuGetLastError cudaGetLastError +#define gpuPeekAtLastError cudaPeekAtLastError +#define gpuGetErrorName cudaGetErrorName +#define gpuGetErrorString cudaGetErrorString +#define gpuGetDeviceProperties cudaGetDeviceProperties +#define gpuStreamDefault cudaStreamDefault +#define gpuGetDevice cudaGetDevice +#define gpuSetDevice cudaSetDevice +#define gpuMalloc cudaMalloc +#define gpuFree cudaFree +#define gpuMemsetAsync cudaMemsetAsync +#define gpuMemset2DAsync cudaMemset2DAsync +#define gpuMemcpyAsync cudaMemcpyAsync +#define gpuMemcpyDeviceToDevice cudaMemcpyDeviceToDevice +#define gpuMemcpyDeviceToHost cudaMemcpyDeviceToHost +#define gpuMemcpyHostToDevice cudaMemcpyHostToDevice +#define gpuStreamQuery cudaStreamQuery +#define gpuSharedMemConfig cudaSharedMemConfig +#define gpuDeviceSetSharedMemConfig cudaDeviceSetSharedMemConfig +#define gpuStreamSynchronize cudaStreamSynchronize +#define gpuDeviceSynchronize cudaDeviceSynchronize +#define gpuMemcpy cudaMemcpy + +#endif + +// gpu_assert can be overridden +#ifndef gpu_assert + +#if defined(EIGEN_HIP_DEVICE_COMPILE) +// HIPCC do not support the use of assert on the GPU side. +#define gpu_assert(COND) +#else +#define gpu_assert(COND) eigen_assert(COND) +#endif + +#endif // gpu_assert + +#endif // EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H diff --git a/Eigen/src/Core/util/GpuHipCudaUndefines.inc b/Eigen/src/Core/util/GpuHipCudaUndefines.inc new file mode 100644 index 0000000..342a323 --- /dev/null +++ b/Eigen/src/Core/util/GpuHipCudaUndefines.inc @@ -0,0 +1,45 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// Copyright (C) 2018 Deven Desai +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#if defined(EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H) + +#ifndef EIGEN_PERMANENTLY_ENABLE_GPU_HIP_CUDA_DEFINES + +#undef gpuStream_t +#undef gpuDeviceProp_t +#undef gpuError_t +#undef gpuSuccess +#undef gpuErrorNotReady +#undef gpuGetDeviceCount +#undef gpuGetErrorString +#undef gpuGetDeviceProperties +#undef gpuStreamDefault +#undef gpuGetDevice +#undef gpuSetDevice +#undef gpuMalloc +#undef gpuFree +#undef gpuMemsetAsync +#undef gpuMemset2DAsync +#undef gpuMemcpyAsync +#undef gpuMemcpyDeviceToDevice +#undef gpuMemcpyDeviceToHost +#undef gpuMemcpyHostToDevice +#undef gpuStreamQuery +#undef gpuSharedMemConfig +#undef gpuDeviceSetSharedMemConfig +#undef gpuStreamSynchronize +#undef gpuDeviceSynchronize +#undef gpuMemcpy + +#endif // EIGEN_PERMANENTLY_ENABLE_GPU_HIP_CUDA_DEFINES + +#undef EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H + +#endif // EIGEN_CORE_GPU_HIP_CUDA_DEFINES_H diff --git a/Eigen/src/Core/util/MaxSizeVector.h b/Eigen/src/Core/util/MaxSizeVector.h new file mode 100644 index 0000000..db5bb89 --- /dev/null +++ b/Eigen/src/Core/util/MaxSizeVector.h @@ -0,0 +1,139 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2014 Benoit Steiner +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_FIXEDSIZEVECTOR_H +#define EIGEN_FIXEDSIZEVECTOR_H + +namespace Eigen { + +/** \class MaxSizeVector + * \ingroup Core_Module + * + * \brief The MaxSizeVector class. + * + * The %MaxSizeVector provides a subset of std::vector functionality. + * + * The goal is to provide basic std::vector operations when using + * std::vector is not an option (e.g. on GPU or when compiling using + * FMA/AVX, as this can cause either compilation failures or illegal + * instruction failures). + * + * Beware: The constructors are not API compatible with these of + * std::vector. + */ +template +class MaxSizeVector { + static const size_t alignment = internal::plain_enum_max(EIGEN_ALIGNOF(T), sizeof(void*)); + + public: + // Construct a new MaxSizeVector, reserve n elements. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE explicit MaxSizeVector(size_t n) + : reserve_(n), size_(0), data_(static_cast(internal::handmade_aligned_malloc(n * sizeof(T), alignment))) {} + + // Construct a new MaxSizeVector, reserve and resize to n. + // Copy the init value to all elements. + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE MaxSizeVector(size_t n, const T& init) + : reserve_(n), size_(n), data_(static_cast(internal::handmade_aligned_malloc(n * sizeof(T), alignment))) { + size_t i = 0; + EIGEN_TRY { + for (; i < size_; ++i) { + new (&data_[i]) T(init); + } + } + EIGEN_CATCH(...) { + // Construction failed, destruct in reverse order: + for (; (i + 1) > 0; --i) { + data_[i - 1].~T(); + } + internal::handmade_aligned_free(data_); + EIGEN_THROW; + } + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE ~MaxSizeVector() { + for (size_t i = size_; i > 0; --i) { + data_[i - 1].~T(); + } + internal::handmade_aligned_free(data_); + } + + void resize(size_t n) { + eigen_assert(n <= reserve_); + for (; size_ < n; ++size_) { + new (&data_[size_]) T; + } + for (; size_ > n; --size_) { + data_[size_ - 1].~T(); + } + eigen_assert(size_ == n); + } + + // Append new elements (up to reserved size). + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void push_back(const T& t) { + eigen_assert(size_ < reserve_); + new (&data_[size_++]) T(t); + } + + // For C++03 compatibility this only takes one argument + template + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void emplace_back(const X& x) { + eigen_assert(size_ < reserve_); + new (&data_[size_++]) T(x); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& operator[](size_t i) const { + eigen_assert(i < size_); + return data_[i]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& operator[](size_t i) { + eigen_assert(i < size_); + return data_[i]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE T& back() { + eigen_assert(size_ > 0); + return data_[size_ - 1]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE const T& back() const { + eigen_assert(size_ > 0); + return data_[size_ - 1]; + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void pop_back() { + eigen_assert(size_ > 0); + data_[--size_].~T(); + } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE size_t size() const { return size_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE bool empty() const { return size_ == 0; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* data() { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* data() const { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* begin() { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr T* end() { return data_ + size_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* begin() const { return data_; } + + EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE constexpr const T* end() const { return data_ + size_; } + + private: + size_t reserve_; + size_t size_; + T* data_; +}; + +} // namespace Eigen + +#endif // EIGEN_FIXEDSIZEVECTOR_H diff --git a/Eigen/src/Core/util/MoreMeta.h b/Eigen/src/Core/util/MoreMeta.h new file mode 100644 index 0000000..6823bca --- /dev/null +++ b/Eigen/src/Core/util/MoreMeta.h @@ -0,0 +1,638 @@ +// This file is part of Eigen, a lightweight C++ template library +// for linear algebra. +// +// Copyright (C) 2008-2015 Gael Guennebaud +// Copyright (C) 2006-2008 Benoit Jacob +// +// This Source Code Form is subject to the terms of the Mozilla +// Public License v. 2.0. If a copy of the MPL was not distributed +// with this file, You can obtain one at http://mozilla.org/MPL/2.0/. + +#ifndef EIGEN_MOREMETA_H +#define EIGEN_MOREMETA_H + +// IWYU pragma: private +#include "../InternalHeaderCheck.h" + +namespace Eigen { + +namespace internal { + +template +struct type_list { + constexpr static int count = sizeof...(tt); +}; + +template +struct type_list { + constexpr static int count = sizeof...(tt) + 1; + typedef t first_type; +}; + +template +struct numeric_list { + constexpr static std::size_t count = sizeof...(nn); +}; + +template +struct numeric_list { + static constexpr std::size_t count = sizeof...(nn) + 1; + static constexpr T first_value = n; +}; + +// Ddoxygen doesn't like the recursive definition of gen_numeric_list. +#ifndef EIGEN_PARSED_BY_DOXYGEN +/* numeric list constructors + * + * equivalencies: + * constructor result + * typename gen_numeric_list::type numeric_list + * typename gen_numeric_list_reversed::type numeric_list + * typename gen_numeric_list_swapped_pair::type numeric_list + * typename gen_numeric_list_repeated::type numeric_list + */ + +template +struct gen_numeric_list : gen_numeric_list {}; + +template +struct gen_numeric_list { + typedef numeric_list type; +}; + +template +struct gen_numeric_list_reversed : gen_numeric_list_reversed {}; +template +struct gen_numeric_list_reversed { + typedef numeric_list type; +}; + +template +struct gen_numeric_list_swapped_pair + : gen_numeric_list_swapped_pair {}; +template +struct gen_numeric_list_swapped_pair { + typedef numeric_list type; +}; + +template +struct gen_numeric_list_repeated : gen_numeric_list_repeated {}; +template +struct gen_numeric_list_repeated { + typedef numeric_list type; +}; +#else +template +struct gen_numeric_list; +#endif // not EIGEN_PARSED_BY_DOXYGEN + +/* list manipulation: concatenate */ + +template +struct concat; + +template +struct concat, type_list> { + typedef type_list type; +}; +template +struct concat, numeric_list> { + typedef numeric_list type; +}; + +template +struct mconcat; +template +struct mconcat { + typedef a type; +}; +template +struct mconcat : concat {}; +template +struct mconcat : concat::type> {}; + +/* list manipulation: extract slices */ + +template +struct take; + +template +struct take> : concat, typename take>::type> {}; + +template +struct take> { + typedef type_list<> type; +}; + +template +struct take<0, type_list> { + typedef type_list<> type; +}; + +template <> +struct take<0, type_list<>> { + typedef type_list<> type; +}; + +template +struct take> + : concat, typename take>::type> {}; + +template +struct take<0, numeric_list> { + typedef numeric_list type; +}; + +template +struct take<0, numeric_list> { + typedef numeric_list type; +}; + +template +struct h_skip_helper_numeric; +template +struct h_skip_helper_numeric : h_skip_helper_numeric {}; +template +struct h_skip_helper_numeric { + typedef numeric_list type; +}; +template +struct h_skip_helper_numeric { + typedef numeric_list type; +}; +template +struct h_skip_helper_numeric { + typedef numeric_list type; +}; + +template +struct h_skip_helper_type; +template +struct h_skip_helper_type : h_skip_helper_type {}; +template +struct h_skip_helper_type<0, t, tt...> { + typedef type_list type; +}; +template +struct h_skip_helper_type { + typedef type_list<> type; +}; +template <> +struct h_skip_helper_type<0> { + typedef type_list<> type; +}; + +template +struct h_skip { + template + constexpr static EIGEN_STRONG_INLINE typename h_skip_helper_numeric::type helper( + numeric_list) { + return typename h_skip_helper_numeric::type(); + } + template + constexpr static EIGEN_STRONG_INLINE typename h_skip_helper_type::type helper(type_list) { + return typename h_skip_helper_type::type(); + } +}; + +template +struct skip { + typedef decltype(h_skip::helper(a())) type; +}; + +template +struct slice : take::type> {}; + +/* list manipulation: retrieve single element from list */ + +template +struct get; + +template +struct get> : get> {}; +template +struct get<0, type_list> { + typedef a type; +}; + +template +struct get> : get> {}; +template +struct get<0, numeric_list> { + constexpr static T value = a; +}; + +template +constexpr T array_get(const numeric_list&) { + return get<(int)n, numeric_list>::value; +} + +/* always get type, regardless of dummy; good for parameter pack expansion */ + +template +struct id_numeric { + typedef t type; +}; +template +struct id_type { + typedef t type; +}; + +/* equality checking, flagged version */ + +template +struct is_same_gf : is_same { + constexpr static int global_flags = 0; +}; + +/* apply_op to list */ + +template class op, typename additional_param, typename... values> +struct h_apply_op_helper { + typedef type_list::type...> type; +}; +template