inverse_mtf-inl.h (2231B)
1 // Copyright (c) the JPEG XL Project Authors. All rights reserved. 2 // 3 // Use of this source code is governed by a BSD-style 4 // license that can be found in the LICENSE file. 5 6 // SIMDified inverse-move-to-front transform. 7 8 #if defined(LIB_JXL_INVERSE_MTF_INL_H_) == defined(HWY_TARGET_TOGGLE) 9 #ifdef LIB_JXL_INVERSE_MTF_INL_H_ 10 #undef LIB_JXL_INVERSE_MTF_INL_H_ 11 #else 12 #define LIB_JXL_INVERSE_MTF_INL_H_ 13 #endif 14 15 #include <hwy/highway.h> 16 17 #include "lib/jxl/sanitizers.h" 18 19 HWY_BEFORE_NAMESPACE(); 20 namespace jxl { 21 namespace HWY_NAMESPACE { 22 23 // These templates are not found via ADL. 24 using hwy::HWY_NAMESPACE::FirstN; 25 using hwy::HWY_NAMESPACE::IfThenElse; 26 using hwy::HWY_NAMESPACE::Load; 27 using hwy::HWY_NAMESPACE::LoadU; 28 using hwy::HWY_NAMESPACE::StoreU; 29 30 inline void MoveToFront(uint8_t* v, uint8_t index) { 31 uint8_t value = v[index]; 32 uint8_t i = index; 33 if (i < 4) { 34 for (; i; --i) v[i] = v[i - 1]; 35 } else { 36 const HWY_CAPPED(uint8_t, 64) d; 37 int tail = i & (Lanes(d) - 1); 38 if (tail) { 39 i -= tail; 40 const auto vec = Load(d, v + i); 41 const auto prev = LoadU(d, v + i + 1); 42 StoreU(IfThenElse(FirstN(d, tail), vec, prev), d, v + i + 1); 43 } 44 while (i) { 45 i -= Lanes(d); 46 const auto vec = Load(d, v + i); 47 StoreU(vec, d, v + i + 1); 48 } 49 } 50 v[0] = value; 51 } 52 53 inline void InverseMoveToFrontTransform(uint8_t* v, int v_len) { 54 HWY_ALIGN uint8_t mtf[256 + 64]; 55 int i; 56 for (i = 0; i < 256; ++i) { 57 mtf[i] = static_cast<uint8_t>(i); 58 } 59 #if JXL_MEMORY_SANITIZER 60 const HWY_CAPPED(uint8_t, 64) d; 61 for (size_t j = 0; j < Lanes(d); ++j) { 62 mtf[256 + j] = 0; 63 } 64 #endif // JXL_MEMORY_SANITIZER 65 for (i = 0; i < v_len; ++i) { 66 uint8_t index = v[i]; 67 v[i] = mtf[index]; 68 if (index) MoveToFront(mtf, index); 69 } 70 } 71 72 // NOLINTNEXTLINE(google-readability-namespace-comments) 73 } // namespace HWY_NAMESPACE 74 } // namespace jxl 75 HWY_AFTER_NAMESPACE(); 76 77 #endif // LIB_JXL_INVERSE_MTF_INL_H_ 78 79 #if HWY_ONCE 80 #ifndef INVERSE_MTF_ONCE 81 #define INVERSE_MTF_ONCE 82 83 namespace jxl { 84 inline void InverseMoveToFrontTransform(uint8_t* v, int v_len) { 85 HWY_STATIC_DISPATCH(InverseMoveToFrontTransform)(v, v_len); 86 } 87 } // namespace jxl 88 89 #endif // INVERSE_MTF_ONCE 90 #endif // HWY_ONCE