commit ff09371267315c39fe0220082943c5834db04ab9
Author: jpegxl-bot <noreply@jpegxl.appspotmail.com>
Date:   Fri Dec 27 18:12:34 2019 +0100

    Update JPEG-XL with latest changes.
    
    This includes all changes up to 2019-12-27 17:38:53 +0100.

diff --git a/.clang-format b/.clang-format
new file mode 100644
index 000000000..a61b61c56
--- /dev/null
+++ b/.clang-format
@@ -0,0 +1,4 @@
+BasedOnStyle: Google
+IncludeCategories:
+  - Regex:           '^<hwy/'
+    Priority:        2
diff --git a/.clang-tidy b/.clang-tidy
new file mode 100644
index 000000000..4e11ff95a
--- /dev/null
+++ b/.clang-tidy
@@ -0,0 +1,56 @@
+# Disabled checks:
+# - google-readability-todo: We don't use the google TODO format.
+#
+# - modernize-deprecated-headers: We don't use std:: versions of the standard
+#   types and functions like size_t or printf, so we should include <stdio.h>
+#   instead <cstdio>.
+# - modernize-return-braced-init-list: this often doesn't improve readability.
+# - modernize-use-auto: is too aggressive towards using auto.
+# - modernize-use-default-member-init: with a mix of constructors and default
+#   member initialization this can be confusing if enforced.
+# - modernize-use-using: typedefs are ok.
+#
+# - readability-else-after-return: It doesn't always improve readability.
+# - readability-static-accessed-through-instance
+#   It is often more useful and readable to access a constant of a passed
+#   variable (like d.N) instead of using the type of the variable that could be
+#   long and complex.
+
+Checks: >-
+  clang-*,
+  -clang-diagnostic-unused-command-line-argument,
+  google-*,
+  modernize-*,
+  performance-*,
+  readability-*,
+  -google-readability-todo,
+  -modernize-deprecated-headers,
+  -modernize-return-braced-init-list,
+  -modernize-use-auto,
+  -modernize-use-default-member-init,
+  -modernize-use-using,
+  -readability-else-after-return,
+  -readability-static-accessed-through-instance,
+
+
+WarningsAsErrors: >-
+  clang-*,
+  clang-analyzer-*,
+  -clang-diagnostic-unused-command-line-argument,
+  google-build-using-namespace,
+  google-explicit-constructor,
+  google-readability-namespace-comments
+
+# We are only interested in the headers from this projects, excluding
+# third_party/ and build/.
+HeaderFilterRegex: '^.*/(jxl|fuif|tools)/.*\.h$'
+
+CheckOptions:
+  - key:             readability-braces-around-statements.ShortStatementLines
+    value:           '2'
+  - key:             google-readability-braces-around-statements.ShortStatementLines
+    value:           '2'
+  - key:             readability-implicit-bool-conversion.AllowPointerConditions
+    value:           '1'
+  - key:             readability-implicit-bool-conversion.AllowIntegerConditions
+    value:           '1'
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 000000000..cc4295c07
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,7 @@
+# Build output directories
+/build
+/build*
+/docker/*.log
+
+# The downloaded corpora files for benchmark.
+/third_party/corpora
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 000000000..ea5cff7aa
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,548 @@
+# Copyright (c) the JPEG XL Project
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+# We define only two stages for development. The "build" stage only compiles the
+# code and doesn't run any target code. The "test" stage runs the code compiled
+# by the previous stage. Stages form dependencies in a block, all the build
+# targets in one stage need to be completed before any of the next stage can
+# start.
+stages:
+  - build
+  - test
+  - deploy
+
+variables:
+  # We need to fetch the third_party dependencies together with the code unless
+  # otherwise noted.
+  GIT_SUBMODULE_STRATEGY: recursive
+
+# A template for running in the generic cloud builders. These are tagged with
+# "linux" and run on shared VMs.
+.linux_builder_template: &linux_builder_template
+  image: &jpegxl-builder gcr.io/jpegxl/jpegxl-builder@sha256:b4744eed15a00e88d6faf54798c8d8bca8b58a581bfdda374cfa8c668ec6c1b2
+  tags:
+    - linux
+  # By default all the workflows run on master and on request. This can be
+  # override by users of this template.
+  only:
+    - master
+    - tags
+    - schedules
+  # Retry on system failures. This can be typically caused by the runner or VM
+  # being evicted.
+  retry:
+    max: 2
+    when:
+      - unknown_failure
+      - api_failure
+      - stuck_or_timeout_failure
+      - runner_system_failure
+
+# A template for building on x86_64 images on the cloud builders.
+.linux_x86_64_template: &linux_x86_64_template
+  <<: *linux_builder_template
+  variables:
+    CC: clang-6.0
+    CXX: clang++-6.0
+
+.linux_i686_template: &linux_i686_template
+  <<: *linux_builder_template
+  variables:
+    CC: clang-6.0
+    CXX: clang++-6.0
+    # This makes cross-compiling for i686 the default when calling ci.sh.
+    BUILD_TARGET: i686-linux-gnu
+    BUILD_DIR: build
+
+# A template for building on aarch64 images on the cloud builders. These may not
+# run on actual aarch64 hardware.
+.linux_aarch64_template: &linux_aarch64_template
+  <<: *linux_builder_template
+  variables:
+    CC: clang-6.0
+    CXX: clang++-6.0
+    # This makes cross-compiling for aarch64 the default when calling ci.sh.
+    BUILD_TARGET: aarch64-linux-gnu
+    BUILD_DIR: build
+    # We need to fetch the third_party dependencies together with the code.
+    GIT_SUBMODULE_STRATEGY: recursive
+
+# A template for cross-compiling for arm-linux-gnueabihf.
+.linux_armhf_template: &linux_armhf_template
+  <<: *linux_builder_template
+  variables:
+    CC: clang-6.0
+    CXX: clang++-6.0
+    # This makes cross-compiling for armhf the default when calling ci.sh.
+    BUILD_TARGET: arm-linux-gnueabihf
+    BUILD_DIR: build
+    # We need to fetch the third_party dependencies together with the code.
+    GIT_SUBMODULE_STRATEGY: recursive
+
+# Common artifacts for the build stage
+.default_build_paths: &default_build_paths
+  - build/libjpegxl.so*
+  - build/tools/cjpegxl
+  - build/tools/djpegxl
+  - build/tools/benchmark_xl
+  - build/tools/decode_and_encode
+  - build/tools/comparison_viewer/compare_codecs
+  - build/tools/comparison_viewer/compare_images
+  - build/tools/viewer/viewer
+  # All the test binaries in the build/ directory and .cmake files used by
+  # ctest, compressed into this file together to save space when uploading.
+  - build/tests.tar.xz
+  # The coverage data produced at compile time.
+  - build/gcno.tar.xz
+
+.linux_x86_64_build_template: &linux_x86_64_build_template
+  <<: *linux_x86_64_template
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+    paths: *default_build_paths
+
+.linux_i686_build_template: &linux_i686_build_template
+  <<: *linux_i686_template
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+    paths: *default_build_paths
+
+.linux_aarch64_build_template: &linux_aarch64_build_template
+  <<: *linux_aarch64_template
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+    paths: *default_build_paths
+
+.linux_armhf_build_template: &linux_armhf_build_template
+  <<: *linux_armhf_template
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+    paths: *default_build_paths
+
+# The default Release build runs always.
+build:x86_64:clang:release:
+  <<: *linux_x86_64_build_template
+  stage: build
+  only:
+    - master
+    - merge_requests
+    - schedules
+    - tags
+  script:
+    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh release -DCMAKE_INSTALL_PREFIX=`pwd`/prefix
+    # Test that the package can be installed.
+    - ninja -C build install
+
+test:x86_64:clang:release:
+  <<: *linux_x86_64_template
+  stage: test
+  only:
+    - master
+    - merge_requests
+    - schedules
+    - tags
+  dependencies:
+    - build:x86_64:clang:release
+  script:
+    - ./ci.sh test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+
+# x86_84 test coverage build
+build:x86_64:clang:coverage:
+  <<: *linux_x86_64_build_template
+  stage: build
+  only:
+    - master
+    - schedules
+    - tags
+  script:
+    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh coverage
+
+test:x86_64:clang:coverage:
+  <<: *linux_x86_64_template
+  stage: test
+  only:
+    - master
+    - schedules
+    - tags
+  dependencies:
+    - build:x86_64:clang:coverage
+  script:
+    - ./ci.sh test
+    - ./ci.sh coverage_report
+  variables:
+    CC: clang-6.0
+    CXX: clang++-6.0
+    # Headers from submodules might be needed when generating the HTML reports.
+    GIT_SUBMODULE_STRATEGY: recursive
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 year
+    paths:
+      - build/coverage.txt
+      - build/coverage*.html
+      - build/coverage.xml
+
+# Updates the "Pages" section in GitLab with the latest coverage report from
+# master.
+pages:
+  <<: *linux_x86_64_template
+  stage: deploy
+  only:
+    - master
+  dependencies:
+    - test:x86_64:clang:coverage
+  script:
+    - mkdir public
+    - mv build/coverage* public/
+    - mv public/coverage.html public/index.html
+    - sed -E "s,GCC Code Coverage Report,Coverage Report for <a href='${CI_PROJECT_URL}/commit/${CI_COMMIT_SHA}'>${CI_COMMIT_SHORT_SHA}</a>," -i public/index.html
+  artifacts:
+    paths:
+      - public
+
+
+# x86_64 Scalar-only release runs only on master and on request.
+build:x86_64:clang:scalar:
+  <<: *linux_x86_64_build_template
+  stage: build
+  only:
+    - master
+    - merge_requests
+    - schedules
+    - tags
+  script:
+    - SKIP_TEST=1 PACK_TEST=1 CMAKE_CXX_FLAGS="-DHWY_STATIC_TARGETS=0 -DHWY_RUNTIME_TARGETS=0" ./ci.sh release
+
+test:x86_64:clang:scalar:
+  <<: *linux_x86_64_template
+  stage: test
+  dependencies:
+    - build:x86_64:clang:scalar
+  script:
+    - ./ci.sh test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+
+# i686 (x86 32-bit) builders
+build:i686:clang:release:
+  <<: *linux_i686_build_template
+  stage: build
+  only:
+    - master
+    - merge_requests
+    - schedules
+    - tags
+  script:
+    # AVX2 on 32-bit binaries hits clang compiler bugs.
+    - CMAKE_FLAGS="-DHWY_DISABLE_AVX2" SKIP_TEST=1 PACK_TEST=1
+        ./ci.sh release -DJPEGXL_ENABLE_TCMALLOC=OFF -DJPEGXL_ENABLE_FUZZERS=OFF
+
+test:i686:clang:release:
+  <<: *linux_i686_template
+  stage: test
+  dependencies:
+    - build:i686:clang:release
+  script:
+    - ./ci.sh test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+
+# aarch64 release runs only on master and on request.
+build:aarch64:clang:release:
+  <<: *linux_aarch64_build_template
+  stage: build
+  only:
+    - master
+    - merge_requests
+    - schedules
+    - tags
+    - web
+  script:
+    - CMAKE_CXX_FLAGS="-DJXL_DISABLE_SLOW_TESTS" SKIP_TEST=1 PACK_TEST=1
+        ./ci.sh release
+
+test:aarch64:clang:release:
+  <<: *linux_aarch64_template
+  stage: test
+  dependencies:
+    - build:aarch64:clang:release
+  script:
+    # LeakSanitizer doesn't work when running under qemu-arm.
+    - ASAN_OPTIONS=detect_leaks=0 ./ci.sh test
+
+# arm release
+build:armhf:clang:release:
+  <<: *linux_armhf_build_template
+  stage: build
+  only:
+    - master
+    - merge_requests
+    - schedules
+    - tags
+    - web
+  script:
+    - CMAKE_CXX_FLAGS="-DJXL_DISABLE_SLOW_TESTS" SKIP_TEST=1 PACK_TEST=1
+        ./ci.sh release
+
+test:armhf:clang:release:
+  <<: *linux_armhf_template
+  stage: test
+  dependencies:
+    - build:armhf:clang:release
+  script:
+    - ./ci.sh test
+
+# arm debug
+build:armhf:clang:debug:
+  <<: *linux_armhf_build_template
+  stage: build
+  only:
+    - merge_requests
+  script:
+    - SKIP_TEST=1 ./ci.sh debug
+
+# "debug" build enabled for merge_requests. This is here to check that
+# debug-only statements still compile.
+build:x86_64:clang:debug:
+  <<: *linux_x86_64_build_template
+  stage: build
+  only:
+    - merge_requests
+  script:
+    - SKIP_TEST=1 ./ci.sh debug
+
+# "asan", "tsan" and "msan" builds only run on master, tags and when requested
+# from the web. Only supported in x86_64 for now.
+build:x86_64:clang:asan:
+  <<: *linux_x86_64_build_template
+  stage: build
+  script:
+    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh asan
+
+test:x86_64:clang:asan:
+  <<: *linux_x86_64_template
+  stage: test
+  dependencies:
+    - build:x86_64:clang:asan
+  script:
+    - ./ci.sh test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+
+# aarch64 asan build and test.
+build:aarch64:clang:asan:
+  <<: *linux_aarch64_build_template
+  stage: build
+  script:
+    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh asan
+
+test:aarch64:clang:asan:
+  <<: *linux_aarch64_template
+  stage: test
+  dependencies:
+    - build:aarch64:clang:asan
+  # Disable asan test on arm since they timeout.
+  only:
+    - tags
+  script:
+    - ./ci.sh test
+
+build:x86_64:clang:tsan:
+  <<: *linux_x86_64_build_template
+  stage: build
+  script:
+    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh tsan
+
+test:x86_64:clang:tsan:
+  <<: *linux_x86_64_template
+  stage: test
+  dependencies:
+    - build:x86_64:clang:tsan
+  script:
+    - ./ci.sh test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+
+build:x86_64:clang:msan:
+  <<: *linux_x86_64_build_template
+  stage: build
+  script:
+    - SKIP_TEST=1 PACK_TEST=1 ./ci.sh msan
+
+test:x86_64:clang:msan:
+  <<: *linux_x86_64_template
+  stage: test
+  dependencies:
+    - build:x86_64:clang:msan
+  script:
+    - ./ci.sh test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+
+# Merge request linter checks.
+build:lint:
+  <<: *linux_x86_64_template
+  stage: build
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  only:
+    - merge_requests
+  script:
+    - LINT_OUTPUT=clang-format.patch ./ci.sh lint
+  allow_failure: true
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 month
+    when: on_failure
+    paths:
+      - clang-format.patch
+
+# Most clang-tidy checks are currently informational only since these are
+# currently broken, however several are marked as errors.
+build:tidy:
+  <<: *linux_x86_64_template
+  stage: build
+  only:
+    - merge_requests
+  script:
+    - CMAKE_BUILD_TYPE="Release" ./ci.sh tidy
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 month
+    when: always
+    paths:
+      - build/clang-tidy.txt
+
+# Faster benchmark over a smaller set of images.
+.benchmark_x86_64_template: &benchmark_x86_64_template
+  <<: *linux_x86_64_template
+  stage: test
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 year
+    paths:
+      - build/benchmark_results/
+
+test:fast_benchmark:release:
+  <<: *benchmark_x86_64_template
+  only:
+    - master
+    - merge_requests
+    - schedules
+    - tags
+  dependencies:
+    - build:x86_64:clang:release
+  script:
+    - STORE_IMAGES=0 ./ci.sh fast_benchmark
+
+test:fast_benchmark:asan:
+  <<: *benchmark_x86_64_template
+  dependencies:
+    - build:x86_64:clang:asan
+  script:
+    - STORE_IMAGES=0 ./ci.sh fast_benchmark
+
+test:fast_benchmark.msan:
+  <<: *benchmark_x86_64_template
+  dependencies:
+    - build:x86_64:clang:msan
+  script:
+    - STORE_IMAGES=0 ./ci.sh fast_benchmark
+
+# This template runs on actual aarch64 hardware.
+.benchmark_aarch64_template: &benchmark_aarch64_template
+  <<: *linux_builder_template
+  image: gcr.io/jpegxl/jpegxl-builder-run-aarch64@sha256:4ff7fe5e3e1d19a8c4604b72a41916173d83ec95d335f7fabd599ad92c438df2
+  stage: test
+  tags:
+    - aarch64
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 year
+    paths:
+      - build/benchmark_results/
+
+test:aarch64:fast_benchmark:release:
+  <<: *benchmark_aarch64_template
+  dependencies:
+    - build:aarch64:clang:release
+  script:
+    # Bind to the big CPUs only.
+    - ./ci.sh cpuset "${RUNNER_CPU_BIG}"
+    - STORE_IMAGES=0 ./ci.sh fast_benchmark
+
+# Benchmark test that runs separately on the big and little CPUs of the runner.
+test:aarch64:arm_benchmark:release:
+  <<: *benchmark_aarch64_template
+  only:
+    - web
+    - master
+  dependencies:
+    - build:aarch64:clang:release
+  script:
+    - ./ci.sh arm_benchmark
+
+# Benchmark ToT on nightly builds. These are scheduled at midnight UTC.
+test:benchmark:release:
+  image: *jpegxl-builder
+  stage: test
+  only:
+    - schedules
+  variables:
+    GIT_SUBMODULE_STRATEGY: none
+  dependencies:
+    - build:x86_64:clang:release
+  script:
+    - STORE_IMAGES=0 ./ci.sh benchmark
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 year
+    paths:
+      - build/benchmark_results/
+
+build:tidy:all:
+  <<: *linux_x86_64_template
+  stage: build
+  only:
+    - master
+  script:
+    - CMAKE_BUILD_TYPE="Release" ./ci.sh tidy all
+  allow_failure: true
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 year
+    when: always
+    paths:
+      - build/clang-tidy.txt
+
+# Emscripten (WASM) build.
+build:ems:all:
+  <<: *linux_x86_64_template
+  stage: build
+  variables:
+    CC: emcc
+    CXX: em++
+    EMSDK: /opt/emsdk
+    EM_CONFIG: /opt/emsdk/.emscripten
+    CMAKE_TOOLCHAIN_FILE: /opt/emsdk/upstream/emscripten/cmake/Modules/Platform/Emscripten.cmake
+  artifacts:
+    name: "$CI_JOB_NAME-$CI_COMMIT_REF_NAME"
+    expire_in: 1 week
+    paths:
+      - build/jpegxl_emcc.*
+  script:
+    - source "${EMSDK}/emsdk_env.sh"
+    - SKIP_TEST=1 ./ci.sh release
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 000000000..8c9e574a0
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,24 @@
+[submodule "third_party/brotli"]
+	path = third_party/brotli
+	url = https://github.com/google/brotli
+[submodule "third_party/lodepng"]
+	path = third_party/lodepng
+	url = https://github.com/lvandeve/lodepng
+[submodule "third_party/lcms"]
+	path = third_party/lcms
+	url = https://github.com/mm2/Little-CMS
+[submodule "third_party/googletest"]
+	path = third_party/googletest
+	url = https://github.com/google/googletest
+[submodule "third_party/gperftools"]
+	path = third_party/gperftools
+	url = https://github.com/gperftools/gperftools
+[submodule "third_party/brunsli"]
+	path = third_party/brunsli
+	url = https://github.com/google/brunsli.git
+[submodule "third_party/sjpeg"]
+	path = third_party/sjpeg
+	url = https://github.com/webmproject/sjpeg.git
+[submodule "third_party/skcms"]
+	path = third_party/skcms
+	url = https://skia.googlesource.com/skcms
diff --git a/CMakeLists.txt b/CMakeLists.txt
new file mode 100644
index 000000000..94d8717bc
--- /dev/null
+++ b/CMakeLists.txt
@@ -0,0 +1,234 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ubuntu bionic ships with cmake 3.10.
+cmake_minimum_required(VERSION 3.10)
+
+# Set compilers to default to Clang is not set.
+if ("$ENV{CC}" STREQUAL "")
+  set(ENV{CC} clang)
+endif()
+if ("$ENV{CXX}" STREQUAL "")
+  set(ENV{CXX} clang++)
+endif()
+
+# Honor VISIBILITY_INLINES_HIDDEN on all types of targets.
+if(POLICY CMP0063)
+  cmake_policy(SET CMP0063 NEW)
+endif()
+# Pass CMAKE_EXE_LINKER_FLAGS to CC and CXX compilers when testing if they work.
+if(POLICY CMP0065)
+  cmake_policy(SET CMP0065 NEW)
+endif()
+
+project(JPEGXL LANGUAGES C CXX)
+
+include(CheckCXXSourceCompiles)
+check_cxx_source_compiles(
+   "int main() {
+      #if !defined(__EMSCRIPTEN__)
+      static_assert(false, \"__EMSCRIPTEN__ is not defined\");
+      #endif
+      return 0;
+    }"
+  JPEGXL_EMSCRIPTEN
+)
+if(JPEGXL_EMSCRIPTEN)
+  message("-- Compiler is EMSCRIPTEN.")
+  set(JPEGXL_NOT_EMSCRIPTEN false)
+  set(CMAKE_STATIC_LIBRARY_SUFFIX ".bc")
+  set(CMAKE_EXECUTABLE_SUFFIX ".html")
+  add_definitions(-DPROFILER_ENABLED=0)
+else()
+  message("-- Compiler is not EMSCRIPTEN.")
+  set(JPEGXL_NOT_EMSCRIPTEN true)
+endif()
+
+### Project build options:
+if(NOT APPLE AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64"
+   AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm")
+  # Enabled by default except on arm64 and Apple builds.
+  set(ENABLE_FUZZERS_DEFAULT true)
+endif()
+if(NOT APPLE AND NOT JPEGXL_EMSCRIPTEN
+   AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64"
+   AND NOT ${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm")
+  # Enabled by default except on arm64 and Apple builds.
+  set(ENABLE_TCMALLOC_DEFAULT true)
+endif()
+
+set(JPEGXL_ENABLE_FUZZERS ${ENABLE_FUZZERS_DEFAULT} CACHE BOOL
+    "Build JPEGXL fuzzer targets")
+set(JPEGXL_ENABLE_BENCHMARK ${JPEGXL_NOT_EMSCRIPTEN} CACHE BOOL
+    "Build JPEGXL benchmark tools")
+set(JPEGXL_ENABLE_SJPEG true CACHE BOOL
+    "Build JPEGXL with support for encoding with sjpeg")
+set(JPEGXL_ENABLE_SKCMS false CACHE BOOL
+    "Where possible, use skcms instead of lcms2")
+set(JPEGXL_ENABLE_VIEWERS ${JPEGXL_NOT_EMSCRIPTEN} CACHE BOOL
+    "Build JPEGXL viewer tools for evaluation")
+set(JPEGXL_ENABLE_TCMALLOC ${ENABLE_TCMALLOC_DEFAULT} CACHE BOOL
+    "Build JPEGXL using gperftools (tcmalloc) allocator.")
+set(JPEGXL_ENABLE_PLUGINS true CACHE BOOL
+    "Build third-party plugings to support JPEG XL in other applications.")
+set(JPEGXL_ENABLE_COVERAGE false CACHE BOOL
+    "Enable code coverage tracking for libjpegxl. This also enables debug and disables optimizations.")
+
+# Require clang for building.
+if(NOT ${CMAKE_C_COMPILER_ID} MATCHES "Clang" OR
+   NOT ${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+  message(FATAL_ERROR "${CMAKE_CXX_COMPILER_ID} compiler is not supported.\n"
+    "Use clang instead:\n  CC=clang CXX=clang++ cmake ..")
+endif()
+if (${CMAKE_C_COMPILER_VERSION} VERSION_LESS 6 OR
+    ${CMAKE_CXX_COMPILER_VERSION} VERSION_LESS 6)
+  message(FATAL_ERROR
+    "Minimum Clang version required is Clang 6, please update.")
+endif()
+
+# CMAKE_EXPORT_COMPILE_COMMANDS is used to generate the compilation database
+# used by clang-tidy.
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
+
+# Global compiler flags for all targets here and in subdirectories.
+add_definitions(
+  # Avoid changing the binary based on the current time and date.
+  -D__DATE__="redacted"
+  -D__TIMESTAMP__="redacted"
+  -D__TIME__="redacted"
+)
+
+# In CMake before 3.12 it is problematic to pass repeated flags like -Xclang.
+# For this reason we place them in CMAKE_CXX_FLAGS instead.
+# See https://gitlab.kitware.com/cmake/cmake/issues/15826
+
+if (NOT "${JPEGXL_EMSCRIPTEN}")
+
+# Pretty colorful messages within reasonable limits.
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+  -Xclang -ferror-limit -Xclang 19 -Xclang -fmessage-length -Xclang 0 \
+  -Xclang -fdiagnostics-show-option -fcolor-diagnostics")
+
+# Machine flags.
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+  -Xclang -mrelax-all \
+  -Xclang -mrelocation-model -Xclang pic \
+  -Xclang -pic-level -Xclang 2 \
+  -Xclang -mconstructor-aliases \
+  -Xclang -mpie-copy-relocations \
+  -Xclang -munwind-tables")
+
+if(WIN32)
+# Not supported by clang-cl, but frame pointers are default on Windows
+else()
+  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fno-omit-frame-pointer")
+endif()
+
+# CPU flags
+if(${CMAKE_SYSTEM_PROCESSOR} MATCHES "x86_64")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+  -mavx2 \
+  -mfma \
+  -Xclang -mprefer-vector-width=128 \
+  -Xclang -target-cpu -Xclang haswell \
+  -Xclang -target-feature -Xclang +avx2")
+elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "AMD64")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /arch:AVX2")
+elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "aarch64")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+  -Xclang -mprefer-vector-width=128 \
+  -Xclang -target-cpu -Xclang cortex-a72")
+elseif(${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} \
+  -mcpu=cortex-a7 -mfpu=neon-vfpv4")
+endif()
+
+endif()  # JPEGXL_EMSCRIPTEN
+
+# Force build with optimizations in release mode.
+set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O2")
+
+add_compile_options(
+  # Ignore this to allow redefining __DATE__ and others.
+  -Wno-builtin-macro-redefined
+
+  # Global warning settings.
+  -Wall
+)
+
+if (NOT WIN32)
+add_compile_options(-Werror)
+endif ()
+
+include(GNUInstallDirs)
+
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED YES)
+
+add_subdirectory(third_party)
+
+set(THREADS_PREFER_PTHREAD_FLAG YES)
+find_package(Threads REQUIRED)
+
+# Enable tests regardless of where are they defined.
+enable_testing()
+include(CTest)
+
+# The jpegxl library definition. This also defines the internal list of flags we
+# use in our project (library, tests and tools): JPEGXL_INTERNAL_FLAGS.
+include(jpegxl.cmake)
+
+if (NOT "${JPEGXL_EMSCRIPTEN}")
+
+# Other libraries outside the core jpegxl library.
+include(jpegxl_extras.cmake)
+include(jpegxl_threads.cmake)
+
+# Plugins for third-party software
+if (${JPEGXL_ENABLE_PLUGINS})
+add_subdirectory(plugins)
+endif ()
+
+# Binary tools
+add_subdirectory(tools)
+
+# Unittests
+cmake_policy(SET CMP0057 NEW)  # https://gitlab.kitware.com/cmake/cmake/issues/18198
+include(GoogleTest)
+
+# Tests for the jpegxl library.
+include(jpegxltests.cmake)
+
+else()  # JPEGXL_EMSCRIPTEN
+
+# WASM API facade.
+add_executable(jpegxl_emcc jxl/jpegxl_emcc.cc)
+target_link_libraries(jpegxl_emcc jpegxl-static)
+set_target_properties(jpegxl_emcc PROPERTIES LINK_FLAGS "\
+  -O3\
+  -s TOTAL_MEMORY=256MB\
+  -s EXPORTED_FUNCTIONS='[\
+    \"_decompress\",\
+    \"_free\",\
+    \"_freeImage\",\
+    \"_getImageWidth\",\
+    \"_getImageHeight\",\
+    \"_getImagePixels\",\
+    \"_malloc\"\
+  ]'\
+")
+
+endif ()  # JPEGXL_EMSCRIPTEN
+
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 000000000..3e3c9f968
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,10 @@
+# How to Contribute
+
+We are currently unable to accept patches to this project, but we'd very much
+appreciate if you open a Gitlab issue for any bug reports/feature requests.
+We will be happy to investigate and hopefully fix any issues.
+
+# Community Guidelines
+
+This project follows
+[Google's Open Source Community Guidelines](https://opensource.google.com/conduct/).
diff --git a/CONTRIBUTORS b/CONTRIBUTORS
new file mode 100644
index 000000000..848096f92
--- /dev/null
+++ b/CONTRIBUTORS
@@ -0,0 +1,23 @@
+# This files lists individuals who made significant contributions to the JPEG XL
+# code base, such as design, adding features, performing experiments, ...
+# Small changes such as a small bugfix or fixing spelling errors are not
+# included. If you'd like to be included in this file thanks to a significant
+# contribution, feel free to send a pull request changing this file.
+Alex Deymo
+Alexander Rhatushnyak
+Evgenii Kliuchnikov
+Iulia-Maria Comșa
+Jan Wassenberg
+Jon Sneyers
+Jyrki Alakuijala
+Krzysztof Potempa
+Lode Vandevenne
+Luca Versari
+Martin Bruse
+Moritz Firsching
+Renata Khasanova
+Robert Obryk
+Sami Boukortt
+Sebastian Gomez-Gonzalez
+Thomas Fischbacher
+Zoltan Szabadka
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 000000000..6b0b1270f
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,203 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
diff --git a/README.OSX.md b/README.OSX.md
new file mode 100644
index 000000000..a2179848b
--- /dev/null
+++ b/README.OSX.md
@@ -0,0 +1,39 @@
+## Disclaimer
+
+OSX builds have "best effort" support, i.e. build might not work at all, some
+tests may fail and some sub-projects are excluded from build.
+
+This manual outlines OSX specific setup. For general building and testing
+instructions see "[README](README.md)" and
+"[Building and Testing changes](doc/building_and_testing.md)".
+
+## Dependencies
+
+[Homebrew](https://brew.sh/) is a popular package manager that can be used for
+installing dependencies.
+
+Make sure that `brew doctor` does not report serious problems and up-to-date
+version of XCode is installed.
+
+Installing (actually, building) `clang` might take a couple hours.
+
+```shell
+brew install llvm
+```
+
+```shell
+brew install clang-format coreutils cmake giflib libjpeg ninja parallel
+```
+
+If `git-clang-format` command is not accessible, run
+
+```shell
+brew link --overwrite clang-format
+```
+
+Before building the project check that `which clang` is
+`/usr/local/opt/llvm/bin/clang`, not the one provided by XCode. If not, update
+`PATH` environment variable.
+
+Also, `export CMAKE_PREFIX_PATH=/usr/local/opt/giflib` might be necessary for
+correct include paths resolving.
diff --git a/README.md b/README.md
new file mode 100644
index 000000000..21e959d6c
--- /dev/null
+++ b/README.md
@@ -0,0 +1,83 @@
+# JPEG XL reference implementation
+
+This repository contains a reference implementation of JPEG XL (encoder and
+decoder). As [previously announced](https://jpeg.org/items/20190803_press.html),
+it is available under a royalty-free and open source license (Apache 2).
+
+**NOTE**
+
+JPEG XL is in the final stages of standardization, but minor changes to the
+codestream are still likely. WARNING: until further notice, do not depend on
+future decoders being able to decode the output of a current encoder.
+
+## Checking out the code
+
+This repository uses git submodules to handle some third party dependencies
+under `third_party/`. To check out these dependencies as well clone the
+repository with `--recursive`:
+
+```shell
+git clone git@gitlab.com:wg1/jpeg-xl.git --recursive
+```
+
+If you didn't check out recursively, and after any update run the following
+command to check out the git submodules.
+
+```shell
+git submodule update --init --recursive
+```
+
+## Minimum build dependencies
+
+Apart from the dependencies in third_party, some of the tools use external
+dependencies that need to be installed in your system first. For a Debian/Ubuntu
+based Linux distribution install:
+
+```shell
+sudo apt install cmake clang-6.0 g++-8 qtbase5-dev libqt5x11extras5-dev \
+  extra-cmake-modules libgif-dev libjpeg-dev ninja-build
+```
+
+For developing changes in JPEG XL, take a look at the
+[Building and Testing changes](doc/building_and_testing.md) guide.
+
+Make sure your default "clang" compiler is at least version 6 running
+
+```bash
+clang --version
+```
+
+If it still shows an old version despite having a clang-6.0 installed, you need
+to update the default clang compiler. In Debian-based systems run:
+
+```bash
+sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-6.0 100
+sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-6.0 100
+```
+
+## Building
+
+The project builds with cmake. We currently support Linux (tested with Debian).
+To build the "release" version, you can use the following helper command:
+
+```shell
+./ci.sh release
+```
+
+This will build and test the project, and leave the binaries in the `build/`
+directory. Check out the `tools` subdirectory for command-line tools that
+interact with the library. You can set the environment variable SKIP_TEST=1 to
+skip the test stage.
+
+There are other build versions with more debug information useful when
+developing. You can read more about it in the
+[Building and Testing changes](doc/building_and_testing.md) guide.
+
+## Documentation
+
+*   [Developing in GitLab](doc/developing_in_gitlab.md)
+*   [XL Overview](doc/xl_overview.md)
+*   [Building and Testing changes](doc/building_and_testing.md)
+*   [Software Contribution Guidelines](doc/guidelines.md)
+*   [JPEG XL committee draft](https://arxiv.org/abs/1908.03565)
+*   [Introductory paper](https://www.spiedigitallibrary.org/proceedings/Download?fullDOI=10.1117%2F12.2529237)
diff --git a/ci.sh b/ci.sh
new file mode 100755
index 000000000..00e03adb6
--- /dev/null
+++ b/ci.sh
@@ -0,0 +1,956 @@
+#!/bin/bash
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Continuous integration helper module. This module is meant to be called from
+# the .gitlab-ci.yml file during the continuous integration build, as well as
+# from the command line for developers.
+
+set -eu
+
+OS=`uname -s`
+
+MYDIR=$(dirname $(realpath "$0"))
+
+### Environment parameters:
+CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE:-RelWithDebInfo}
+CMAKE_PREFIX_PATH=${CMAKE_PREFIX_PATH:-}
+SKIP_TEST="${SKIP_TEST:-0}"
+BUILD_TARGET="${BUILD_TARGET:-}"
+if [[ -n "${BUILD_TARGET}" ]]; then
+  BUILD_DIR="${BUILD_DIR:-${MYDIR}/build-${BUILD_TARGET%%-*}}"
+else
+  BUILD_DIR="${BUILD_DIR:-${MYDIR}/build}"
+fi
+# Whether we should post a message in the MR when the build fails.
+POST_MESSAGE_ON_ERROR="${POST_MESSAGE_ON_ERROR:-1}"
+
+# Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS
+CMAKE_FLAGS=${CMAKE_FLAGS:-}
+CMAKE_C_FLAGS=${CMAKE_C_FLAGS:-${CMAKE_FLAGS}}
+CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS:-${CMAKE_FLAGS}}
+CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-}
+CMAKE_MODULE_LINKER_FLAGS=${CMAKE_MODULE_LINKER_FLAGS:-}
+CMAKE_SHARED_LINKER_FLAGS=${CMAKE_SHARED_LINKER_FLAGS:-}
+CMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE:-}
+
+# Version inferred from the CI variables.
+CI_COMMIT_SHA=${CI_COMMIT_SHA:-}
+JPEGXL_VERSION=${JPEGXL_VERSION:-${CI_COMMIT_SHA:0:8}}
+
+# Benchmark parameters
+STORE_IMAGES=${STORE_IMAGES:-1}
+BENCHMARK_CORPORA="${MYDIR}/third_party/corpora"
+
+# Local flags passed to sanitizers.
+UBSAN_FLAGS=(
+  -fsanitize=alignment
+  -fsanitize=bool
+  -fsanitize=bounds
+  -fsanitize=builtin
+  -fsanitize=enum
+  -fsanitize=float-cast-overflow
+  -fsanitize=float-divide-by-zero
+  -fsanitize=integer-divide-by-zero
+  -fsanitize=null
+  -fsanitize=object-size
+  -fsanitize=pointer-overflow
+  -fsanitize=return
+  -fsanitize=returns-nonnull-attribute
+  -fsanitize=shift-base
+  -fsanitize=shift-exponent
+  -fsanitize=unreachable
+  -fsanitize=vla-bound
+
+  -fno-sanitize-recover=undefined
+  # Brunsli uses unaligned accesses to uint32_t, so alignment is just a warning.
+  -fsanitize-recover=alignment
+)
+# -fsanitize=function doesn't work on aarch64 and arm.
+if [[ "${BUILD_TARGET%%-*}" != "aarch64" &&
+    "${BUILD_TARGET%%-*}" != "arm" ]]; then
+  UBSAN_FLAGS+=(
+    -fsanitize=function
+  )
+fi
+if [[ "${BUILD_TARGET%%-*}" != "arm" ]]; then
+  UBSAN_FLAGS+=(
+    -fsanitize=signed-integer-overflow
+  )
+fi
+
+CLANG_TIDY_BIN=$(which clang-tidy-6.0 clang-tidy-7 clang-tidy-8 | head -n 1)
+# Default to "cat" if "colordiff" is not installed or if stdout is not a tty.
+if [[ -t 1 ]]; then
+  COLORDIFF_BIN=$(which colordiff cat | head -n 1)
+else
+  COLORDIFF_BIN="cat"
+fi
+
+CLANG_VERSION="${CLANG_VERSION:-}"
+# Detect the clang version suffix and store it in CLANG_VERSION. For example,
+# "6.0" for clang 6 or "7" for clang 7.
+detect_clang_version() {
+  if [[ -n "${CLANG_VERSION}" ]]; then
+    return 0
+  fi
+  local clang_version=$("${CC:-clang}" --version | head -n1)
+  local llvm_tag
+  case "${clang_version}" in
+    "clang version 6."*)
+      CLANG_VERSION="6.0"
+      ;;
+    "clang version 7."*)
+      CLANG_VERSION="7"
+      ;;
+    "clang version 8."*)
+      CLANG_VERSION="8"
+      ;;
+    "clang version 9."*)
+      CLANG_VERSION="9"
+      ;;
+    *)
+      echo "Unknown clang version: ${clang_version}" >&2
+      return 1
+  esac
+}
+
+# Temporary files cleanup hooks.
+CLEANUP_FILES=()
+cleanup() {
+  if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
+    rm -fr "${CLEANUP_FILES[@]}"
+  fi
+}
+
+# Executed on exit.
+on_exit() {
+  local retcode="$1"
+  # Always cleanup the CLEANUP_FILES.
+  cleanup
+
+  # Post a message in the MR when requested with POST_MESSAGE_ON_ERROR but only
+  # if the run failed and we are not running from a MR pipeline.
+  if [[ ${retcode} -ne 0 && -n "${CI_BUILD_NAME:-}" &&
+        -n "${POST_MESSAGE_ON_ERROR}" && -z "${CI_MERGE_REQUEST_ID:-}" &&
+        "${CI_BUILD_REF_NAME}" = "master" ]]; then
+    load_mr_vars_from_commit
+    { set +xeu; } 2>/dev/null
+    local message="**Run ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} failed.**
+
+Check the output of the job at ${CI_JOB_URL:-} to see if this was your problem.
+If it was, please rollback this change or fix the problem ASAP, broken builds
+slow down development. Check if the error already existed in the previous build
+as well.
+
+Pipeline: ${CI_PIPELINE_URL}
+
+Previous build commit: ${CI_COMMIT_BEFORE_SHA}
+"
+    cmd_post_mr_comment "${message}"
+  fi
+}
+
+trap 'retcode=$?; { set +x; } 2>/dev/null; on_exit ${retcode}' INT TERM EXIT
+
+
+# These variables are populated when calling merge_request_commits().
+
+# The current hash at the top of the current branch or merge request branch (if
+# running from a merge request pipeline).
+MR_HEAD_SHA=""
+# The common ancestor between the current commit and the tracked branch, such
+# as master. This includes a list
+MR_ANCESTOR_SHA=""
+
+# Populate MR_HEAD_SHA and MR_ANCESTOR_SHA.
+merge_request_commits() {
+  { set +x; } 2>/dev/null
+  # CI_BUILD_REF is the reference currently being build in the CI workflow.
+  MR_HEAD_SHA=$(git -C "${MYDIR}" rev-parse -q "${CI_BUILD_REF:-HEAD}")
+  if [[ -z "${CI_MERGE_REQUEST_IID:-}" ]]; then
+    # We are in a local branch, not a merge request.
+    MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q HEAD@{upstream} || true)
+  else
+    # Merge request pipeline in CI. In this case the upstream is called "origin"
+    # but it refers to the forked project that's the source of the merge
+    # request. We need to get the target of the merge request, for which we need
+    # to query that repository using our CI_JOB_TOKEN.
+    echo "machine gitlab.com login gitlab-ci-token password ${CI_JOB_TOKEN}" \
+      >> "${HOME}/.netrc"
+    git -C "${MYDIR}" fetch "${CI_MERGE_REQUEST_PROJECT_URL}" \
+      "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}"
+    MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q FETCH_HEAD)
+  fi
+  if [[ -z "${MR_ANCESTOR_SHA}" ]]; then
+    echo "Warning, not tracking any branch, using the last commit in HEAD.">&2
+    # This prints the return value with just HEAD.
+    MR_ANCESTOR_SHA=$(git -C "${MYDIR}" rev-parse -q "${MR_HEAD_SHA}^")
+  else
+    MR_ANCESTOR_SHA=$(git -C "${MYDIR}" merge-base --all \
+      "${MR_ANCESTOR_SHA}" "${MR_HEAD_SHA}")
+  fi
+  set -x
+}
+
+# Load the MR iid from the landed commit message when running not from a
+# merge request workflow. This is useful to post back results at the merge
+# request when running pipelines from master.
+load_mr_vars_from_commit() {
+  { set +x; } 2>/dev/null
+  if [[ -z "${CI_MERGE_REQUEST_IID:-}" ]]; then
+    local mr_iid=$(git rev-list --format=%B --max-count=1 HEAD |
+      grep -F "${CI_PROJECT_URL}/merge_requests" | head -n 1)
+    # mr_iid contains a string like this if it matched:
+    #  Part-of: <https://gitlab.com/wg1/jpeg-xlm/merge_requests/123456>
+    if [[ -n "${mr_iid}" ]]; then
+      mr_iid=$(echo "${mr_iid}" |
+        sed -E 's,^.*merge_requests/([0-9]+)>.*$,\1,')
+      CI_MERGE_REQUEST_IID="${mr_iid}"
+      CI_MERGE_REQUEST_PROJECT_ID=${CI_PROJECT_ID}
+    fi
+  fi
+  set -x
+}
+
+# Posts a comment to the current merge request.
+cmd_post_mr_comment() {
+  { set +x; } 2>/dev/null
+  local comment="$1"
+  if [[ -n "${BOT_TOKEN:-}" && -n "${CI_MERGE_REQUEST_IID:-}" ]]; then
+    local url="${CI_API_V4_URL}/projects/${CI_MERGE_REQUEST_PROJECT_ID}/merge_requests/${CI_MERGE_REQUEST_IID}/notes"
+    curl -X POST -g \
+      -H "PRIVATE-TOKEN: ${BOT_TOKEN}" \
+      --data-urlencode "body=${comment}" \
+      --output /dev/null \
+      "${url}"
+  fi
+  set -x
+}
+
+cmake_configure() {
+  local args=(
+    -B"${BUILD_DIR}" -H"${MYDIR}"
+    -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}"
+    -G Ninja
+    -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}"
+    -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}"
+    -DCMAKE_TOOLCHAIN_FILE="${CMAKE_TOOLCHAIN_FILE}"
+    -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}"
+    -DCMAKE_MODULE_LINKER_FLAGS="${CMAKE_MODULE_LINKER_FLAGS}"
+    -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}"
+    -DCMAKE_PREFIX_PATH="${CMAKE_PREFIX_PATH}"
+    -DJPEGXL_VERSION="${JPEGXL_VERSION}"
+  )
+  if [[ -n "${BUILD_TARGET}" ]]; then
+    # If set, BUILD_TARGET must be the target triplet such as
+    # x86_64-unknown-linux-gnu.
+    args+=(
+      -DCMAKE_C_COMPILER_TARGET="${BUILD_TARGET}"
+      -DCMAKE_CXX_COMPILER_TARGET="${BUILD_TARGET}"
+      # Only the first element of the target triplet.
+      -DCMAKE_SYSTEM_PROCESSOR="${BUILD_TARGET%%-*}"
+      -DCMAKE_SYSTEM_NAME=Linux
+      # These are needed to make googletest work when cross-compiling.
+      -DCMAKE_CROSSCOMPILING=1
+      -DHAVE_STD_REGEX=0
+      -DHAVE_POSIX_REGEX=0
+      -DHAVE_GNU_POSIX_REGEX=0
+      -DHAVE_STEADY_CLOCK=0
+      -DHAVE_THREAD_SAFETY_ATTRIBUTES=0
+    )
+    # Use pkg-config for the target:
+    local pkg_config=$(which "${BUILD_TARGET}-pkg-config" || true)
+    if [[ -n "${pkg_config}" ]]; then
+      args+=(-DPKG_CONFIG_EXECUTABLE="${pkg_config}")
+    fi
+  fi
+  cmake "${args[@]}" "$@"
+}
+
+cmake_build_and_test() {
+  # gtest_discover_tests() runs the test binaries to discover the list of tests
+  # at build time, which fails under qemu.
+  ASAN_OPTIONS=detect_leaks=0 cmake --build "${BUILD_DIR}"
+  # Pack test binaries if requested.
+  if [[ "${PACK_TEST:-}" == "1" ]]; then
+    (cd "${BUILD_DIR}"
+     find -name '*.cmake' -a '!' -path '*CMakeFiles*'
+     find -type d -name tests -a '!' -path '*CMakeFiles*'
+    ) | tar -C "${BUILD_DIR}" -cf "${BUILD_DIR}/tests.tar.xz" -T - \
+      --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
+    du -h "${BUILD_DIR}/tests.tar.xz"
+    # Pack coverage data if also available.
+    touch "${BUILD_DIR}/gcno.sentinel"
+    (cd "${BUILD_DIR}"; echo gcno.sentinel; find -name '*gcno') | \
+      tar -C "${BUILD_DIR}" -cvf "${BUILD_DIR}/gcno.tar.xz" -T - \
+        --use-compress-program="xz --threads=$(nproc --all || echo 1) -6"
+  fi
+
+  if [[ "${SKIP_TEST}" -ne "1" ]]; then
+    (cd "${BUILD_DIR}"
+     export UBSAN_OPTIONS=print_stacktrace=1
+     ctest -j $(nproc --all || echo 1) --output-on-failure)
+  fi
+}
+
+# Configure the build to strip unused functions. This considerably reduces the
+# output size, specially for tests which only use a small part of the whole
+# library.
+strip_dead_code() {
+  # Emscripten does tree shaking without any extra flags.
+  if [[ "${CMAKE_TOOLCHAIN_FILE##*/}" == "Emscripten.cmake" ]]; then
+    return 0
+  fi
+  # -ffunction-sections, -fdata-sections and -Wl,--gc-sections effectively
+  # discard all unreachable code, reducing the code size. For this to work, we
+  # need to also pass --no-export-dynamic to prevent it from exporting all the
+  # internal symbols (like functions) making them all reachable and thus not a
+  # candidate for removal.
+  CMAKE_CXX_FLAGS+=" -ffunction-sections -fdata-sections"
+  CMAKE_C_FLAGS+=" -ffunction-sections -fdata-sections"
+  if [[ "${OS}" == "Darwin" ]]; then
+    CMAKE_EXE_LINKER_FLAGS+=" -dead_strip"
+    CMAKE_SHARED_LINKER_FLAGS+=" -dead_strip"
+  else
+    CMAKE_EXE_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
+    CMAKE_SHARED_LINKER_FLAGS+=" -Wl,--gc-sections -Wl,--no-export-dynamic"
+  fi
+}
+
+### Externally visible commands
+
+cmd_debug() {
+  CMAKE_BUILD_TYPE="Debug"
+  cmake_configure "$@"
+  cmake_build_and_test
+}
+
+cmd_release() {
+  CMAKE_BUILD_TYPE="Release"
+  strip_dead_code
+  cmake_configure "$@"
+  cmake_build_and_test
+}
+
+cmd_opt() {
+  CMAKE_BUILD_TYPE="RelWithDebInfo"
+  CMAKE_CXX_FLAGS+=" -DJXL_DEBUG_WARNING -DJXL_DEBUG_ON_ERROR"
+  cmake_configure "$@"
+  cmake_build_and_test
+}
+
+cmd_coverage() {
+  cmd_release -DJPEGXL_ENABLE_COVERAGE=ON "$@"
+
+  if [[ "${SKIP_TEST}" -ne "1" ]]; then
+    # If we didn't run the test we also don't print a coverage report.
+    cmd_coverage_report
+  fi
+}
+
+cmd_coverage_report() {
+  detect_clang_version
+  LLVM_COV=$(which "llvm-cov-${CLANG_VERSION}")
+  local real_build_dir=$(realpath ${BUILD_DIR})
+  local gcovr_args=(
+    -r "${real_build_dir}"
+    --gcov-executable "${LLVM_COV} gcov"
+    # Only print coverage information for the jxl and fuif directories. The rest
+    # is not part of the code under test.
+    --filter '.*jxl/.*'
+    --exclude '.*_test.cc'
+    --object-directory "${real_build_dir}"
+  )
+
+  (
+   cd "${real_build_dir}"
+    gcovr "${gcovr_args[@]}" --html --html-details \
+      --output="${real_build_dir}/coverage.html"
+    gcovr "${gcovr_args[@]}" --print-summary |
+      tee "${real_build_dir}/coverage.txt"
+    gcovr "${gcovr_args[@]}" --xml --output="${real_build_dir}/coverage.xml"
+  )
+}
+
+cmd_test() {
+  # Unpack tests if needed.
+  if [[ -e "${BUILD_DIR}/tests.tar.xz" && ! -d "${BUILD_DIR}/tests" ]]; then
+    tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/tests.tar.xz"
+  fi
+  if [[ -e "${BUILD_DIR}/gcno.tar.xz" && ! -d "${BUILD_DIR}/gcno.sentinel" ]]; then
+    tar -C "${BUILD_DIR}" -Jxvf "${BUILD_DIR}/gcno.tar.xz"
+  fi
+  (cd "${BUILD_DIR}"
+   export UBSAN_OPTIONS=print_stacktrace=1
+   ctest -j $(nproc --all || echo 1) --output-on-failure)
+}
+
+cmd_asan() {
+  detect_clang_version
+  LLVM_SYMBOLIZER=$(which llvm-symbolizer "llvm-symbolizer-${CLANG_VERSION}" | \
+    head -n1)
+  LLVM_SYMBOLIZER="$(realpath "${LLVM_SYMBOLIZER}" || true)"
+
+  CMAKE_C_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
+    -fsanitize=address ${UBSAN_FLAGS[@]}"
+  CMAKE_CXX_FLAGS+=" -DJXL_ENABLE_ASSERT=1 -g -DADDRESS_SANITIZER \
+    -fsanitize=address ${UBSAN_FLAGS[@]}"
+  strip_dead_code
+  cmake_configure "$@"
+  export ASAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
+  export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
+  cmake_build_and_test
+}
+
+cmd_tsan() {
+  local tsan_args=(
+    -DJXL_ENABLE_ASSERT=1
+    -g
+    -DTHREAD_SANITIZER
+    ${UBSAN_FLAGS[@]}
+    -fsanitize=thread
+  )
+  CMAKE_C_FLAGS+=" ${tsan_args[@]}"
+  CMAKE_CXX_FLAGS+=" ${tsan_args[@]}"
+
+  CMAKE_BUILD_TYPE="RelWithDebInfo"
+  cmake_configure "$@" -DJPEGXL_ENABLE_TCMALLOC=OFF
+  cmake_build_and_test
+}
+
+cmd_msan() {
+  detect_clang_version
+  local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
+  if [[ ! -d "${msan_prefix}" || -e "${msan_prefix}/lib/libc++abi.a" ]]; then
+    # Install msan libraries for this version if needed or if an older version
+    # with libc++abi was installed.
+    cmd_msan_install
+  fi
+
+  local msan_c_flags=(
+    -fsanitize=memory
+    -fno-omit-frame-pointer
+    -fsanitize-memory-track-origins
+
+    -DJXL_ENABLE_ASSERT=1
+    -g
+    -DMEMORY_SANITIZER
+
+    # Force gtest to not use the cxxbai.
+    -DGTEST_HAS_CXXABI_H_=0
+  )
+  local msan_cxx_flags=(
+    "${msan_c_flags[@]}"
+
+    # Some C++ sources don't use the std at all, so the -stdlib=libc++ is unused
+    # in those cases. Ignore the warning.
+    -Wno-unused-command-line-argument
+    -stdlib=libc++
+
+    # We include the libc++ from the msan directory instead, so we don't want
+    # the std includes.
+    -nostdinc++
+    -cxx-isystem"${msan_prefix}/include/c++/v1"
+  )
+
+  local msan_linker_flags=(
+    -L"${msan_prefix}"/lib
+    -Wl,-rpath -Wl,"${msan_prefix}"/lib/
+  )
+
+  LLVM_SYMBOLIZER=$(which llvm-symbolizer "llvm-symbolizer-${CLANG_VERSION}" | \
+    head -n1)
+  LLVM_SYMBOLIZER="$(realpath "${LLVM_SYMBOLIZER}" || true)"
+
+  CMAKE_C_FLAGS+=" ${msan_c_flags[@]} ${UBSAN_FLAGS[@]}"
+  CMAKE_CXX_FLAGS+=" ${msan_cxx_flags[@]} ${UBSAN_FLAGS[@]}"
+  CMAKE_EXE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
+  CMAKE_MODULE_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
+  CMAKE_SHARED_LINKER_FLAGS+=" ${msan_linker_flags[@]}"
+  strip_dead_code
+  cmake_configure "$@" \
+    -DCMAKE_CROSSCOMPILING=1 -DRUN_HAVE_STD_REGEX=0 -DRUN_HAVE_POSIX_REGEX=0
+  export MSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
+  export UBSAN_SYMBOLIZER_PATH="${LLVM_SYMBOLIZER}"
+  cmake_build_and_test
+}
+
+# Install libc++ libraries compiled with msan in the msan_prefix for the current
+# compiler version.
+cmd_msan_install() {
+  local tmpdir=$(mktemp -d)
+  CLEANUP_FILES+=("${tmpdir}")
+  # Detect the llvm to install:
+  export CC="${CC:-clang}"
+  export CXX="${CXX:-clang++}"
+  detect_clang_version
+  local llvm_tag
+  case "${CLANG_VERSION}" in
+    "6.0")
+      llvm_tag="llvmorg-6.0.1"
+      ;;
+    "7")
+      llvm_tag="llvmorg-7.0.1"
+      ;;
+    "8")
+      llvm_tag="llvmorg-8.0.0"
+      ;;
+    "9")
+      llvm_tag="llvmorg-9.0.0"
+      ;;
+    *)
+      echo "Unknown clang version: ${CLANG_VERSION}" >&2
+      return 1
+  esac
+  local llvm_targz="${tmpdir}/${llvm_tag}.tar.gz"
+  curl -L --show-error -o "${llvm_targz}" \
+    "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz"
+  tar -C "${tmpdir}" -zxf "${llvm_targz}"
+  local llvm_root="${tmpdir}/llvm-project-${llvm_tag}"
+
+  local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
+  rm -rf "${msan_prefix}"
+
+  declare -A CMAKE_EXTRAS
+  CMAKE_EXTRAS[libcxx]="\
+    -DLIBCXX_CXX_ABI=libstdc++ \
+    -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON"
+
+  for project in libcxx; do
+    local proj_build="${tmpdir}/build-${project}"
+    local proj_dir="${llvm_root}/${project}"
+    mkdir -p "${proj_build}"
+    cmake -B"${proj_build}" -H"${proj_dir}" \
+      -G Ninja \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_USE_SANITIZER=Memory \
+      -DLLVM_PATH="${llvm_root}/llvm" \
+      -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \
+                            head -n1)" \
+      -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
+      -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \
+      -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \
+      -DCMAKE_SHARED_LINKER_FLAGS="${CMAKE_SHARED_LINKER_FLAGS}" \
+      -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \
+      ${CMAKE_EXTRAS[${project}]}
+    cmake --build "${proj_build}"
+    ninja -C "${proj_build}" install
+  done
+}
+
+cmd_fast_benchmark() {
+  local small_corpus_tar="${BENCHMARK_CORPORA}/jyrki-full.tar"
+  mkdir -p "${BENCHMARK_CORPORA}"
+  curl --show-error -o "${small_corpus_tar}" -z "${small_corpus_tar}" \
+    "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/jyrki-full.tar"
+
+  local tmpdir=$(mktemp -d)
+  CLEANUP_FILES+=("${tmpdir}")
+  tar -xf "${small_corpus_tar}" -C "${tmpdir}"
+
+  run_benchmark "${tmpdir}" 1048576
+}
+
+cmd_benchmark() {
+  local nikon_corpus_tar="${BENCHMARK_CORPORA}/nikon-subset.tar"
+  mkdir -p "${BENCHMARK_CORPORA}"
+  curl --show-error -o "${nikon_corpus_tar}" -z "${nikon_corpus_tar}" \
+    "https://storage.googleapis.com/artifacts.jpegxl.appspot.com/corpora/nikon-subset.tar"
+
+  local tmpdir=$(mktemp -d)
+  CLEANUP_FILES+=("${tmpdir}")
+  tar -xvf "${nikon_corpus_tar}" -C "${tmpdir}"
+
+  local sem_id="jpegxl_benchmark-$$"
+  local nprocs=$(nproc --all || echo 1)
+  images=()
+  local filename
+  while IFS= read -r filename; do
+    # This removes the './'
+    filename="${filename:2}"
+    local mode
+    if [[ "${filename:0:4}" == "srgb" ]]; then
+      mode="RGB_D65_SRG_Rel_SRG"
+    elif [[ "${filename:0:5}" == "adobe" ]]; then
+      mode="RGB_D65_Ado_Rel_Ado"
+    else
+      echo "Unknown image colorspace: ${filename}" >&2
+      exit 1
+    fi
+    png_filename="${filename%.ppm}.png"
+    png_filename=$(echo "${png_filename}" | tr '/' '_')
+    sem --bg --id "${sem_id}" -j"${nprocs}" -- \
+      "${BUILD_DIR}/tools/decode_and_encode" \
+        "${tmpdir}/${filename}" "${mode}" "${tmpdir}/${png_filename}"
+    images+=( "${png_filename}" )
+  done < <(cd "${tmpdir}"; find . -name '*.ppm' -type f)
+  sem --id "${sem_id}" --wait
+
+  # We need about 10 GiB per thread on these images.
+  run_benchmark "${tmpdir}" 10485760
+}
+
+get_mem_available() {
+  if [[ "${OS}" == "Darwin" ]]; then
+    echo $(vm_stat | grep -F 'Pages free:' | awk '{print $3 * 4}')
+  else
+    echo $(grep -F MemAvailable: /proc/meminfo | awk '{print $2}')
+  fi
+}
+
+run_benchmark() {
+  local src_img_dir="$1"
+  local mem_per_thread="${2:-10485760}"
+
+  local output_dir="${BUILD_DIR}/benchmark_results"
+  mkdir -p "${output_dir}"
+
+  # The memory available at the beginning of the benchmark run in kB. The number
+  # of threads depends on the available memory, and the passed memory per
+  # thread. We also add a 2 GiB of constant memory.
+  local mem_available="$(get_mem_available)"
+  # Check that we actually have a MemAvailable value.
+  [[ -n "${mem_available}" ]]
+  local num_threads=$(( (${mem_available} - 1048576) / ${mem_per_thread} ))
+  if [[ ${num_threads} -le 0 ]]; then
+    num_threads=1
+  fi
+
+  local benchmark_args=(
+    --input "${src_img_dir}/*.png"
+    --codec=jpeg:yuv420:q85,webp:q80,jxl:fast:d1,jxl:fast:d1:downsampling=8,jxl:fast:d4,jxl:fast:d4:downsampling=8,jxl:mg:nl,jxl:mg,jxl:mg:P7,jxl:mg:q80
+    --output_dir "${output_dir}"
+    --noprofiler --show_progress
+    --num_threads="${num_threads}"
+  )
+  if [[ "${STORE_IMAGES}" == "1" ]]; then
+    benchmark_args+=(--save_decompressed --save_compressed)
+  fi
+  "${BUILD_DIR}/tools/benchmark_xl" "${benchmark_args[@]}" | \
+     tee "${output_dir}/results.txt"
+  # Check error code for benckmark_xl command. This will exit if not.
+  [[ "${PIPESTATUS[0]}" == 0 ]]
+
+  if [[ -n "${CI_BUILD_NAME:-}" ]]; then
+    { set +x; } 2>/dev/null
+    local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}):
+
+$(cat "${output_dir}/results.txt")
+"
+    cmd_post_mr_comment "${message}"
+    set -x
+  fi
+}
+
+# Helper function to wait for the CPU temperature to cool down on ARM.
+wait_for_temp() {
+  { set +x; } 2>/dev/null
+  local temp_limit=${1:-37000}
+  if [[ -z "${THERMAL_FILE:-}" ]]; then
+    echo "Must define the THERMAL_FILE with the thermal_zoneX/temp file" \
+      "to read the temperature from. This is normally set in the runner." >&2
+    exit 1
+  fi
+  local org_temp=$(cat "${THERMAL_FILE}")
+  if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
+    echo -n "Waiting for temp to get down from ${org_temp}... "
+  fi
+  local temp="${org_temp}"
+  while [[ "${temp}" -ge "${temp_limit}" ]]; do
+    sleep 1
+    temp=$(cat "${THERMAL_FILE}")
+  done
+  if [[ "${org_temp}" -ge "${temp_limit}" ]]; then
+    echo "Done, temp=${temp}"
+  fi
+  set -x
+}
+
+# Helper function to set the cpuset restriction of the current process.
+cmd_cpuset() {
+  local newset="$1"
+  local mycpuset=$(cat /proc/self/cpuset)
+  mycpuset="/dev/cpuset${mycpuset}"
+  # Check that the directory exists:
+  [[ -d "${mycpuset}" ]]
+  echo "${newset}" >"${mycpuset}/cpuset.cpus"
+}
+
+# Run benchmarks on ARM for the big and little CPUs.
+cmd_arm_benchmark() {
+  local benchmarks=(
+    # Lossy options:
+    "--adaptive_reconstruction=1 --distance=1.0 --speed=fast"
+    "--adaptive_reconstruction=0 --distance=1.0 --speed=fast"
+    "--adaptive_reconstruction=1 --distance=8.0 --speed=fast"
+    "--adaptive_reconstruction=0 --distance=8.0 --speed=fast"
+    "--modular-group -Q 90"
+    "--modular-group -Q 50"
+    # Lossless options:
+    "--modular-group"
+    "--modular-group -E 0 -I 0 -A"
+    "--modular-group -P 5 -A"
+    "--modular-group --responsive=1"
+    # Near-lossless options:
+    "--adaptive_reconstruction=0 --distance=0.3 --speed=fast"
+    "--modular-group -N 3 -B 11"
+    "--modular-group -Q 97"
+  )
+
+  local images=(
+    "third_party/testdata/imagecompression.info/flower_foveon.png"
+  )
+
+  local cpu_confs=(
+    "${RUNNER_CPU_LITTLE}"
+    "${RUNNER_CPU_BIG}"
+    # The CPU description is something like 3-7, so these configurations only
+    # take the first CPU of the group.
+    "${RUNNER_CPU_LITTLE%%-*}"
+    "${RUNNER_CPU_BIG%%-*}"
+  )
+  local output_dir="${BUILD_DIR}/benchmark_results"
+  mkdir -p "${output_dir}"
+  local runs_file="${output_dir}/runs.txt"
+
+  if [[ ! -e "${runs_file}" ]]; then
+    echo -e "flags\tsrc_img\tsrc size\tsrc pixels\tcpuset\tenc size (B)\tdec speed (MP/s)" |
+      tee -a "${runs_file}"
+  fi
+
+  local flags
+  local src_img
+  for src_img in "${images[@]}"; do
+    local src_img_hash=$(sha1sum "${src_img}" | cut -f 1 -d ' ')
+    for flags in "${benchmarks[@]}"; do
+      cmd_cpuset "${RUNNER_CPU_ALL}"
+      # Encoding step.
+      local enc_file_hash="$flags || ${src_img} || ${src_img_hash}"
+      enc_file_hash=$(echo "${enc_file_hash}" | sha1sum | cut -f 1 -d ' ')
+
+      local enc_file="${BUILD_DIR}/${enc_file_hash}.bin"
+      if [[ ! -f "${enc_file}" ]]; then
+        "${BUILD_DIR}/tools/cjpegxl" ${flags} "${src_img}" "${enc_file}.tmp"
+        mv "${enc_file}.tmp" "${enc_file}"
+      fi
+      local enc_size=$(stat -c "%s" "${enc_file}")
+
+      for cpu_conf in "${cpu_confs[@]}"; do
+        cmd_cpuset "${cpu_conf}"
+        echo "Decoding with: img=${src_img} cpus=${cpu_conf} enc_flags=${flags}"
+
+        local dec_output
+        wait_for_temp
+        dec_output=$("${BUILD_DIR}/tools/djpegxl" "${enc_file}" \
+          --num_reps=5 2>&1 | grep -F "MP/s")
+        local img_size=$(echo "${dec_output}" | cut -f 1 -d ',')
+        local img_size_x=$(echo "${img_size}" | cut -f 1 -d ' ')
+        local img_size_y=$(echo "${img_size}" | cut -f 3 -d ' ')
+        local img_size_px=$(( ${img_size_x} * ${img_size_y} ))
+        local dec_speed=$(echo "${dec_output}" | cut -f 5 -d ' ')
+
+        # Record entry in a tab-separated file.
+        echo -e "${flags}\t${src_img}\t${img_size}\t${img_size_px}\t${cpu_conf}\t${enc_size}\t${dec_speed}" |
+          tee -a "${runs_file}"
+      done
+    done
+  done
+  cmd_cpuset "${RUNNER_CPU_ALL}"
+  cat "${runs_file}"
+
+  if [[ -n "${CI_BUILD_NAME:-}" ]]; then
+    load_mr_vars_from_commit
+    { set +x; } 2>/dev/null
+    local message="Results for ${CI_BUILD_NAME} @ ${CI_COMMIT_SHORT_SHA} (job ${CI_JOB_URL:-}):
+
+\`\`\`
+$(column -t -s "	" "${runs_file}")
+\`\`\`
+"
+    cmd_post_mr_comment "${message}"
+    set -x
+  fi
+}
+
+# Runs the linter (clang-format) on the pending CLs.
+cmd_lint() {
+  merge_request_commits
+  # { set +x; } 2>/dev/null
+  local versions=(${1:-6.0 7 8 9})
+  local clang_format_bins=("${versions[@]/#/clang-format-}" clang-format)
+  local tmpdir=$(mktemp -d)
+  CLEANUP_FILES+=("${tmpdir}")
+
+  local installed=()
+  local clang_patch
+  local clang_format
+  for clang_format in "${clang_format_bins[@]}"; do
+    if ! which "${clang_format}" >/dev/null; then
+      continue
+    fi
+    installed+=("${clang_format}")
+    local tmppatch="${tmpdir}/${clang_format}.patch"
+    # We include in this linter all the changes including the uncommited changes
+    # to avoid printing changes already applied.
+    set -x
+    git -C "${MYDIR}" "${clang_format}" --binary "${clang_format}" \
+      --style=file --diff "${MR_ANCESTOR_SHA}" -- >"${tmppatch}"
+    { set +x; } 2>/dev/null
+
+    if grep -E '^--- ' "${tmppatch}">/dev/null; then
+      if [[ -n "${LINT_OUTPUT:-}" ]]; then
+        cp "${tmppatch}" "${LINT_OUTPUT}"
+      fi
+      clang_patch="${tmppatch}"
+    else
+      echo "clang-format check OK" >&2
+      return 0
+    fi
+  done
+
+  if [[ ${#installed[@]} -eq 0 ]]; then
+    echo "You must install clang-format for \"git clang-format\"" >&2
+    exit 1
+  fi
+
+  # clang-format is installed but found problems.
+  echo "clang-format findings:" >&2
+  "${COLORDIFF_BIN}" < "${clang_patch}"
+
+  echo "clang-format found issues in your patches from ${MR_ANCESTOR_SHA}" \
+    "to the current patch. Run \`./ci.sh lint | patch -p1\` from the base" \
+    "directory to apply them." >&2
+  exit 1
+}
+
+# Runs clang-tidy on the pending CLs. If the "all" argument is passed it runs
+# clang-tidy over all the source files instead.
+cmd_tidy() {
+  local what="${1:-}"
+
+  if [[ -z "${CLANG_TIDY_BIN}" ]]; then
+    echo "ERROR: You must install clang-tidy-6.0 or newer to use ci.sh tidy" >&2
+    exit 1
+  fi
+
+  local git_args=()
+  if [[ "${what}" == "all" ]]; then
+    git_args=(ls-files)
+    shift
+  else
+    merge_request_commits
+    git_args=(
+        diff-tree --no-commit-id --name-only -r "${MR_ANCESTOR_SHA}"
+        "${MR_HEAD_SHA}"
+    )
+  fi
+
+  # Clang-tidy needs the compilation database generated by cmake.
+  if [[ ! -e "${BUILD_DIR}/compile_commands.json" ]]; then
+    # Generate the build options in debug mode, since we need the debug asserts
+    # enabled for the clang-tidy analyzer to use them.
+    CMAKE_BUILD_TYPE="Debug"
+    cmake_configure
+  fi
+
+  cd "${MYDIR}"
+  local nprocs=$(nproc --all || echo 1)
+  local ret=0
+  if ! parallel -j"${nprocs}" --keep-order -- \
+      "${CLANG_TIDY_BIN}" -p "${BUILD_DIR}" -format-style=file -quiet "$@" {} \
+      < <(git "${git_args[@]}" | grep -E '(\.cc|\.cpp)$') \
+      >"${BUILD_DIR}/clang-tidy.txt"; then
+    ret=1
+  fi
+  { set +x; } 2>/dev/null
+  echo "Findings statistics:" >&2
+  grep -E ' \[[A-Za-z\.,\-]+\]' -o "${BUILD_DIR}/clang-tidy.txt" | sort \
+    | uniq -c >&2
+
+  if [[ $ret -ne 0 ]]; then
+    cat >&2 <<EOF
+Errors found, see ${BUILD_DIR}/clang-tidy.txt for details.
+To automatically fix them, run:
+
+  SKIP_TEST=1 ./ci.sh debug
+  ${CLANG_TIDY_BIN} -p ${BUILD_DIR} -fix -format-style=file -quiet $@ \$(git ${git_args[@]} | grep -E '(\.cc|\.cpp)\$')
+EOF
+  fi
+
+  return ${ret}
+}
+
+main() {
+  local cmd="${1:-}"
+  if [[ -z "${cmd}" ]]; then
+    cat >&2 <<EOF
+Use: $0 CMD
+
+Where cmd is one of:
+ opt       Build and test a Release with symbols build.
+ debug     Build and test a Debug build (NDEBUG is not defined).
+ release   Build and test a striped Release binary without debug information.
+ asan      Build and test an ASan (AddressSanitizer) build.
+ msan      Build and test an MSan (MemorySanitizer) build. Needs to have msan
+           c++ libs installed with msan_install first.
+ tsan      Build and test a TSan (ThreadSanitizer) build.
+ test      Run the tests build by opt, debug, release, asan or msan. Useful when
+           building with SKIP_TEST=1.
+ benchmark Run the benchmark over the default corpus.
+ fast_benchmark Run the benchmark over the small corpus.
+
+ coverage  Buils and run tests with coverage support. Runs coverage_report as
+           well.
+ coverage_report Generate HTML, XML and text coverage report after a coverage
+           run.
+
+ lint      Run the linter checks on the current commit or merge request.
+ tidy      Run clang-tidy on the current commit or merge request.
+
+ msan_install Install the libc++ libraries required to build in msan mode. This
+              needs to be done once.
+
+You can pass some optional environment variables as well:
+ - BUILD_DIR: The output build directory (by default "$$repo/build")
+ - BUILD_TARGET: The target triplet used when cross-compiling.
+ - CMAKE_FLAGS: Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS.
+ - CMAKE_PREFIX_PATH: Installation prefixes to be searched by the find_package.
+ - LINT_OUTPUT: Path to the output patch from the "lint" command.
+ - SKIP_TEST=1: Skip the test stage.
+ - STORE_IMAGES=0: Makes the benchmark discard the computed images.
+
+These optional environment variables are forwarded to the cmake call as
+parameters:
+ - CMAKE_C_FLAGS
+ - CMAKE_CXX_FLAGS
+ - CMAKE_EXE_LINKER_FLAGS
+ - CMAKE_MODULE_LINKER_FLAGS
+ - CMAKE_SHARED_LINKER_FLAGS
+ - CMAKE_TOOLCHAIN_FILE
+ - CMAKE_BUILD_TYPE
+
+Example:
+  BUILD_DIR=/tmp/build $0 opt
+EOF
+    exit 1
+  fi
+
+  cmd="cmd_${cmd}"
+  shift
+  set -x
+  "${cmd}" "$@"
+}
+
+main "$@"
diff --git a/doc/building_and_testing.md b/doc/building_and_testing.md
new file mode 100644
index 000000000..c060aeb17
--- /dev/null
+++ b/doc/building_and_testing.md
@@ -0,0 +1,181 @@
+# Building and Testing
+
+## Basic building
+
+If you only want to build the project but not modify it you can run:
+
+```bash
+./ci.sh release
+```
+
+Running `./ci.sh` with no parameters shows a list of available commands. For
+example, you can run `opt` for optimized developer builds with symbols or
+`debug` for debug builds which do not have NDEBUG defined and therefore include
+more runtime debug information.
+
+## Sending patches
+
+Before sending a patch, make sure your patch conforms to the
+[project guidelines](guidelines.md) and test it following these steps.
+
+You will need to install a few other dependencies for verifying these patches,
+which are used by the `./ci.sh` helper command.
+
+```bash
+sudo apt install clang-format-6.0 clang-tidy-6.0 curl parallel
+```
+
+### Build and unittest checks
+
+Your patch must build in the "release" and "debug" configurations at least, and
+pass all the tests. The "opt" configuration will be tested in the Merge Request
+pipeline which is equivalent to a "release" build with debug information.
+
+```bash
+./ci.sh opt
+```
+
+All of the tests must pass. If you added new functions, you should also add
+tests to these functions in the same commit.
+
+### Cross-compiling
+
+To compile the code for an architecture different than the one you are running
+you can pass a
+[toolchain file](https://cmake.org/cmake/help/latest/manual/cmake-toolchains.7.html)
+to cmake if you have one for your target, or you can use the `BUILD_TARGET`
+environment variable in `./ci.sh`. This assumes that you already have a
+cross-compiling environment set up and the library dependencies are already
+installed for the target architecture as well. Alternatively, you can use one of
+the [jpegxl docker containers](developing_in_docker.md) already configured to
+cross-compile and run for other architectures.
+
+For example, to compile for the `aarch64-linux-gnu` target triplet you can run:
+
+```bash
+BUILD_TARGET=aarch64-linux-gnu ./ci.sh release
+```
+
+### Format checks (lint)
+
+```bash
+./ci.sh lint
+```
+
+Linter checks will verify that the format of your patch conforms to the project
+style (see [guidelines](guidelines.md)). For this, we run clang-format only on
+the lines that were changed by your commits.
+
+If your local git branch is tracking `origin/master` and you landed a few
+commits in your branch, running this lint command will check all the changes
+made from the common ancestor with `origin/master` to the latest changes,
+including uncommitted changes. The output of the program will show the patch
+that should be applied to fix your commits. You can apply these changes with the
+following command from the base directory of the git checkout:
+
+```bash
+./ci.sh lint | patch -p1
+```
+
+### Programming errors (tidy)
+
+```bash
+./ci.sh tidy
+```
+
+clang-tidy is a tool to check common programming errors in C++, and other valid
+C++ constructions that are discouraged by the style guide or otherwise dangerous
+and may constitute a bug.
+
+To run clang-tidy on the files changed by your changes you can run `./ci.sh
+tidy`. Note that this will report all the problems encountered in any file that
+was modified by one of your commits, not just on the lines that your commits
+modified.
+
+## Testing
+
+All of the `./ci.sh` workflows like `release`, `opt`, etc will run the test
+after building. It is possible to manually run the tests, for example:
+
+```bash
+cd build/
+ctest -j32 --output-on-failure
+```
+
+It is also possible for faster iteration to run a specific test binary directly.
+googletest binaries support a glob filter to match the name of the test, for
+example `TransposeTest*` will execute all the test names that start with
+`TransposeTest`.
+
+```bash
+build/dct_test --gtest_filter=TransposeTest.Transpose8
+
+```
+
+If you want to run multiple tests from different binaries, you can pass a
+regular expression filter to ctest. Note that this is a regular expression
+instead of a glob match like in the `gtest_filter`.
+
+```bash
+cd build/
+ctest -j32 --output-on-failure -R '.*Transpose.*'
+```
+
+### Address Sanitizer (asan)
+
+```bash
+./ci.sh asan
+```
+
+ASan builds allow to check for invalid address usages, such as use-after-free.
+To perform these checks, as well as other undefined behavior checks we only need
+to build and run the unittests with ASan enabled which can be easily achieved
+with the command above. If you want to have the ASan build files separated from
+your regular `build/` directory to quickly switch between asan and regular
+builds, you can pass the build directory target as follows:
+
+```bash
+BUILD_DIR=build-asan ./ci.sh asan
+```
+
+### Memory Sanitizer (msan)
+
+MSan allows to check for invalid memory accesses at runtime, such as using an
+uninitialized value which likely means that there is a bug. To run these checks,
+a specially compiled version of the project and tests is needed.
+
+For building with MSan, you need to build a version of libc++ with
+`-fsanitize=memory` so we can link against it from the MSan build. Also, having
+an `llvm-symbolizer` installed is very helpful to obtain stack traces that
+include the symbols (functions and line numbers). To install `llvm-symbolizer`
+on a Debian-based system run:
+
+```bash
+sudo apt install llvm # or llvm-6.0, llvm-7, etc for a specific version.
+```
+
+To install a version of libc++ compiled with `-fsanitize=memory` you can use the
+`./ci.sh msan_install` command helper. This will download, compile and install
+libc++ and libc++abi in the `${HOME}/.msan` directory to be used later.
+
+After this is set up, you can build the project using the following command:
+
+```bash
+./ci.sh msan
+```
+
+This command by default uses the `build` directory to store the cmake and object
+files. If you want to have a separate build directory configured with msan you
+can for example call:
+
+```bash
+BUILD_DIR=build-msan ./ci.sh msan
+```
+
+### Benchmark
+
+To run the benchmark:
+
+```shell
+build/tools/benchmark_xl --input "/path/*.png" --codec jxl:d1,jxl:d2,jxl:d4
+```
diff --git a/doc/developing_in_docker.md b/doc/developing_in_docker.md
new file mode 100644
index 000000000..906a971e8
--- /dev/null
+++ b/doc/developing_in_docker.md
@@ -0,0 +1,90 @@
+# Developing in docker
+
+Docker allows to run software in a packaged container, isolated from your host
+system. This is useful to run the code in a standard environment instead of
+dealing with problems caused by different building environments during
+development.
+
+Read over the [docker install instructions](https://docs.docker.com/install/) to
+get docker installed in your computer.
+
+## Basic building
+
+The image used by our builders is an ubuntu-bionic image with all the required
+dependencies and build tools installed. You can pull this image from
+`gcr.io/jpegxl/jpegxl-builder`.
+
+```bash
+sudo docker pull gcr.io/jpegxl/jpegxl-builder
+```
+
+## Cross-compiling environments
+
+We have installed the required cross-compiling tools in the main docker image
+`jpegxl-builder`. This allows to compile for other architectures such as arm
+and run the tests emulated under qemu.
+
+To use the main docker image you can run the following command:
+```bash
+sudo docker run -it --rm \
+  --user $(id -u):$(id -g) \
+  -v $HOME/jpegxl:/jpegxl \
+  gcr.io/jpegxl/jpegxl-builder bash
+```
+
+This creates and runs a container that will be deleted after you exit from this
+terminal (`--rm` flag).
+
+The `-v` flag is to map the directory containing your jpegxl checkout in your
+host (assumed to be at `$HOME/jpegxl`) to a directory inside the container at
+/jpegxl. This means that whenever you make changes to the code from your host
+using your favorite editor they are also changed in the container, since the
+directory is simply mounted into the container.
+
+On OSX, the path must be one of those whitelisted/shared with Docker. $HOME
+(which is a subdirectory of /Users/) is known to work with the factory-default
+settings of Docker.
+
+**Quick start:** To compile and run unit tests,
+
+````bash
+CC=clang-6.0 CXX=clang++-6.0 ./ci.sh opt
+```
+Note that the default "clang" compiler is not installed on the image, hence we
+specify clang-6.0. If a build/ directory already exists and was configured for
+a different compiler, cmake will complain. This can be avoided by renaming any
+existing build/ directory. On OSX, "cannot find name for group ID" can be
+ignored.
+
+The docker container already has several `qemu-*-static` binaries (such as
+`qemu-aarch64-static`) that emulate  other architectures on x86_64. These qemu
+binaries are automatically used when running a foreign architecture program in
+the container only if you have `binfmt` installed and configured to use the
+binaries from `/usr/bin/qemu-*-static` in the *host*. This is the default in
+Ubuntu/Debian. If you have these configured somewhere else, symlink that
+location to the `/usr/bin/qemu-*-static` inside the docker before running. To
+install binfmt support in your Ubuntu host run outside the container:
+
+```bash
+sudo apt install binfmt-support
+```
+
+Then to cross-compile and run unit tests execute the following commands:
+
+```bash
+cd /jpegxl
+export BUILD_TARGET=aarch64-linux-gnu CC=clang-6.0 CXX=clang++-6.0
+./ci.sh release
+```
+
+The `BUILD_TARGET=aarch64-linux-gnu` environment variable tells the `ci.sh`
+script to cross-compile for that target. This also changes the default
+`BUILD_DIR` to `build-aarch64` since you never want to mix them with the `build`
+of your host. You can also explicitly set a `BUILD_DIR` environment variable and
+that will be used instead. The list of supported BUILD_TARGET values for this
+container is:
+
+*    *the empty string* (for native x86_64 support)
+*    aarch64-linux-gnu
+*    arm-linux-gnueabihf
+*    i686-linux-gnu
diff --git a/doc/developing_in_gitlab.md b/doc/developing_in_gitlab.md
new file mode 100644
index 000000000..6ee53ef9d
--- /dev/null
+++ b/doc/developing_in_gitlab.md
@@ -0,0 +1,306 @@
+# Developing in GitLab
+
+This document describes the development steps related to handling the git
+repository.
+
+## Initial setup
+
+You need to perform these steps at least once. After that, the "Life of a Merge"
+section describes the common everyday workflows.
+
+### Configure your SSH access
+
+The easiest way to configure access to a private GitLab repo is to use SSH keys.
+For that you need an SSH private and public key, ideally a strong one. You can
+use different keys for different sites if you want. In this example, we will
+create one for using in GitLab only.
+
+Create the `~/.ssh/id_rsa_gitlab` file executing the following. (Here and
+elsewhere, {{X}} are placeholders for your email/username)
+
+```bash
+ssh-keygen -t rsa -b 4096 -C "{{EMAIL}}" -f ~/.ssh/id_rsa_gitlab
+```
+
+Go to your [SSH settings in GitLab](https://gitlab.com/profile/keys) and paste
+the contents of your *public key* (the one ending in `.pub`), that would be the
+output of this command:
+
+```bash
+cat ~/.ssh/id_rsa_gitlab.pub
+```
+
+To use a specific key when SSHing to the gitlab.com domain, you can add this
+snippet of config to your .ssh/config file executing the following.
+
+```bash
+cat >> ~/.ssh/config <<EOF
+
+Host gitlab.com
+  Hostname gitlab.com
+  IdentityFile ~/.ssh/id_rsa_gitlab
+  IdentitiesOnly yes
+EOF
+```
+
+The `IdentitiesOnly yes` part forces to only use the provided IdentityFile when
+talking to GitLab.
+
+### Checkout the JPEG XL code from GitLab
+
+The JPEG XL code is located in
+[this repo](https://gitlab.com/wg1/jpeg-xl). Clone it using the SSH
+version of the URL.
+
+```bash
+git clone git@gitlab.com:wg1/jpeg-xl.git --recursive ~/jpeg-xl
+```
+
+### Fork your private copy
+
+The normal developer workflow in GitLab involves creating your own fork of a
+repository and uploading your own changes there. From your own copy you can
+request merges *to* the upstream repository directly, there's no need to create
+a branch in the upstream repository.
+
+Navigate to the [JPEG XL GitLab repo](https://gitlab.com/wg1/jpeg-xl) and
+click "Fork" on the top right corner. After creating the Fork you will have an
+exact copy of the repository which should remain private. Then add this new copy
+to your existing checkout run:
+
+```bash
+cd ~/jpeg-xl
+git remote set-url --push origin git@gitlab.com:{{USERNAME}}/jpeg-xl.git
+```
+
+{{USERNAME}} denotes your GitLab username. This will make the "git push origin"
+command push to your fork, which then allows you to create merge requests.
+
+## Life of a Merge
+
+All the commands here assume you are in a git checkout as setup here.
+
+### Sync to the latest version
+
+```bash
+git fetch origin
+```
+
+The last upstream version is now on `origin/master` and none of your local
+branches have been modified by this command.
+
+### Start a new branch
+
+To start a new change you need a local branch. Each branch will represent a list
+of individual commits which can then be requested to be merged as a single merge
+request. So in general one branch is one code review, but each branch can have
+multiple individual commits in it.
+
+```bash
+git checkout origin/master -b mybranch
+```
+
+This will create a new branch `mybranch` tracking `origin/master`. A branch can
+track any remove or local branch, which is used by some tools. Running `git
+branch -vv` will show all the branches you have have, what are they tracking and
+how many commits are ahead or behind. If you create a branch without tracking
+any other, you can add or change the tracking branch of the current branch
+running `git branch --set-upstream-to=...`.
+
+### Add changes to your branch
+
+Follow any of the many online tutorials, for example
+[The basics](https://git-scm.com/book/en/v2/Git-Basics-Getting-a-Git-Repository)
+chapter from the https://git-scm.com/doc website is a good starting guide.
+Create, change or delete files and do a git commit with a message.
+
+The commit message is required. A commit message should follow the 50/72 rule:
+
+*   First line is 50 characters or less.
+*   Then a blank line.
+*   Remaining text should be wrapped at 72 characters.
+
+The first line should identify your commit, since that's what most tools will
+show to the user. First lines like "Some fixes" are not useful. Explain what the
+commit contains and why.
+
+### Upload your changes for review
+
+The first step is a local review of your changes to see what will you be sending
+for review. `gitg` is a nice Gtk UI for reviewing your local changes, or `tig`
+for similar ncurses console-based interface. Otherwise, from the terminal you
+can run:
+
+```bash
+git branch -vv
+```
+
+To show the current status of your local branches. In particular, since your
+branch is tracking origin/master (as seen in the output) git will tell you that
+you are one commit ahead of the tracking branch.
+
+```
+* mybranch       e74ae1a [origin/master: ahead 1] Improved decoding speed by 40%
+```
+
+It is a good idea before uploading to sync again with upstream (`git fetch
+origin`) and then run `git branch -vv` to check whether there are new changes
+upstream. If that is the case, you will see a "behind" flag in the output:
+
+```
+* mybranch       e74ae1a [origin/master: ahead 1, behind 2] Improved decoding speed by 40%
+```
+
+To sync your changes on top of the latest changes in upstream you need to
+rebase:
+
+```bash
+git rebase
+```
+
+This will by default rebase your current branch changes on top of the tracking
+branch. In this case, this will try to apply the current commit on top of the
+latest origin/master (which has 2 more commits than the ones we have in our
+branch) and your branch will now include that. There could be conflicts that you
+have to deal with. A shortcut to do both fetch and rebase is to run `git pull
+-r`, where the `-r` stands for "rebase" and will rebase the local commits on top
+of the remote ones.
+
+Before uploading a patch, make sure your patch conforms to the
+[project guidelines](guidelines.md) and it
+[builds and passes tests](building_and_testing.md).
+
+Once you are ready to send your branch for review, upload it to *your* fork:
+
+```bash
+git push origin mybranch
+```
+
+This will push your local branch "mybranch" to a remote in your fork called
+"mybranch". The name can be anything, but keep in mind that it is public. A link
+to the URL to create a merge request will be displayed.
+
+```
+Enumerating objects: 627, done.
+Counting objects: 100% (627/627), done.
+Delta compression using up to 56 threads
+Compressing objects: 100% (388/388), done.
+Writing objects: 100% (389/389), 10.71 MiB | 8.34 MiB/s, done.
+Total 389 (delta 236), reused 0 (delta 0)
+remote: Resolving deltas: 100% (236/236), completed with 225 local objects.
+remote:
+remote: To create a merge request for mybranch, visit:
+remote:   https://gitlab.com/{{USERNAME}}/jpeg-xl/merge_requests/new?merge_request%5Bsource_branch%5D=mybranch
+remote:
+To gitlab.com:{{USERNAME}}/jpeg-xl.git
+ * [new branch]      mybranch -> mybranch
+```
+
+### Updating submodules
+
+The repository uses submodules for external library dependencies in
+third_party. Each submodule points to a particular external commit of the
+external repository by the hash code of that external commit. Just like
+regular source code files, this hash code is part of the current branch and
+jpeg xl commit you have checked out.
+
+When changing branches or when doing `git rebase`, git will unfortunately
+*not* automatically set those hashes to the ones of the branch or jpeg xl
+commit you changed to nor set the source files of the third_party submodules
+to the new state. That is, even though git will have updated the jpeg xl
+source code files on your disk to the new ones, it will leave the submodule
+hashes and the files in third_party in your workspace to the ones they were
+before you changed branches. This will show up in a git diff because this
+is seen as a change compared to the branch you switched to. The git diff shows
+the difference in hash codes (as if you are changing to the old ones), it does
+not show changes in files inside the third_party directory.
+
+This mismatch can cause at least two problems:
+
+*) the jpeg xl codebase may not compile due to third_party library version
+mismatch if e.g. API changed or a submodule was added/removed.
+
+*) when using `commit -a` your commit, which may be a technical change
+unrelated to submodule changes, will unintentially contain a change to the
+submodules hash code, which is undesired unless you actually want to change
+the version of third_party libraries.
+
+To resolve this, the submodules must be updated manually with
+the following command after those actions (at least when the submodules
+changed):
+
+```
+git submodule update --init --recursive
+```
+
+Here, the init flag ensures new modules get added when encessary and the
+recursive flag is required for the submodules depending on other submodules.
+
+If you checkout a different branch, you can spot that submodules changed
+when it shows a message similar to this:
+
+```
+M       third_party/brotli
+M       third_party/lcms
+```
+
+If you do a rebase you may end up in a harder to solve situation, where
+`git submodule update --init --recursive` itself fails with errors such as:
+
+```
+Unable to checkout '35ef5c554d888bef217d449346067de05e269b30' in submodule path 'third_party/brunsli/third_party/brotli'
+Unable to checkout '35ef5c554d888bef217d449346067de05e269b30' in submodule path 'third_party/brotli'
+```
+
+In that case, you can use the force flag:
+
+```
+git submodule update --init --recursive --force
+```
+
+### Iterating changes in your merge request
+
+To address reviewer changes you need to amend the local changes in your branch
+first. Make the changes you need in your commit locally by running `git commit
+--amend file1 file2 file3 ...` or `git commit --amend -a` to amend all the
+changes from all the staged files.
+
+Once you have the new version of the "mybranch" branch to re-upload, you need to
+force push it to the same branch in your fork. Since you are pushing a different
+version of the same commit (as opposed to another commit on top of the existing
+ones), you need to force the operation to replace the old version.
+
+```bash
+git push origin mybranch --force
+```
+
+The merge request should now be updated with the new changes.
+
+### Merging your changes
+
+We use "rebase" as a merge policy, which means that there a no "merge" commits
+(commits with more than one parent) but instead only a linear history of
+changes. If other commits landed in the master branch since you last synced you
+need to `git fetch`, `git rebase` and push again your changes which need to go
+through the pipeline again to verify that all the tests pass again after
+including the latests changes. Instead of doing this manually, you can _Assign_
+the patch to the user @jpegxl-bot which will do this for you. In order for the
+bot to manage your Merge Request, you need to add it to your fork as a
+Developer.
+
+### Trying locally a pending merge request.
+
+If you want to review in your computer a pending merge request proposed by
+another user you can fetch the merge request commit with the following command,
+replacing `NNNN` with the merge request number:
+
+```bash
+git fetch origin refs/merge-requests/NNNN/head
+git checkout FETCH_HEAD
+```
+
+The first command will add to your local git repository the remote commit for
+the pending merge request and store a temporary reference called `FETCH_HEAD`.
+The second command then checks out that reference. From this point you can
+review the files in your computer, create a local branch for this FETCH_HEAD or
+build on top of it.
diff --git a/doc/guidelines.md b/doc/guidelines.md
new file mode 100644
index 000000000..d44594217
--- /dev/null
+++ b/doc/guidelines.md
@@ -0,0 +1,67 @@
+# Guidelines for JPEG-XL software
+
+The following rules govern experimental software contributed for JPEG XL. Merge
+requests not complying with these rules will not be accepted by the software
+coordinators.
+
+## License
+
+The contribution shall be licensed under the Apache 2 license, with each file
+beginning with the following header:
+
+```
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+```
+
+## Language
+
+The software shall be compilable as C++ without custom preprocessors, and be
+sent in the preferred representation for reading/development (without
+obfuscation).
+
+In general, please follow
+[guidelines on features and style](https://google.github.io/styleguide/cppguide.html)
+for newly added code.
+
+In particular:
+
+*   The code shall compile under clang++ (version 6) without any warnings.
+*   Exceptions and RTTI shall not be used.
+*   Features from C++11 may be used if desired, but not C++14 or later.
+*   Integer types shall be fixed-size types from stdint.h (e.g. uint32_t), or size_t, or [u]intptr_t.
+*   NULL shall not be used (use nullptr instead).
+*   The code shall be formatted by running clang-format.
+*   Macro names shall be UPPERCASE_WITH_UNDERSCORES.
+*   External dependencies should be minimized; large dependencies such as Qt or Boost are not allowed.
+*   Headers shall be surrounded by include guards of the form
+
+    ```
+    #ifndef FILENAME_H_
+    #define FILENAME_H_
+    #endif // FILENAME_H_
+
+    ```
+
+## Acceptance test
+
+Before merging, the pipeline shall pass.
+
+Note that tests can also be run locally from the cmake build/ directory:
+`ctest -j50 --output-on-failure`   (or some other number of parallel tasks)
+
+The code shall not involve race conditions nor accesses to undefined memory.
+In particular, before merging results of a core experiment, tests shall be run
+in three configurations: compiling with -fsanitize=address, -fsanitize=memory
+and -fsanitize=thread.
diff --git a/doc/tables/adobe.md b/doc/tables/adobe.md
new file mode 100644
index 000000000..f3beef75a
--- /dev/null
+++ b/doc/tables/adobe.md
@@ -0,0 +1,6 @@
+#### Table M.8 – "Adobe" marker template
+
+```
+0xEE, 0x00, 0x0E, 0x41, 0x64, 0x6F, 0x62, 0x65, 0x00, 0x64, 0x00, 0x00, 0x00, 0x00, 0x01
+```
+
diff --git a/doc/tables/all_tables.pdf b/doc/tables/all_tables.pdf
new file mode 100644
index 000000000..be1a9af14
Binary files /dev/null and b/doc/tables/all_tables.pdf differ
diff --git a/doc/tables/all_tables.sh b/doc/tables/all_tables.sh
new file mode 100755
index 000000000..cfd289a16
--- /dev/null
+++ b/doc/tables/all_tables.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cat dct_gen.md \
+    is_zero_base.md num_nonzeros_base.md brn_proto.md app0.md icc.md ducky.md \
+    adobe.md stock_counts.md stock_values.md symbol_order.md stock_quant.md \
+    quant.md freq_context.md num_nonzero_context.md nonzero_buckets.md \
+    > all_tables.md
diff --git a/doc/tables/app0.md b/doc/tables/app0.md
new file mode 100644
index 000000000..266f21047
--- /dev/null
+++ b/doc/tables/app0.md
@@ -0,0 +1,6 @@
+#### Table M.4 – APP0 template
+
+```
+0xE0, 0x00, 0x10, 0x4A, 0x46, 0x49, 0x46, 0x00, 0x01, 0x01, 0x00, 0x00, 0x01, 0x00, 0x01, 0x00, 0x00
+```
+
diff --git a/doc/tables/brn_proto.md b/doc/tables/brn_proto.md
new file mode 100644
index 000000000..b5f80b6a7
--- /dev/null
+++ b/doc/tables/brn_proto.md
@@ -0,0 +1,23 @@
+#### Table M.3 – Protocol Buffer descriptor of top-level structure of losslessly compressed JPEG stream
+
+```protobuf
+message Header {
+  optional uint64 width = 1;
+  optional uint64 height = 2;
+  required uint64 version_and_component_count_code = 3;
+  optional uint64 subsampling_code = 4;
+}
+
+message Jpeg {
+  required bytes signature = 1;
+  required Header header = 2;
+  optional bytes meta_data = 3;
+  optional bytes jpeg1_internals = 4;
+  optional bytes quant_data = 5;
+  optional bytes histogram_data = 6;
+  optional bytes dc_data = 7;
+  optional bytes ac_data = 8;
+  optional bytes original_jpg = 9;
+}
+```
+
diff --git a/doc/tables/dct_gen.md b/doc/tables/dct_gen.md
new file mode 100644
index 000000000..f3b59e248
--- /dev/null
+++ b/doc/tables/dct_gen.md
@@ -0,0 +1,241 @@
+#### Electronic Insert I.1 – DCT-II / DCT-III code generator
+
+```python
+#######################################################################
+# DCT-II / DCT-III generator
+#
+# Based on:
+#  "A low multiplicative complexity fast recursive DCT-2 algorithm"
+#  by Maxim Vashkevich and Alexander Petrovsky / arXiv / 20 Jul 2012
+#######################################################################
+
+import math
+import sys
+N = 8
+
+#######################################################################
+# Base transforms / generators
+#######################################################################
+
+CNTR = 0
+def makeTmp():
+  global CNTR
+  result = "t{:02d}".format(CNTR)
+  CNTR = CNTR + 1
+  return result
+
+def makeVar(i):
+  return "i{:02d}".format(i)
+
+def add(x, y):
+  tmp = makeTmp()
+  print(tmp + " = " + x + " + " + y + ";")
+  return tmp
+
+def sub(x, y):
+  tmp = makeTmp()
+  print(tmp + " = " + x + " - " + y + ";")
+  return tmp
+
+def mul(x, c):
+  tmp = makeTmp()
+  print(tmp + " = " + x + " * " + c + ";")
+  return tmp
+
+# 2.0 * math.cos((a + 0.0) / (b + 0.0) * math.pi)
+def C2(a, b):
+  return "c_c2_" + str(a) + "_" + str(b)
+
+# 1.0 / C2(a, b)
+def iC2(a, b):
+  return "c_ic2_" + str(a) + "_" + str(b)
+
+#######################################################################
+# Utilities
+#######################################################################
+
+# Generate identity matrix. Usually this matrix is passed to
+# DCT algorithm to generate "basis" vectors of the transform.
+def makeVars():
+  return [makeVar(i) for i in range(N)]
+
+# Split list of variables info halves.
+def split(x):
+  m = len(x)
+  m2 = m // 2
+  return (x[0 : m2], x[m2 : m])
+
+# Make a list of variables in a reverse order.
+def reverse(varz):
+  m = len(varz)
+  result = [0] * m
+  for i in range(m):
+    result[i] = varz[m - 1 - i]
+  return result
+
+# Apply permutation
+def permute(x, p):
+ return [x[p[i]] for i in range(len(p))]
+
+def transposePermutation(p):
+  n = len(p)
+  result = [0] * n
+  for i in range(n):
+    result[p[i]] = i
+  return result
+
+# See paper. Split even-odd elements.
+def P(n):
+  if n == 1:
+    return [0]
+  n2 = n // 2
+  return [2 * i for i in range(n2)] + [2 * i + 1 for i in range(n2)]
+
+# See paper. Interleave first and second half.
+def Pt(n):
+  return transposePermutation(P(n))
+
+#######################################################################
+# Scheme
+#######################################################################
+
+def B2(x):
+  n = len(x)
+  n2 = n // 2
+  if n == 1:
+    raise "ooops"
+  (top, bottom) = split(x)
+  bottom = reverse(bottom)
+  t = [add(top[i], bottom[i]) for i in range(n2)]
+  b = [sub(top[i], bottom[i]) for i in range(n2)]
+  return t + b
+
+def iB2(x):
+  n = len(x)
+  n2 = n // 2
+  if n == 1:
+    raise "ooops"
+  (top, bottom) = split(x)
+  t = [add(top[i], bottom[i]) for i in range(n2)]
+  b = [sub(top[i], bottom[i]) for i in range(n2)]
+  return t + reverse(b)
+
+def B4(x, rn):
+  n = len(x)
+  n2 = n // 2
+  if n == 1:
+    raise "ooops"
+  (top, bottom) = split(x)
+  rbottom = reverse(bottom)
+  t = [sub(top[i], rbottom[i]) for i in range(n2)]
+  b = [mul(bottom[i], C2(rn, 2 * N)) for i in range(n2)]
+  top = [add(t[i], b[i]) for i in range(n2)]
+  bottom = [sub(t[i], b[i]) for i in range(n2)]
+  return top + bottom
+
+def iB4(x, rn):
+  n = len(x)
+  n2 = n // 2
+  if n == 1:
+    raise "ooops"
+  (top, bottom) = split(x)
+  t = [add(top[i], bottom[i]) for i in range(n2)]
+  b = [sub(top[i], bottom[i]) for i in range(n2)]
+  bottom = [mul(b[i], iC2(rn, 2 * N)) for i in range(n2)]
+  rbottom = reverse(bottom)
+  top = [add(t[i], rbottom[i]) for i in range(n2)]
+  return top + bottom
+
+def P4(n):
+  if n == 1:
+    return [0]
+  if n == 2:
+    return [0, 1]
+  n2 = n // 2
+  result = [0] * n
+  tc = 0
+  bc = 0
+  i = 0
+  result[i] = tc; tc = tc + 1; i = i + 1
+  turn = True
+  while i < n - 1:
+    if turn:
+      result[i] = n2 + bc; bc = bc + 1; i = i + 1
+      result[i] = n2 + bc; bc = bc + 1; i = i + 1
+    else:
+      result[i] = tc; tc = tc + 1; i = i + 1
+      result[i] = tc; tc = tc + 1; i = i + 1
+    turn = not turn
+  result[i] = tc; tc = tc + 1; i = i + 1
+  return result
+
+def iP4(n):
+  return transposePermutation(P4(n))
+
+def d2n(x):
+  n = len(x)
+  if n == 1:
+    return x
+  y = B2(x)
+  (top, bottom) = split(y)
+  return permute(d2n(top) + d4n(bottom, N // 2), Pt(n))
+
+def id2n(x):
+  n = len(x)
+  if n == 1:
+    return x
+  (top, bottom) = split(permute(x, P(n)))
+  return iB2(id2n(top) + id4n(bottom, N // 2))
+
+def d4n(x, rn):
+  n = len(x)
+  if n == 1:
+    return x
+  y = B4(x, rn)
+  (top, bottom) = split(y)
+  rn2 = rn // 2
+  return permute(d4n(top, rn2) + d4n(bottom, N - rn2), P4(n))
+
+def id4n(x, rn):
+  n = len(x)
+  if n == 1:
+    return x
+  (top, bottom) = split(permute(x, iP4(n)))
+  rn2 = rn // 2
+  y = id4n(top, rn2) + id4n(bottom, N -rn2)
+  return iB4(y, rn)
+
+#######################################################################
+# Main.
+#######################################################################
+
+def help():
+  print("Usage: %s [N [T]]" % sys.argv[0])
+  print("  N should be the power of 2, default is 8")
+  print("  T is one of {2, 3}, default is 2")
+  sys.exit()
+
+def parseInt(s):
+  try:
+    return int(s)
+  except ValueError:
+    help()
+
+if __name__ == "__main__":
+  if len(sys.argv) < 1 or len(sys.argv) > 3: help()
+  if len(sys.argv) >= 2:
+    N = parseInt(sys.argv[1])
+    if (N & (N - 1)) != 0: help()
+  type = 0
+  if len(sys.argv) >= 3:
+    typeOption = sys.argv[2]
+    if len(typeOption) != 1: help()
+    type = "23".index(typeOption)
+    if type == -1: help()
+  if type == 0:
+    vars = d2n(makeVars())
+  else:  # type == 1
+    vars = id2n(makeVars())
+  print("Output vector: " + str(vars))
+```
+
diff --git a/doc/tables/ducky.md b/doc/tables/ducky.md
new file mode 100644
index 000000000..307f68804
--- /dev/null
+++ b/doc/tables/ducky.md
@@ -0,0 +1,6 @@
+#### Table M.7 – "Ducky" marker template
+
+```
+0xEC, 0x00, 0x11, 0x44, 0x75, 0x63, 0x6B, 0x79, 0x00, 0x01, 0x00, 0x04, 0x00, 0x00, 0x00, 0x64, 0x00, 0x00
+```
+
diff --git a/doc/tables/freq_context.md b/doc/tables/freq_context.md
new file mode 100644
index 000000000..3e218fb1b
--- /dev/null
+++ b/doc/tables/freq_context.md
@@ -0,0 +1,54 @@
+#### Table M.15 – freq_context
+
+`scheme == 0`:
+```
+0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+```
+
+`scheme == 1`:
+```
+0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
+1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0
+```
+
+`scheme == 2`:
+```
+0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+2, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
+3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 1, 1, 1
+```
+
+`scheme == 3`:
+```
+0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,
+6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,
+7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 2, 2, 2
+```
+
+`scheme == 4`:
+```
+ 0,  1,  2,  3,  4,  4,  5,  5,  6,  6,  7,  7,  8,  8,  8,  8,  9,  9,
+ 9,  9, 10, 10, 10, 10, 11, 11, 11, 11, 12, 12, 12, 12, 13, 13, 13, 13,
+13, 13, 13, 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15,
+15, 15, 15, 15, 15, 15, 15, 15, 15, 15
+```
+
+`scheme == 5`:
+```
+ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 16,
+17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22, 23, 23, 24, 24, 24, 24,
+25, 25, 25, 25, 26, 26, 26, 26, 27, 27, 27, 27, 28, 28, 28, 28, 29, 29,
+29, 29, 30, 30, 30, 30, 31, 31, 31, 31
+```
+
+`scheme == 6`:
+```
+ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,
+18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,
+36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,
+54, 55, 56, 57, 58, 59, 60, 61, 62, 63
+```
+
diff --git a/doc/tables/icc.md b/doc/tables/icc.md
new file mode 100644
index 000000000..1f3b4cde3
--- /dev/null
+++ b/doc/tables/icc.md
@@ -0,0 +1,6 @@
+#### Table M.6 – common ICC profile template
+
+```
+0xE2, 0x0C, 0x58, 0x49, 0x43, 0x43, 0x5F, 0x50, 0x52, 0x4F, 0x46, 0x49, 0x4C, 0x45, 0x00, 0x01, 0x01, 0x00, 0x00, 0x0C, 0x48, 0x4C, 0x69, 0x6E, 0x6F, 0x02, 0x10, 0x00, 0x00, 0x6D, 0x6E, 0x74, 0x72, 0x52, 0x47, 0x42, 0x20, 0x58, 0x59, 0x5A, 0x20, 0x07, 0xCE, 0x00, 0x02, 0x00, 0x09, 0x00, 0x06, 0x00, 0x31, 0x00, 0x00, 0x61, 0x63, 0x73, 0x70, 0x4D, 0x53, 0x46, 0x54, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x43, 0x20, 0x73, 0x52, 0x47, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xF6, 0xD6, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xD3, 0x2D, 0x48, 0x50, 0x20, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x11, 0x63, 0x70, 0x72, 0x74, 0x00, 0x00, 0x01, 0x50, 0x00, 0x00, 0x00, 0x33, 0x64, 0x65, 0x73, 0x63, 0x00, 0x00, 0x01, 0x84, 0x00, 0x00, 0x00, 0x6C, 0x77, 0x74, 0x70, 0x74, 0x00, 0x00, 0x01, 0xF0, 0x00, 0x00, 0x00, 0x14, 0x62, 0x6B, 0x70, 0x74, 0x00, 0x00, 0x02, 0x04, 0x00, 0x00, 0x00, 0x14, 0x72, 0x58, 0x59, 0x5A, 0x00, 0x00, 0x02, 0x18, 0x00, 0x00, 0x00, 0x14, 0x67, 0x58, 0x59, 0x5A, 0x00, 0x00, 0x02, 0x2C, 0x00, 0x00, 0x00, 0x14, 0x62, 0x58, 0x59, 0x5A, 0x00, 0x00, 0x02, 0x40, 0x00, 0x00, 0x00, 0x14, 0x64, 0x6D, 0x6E, 0x64, 0x00, 0x00, 0x02, 0x54, 0x00, 0x00, 0x00, 0x70, 0x64, 0x6D, 0x64, 0x64, 0x00, 0x00, 0x02, 0xC4, 0x00, 0x00, 0x00, 0x88, 0x76, 0x75, 0x65, 0x64, 0x00, 0x00, 0x03, 0x4C, 0x00, 0x00, 0x00, 0x86, 0x76, 0x69, 0x65, 0x77, 0x00, 0x00, 0x03, 0xD4, 0x00, 0x00, 0x00, 0x24, 0x6C, 0x75, 0x6D, 0x69, 0x00, 0x00, 0x03, 0xF8, 0x00, 0x00, 0x00, 0x14, 0x6D, 0x65, 0x61, 0x73, 0x00, 0x00, 0x04, 0x0C, 0x00, 0x00, 0x00, 0x24, 0x74, 0x65, 0x63, 0x68, 0x00, 0x00, 0x04, 0x30, 0x00, 0x00, 0x00, 0x0C, 0x72, 0x54, 0x52, 0x43, 0x00, 0x00, 0x04, 0x3C, 0x00, 0x00, 0x08, 0x0C, 0x67, 0x54, 0x52, 0x43, 0x00, 0x00, 0x04, 0x3C, 0x00, 0x00, 0x08, 0x0C, 0x62, 0x54, 0x52, 0x43, 0x00, 0x00, 0x04, 0x3C, 0x00, 0x00, 0x08, 0x0C, 0x74, 0x65, 0x78, 0x74, 0x00, 0x00, 0x00, 0x00, 0x43, 0x6F, 0x70, 0x79, 0x72, 0x69, 0x67, 0x68, 0x74, 0x20, 0x28, 0x63, 0x29, 0x20, 0x31, 0x39, 0x39, 0x38, 0x20, 0x48, 0x65, 0x77, 0x6C, 0x65, 0x74, 0x74, 0x2D, 0x50, 0x61, 0x63, 0x6B, 0x61, 0x72, 0x64, 0x20, 0x43, 0x6F, 0x6D, 0x70, 0x61, 0x6E, 0x79, 0x00, 0x00, 0x64, 0x65, 0x73, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x73, 0x52, 0x47, 0x42, 0x20, 0x49, 0x45, 0x43, 0x36, 0x31, 0x39, 0x36, 0x36, 0x2D, 0x32, 0x2E, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x73, 0x52, 0x47, 0x42, 0x20, 0x49, 0x45, 0x43, 0x36, 0x31, 0x39, 0x36, 0x36, 0x2D, 0x32, 0x2E, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x59, 0x5A, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xF3, 0x51, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x16, 0xCC, 0x58, 0x59, 0x5A, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x58, 0x59, 0x5A, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x6F, 0xA2, 0x00, 0x00, 0x38, 0xF5, 0x00, 0x00, 0x03, 0x90, 0x58, 0x59, 0x5A, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x99, 0x00, 0x00, 0xB7, 0x85, 0x00, 0x00, 0x18, 0xDA, 0x58, 0x59, 0x5A, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0xA0, 0x00, 0x00, 0x0F, 0x84, 0x00, 0x00, 0xB6, 0xCF, 0x64, 0x65, 0x73, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x49, 0x45, 0x43, 0x20, 0x68, 0x74, 0x74, 0x70, 0x3A, 0x2F, 0x2F, 0x77, 0x77, 0x77, 0x2E, 0x69, 0x65, 0x63, 0x2E, 0x63, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x49, 0x45, 0x43, 0x20, 0x68, 0x74, 0x74, 0x70, 0x3A, 0x2F, 0x2F, 0x77, 0x77, 0x77, 0x2E, 0x69, 0x65, 0x63, 0x2E, 0x63, 0x68, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x65, 0x73, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2E, 0x49, 0x45, 0x43, 0x20, 0x36, 0x31, 0x39, 0x36, 0x36, 0x2D, 0x32, 0x2E, 0x31, 0x20, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6C, 0x74, 0x20, 0x52, 0x47, 0x42, 0x20, 0x63, 0x6F, 0x6C, 0x6F, 0x75, 0x72, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x2D, 0x20, 0x73, 0x52, 0x47, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2E, 0x49, 0x45, 0x43, 0x20, 0x36, 0x31, 0x39, 0x36, 0x36, 0x2D, 0x32, 0x2E, 0x31, 0x20, 0x44, 0x65, 0x66, 0x61, 0x75, 0x6C, 0x74, 0x20, 0x52, 0x47, 0x42, 0x20, 0x63, 0x6F, 0x6C, 0x6F, 0x75, 0x72, 0x20, 0x73, 0x70, 0x61, 0x63, 0x65, 0x20, 0x2D, 0x20, 0x73, 0x52, 0x47, 0x42, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x65, 0x73, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2C, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x20, 0x56, 0x69, 0x65, 0x77, 0x69, 0x6E, 0x67, 0x20, 0x43, 0x6F, 0x6E, 0x64, 0x69, 0x74, 0x69, 0x6F, 0x6E, 0x20, 0x69, 0x6E, 0x20, 0x49, 0x45, 0x43, 0x36, 0x31, 0x39, 0x36, 0x36, 0x2D, 0x32, 0x2E, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2C, 0x52, 0x65, 0x66, 0x65, 0x72, 0x65, 0x6E, 0x63, 0x65, 0x20, 0x56, 0x69, 0x65, 0x77, 0x69, 0x6E, 0x67, 0x20, 0x43, 0x6F, 0x6E, 0x64, 0x69, 0x74, 0x69, 0x6F, 0x6E, 0x20, 0x69, 0x6E, 0x20, 0x49, 0x45, 0x43, 0x36, 0x31, 0x39, 0x36, 0x36, 0x2D, 0x32, 0x2E, 0x31, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x69, 0x65, 0x77, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0xA4, 0xFE, 0x00, 0x14, 0x5F, 0x2E, 0x00, 0x10, 0xCF, 0x14, 0x00, 0x03, 0xED, 0xCC, 0x00, 0x04, 0x13, 0x0B, 0x00, 0x03, 0x5C, 0x9E, 0x00, 0x00, 0x00, 0x01, 0x58, 0x59, 0x5A, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4C, 0x09, 0x56, 0x00, 0x50, 0x00, 0x00, 0x00, 0x57, 0x1F, 0xE7, 0x6D, 0x65, 0x61, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x8F, 0x00, 0x00, 0x00, 0x02, 0x73, 0x69, 0x67, 0x20, 0x00, 0x00, 0x00, 0x00, 0x43, 0x52, 0x54, 0x20, 0x63, 0x75, 0x72, 0x76, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x0A, 0x00, 0x0F, 0x00, 0x14, 0x00, 0x19, 0x00, 0x1E, 0x00, 0x23, 0x00, 0x28, 0x00, 0x2D, 0x00, 0x32, 0x00, 0x37, 0x00, 0x3B, 0x00, 0x40, 0x00, 0x45, 0x00, 0x4A, 0x00, 0x4F, 0x00, 0x54, 0x00, 0x59, 0x00, 0x5E, 0x00, 0x63, 0x00, 0x68, 0x00, 0x6D, 0x00, 0x72, 0x00, 0x77, 0x00, 0x7C, 0x00, 0x81, 0x00, 0x86, 0x00, 0x8B, 0x00, 0x90, 0x00, 0x95, 0x00, 0x9A, 0x00, 0x9F, 0x00, 0xA4, 0x00, 0xA9, 0x00, 0xAE, 0x00, 0xB2, 0x00, 0xB7, 0x00, 0xBC, 0x00, 0xC1, 0x00, 0xC6, 0x00, 0xCB, 0x00, 0xD0, 0x00, 0xD5, 0x00, 0xDB, 0x00, 0xE0, 0x00, 0xE5, 0x00, 0xEB, 0x00, 0xF0, 0x00, 0xF6, 0x00, 0xFB, 0x01, 0x01, 0x01, 0x07, 0x01, 0x0D, 0x01, 0x13, 0x01, 0x19, 0x01, 0x1F, 0x01, 0x25, 0x01, 0x2B, 0x01, 0x32, 0x01, 0x38, 0x01, 0x3E, 0x01, 0x45, 0x01, 0x4C, 0x01, 0x52, 0x01, 0x59, 0x01, 0x60, 0x01, 0x67, 0x01, 0x6E, 0x01, 0x75, 0x01, 0x7C, 0x01, 0x83, 0x01, 0x8B, 0x01, 0x92, 0x01, 0x9A, 0x01, 0xA1, 0x01, 0xA9, 0x01, 0xB1, 0x01, 0xB9, 0x01, 0xC1, 0x01, 0xC9, 0x01, 0xD1, 0x01, 0xD9, 0x01, 0xE1, 0x01, 0xE9, 0x01, 0xF2, 0x01, 0xFA, 0x02, 0x03, 0x02, 0x0C, 0x02, 0x14, 0x02, 0x1D, 0x02, 0x26, 0x02, 0x2F, 0x02, 0x38, 0x02, 0x41, 0x02, 0x4B, 0x02, 0x54, 0x02, 0x5D, 0x02, 0x67, 0x02, 0x71, 0x02, 0x7A, 0x02, 0x84, 0x02, 0x8E, 0x02, 0x98, 0x02, 0xA2, 0x02, 0xAC, 0x02, 0xB6, 0x02, 0xC1, 0x02, 0xCB, 0x02, 0xD5, 0x02, 0xE0, 0x02, 0xEB, 0x02, 0xF5, 0x03, 0x00, 0x03, 0x0B, 0x03, 0x16, 0x03, 0x21, 0x03, 0x2D, 0x03, 0x38, 0x03, 0x43, 0x03, 0x4F, 0x03, 0x5A, 0x03, 0x66, 0x03, 0x72, 0x03, 0x7E, 0x03, 0x8A, 0x03, 0x96, 0x03, 0xA2, 0x03, 0xAE, 0x03, 0xBA, 0x03, 0xC7, 0x03, 0xD3, 0x03, 0xE0, 0x03, 0xEC, 0x03, 0xF9, 0x04, 0x06, 0x04, 0x13, 0x04, 0x20, 0x04, 0x2D, 0x04, 0x3B, 0x04, 0x48, 0x04, 0x55, 0x04, 0x63, 0x04, 0x71, 0x04, 0x7E, 0x04, 0x8C, 0x04, 0x9A, 0x04, 0xA8, 0x04, 0xB6, 0x04, 0xC4, 0x04, 0xD3, 0x04, 0xE1, 0x04, 0xF0, 0x04, 0xFE, 0x05, 0x0D, 0x05, 0x1C, 0x05, 0x2B, 0x05, 0x3A, 0x05, 0x49, 0x05, 0x58, 0x05, 0x67, 0x05, 0x77, 0x05, 0x86, 0x05, 0x96, 0x05, 0xA6, 0x05, 0xB5, 0x05, 0xC5, 0x05, 0xD5, 0x05, 0xE5, 0x05, 0xF6, 0x06, 0x06, 0x06, 0x16, 0x06, 0x27, 0x06, 0x37, 0x06, 0x48, 0x06, 0x59, 0x06, 0x6A, 0x06, 0x7B, 0x06, 0x8C, 0x06, 0x9D, 0x06, 0xAF, 0x06, 0xC0, 0x06, 0xD1, 0x06, 0xE3, 0x06, 0xF5, 0x07, 0x07, 0x07, 0x19, 0x07, 0x2B, 0x07, 0x3D, 0x07, 0x4F, 0x07, 0x61, 0x07, 0x74, 0x07, 0x86, 0x07, 0x99, 0x07, 0xAC, 0x07, 0xBF, 0x07, 0xD2, 0x07, 0xE5, 0x07, 0xF8, 0x08, 0x0B, 0x08, 0x1F, 0x08, 0x32, 0x08, 0x46, 0x08, 0x5A, 0x08, 0x6E, 0x08, 0x82, 0x08, 0x96, 0x08, 0xAA, 0x08, 0xBE, 0x08, 0xD2, 0x08, 0xE7, 0x08, 0xFB, 0x09, 0x10, 0x09, 0x25, 0x09, 0x3A, 0x09, 0x4F, 0x09, 0x64, 0x09, 0x79, 0x09, 0x8F, 0x09, 0xA4, 0x09, 0xBA, 0x09, 0xCF, 0x09, 0xE5, 0x09, 0xFB, 0x0A, 0x11, 0x0A, 0x27, 0x0A, 0x3D, 0x0A, 0x54, 0x0A, 0x6A, 0x0A, 0x81, 0x0A, 0x98, 0x0A, 0xAE, 0x0A, 0xC5, 0x0A, 0xDC, 0x0A, 0xF3, 0x0B, 0x0B, 0x0B, 0x22, 0x0B, 0x39, 0x0B, 0x51, 0x0B, 0x69, 0x0B, 0x80, 0x0B, 0x98, 0x0B, 0xB0, 0x0B, 0xC8, 0x0B, 0xE1, 0x0B, 0xF9, 0x0C, 0x12, 0x0C, 0x2A, 0x0C, 0x43, 0x0C, 0x5C, 0x0C, 0x75, 0x0C, 0x8E, 0x0C, 0xA7, 0x0C, 0xC0, 0x0C, 0xD9, 0x0C, 0xF3, 0x0D, 0x0D, 0x0D, 0x26, 0x0D, 0x40, 0x0D, 0x5A, 0x0D, 0x74, 0x0D, 0x8E, 0x0D, 0xA9, 0x0D, 0xC3, 0x0D, 0xDE, 0x0D, 0xF8, 0x0E, 0x13, 0x0E, 0x2E, 0x0E, 0x49, 0x0E, 0x64, 0x0E, 0x7F, 0x0E, 0x9B, 0x0E, 0xB6, 0x0E, 0xD2, 0x0E, 0xEE, 0x0F, 0x09, 0x0F, 0x25, 0x0F, 0x41, 0x0F, 0x5E, 0x0F, 0x7A, 0x0F, 0x96, 0x0F, 0xB3, 0x0F, 0xCF, 0x0F, 0xEC, 0x10, 0x09, 0x10, 0x26, 0x10, 0x43, 0x10, 0x61, 0x10, 0x7E, 0x10, 0x9B, 0x10, 0xB9, 0x10, 0xD7, 0x10, 0xF5, 0x11, 0x13, 0x11, 0x31, 0x11, 0x4F, 0x11, 0x6D, 0x11, 0x8C, 0x11, 0xAA, 0x11, 0xC9, 0x11, 0xE8, 0x12, 0x07, 0x12, 0x26, 0x12, 0x45, 0x12, 0x64, 0x12, 0x84, 0x12, 0xA3, 0x12, 0xC3, 0x12, 0xE3, 0x13, 0x03, 0x13, 0x23, 0x13, 0x43, 0x13, 0x63, 0x13, 0x83, 0x13, 0xA4, 0x13, 0xC5, 0x13, 0xE5, 0x14, 0x06, 0x14, 0x27, 0x14, 0x49, 0x14, 0x6A, 0x14, 0x8B, 0x14, 0xAD, 0x14, 0xCE, 0x14, 0xF0, 0x15, 0x12, 0x15, 0x34, 0x15, 0x56, 0x15, 0x78, 0x15, 0x9B, 0x15, 0xBD, 0x15, 0xE0, 0x16, 0x03, 0x16, 0x26, 0x16, 0x49, 0x16, 0x6C, 0x16, 0x8F, 0x16, 0xB2, 0x16, 0xD6, 0x16, 0xFA, 0x17, 0x1D, 0x17, 0x41, 0x17, 0x65, 0x17, 0x89, 0x17, 0xAE, 0x17, 0xD2, 0x17, 0xF7, 0x18, 0x1B, 0x18, 0x40, 0x18, 0x65, 0x18, 0x8A, 0x18, 0xAF, 0x18, 0xD5, 0x18, 0xFA, 0x19, 0x20, 0x19, 0x45, 0x19, 0x6B, 0x19, 0x91, 0x19, 0xB7, 0x19, 0xDD, 0x1A, 0x04, 0x1A, 0x2A, 0x1A, 0x51, 0x1A, 0x77, 0x1A, 0x9E, 0x1A, 0xC5, 0x1A, 0xEC, 0x1B, 0x14, 0x1B, 0x3B, 0x1B, 0x63, 0x1B, 0x8A, 0x1B, 0xB2, 0x1B, 0xDA, 0x1C, 0x02, 0x1C, 0x2A, 0x1C, 0x52, 0x1C, 0x7B, 0x1C, 0xA3, 0x1C, 0xCC, 0x1C, 0xF5, 0x1D, 0x1E, 0x1D, 0x47, 0x1D, 0x70, 0x1D, 0x99, 0x1D, 0xC3, 0x1D, 0xEC, 0x1E, 0x16, 0x1E, 0x40, 0x1E, 0x6A, 0x1E, 0x94, 0x1E, 0xBE, 0x1E, 0xE9, 0x1F, 0x13, 0x1F, 0x3E, 0x1F, 0x69, 0x1F, 0x94, 0x1F, 0xBF, 0x1F, 0xEA, 0x20, 0x15, 0x20, 0x41, 0x20, 0x6C, 0x20, 0x98, 0x20, 0xC4, 0x20, 0xF0, 0x21, 0x1C, 0x21, 0x48, 0x21, 0x75, 0x21, 0xA1, 0x21, 0xCE, 0x21, 0xFB, 0x22, 0x27, 0x22, 0x55, 0x22, 0x82, 0x22, 0xAF, 0x22, 0xDD, 0x23, 0x0A, 0x23, 0x38, 0x23, 0x66, 0x23, 0x94, 0x23, 0xC2, 0x23, 0xF0, 0x24, 0x1F, 0x24, 0x4D, 0x24, 0x7C, 0x24, 0xAB, 0x24, 0xDA, 0x25, 0x09, 0x25, 0x38, 0x25, 0x68, 0x25, 0x97, 0x25, 0xC7, 0x25, 0xF7, 0x26, 0x27, 0x26, 0x57, 0x26, 0x87, 0x26, 0xB7, 0x26, 0xE8, 0x27, 0x18, 0x27, 0x49, 0x27, 0x7A, 0x27, 0xAB, 0x27, 0xDC, 0x28, 0x0D, 0x28, 0x3F, 0x28, 0x71, 0x28, 0xA2, 0x28, 0xD4, 0x29, 0x06, 0x29, 0x38, 0x29, 0x6B, 0x29, 0x9D, 0x29, 0xD0, 0x2A, 0x02, 0x2A, 0x35, 0x2A, 0x68, 0x2A, 0x9B, 0x2A, 0xCF, 0x2B, 0x02, 0x2B, 0x36, 0x2B, 0x69, 0x2B, 0x9D, 0x2B, 0xD1, 0x2C, 0x05, 0x2C, 0x39, 0x2C, 0x6E, 0x2C, 0xA2, 0x2C, 0xD7, 0x2D, 0x0C, 0x2D, 0x41, 0x2D, 0x76, 0x2D, 0xAB, 0x2D, 0xE1, 0x2E, 0x16, 0x2E, 0x4C, 0x2E, 0x82, 0x2E, 0xB7, 0x2E, 0xEE, 0x2F, 0x24, 0x2F, 0x5A, 0x2F, 0x91, 0x2F, 0xC7, 0x2F, 0xFE, 0x30, 0x35, 0x30, 0x6C, 0x30, 0xA4, 0x30, 0xDB, 0x31, 0x12, 0x31, 0x4A, 0x31, 0x82, 0x31, 0xBA, 0x31, 0xF2, 0x32, 0x2A, 0x32, 0x63, 0x32, 0x9B, 0x32, 0xD4, 0x33, 0x0D, 0x33, 0x46, 0x33, 0x7F, 0x33, 0xB8, 0x33, 0xF1, 0x34, 0x2B, 0x34, 0x65, 0x34, 0x9E, 0x34, 0xD8, 0x35, 0x13, 0x35, 0x4D, 0x35, 0x87, 0x35, 0xC2, 0x35, 0xFD, 0x36, 0x37, 0x36, 0x72, 0x36, 0xAE, 0x36, 0xE9, 0x37, 0x24, 0x37, 0x60, 0x37, 0x9C, 0x37, 0xD7, 0x38, 0x14, 0x38, 0x50, 0x38, 0x8C, 0x38, 0xC8, 0x39, 0x05, 0x39, 0x42, 0x39, 0x7F, 0x39, 0xBC, 0x39, 0xF9, 0x3A, 0x36, 0x3A, 0x74, 0x3A, 0xB2, 0x3A, 0xEF, 0x3B, 0x2D, 0x3B, 0x6B, 0x3B, 0xAA, 0x3B, 0xE8, 0x3C, 0x27, 0x3C, 0x65, 0x3C, 0xA4, 0x3C, 0xE3, 0x3D, 0x22, 0x3D, 0x61, 0x3D, 0xA1, 0x3D, 0xE0, 0x3E, 0x20, 0x3E, 0x60, 0x3E, 0xA0, 0x3E, 0xE0, 0x3F, 0x21, 0x3F, 0x61, 0x3F, 0xA2, 0x3F, 0xE2, 0x40, 0x23, 0x40, 0x64, 0x40, 0xA6, 0x40, 0xE7, 0x41, 0x29, 0x41, 0x6A, 0x41, 0xAC, 0x41, 0xEE, 0x42, 0x30, 0x42, 0x72, 0x42, 0xB5, 0x42, 0xF7, 0x43, 0x3A, 0x43, 0x7D, 0x43, 0xC0, 0x44, 0x03, 0x44, 0x47, 0x44, 0x8A, 0x44, 0xCE, 0x45, 0x12, 0x45, 0x55, 0x45, 0x9A, 0x45, 0xDE, 0x46, 0x22, 0x46, 0x67, 0x46, 0xAB, 0x46, 0xF0, 0x47, 0x35, 0x47, 0x7B, 0x47, 0xC0, 0x48, 0x05, 0x48, 0x4B, 0x48, 0x91, 0x48, 0xD7, 0x49, 0x1D, 0x49, 0x63, 0x49, 0xA9, 0x49, 0xF0, 0x4A, 0x37, 0x4A, 0x7D, 0x4A, 0xC4, 0x4B, 0x0C, 0x4B, 0x53, 0x4B, 0x9A, 0x4B, 0xE2, 0x4C, 0x2A, 0x4C, 0x72, 0x4C, 0xBA, 0x4D, 0x02, 0x4D, 0x4A, 0x4D, 0x93, 0x4D, 0xDC, 0x4E, 0x25, 0x4E, 0x6E, 0x4E, 0xB7, 0x4F, 0x00, 0x4F, 0x49, 0x4F, 0x93, 0x4F, 0xDD, 0x50, 0x27, 0x50, 0x71, 0x50, 0xBB, 0x51, 0x06, 0x51, 0x50, 0x51, 0x9B, 0x51, 0xE6, 0x52, 0x31, 0x52, 0x7C, 0x52, 0xC7, 0x53, 0x13, 0x53, 0x5F, 0x53, 0xAA, 0x53, 0xF6, 0x54, 0x42, 0x54, 0x8F, 0x54, 0xDB, 0x55, 0x28, 0x55, 0x75, 0x55, 0xC2, 0x56, 0x0F, 0x56, 0x5C, 0x56, 0xA9, 0x56, 0xF7, 0x57, 0x44, 0x57, 0x92, 0x57, 0xE0, 0x58, 0x2F, 0x58, 0x7D, 0x58, 0xCB, 0x59, 0x1A, 0x59, 0x69, 0x59, 0xB8, 0x5A, 0x07, 0x5A, 0x56, 0x5A, 0xA6, 0x5A, 0xF5, 0x5B, 0x45, 0x5B, 0x95, 0x5B, 0xE5, 0x5C, 0x35, 0x5C, 0x86, 0x5C, 0xD6, 0x5D, 0x27, 0x5D, 0x78, 0x5D, 0xC9, 0x5E, 0x1A, 0x5E, 0x6C, 0x5E, 0xBD, 0x5F, 0x0F, 0x5F, 0x61, 0x5F, 0xB3, 0x60, 0x05, 0x60, 0x57, 0x60, 0xAA, 0x60, 0xFC, 0x61, 0x4F, 0x61, 0xA2, 0x61, 0xF5, 0x62, 0x49, 0x62, 0x9C, 0x62, 0xF0, 0x63, 0x43, 0x63, 0x97, 0x63, 0xEB, 0x64, 0x40, 0x64, 0x94, 0x64, 0xE9, 0x65, 0x3D, 0x65, 0x92, 0x65, 0xE7, 0x66, 0x3D, 0x66, 0x92, 0x66, 0xE8, 0x67, 0x3D, 0x67, 0x93, 0x67, 0xE9, 0x68, 0x3F, 0x68, 0x96, 0x68, 0xEC, 0x69, 0x43, 0x69, 0x9A, 0x69, 0xF1, 0x6A, 0x48, 0x6A, 0x9F, 0x6A, 0xF7, 0x6B, 0x4F, 0x6B, 0xA7, 0x6B, 0xFF, 0x6C, 0x57, 0x6C, 0xAF, 0x6D, 0x08, 0x6D, 0x60, 0x6D, 0xB9, 0x6E, 0x12, 0x6E, 0x6B, 0x6E, 0xC4, 0x6F, 0x1E, 0x6F, 0x78, 0x6F, 0xD1, 0x70, 0x2B, 0x70, 0x86, 0x70, 0xE0, 0x71, 0x3A, 0x71, 0x95, 0x71, 0xF0, 0x72, 0x4B, 0x72, 0xA6, 0x73, 0x01, 0x73, 0x5D, 0x73, 0xB8, 0x74, 0x14, 0x74, 0x70, 0x74, 0xCC, 0x75, 0x28, 0x75, 0x85, 0x75, 0xE1, 0x76, 0x3E, 0x76, 0x9B, 0x76, 0xF8, 0x77, 0x56, 0x77, 0xB3, 0x78, 0x11, 0x78, 0x6E, 0x78, 0xCC, 0x79, 0x2A, 0x79, 0x89, 0x79, 0xE7, 0x7A, 0x46, 0x7A, 0xA5, 0x7B, 0x04, 0x7B, 0x63, 0x7B, 0xC2, 0x7C, 0x21, 0x7C, 0x81, 0x7C, 0xE1, 0x7D, 0x41, 0x7D, 0xA1, 0x7E, 0x01, 0x7E, 0x62, 0x7E, 0xC2, 0x7F, 0x23, 0x7F, 0x84, 0x7F, 0xE5, 0x80, 0x47, 0x80, 0xA8, 0x81, 0x0A, 0x81, 0x6B, 0x81, 0xCD, 0x82, 0x30, 0x82, 0x92, 0x82, 0xF4, 0x83, 0x57, 0x83, 0xBA, 0x84, 0x1D, 0x84, 0x80, 0x84, 0xE3, 0x85, 0x47, 0x85, 0xAB, 0x86, 0x0E, 0x86, 0x72, 0x86, 0xD7, 0x87, 0x3B, 0x87, 0x9F, 0x88, 0x04, 0x88, 0x69, 0x88, 0xCE, 0x89, 0x33, 0x89, 0x99, 0x89, 0xFE, 0x8A, 0x64, 0x8A, 0xCA, 0x8B, 0x30, 0x8B, 0x96, 0x8B, 0xFC, 0x8C, 0x63, 0x8C, 0xCA, 0x8D, 0x31, 0x8D, 0x98, 0x8D, 0xFF, 0x8E, 0x66, 0x8E, 0xCE, 0x8F, 0x36, 0x8F, 0x9E, 0x90, 0x06, 0x90, 0x6E, 0x90, 0xD6, 0x91, 0x3F, 0x91, 0xA8, 0x92, 0x11, 0x92, 0x7A, 0x92, 0xE3, 0x93, 0x4D, 0x93, 0xB6, 0x94, 0x20, 0x94, 0x8A, 0x94, 0xF4, 0x95, 0x5F, 0x95, 0xC9, 0x96, 0x34, 0x96, 0x9F, 0x97, 0x0A, 0x97, 0x75, 0x97, 0xE0, 0x98, 0x4C, 0x98, 0xB8, 0x99, 0x24, 0x99, 0x90, 0x99, 0xFC, 0x9A, 0x68, 0x9A, 0xD5, 0x9B, 0x42, 0x9B, 0xAF, 0x9C, 0x1C, 0x9C, 0x89, 0x9C, 0xF7, 0x9D, 0x64, 0x9D, 0xD2, 0x9E, 0x40, 0x9E, 0xAE, 0x9F, 0x1D, 0x9F, 0x8B, 0x9F, 0xFA, 0xA0, 0x69, 0xA0, 0xD8, 0xA1, 0x47, 0xA1, 0xB6, 0xA2, 0x26, 0xA2, 0x96, 0xA3, 0x06, 0xA3, 0x76, 0xA3, 0xE6, 0xA4, 0x56, 0xA4, 0xC7, 0xA5, 0x38, 0xA5, 0xA9, 0xA6, 0x1A, 0xA6, 0x8B, 0xA6, 0xFD, 0xA7, 0x6E, 0xA7, 0xE0, 0xA8, 0x52, 0xA8, 0xC4, 0xA9, 0x37, 0xA9, 0xA9, 0xAA, 0x1C, 0xAA, 0x8F, 0xAB, 0x02, 0xAB, 0x75, 0xAB, 0xE9, 0xAC, 0x5C, 0xAC, 0xD0, 0xAD, 0x44, 0xAD, 0xB8, 0xAE, 0x2D, 0xAE, 0xA1, 0xAF, 0x16, 0xAF, 0x8B, 0xB0, 0x00, 0xB0, 0x75, 0xB0, 0xEA, 0xB1, 0x60, 0xB1, 0xD6, 0xB2, 0x4B, 0xB2, 0xC2, 0xB3, 0x38, 0xB3, 0xAE, 0xB4, 0x25, 0xB4, 0x9C, 0xB5, 0x13, 0xB5, 0x8A, 0xB6, 0x01, 0xB6, 0x79, 0xB6, 0xF0, 0xB7, 0x68, 0xB7, 0xE0, 0xB8, 0x59, 0xB8, 0xD1, 0xB9, 0x4A, 0xB9, 0xC2, 0xBA, 0x3B, 0xBA, 0xB5, 0xBB, 0x2E, 0xBB, 0xA7, 0xBC, 0x21, 0xBC, 0x9B, 0xBD, 0x15, 0xBD, 0x8F, 0xBE, 0x0A, 0xBE, 0x84, 0xBE, 0xFF, 0xBF, 0x7A, 0xBF, 0xF5, 0xC0, 0x70, 0xC0, 0xEC, 0xC1, 0x67, 0xC1, 0xE3, 0xC2, 0x5F, 0xC2, 0xDB, 0xC3, 0x58, 0xC3, 0xD4, 0xC4, 0x51, 0xC4, 0xCE, 0xC5, 0x4B, 0xC5, 0xC8, 0xC6, 0x46, 0xC6, 0xC3, 0xC7, 0x41, 0xC7, 0xBF, 0xC8, 0x3D, 0xC8, 0xBC, 0xC9, 0x3A, 0xC9, 0xB9, 0xCA, 0x38, 0xCA, 0xB7, 0xCB, 0x36, 0xCB, 0xB6, 0xCC, 0x35, 0xCC, 0xB5, 0xCD, 0x35, 0xCD, 0xB5, 0xCE, 0x36, 0xCE, 0xB6, 0xCF, 0x37, 0xCF, 0xB8, 0xD0, 0x39, 0xD0, 0xBA, 0xD1, 0x3C, 0xD1, 0xBE, 0xD2, 0x3F, 0xD2, 0xC1, 0xD3, 0x44, 0xD3, 0xC6, 0xD4, 0x49, 0xD4, 0xCB, 0xD5, 0x4E, 0xD5, 0xD1, 0xD6, 0x55, 0xD6, 0xD8, 0xD7, 0x5C, 0xD7, 0xE0, 0xD8, 0x64, 0xD8, 0xE8, 0xD9, 0x6C, 0xD9, 0xF1, 0xDA, 0x76, 0xDA, 0xFB, 0xDB, 0x80, 0xDC, 0x05, 0xDC, 0x8A, 0xDD, 0x10, 0xDD, 0x96, 0xDE, 0x1C, 0xDE, 0xA2, 0xDF, 0x29, 0xDF, 0xAF, 0xE0, 0x36, 0xE0, 0xBD, 0xE1, 0x44, 0xE1, 0xCC, 0xE2, 0x53, 0xE2, 0xDB, 0xE3, 0x63, 0xE3, 0xEB, 0xE4, 0x73, 0xE4, 0xFC, 0xE5, 0x84, 0xE6, 0x0D, 0xE6, 0x96, 0xE7, 0x1F, 0xE7, 0xA9, 0xE8, 0x32, 0xE8, 0xBC, 0xE9, 0x46, 0xE9, 0xD0, 0xEA, 0x5B, 0xEA, 0xE5, 0xEB, 0x70, 0xEB, 0xFB, 0xEC, 0x86, 0xED, 0x11, 0xED, 0x9C, 0xEE, 0x28, 0xEE, 0xB4, 0xEF, 0x40, 0xEF, 0xCC, 0xF0, 0x58, 0xF0, 0xE5, 0xF1, 0x72, 0xF1, 0xFF, 0xF2, 0x8C, 0xF3, 0x19, 0xF3, 0xA7, 0xF4, 0x34, 0xF4, 0xC2, 0xF5, 0x50, 0xF5, 0xDE, 0xF6, 0x6D, 0xF6, 0xFB, 0xF7, 0x8A, 0xF8, 0x19, 0xF8, 0xA8, 0xF9, 0x38, 0xF9, 0xC7, 0xFA, 0x57, 0xFA, 0xE7, 0xFB, 0x77, 0xFC, 0x07, 0xFC, 0x98, 0xFD, 0x29, 0xFD, 0xBA, 0xFE, 0x4B, 0xFE, 0xDC, 0xFF, 0x6D, 0xFF, 0xFF
+```
+
diff --git a/doc/tables/is_zero_base.md b/doc/tables/is_zero_base.md
new file mode 100644
index 000000000..7e2d081f3
--- /dev/null
+++ b/doc/tables/is_zero_base.md
@@ -0,0 +1,9 @@
+#### Table M.1 – is_zero_base table
+
+```
+228, 216, 216, 195, 192, 189, 182, 184, 179, 176, 171, 168, 166, 159,
+156, 151, 151, 150, 150, 146, 144, 138, 138, 137, 135, 131, 127, 126,
+124, 123, 124, 123, 122, 121, 118, 117, 114, 115, 116, 116, 115, 115,
+114, 111, 111, 111, 112, 111, 110, 110, 110, 111, 111, 114, 110, 111,
+112, 113, 116, 120, 126, 131, 147, 160
+```
diff --git a/doc/tables/markdown-pdf.css b/doc/tables/markdown-pdf.css
new file mode 100644
index 000000000..c1efc1cc2
--- /dev/null
+++ b/doc/tables/markdown-pdf.css
@@ -0,0 +1,36 @@
+/*
+ settings.json:
+    "markdown-pdf.styles": ["markdown-pdf.css",],
+    "markdown-pdf.format": "Letter",
+    "markdown-pdf.margin.top": "1in",
+    "markdown-pdf.margin.bottom": "1in",
+    "markdown-pdf.margin.left": "1in",
+    "markdown-pdf.margin.right": "1in",
+    "markdown-pdf.stylesRelativePathFile" : true,
+    "markdown-pdf.displayHeaderFooter": false,
+ */
+
+body {
+  font-family: "Times";
+  font-size: 10pt;
+  padding: 0;
+}
+
+h4 {
+  font-family: "Times New Roman";
+  font-size: 10pt;
+  font-weight: bold;
+}
+
+code {
+  font-family: Consolas, "Source Code Pro";
+  font-size: 10pt;
+}
+
+pre.hljs code > div {
+  padding: 0px;
+}
+
+:not(pre):not(.hljs) > code {
+  color: #4d4d4c;
+}
diff --git a/doc/tables/nonzero_buckets.md b/doc/tables/nonzero_buckets.md
new file mode 100644
index 000000000..77a5a396e
--- /dev/null
+++ b/doc/tables/nonzero_buckets.md
@@ -0,0 +1,9 @@
+#### Table M.17 – nonzero_buckets
+
+```
+  0,  1,  2,  3,  4,  4,  5,  5,  5,  6,  6,  6,  6,  7,  7,  7,  7,
+  7,  7,  7,  7,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  8,  9,  9,
+  9,  9,  9,  9,  9,  9,  9,  9,  9,  9,  9, 10, 10, 10, 10, 10, 10,
+ 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10
+```
+
diff --git a/doc/tables/num_nonzero_context.md b/doc/tables/num_nonzero_context.md
new file mode 100644
index 000000000..b73d48c38
--- /dev/null
+++ b/doc/tables/num_nonzero_context.md
@@ -0,0 +1,60 @@
+#### Table M.16 – num_nonzero_context
+
+`scheme == 0`:
+```
+0, 1, 1, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5,
+6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, 7, 7, 7, 7,
+7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7
+```
+
+`scheme == 1`:
+```
+ 0,  2,  2,  4,  4,  4,  6,  6,  6,  6,  8,  8,  8,  8,  8,  8, 10, 10,
+10, 10, 10, 10, 10, 10, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12,
+12, 12, 12, 12, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14, 14,
+14, 14, 14, 14, 14, 14, 14, 14, 14, 14
+```
+
+`scheme == 2`:
+```
+ 0,  4,  4,  8,  8,  8, 12, 12, 12, 12, 16, 16, 16, 16, 16, 16, 20, 20,
+20, 20, 20, 20, 20, 20, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24, 24,
+24, 24, 24, 24, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28, 28,
+28, 28, 28, 28, 28, 28, 28, 28, 28, 28
+```
+
+`scheme == 3`:
+```
+ 0,  8,  8, 16, 16, 16, 24, 24, 24, 24, 32, 32, 32, 32, 32, 32, 40, 40,
+40, 40, 40, 40, 40, 40, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48,
+48, 48, 48, 48, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55,
+55, 55, 55, 55, 55, 55, 55, 55, 55, 55
+```
+
+`scheme == 4`:
+```
+  0,  16,  16,  32,  32,  32,  48,  48,  48,  48,  64,  64,  64,  64,
+ 64,  64,  80,  80,  80,  80,  80,  80,  80,  80,  95,  95,  95,  95,
+ 95,  95,  95,  95,  95,  95,  95,  95,  95,  95,  95,  95, 109, 109,
+109, 109, 109, 109, 109, 109, 109, 109, 109, 109, 109, 109, 109, 109,
+109, 109, 109, 109, 109, 109, 109, 109
+```
+
+`scheme == 5`:
+```
+  0,  32,  32,  64,  64,  64,  96,  96,  96,  96, 127, 127, 127, 127,
+127, 127, 157, 157, 157, 157, 157, 157, 157, 157, 185, 185, 185, 185,
+185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 185, 211, 211,
+211, 211, 211, 211, 211, 211, 211, 211, 211, 211, 211, 211, 211, 211,
+211, 211, 211, 211, 211, 211, 211, 211
+```
+
+`scheme == 6`:
+```
+  0,  64,  64, 127, 127, 127, 188, 188, 188, 188, 246, 246, 246, 246,
+246, 246, 300, 300, 300, 300, 300, 300, 300, 300, 348, 348, 348, 348,
+348, 348, 348, 348, 348, 348, 348, 348, 348, 348, 348, 348, 388, 388,
+388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388, 388,
+388, 388, 388, 388, 388, 388, 388, 388
+```
+
diff --git a/doc/tables/num_nonzeros_base.md b/doc/tables/num_nonzeros_base.md
new file mode 100644
index 000000000..165c7389c
--- /dev/null
+++ b/doc/tables/num_nonzeros_base.md
@@ -0,0 +1,258 @@
+#### Table M.2 – num_nonzeros_base table
+
+```
+251, 252, 117, 249, 161, 136,  83, 238, 184, 126, 137, 129, 140, 119,
+ 70, 213, 160, 175, 174, 130, 166, 134, 122, 125, 131, 144, 136, 133,
+139, 123,  79, 216, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+254, 252, 174, 232, 189, 155, 122, 177, 204, 173, 146, 149, 141, 133,
+103, 109, 167, 187, 168, 142, 154, 147, 125, 139, 144, 138, 138, 153,
+141, 133,  90, 121, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+251, 240, 197, 176, 184, 177, 114,  89, 194, 165, 153, 161, 158, 136,
+ 92,  95, 123, 171, 160, 140, 148, 136, 129, 139, 145, 136, 143, 134,
+138, 124,  92, 154, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+247, 220, 201, 110, 194, 176, 147,  59, 175, 171, 156, 157, 152, 146,
+115, 114,  88, 151, 164, 141, 153, 135, 141, 131, 146, 139, 140, 145,
+138, 137, 112, 184, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+238, 179, 203,  63, 194, 173, 149,  71, 139, 169, 154, 159, 150, 146,
+117, 143,  78, 122, 152, 137, 149, 138, 138, 133, 134, 142, 142, 142,
+148, 128, 118, 199, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+227, 127, 200,  44, 192, 170, 148, 100, 102, 161, 156, 153, 148, 149,
+124, 160,  88, 101, 134, 132, 149, 145, 134, 134, 136, 141, 138, 142,
+144, 137, 116, 208, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+214,  86, 195,  44, 187, 163, 148, 126,  81, 147, 156, 152, 150, 144,
+121, 172,  96,  95, 117, 122, 145, 152, 136, 133, 135, 135, 131, 142,
+141, 135, 114, 217, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+198,  56, 191,  54, 171, 162, 147, 144,  74, 128, 152, 149, 150, 142,
+119, 177, 101, 100, 106, 111, 135, 154, 136, 137, 136, 132, 133, 142,
+144, 130, 117, 222, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+176,  40, 189,  73, 147, 159, 148, 152,  79, 106, 147, 149, 151, 139,
+123, 188, 108, 110, 106,  97, 125, 151, 137, 138, 135, 135, 134, 136,
+140, 131, 116, 221, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+148,  33, 185,  88, 117, 158, 145, 163,  95,  91, 137, 146, 150, 140,
+120, 197, 115, 116, 114,  92, 114, 144, 130, 133, 132, 133, 129, 140,
+138, 130, 111, 224, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+117,  31, 180, 104,  93, 150, 143, 166,  99,  85, 124, 139, 148, 142,
+118, 201, 105, 120, 120,  90, 107, 135, 127, 130, 131, 131, 132, 140,
+142, 133, 114, 229, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+ 87,  35, 170, 110,  78, 141, 144, 176, 106,  90, 112, 132, 143, 138,
+119, 204, 111, 121, 125,  90, 105, 131, 124, 122, 129, 128, 129, 137,
+138, 133, 114, 227, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+ 63,  42, 159, 123,  73, 127, 142, 191, 105,  91, 105, 123, 139, 137,
+120, 209, 117, 110, 122,  98, 110, 125, 115, 123, 122, 126, 128, 134,
+141, 129, 113, 229, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+ 45,  53, 146, 135,  71, 114, 138, 193, 100,  98,  98, 113, 133, 135,
+118, 222, 113, 111, 139, 103, 107, 126, 111, 119, 121, 122, 127, 135,
+141, 128, 114, 242, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+ 33,  60, 132, 138,  75, 100, 134, 203, 112,  99,  98, 105, 126, 131,
+115, 229, 107,  93, 121, 106, 108, 122, 106, 109, 114, 116, 127, 133,
+143, 128, 110, 242, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+ 24,  70, 118, 134,  76,  87, 130, 201, 110,  96,  99,  97, 119, 130,
+111, 229,  97, 104, 125, 102, 112, 125, 101, 109, 113, 114, 125, 129,
+142, 127, 112, 241, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+ 17,  65, 100, 121,  80,  75, 124, 174, 117, 100,  94,  93, 114, 128,
+110, 216, 103,  94, 113, 122, 118, 126, 113, 108, 105, 108, 122, 128,
+141, 125, 113, 238, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+ 12,  70,  82, 132,  78,  65, 118, 155, 136, 103,  97,  89, 106, 124,
+111, 215, 115, 123, 129,  99, 104, 127, 110, 108, 101, 109, 118, 126,
+136, 123, 110, 233, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  8,  66,  61, 117,  91,  59, 108, 195, 101, 112,  99,  99,  99, 116,
+106, 230, 127,  99, 144, 101, 118, 137, 117, 111, 106, 104, 116, 121,
+134, 122, 110, 223, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  6,  78,  42, 146, 101,  54,  94, 201, 116, 102, 110,  94,  92, 108,
+103, 214, 108, 111, 127, 102, 121, 132, 120, 121,  95,  98, 110, 121,
+129, 117, 107, 235, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  5,  93,  29, 145, 102,  52,  77, 216, 108, 115, 108, 102,  89,  97,
+ 94, 229,  89, 103, 139, 120, 103, 151, 102, 100,  97,  96,  99, 111,
+125, 116, 104, 242, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  4, 105,  21, 145, 100,  54,  64, 217, 100, 122, 128,  87,  88,  91,
+ 87, 230, 112,  80, 148,  95, 146, 123,  96, 140,  90,  91,  98, 106,
+122, 111, 100, 249, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  4, 130,  14, 142, 104,  56,  51, 208, 116, 135, 100,  89,  82,  84,
+ 75, 239,  85,  85, 122, 125,  94, 144, 151, 136,  92,  97, 104, 109,
+113, 110,  91, 246, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  3, 126,   9, 172, 105,  57,  39, 219,  95, 120, 118,  96,  93,  75,
+ 66, 241, 102, 134,  96, 156, 146, 162, 130, 112,  82,  89,  97, 101,
+116, 103,  82, 254, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  3, 149,   7, 182, 122,  54,  29, 224, 103, 100, 113,  96,  90,  74,
+ 55, 250, 127,  94, 118,  93, 135, 160, 113, 130,  95, 117, 106,  96,
+111,  97,  77, 242, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  3, 150,   4, 170, 138,  59,  20, 229,  91, 150, 107,  98,  92,  68,
+ 48, 245, 113,  64, 114, 111, 134, 127, 102, 104,  85, 118, 103, 107,
+102,  91,  72, 245, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  3, 171,   3, 165, 137,  62,  14, 211,  96, 127, 132, 121,  95,  62,
+ 37, 248, 102,  57, 144,  85, 127, 191, 102,  97, 127, 104,  91, 102,
+107,  81,  64, 254, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  2, 166,   2, 196, 122,  65,  10, 243, 102,  93, 117,  92,  96,  63,
+ 29, 251, 169, 159, 149,  96,  91, 139, 157,  40, 100,  89, 120,  92,
+109,  79,  58, 247, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  2, 176,   2, 189, 118,  48,   7, 219,  68,  43, 109,  96, 129,  75,
+ 19, 254,   2,   3, 185,   6, 102, 127, 127, 127,   1, 131,  83,  99,
+107,  80,  45, 254, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  1, 205,   2, 208,  64,  89,   4, 223,  29, 169,  29, 123, 118,  76,
+ 11, 240, 202, 243,  65,   6,  12, 243,  96,  55, 102, 102, 114, 102,
+107,  74,  31, 247, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  1, 216,   1, 214, 127,  94,   2, 234, 145,   3, 127, 106, 155,  80,
+  4, 247,   4,  65,  86, 127, 127, 127, 127, 102, 127, 143, 143, 108,
+113,  80,  16, 216, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
+```
+  2, 199,   1, 222,  93,  94,   1, 232,   2,  65,  74, 139, 201,  48,
+  2, 254, 169, 127,  52, 243, 251, 249, 102,  86, 202, 153,  65,  65,
+146,  69,   8, 238, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128, 128,
+128, 128, 128, 128, 128, 128, 128
+```
+
diff --git a/doc/tables/quant.md b/doc/tables/quant.md
new file mode 100644
index 000000000..1fb80d754
--- /dev/null
+++ b/doc/tables/quant.md
@@ -0,0 +1,19 @@
+#### Table M.13 – template quant tables
+
+`is_luma == true`:
+```
+ 16,  11,  10,  16, 24, 40, 51,  61,  12,  12,  14, 19, 26, 58, 60,
+ 55,  14,  13,  16, 24, 40, 57,  69,  56,  14,  17, 22, 29, 51, 87,
+ 80,  62,  18,  22, 37, 56, 68, 109, 103,  77,  24, 35, 55, 64, 81,
+104, 113,  92,  49, 64, 78, 87, 103, 121, 120, 101, 72, 92, 95, 98,
+112, 100, 103,  99
+```
+
+`is_luma == false`:
+```
+17, 18, 24, 47, 99, 99, 99, 99, 18, 21, 26, 66, 99, 99, 99, 99, 24, 26,
+56, 99, 99, 99, 99, 99, 47, 66, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99,
+99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99,
+99, 99, 99, 99, 99, 99, 99, 99, 99, 99
+```
+
diff --git a/doc/tables/stock_counts.md b/doc/tables/stock_counts.md
new file mode 100644
index 000000000..6e8e4458c
--- /dev/null
+++ b/doc/tables/stock_counts.md
@@ -0,0 +1,22 @@
+#### Table M.9 – stock counts arrays
+
+`is_ac == 0`, `stock_index == 0`:
+```
+0, 0, 3, 1, 1, 1, 1, 1, 1, 1, 1, 2, 0, 0, 0, 0, 0
+```
+
+`is_ac == 0`, `stock_index == 1`:
+```
+0, 0, 1, 5, 1, 1, 1, 1, 1, 2, 0, 0, 0, 0, 0, 0, 0
+```
+
+`is_ac == 1`, `stock_index == 0`:
+```
+0, 0, 2, 1, 3, 3, 2, 4, 3, 5, 5, 4, 4, 0, 0, 1, 126
+```
+
+`is_ac == 1`, `stock_index == 1`:
+```
+0, 0, 2, 1, 2, 4, 4, 3, 4, 7, 5, 4, 4, 0, 1, 2, 120
+```
+
diff --git a/doc/tables/stock_quant.md b/doc/tables/stock_quant.md
new file mode 100644
index 000000000..b32fd3c7d
--- /dev/null
+++ b/doc/tables/stock_quant.md
@@ -0,0 +1,130 @@
+#### Table M.12 – stock quant tables
+
+`is_luma == true`, `stock_index == 0`:
+```
+ 3,  2,  2,  3,  5,  8, 10, 12,  2,  2,  3,  4,  5, 12, 12, 11,  3,  3,
+ 3,  5,  8, 11, 14, 11,  3,  3,  4,  6, 10, 17, 16, 12,  4,  4,  7, 11,
+14, 22, 21, 15,  5,  7, 11, 13, 16, 21, 23, 18, 10, 13, 16, 17, 21, 24,
+24, 20, 14, 18, 19, 20, 22, 20, 21, 20
+```
+
+`is_luma == true`, `stock_index == 1`:
+```
+ 8,  6,  5,  8, 12, 20, 26, 31,  6,  6,  7, 10, 13, 29, 30, 28,  7,  7,
+ 8, 12, 20, 29, 35, 28,  7,  9, 11, 15, 26, 44, 40, 31,  9, 11, 19, 28,
+34, 55, 52, 39, 12, 18, 28, 32, 41, 52, 57, 46, 25, 32, 39, 44, 52, 61,
+60, 51, 36, 46, 48, 49, 56, 50, 52, 50
+```
+
+`is_luma == true`, `stock_index == 2`:
+```
+ 6,  4,  4,  6, 10, 16, 20, 24,  5,  5,  6,  8, 10, 23, 24, 22,  6,  5,
+ 6, 10, 16, 23, 28, 22,  6,  7,  9, 12, 20, 35, 32, 25,  7,  9, 15, 22,
+27, 44, 41, 31, 10, 14, 22, 26, 32, 42, 45, 37, 20, 26, 31, 35, 41, 48,
+48, 40, 29, 37, 38, 39, 45, 40, 41, 40
+```
+
+`is_luma == true`, `stock_index == 3`:
+```
+ 5,  3,  3,  5,  7, 12, 15, 18,  4,  4,  4,  6,  8, 17, 18, 17,  4,  4,
+ 5,  7, 12, 17, 21, 17,  4,  5,  7,  9, 15, 26, 24, 19,  5,  7, 11, 17,
+20, 33, 31, 23,  7, 11, 17, 19, 24, 31, 34, 28, 15, 19, 23, 26, 31, 36,
+36, 30, 22, 28, 29, 29, 34, 30, 31, 30
+```
+
+`is_luma == true`, `stock_index == 4`:
+```
+ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1
+```
+
+`is_luma == true`, `stock_index == 5`:
+```
+ 2,  1,  1,  2,  2,  4,  5,  6,  1,  1,  1,  2,  3,  6,  6,  6,  1,  1,
+ 2,  2,  4,  6,  7,  6,  1,  2,  2,  3,  5,  9,  8,  6,  2,  2,  4,  6,
+ 7, 11, 10,  8,  2,  4,  6,  6,  8, 10, 11,  9,  5,  6,  8,  9, 10, 12,
+12, 10,  7,  9, 10, 10, 11, 10, 10, 10
+```
+
+`is_luma == true`, `stock_index == 6`:
+```
+ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+ 1,  1,  1,  1,  1,  2,  1,  1,  1,  1,  1,  1,  2,  2,  1,  1,  1,  1,
+ 1,  2,  2,  3,  1,  1,  1,  1,  2,  2,  3,  3,  1,  1,  1,  2,  2,  3,
+ 3,  3,  1,  1,  2,  2,  3,  3,  3,  3
+```
+
+`is_luma == true`, `stock_index == 7`:
+```
+10,  7,  6, 10, 14, 24, 31, 37,  7,  7,  8, 11, 16, 35, 36, 33,  8,  8,
+10, 14, 24, 34, 41, 34,  8, 10, 13, 17, 31, 52, 48, 37, 11, 13, 22, 34,
+41, 65, 62, 46, 14, 21, 33, 38, 49, 62, 68, 55, 29, 38, 47, 52, 62, 73,
+72, 61, 43, 55, 57, 59, 67, 60, 62, 59
+```
+
+`is_luma == false`, `stock_index == 0`:
+```
+ 9,  9,  9, 12, 11, 12, 24, 13, 13, 24, 50, 33, 28, 33, 50, 50, 50, 50,
+50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50,
+50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50,
+50, 50, 50, 50, 50, 50, 50, 50, 50, 50
+```
+
+`is_luma == false`, `stock_index == 1`:
+```
+ 3,  4,  5,  9, 20, 20, 20, 20,  4,  4,  5, 13, 20, 20, 20, 20,  5,  5,
+11, 20, 20, 20, 20, 20,  9, 13, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
+20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20, 20,
+20, 20, 20, 20, 20, 20, 20, 20, 20, 20
+```
+
+`is_luma == false`, `stock_index == 2`:
+```
+ 9,  9, 12, 24, 50, 50, 50, 50,  9, 11, 13, 33, 50, 50, 50, 50, 12, 13,
+28, 50, 50, 50, 50, 50, 24, 33, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50,
+50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50, 50,
+50, 50, 50, 50, 50, 50, 50, 50, 50, 50
+```
+
+`is_luma == false`, `stock_index == 3`:
+```
+ 5,  5,  7, 14, 30, 30, 30, 30,  5,  6,  8, 20, 30, 30, 30, 30,  7,  8,
+17, 30, 30, 30, 30, 30, 14, 20, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30,
+30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30, 30,
+30, 30, 30, 30, 30, 30, 30, 30, 30, 30
+```
+
+`is_luma == false`, `stock_index == 4`:
+```
+ 7,  7, 10, 19, 40, 40, 40, 40,  7,  8, 10, 26, 40, 40, 40, 40, 10, 10,
+22, 40, 40, 40, 40, 40, 19, 26, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40, 40,
+40, 40, 40, 40, 40, 40, 40, 40, 40, 40
+```
+
+`is_luma == false`, `stock_index == 5`:
+```
+ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,  1,
+ 1,  1,  1,  1,  1,  1,  1,  1,  1,  1
+```
+
+`is_luma == false`, `stock_index == 6`:
+```
+ 2,  2,  2,  5, 10, 10, 10, 10,  2,  2,  3,  7, 10, 10, 10, 10,  2,  3,
+ 6, 10, 10, 10, 10, 10,  5,  7, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
+10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10, 10,
+10, 10, 10, 10, 10, 10, 10, 10, 10, 10
+```
+
+`is_luma == false`, `stock_index == 7`:
+```
+10, 11, 14, 28, 59, 59, 59, 59, 11, 13, 16, 40, 59, 59, 59, 59, 14, 16,
+34, 59, 59, 59, 59, 59, 28, 40, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59, 59,
+59, 59, 59, 59, 59, 59, 59, 59, 59, 59
+```
+
diff --git a/doc/tables/stock_values.md b/doc/tables/stock_values.md
new file mode 100644
index 000000000..8e67cff1e
--- /dev/null
+++ b/doc/tables/stock_values.md
@@ -0,0 +1,44 @@
+#### Table M.10 – stock values arrays
+
+`is_ac == 0`, `stock_index == 0`:
+```
+0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 256
+```
+
+`is_ac == 0`, `stock_index == 1`:
+```
+0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 256
+```
+
+`is_ac == 1`, `stock_index == 0`:
+```
+  1,   2,   3,   0,   4,  17,   5,  18,  33,  49,  65,   6,  19,  81,
+ 97,   7,  34, 113,  20,  50, 129, 145, 161,   8,  35,  66, 177, 193,
+ 21,  82, 209, 240,  36,  51,  98, 114, 130,   9,  10,  22,  23,  24,
+ 25,  26,  37,  38,  39,  40,  41,  42,  52,  53,  54,  55,  56,  57,
+ 58,  67,  68,  69,  70,  71,  72,  73,  74,  83,  84,  85,  86,  87,
+ 88,  89,  90,  99, 100, 101, 102, 103, 104, 105, 106, 115, 116, 117,
+118, 119, 120, 121, 122, 131, 132, 133, 134, 135, 136, 137, 138, 146,
+147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, 165, 166, 167,
+168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186, 194, 195,
+196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214, 215, 216,
+217, 218, 225, 226, 227, 228, 229, 230, 231, 232, 233, 234, 241, 242,
+243, 244, 245, 246, 247, 248, 249, 250, 256
+```
+
+`is_ac == 1`, `stock_index == 1`:
+```
+  0,   1,   2,   3,  17,   4,   5,  33,  49,   6,  18,  65,  81,   7,
+ 97, 113,  19,  34,  50, 129,   8,  20,  66, 145, 161, 177, 193,   9,
+ 35,  51,  82, 240,  21,  98, 114, 209,  10,  22,  36,  52, 225,  37,
+241,  23,  24,  25,  26,  38,  39,  40,  41,  42,  53,  54,  55,  56,
+ 57,  58,  67,  68,  69,  70,  71,  72,  73,  74,  83,  84,  85,  86,
+ 87,  88,  89,  90,  99, 100, 101, 102, 103, 104, 105, 106, 115, 116,
+117, 118, 119, 120, 121, 122, 130, 131, 132, 133, 134, 135, 136, 137,
+138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, 165,
+166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186,
+194, 195, 196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214,
+215, 216, 217, 218, 226, 227, 228, 229, 230, 231, 232, 233, 234, 242,
+243, 244, 245, 246, 247, 248, 249, 250, 256
+```
+
diff --git a/doc/tables/symbol_order.md b/doc/tables/symbol_order.md
new file mode 100644
index 000000000..a196c0f52
--- /dev/null
+++ b/doc/tables/symbol_order.md
@@ -0,0 +1,30 @@
+#### Table M.11 – predefined symbol order
+
+`is_ac == 0`:
+```
+0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
+```
+
+`is_ac == 1`:
+```
+  1,   0,   2,   3,  17,   4,   5,  33,  18,  49,  65,   6,  81,  19,
+ 97,   7,  34, 113,  50, 129,  20, 145, 161,   8,  35,  66, 177, 193,
+ 21,  82, 209, 240,  36,  51,  98, 114,   9, 130,  10,  22,  52, 225,
+ 23,  37, 241,  24,  25,  26,  38,  39,  40,  41,  42,  53,  54,  55,
+ 56,  57,  58,  67,  68,  69,  70,  71,  72,  73,  74,  83,  84,  85,
+ 86,  87,  88,  89,  90,  99, 100, 101, 102, 103, 104, 105, 106, 115,
+116, 117, 118, 119, 120, 121, 122, 131, 132, 133, 134, 135, 136, 137,
+138, 146, 147, 148, 149, 150, 151, 152, 153, 154, 162, 163, 164, 165,
+166, 167, 168, 169, 170, 178, 179, 180, 181, 182, 183, 184, 185, 186,
+194, 195, 196, 197, 198, 199, 200, 201, 202, 210, 211, 212, 213, 214,
+215, 216, 217, 218, 226, 227, 228, 229, 230, 231, 232, 233, 234, 242,
+243, 244, 245, 246, 247, 248, 249, 250,  16,  32,  48,  64,  80,  96,
+112, 128, 144, 160, 176, 192, 208,  11,  12,  13,  14,  15,  27,  28,
+ 29,  30,  31,  43,  44,  45,  46,  47,  59,  60,  61,  62,  63,  75,
+ 76,  77,  78,  79,  91,  92,  93,  94,  95, 107, 108, 109, 110, 111,
+123, 124, 125, 126, 127, 139, 140, 141, 142, 143, 155, 156, 157, 158,
+159, 171, 172, 173, 174, 175, 187, 188, 189, 190, 191, 203, 204, 205,
+206, 207, 219, 220, 221, 222, 223, 224, 235, 236, 237, 238, 239, 251,
+252, 253, 254, 255
+```
+
diff --git a/doc/xl_overview.md b/doc/xl_overview.md
new file mode 100644
index 000000000..59e0c252d
--- /dev/null
+++ b/doc/xl_overview.md
@@ -0,0 +1,181 @@
+# XL Overview
+
+## Requirements
+
+JPEG XL was designed for two main requirements:
+
+*   high quality: visually lossless at reasonable bitrates;
+*   decoding speed: multithreaded decoding should be able to reach around
+    400 Megapixel/s on large images.
+
+These goals apply to various types of images, including HDR content, whose
+support is made possible by full-precision (float32) computations and extensive
+support of color spaces and transfer functions.
+
+High performance is achieved by designing the format with careful consideration
+of memory bandwidth usage and ease of SIMD/GPU implementation.
+
+The full requirements for JPEG XL are listed in document wg1m82079.
+
+## General architecture
+
+The architecture follows the traditional block transform model with improvements
+in the individual components. For a quick overview, we sketch a "block diagram"
+of the lossy format decoder in the form of module names in **bold** followed by
+a brief description. Note that post-processing modules in [brackets] are
+optional - they are unnecessary or even counterproductive at very high quality
+settings.
+
+**Header**: decode metadata (e.g. image dimensions) from compressed fields
+(smaller than Exp-Golomb thanks to per-field encodings). The compression and
+small number of required fields enables very compact headers - much smaller than
+JFIF and HEVC. The container supports multiple images (e.g. animations/bursts)
+and passes (progressive).
+
+**Bitstream**: decode transform coefficient residuals using rANS-encoded
+<#bits,bits> symbols
+
+**Dequantize**: from adaptive quant map side information, plus chroma from luma
+
+**DC prediction**: expand DC residuals using adaptive (history-based) predictors
+
+**Chroma from luma**: restore predicted X from B and Y from B
+
+**IDCT:** 2x2..32x32, floating-point
+
+**[Gaborish]**: additional deblocking convolution with 3x3 kernel
+
+**[Edge preserving filter]**: nonlinear adaptive smoothing controlled by side
+information
+
+**[Noise injection]**: add perceptually pleasing noise according to a per-image
+noise model
+
+**Color space conversion**: from perceptual opsin XYB to linear RGB
+
+**[Converting to other color spaces via ICC]**
+
+The encoder is basically the reverse:
+
+**Color space conversion**: from linear RGB to perceptual opsin XYB
+
+**[Noise estimation]**: compute a noise model for the image
+
+**[Gaborish]**: sharpening to counteract the blurring on the decoder side
+
+**DCT**: transform sizes communicated via per-block side information
+
+**Chroma from luma**: find the best multipliers of Y for X and B channels of
+entire image
+
+**Adaptive quantization**: iterative search for quant map that yields the best
+perceived restoration
+
+**Quantize**: store 16-bit prediction residuals
+
+**DC prediction**: store residuals (prediction happens in quantized space)
+
+**Entropy coding**: rANS and context modeling with clustering
+
+
+# File Structure
+
+A codestream begins with a `FileHeader` followed by one or more "passes"
+(= scans: e.g. DC or AC_LF) which are then added together (summing the
+respective color components in Opsin space) to form the final image. There is no
+limit to the number of passes, so an encoder could choose to send salient parts
+first, followed by arbitrary decompositions of the final image (in terms of
+resolution, bit depth, quality or spatial location).
+
+Each pass contains groups of AC and DC data. A group is a subset of pixels that
+can be decoded in parallel. DC groups contain 256x256 DCs (from 2048x2048 input
+pixels), AC groups cover 256x256 input pixels.
+
+Each pass starts with a table of contents (sizes of each of their DC+AC
+groups), which enables parallel decoding and/or the decoding of a subset.
+However, there is no higher-level TOC of passes, as that would prevent
+appending additional images and could be too constraining for the encoder.
+
+
+## Lossless
+
+JPEG XL supports tools for lossless coding designed by Alexander Rhatushnyak and
+Jon Sneyers. They are about 60-75% of size of PNG, and smaller than WebP
+lossless for photos.
+
+An adaptive predictor computes 4 from the NW, N, NE and W pixels and combines
+them with weights based on previous errors. The error value is encoded in a
+bucket chosen based on a heuristic max error. The result is entropy-coded using
+the ANS encoder.
+
+## Current Reference Implementation
+
+### Conventions
+
+The software is written in C++ and built using CMake 3.6 or later.
+
+Error handling is done by having functions return values of type `jxl::Status`
+(a thin wrapper around bool which checks that it is not ignored). A convenience
+macro named `JXL_RETURN_IF_ERROR` makes this more convenient by automatically
+forwarding errors, and another macro named `JXL_FAILURE` exits with an error
+message if reached, with no effect in optimized builds.
+
+To diagnose the cause of encoder/decoder failures (which often only result in a
+generic "decode failed" message), build using the following command:
+
+```bash
+CMAKE_FLAGS="-DJXL_CRASH_ON_ERROR" ./ci.sh opt
+```
+
+In such builds, the first JXL_FAILURE will print a message identifying where the
+problem is and the program will exit immediately afterwards.
+
+### Architecture
+
+Getting back to the earlier block diagram:
+
+**Header** handling is implemented in `headers.h` and `field*`.
+
+**Bitstream**: `entropy_coder.h`, `dec_ans_*`.
+
+**(De)quantize**: `quantizer.h`.
+
+**DC prediction**: `predictor.h`.
+
+**Chroma from luma**: `chroma_from_luma.h`
+
+**(I)DCT**: `dct.h`. Instead of operating directly on blocks of memory, the
+functions operate on thin wrappers (declared in `block.h`) which can handle
+blocks spread across multiple image lines.
+
+**DCT size selection**: `ac_strategy.cc`
+
+**[Gaborish]**: `gaborish.h`.
+
+**[Edge preserving filter]**: `epf.h`
+
+**[Noise injection]**: `noise*` (currently disabled)
+
+**Color space conversion**: `color_*`, `dec_xyb.h`.
+
+## Decoder overview
+
+After decoding headers, the decoder begins processing frames (`dec_frame.cc`).
+
+For each pass, it will read the DC group table of contents (TOC) and start
+decoding, dequantizing and restoring color correlation of each DC group
+(covering 2048x2048 pixels in the input image) in parallel
+(`compressed_dc.cc`). The DC is split into parts corresponding to each AC group
+(with 1px of extra border); the AC group TOC is read and each AC group (256x256
+pixels) is processed in parallel (`dec_group.cc`).
+
+In each AC group, the decoder reads per-block side information indicating the
+kind of DCT transform; this is followed by the quantization field. Then, AC
+coefficients are read, dequantized and have color correlation restored on a
+tile per tile basis for better locality.
+
+After all the groups are read, postprocessing is applied: Gaborish smoothing
+and edge preserving filter, to reduce blocking and other artifacts.
+
+Finally, the image is converted back from the XYB color space
+(`dec_xyb.cc`) and saved to the output image (`codec_*.cc`).
diff --git a/docker/Dockerfile.jpegxl-builder b/docker/Dockerfile.jpegxl-builder
new file mode 100644
index 000000000..552fadbe4
--- /dev/null
+++ b/docker/Dockerfile.jpegxl-builder
@@ -0,0 +1,18 @@
+# Copyright (c) the JPEG XL Project
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+# Build an Ubuntu-based docker image with the installed software needed to
+# develop and test JPEG XL.
+
+FROM ubuntu:bionic
+
+# Set a prompt for when using it locally.
+ENV PS1="\[\033[01;33m\]\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ "
+
+COPY scripts /jpegxl_scripts
+
+RUN /jpegxl_scripts/jpegxl_builder.sh && \
+  rm -rf /jpegxl_scripts
diff --git a/docker/Dockerfile.jpegxl-builder-run-aarch64 b/docker/Dockerfile.jpegxl-builder-run-aarch64
new file mode 100644
index 000000000..f904cfbf3
--- /dev/null
+++ b/docker/Dockerfile.jpegxl-builder-run-aarch64
@@ -0,0 +1,30 @@
+# Copyright (c) the JPEG XL Project
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+# Build an Ubuntu-based docker image for aarch64 with the installed software
+# needed to run JPEG XL. This is only useful when running on actual aarch64
+# hardware.
+
+FROM arm64v8/ubuntu:bionic
+
+# Set a prompt for when using it locally.
+ENV PS1="\[\033[01;33m\]\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ "
+
+RUN set -ex; \
+  apt-get update -y; \
+  apt-get install -y \
+    bsdmainutils \
+    cmake \
+    curl \
+    extra-cmake-modules \
+    git \
+    libjpeg8 \
+    libgif7 \
+    libpng16-16 \
+    libqt5x11extras5 \
+    libsdl2-2.0-0 \
+    parallel; \
+  rm -rf /var/lib/apt/lists/*;
diff --git a/docker/README.md b/docker/README.md
new file mode 100644
index 000000000..874df1cb8
--- /dev/null
+++ b/docker/README.md
@@ -0,0 +1,7 @@
+### Docker container infrastructure for JPEG XL
+
+This directory contains the requirements to build a docker image for the
+JPEG XL project builder.
+
+Docker images need to be created and upload manually. See ./build.sh for
+details.
diff --git a/docker/build.sh b/docker/build.sh
new file mode 100755
index 000000000..d0909957b
--- /dev/null
+++ b/docker/build.sh
@@ -0,0 +1,89 @@
+#!/bin/bash
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+MYDIR=$(dirname $(realpath "$0"))
+
+declare -a TARGETS
+
+load_targets() {
+  for f in $(find -maxdepth 1 -name 'Dockerfile.*' | sort); do
+    local target="${f#*Dockerfile.}"
+    TARGETS+=("${target}")
+  done
+}
+
+usage() {
+    cat >&2 <<EOF
+Use: $1 [targets]
+
+Available targets:
+  * all
+EOF
+  for target in "${TARGETS[@]}"; do
+    echo "  * ${target}" >&2
+  done
+}
+
+build_target() {
+  local target="$1"
+
+  local dockerfile="${MYDIR}/Dockerfile.${target}"
+  # JPEG XL builder images are stored in the gcr.io/jpegxl project.
+  local tag="gcr.io/jpegxl/${target}"
+
+  echo "Building ${target}"
+  if ! sudo docker build -t "${tag}" -f "${dockerfile}" "${MYDIR}" >"${target}.log" 2>&1; then
+    echo "${target} failed. See ${target}.log" >&2
+  else
+    echo "Done, to upload image run:" >&2
+    echo "  sudo docker push ${tag}"
+    if [[ "${JPEGXL_PUSH:-}" == "1" ]]; then
+      echo "sudo docker push ${tag}" >&2
+      sudo docker push "${tag}"
+      # The RepoDigest is only created after it is pushed.
+      local fulltag=$(sudo docker inspect --format="{{.RepoDigests}}" "${tag}")
+      fulltag="${fulltag#[}"
+      fulltag="${fulltag%]}"
+      echo "Updating .gitlab-ci.yml to ${fulltag}" >&2
+      sed -E "s;${tag}@sha256:[0-9a-f]+;${fulltag};" \
+        -i "${MYDIR}/../.gitlab-ci.yml"
+    fi
+  fi
+}
+
+main() {
+  cd "${MYDIR}"
+  local target="${1:-}"
+
+  load_targets
+  if [[ -z "${target}" ]]; then
+    usage $0
+    exit 1
+  fi
+
+  if [[ "${target}" == "all" ]]; then
+    for target in "${TARGETS[@]}"; do
+      build_target "${target}"
+    done
+  else
+    for target in "$@"; do
+      build_target "${target}"
+    done
+  fi
+}
+
+main "$@"
diff --git a/docker/scripts/emsdk_install.sh b/docker/scripts/emsdk_install.sh
new file mode 100755
index 000000000..667a1ccf0
--- /dev/null
+++ b/docker/scripts/emsdk_install.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+EMSDK_URL="https://github.com/emscripten-core/emsdk/archive/master.tar.gz"
+EMSDK_DIR="/opt/emsdk"
+
+EMSDK_RELEASE="1.39.4"  # Alternatively, we can use "latest"
+
+set -eu -x
+
+# Temporary files cleanup hooks.
+CLEANUP_FILES=()
+cleanup() {
+  if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
+    rm -fr "${CLEANUP_FILES[@]}"
+  fi
+}
+trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT
+
+main() {
+  local workdir=$(mktemp -d --suffix=qemu)
+  CLEANUP_FILES+=("${workdir}")
+
+  local emsdktar="${workdir}/emsdk.tar.gz"
+  curl --output "${emsdktar}" "${EMSDK_URL}" --location
+  mkdir -p "${EMSDK_DIR}"
+  tar -zxf "${emsdktar}" -C "${EMSDK_DIR}" --strip-components=1
+
+  cd "${EMSDK_DIR}"
+  ./emsdk install --shallow "${EMSDK_RELEASE}"
+  ./emsdk activate --embedded "${EMSDK_RELEASE}"
+}
+
+main "$@"
diff --git a/docker/scripts/jpegxl_builder.sh b/docker/scripts/jpegxl_builder.sh
new file mode 100755
index 000000000..b495d3a82
--- /dev/null
+++ b/docker/scripts/jpegxl_builder.sh
@@ -0,0 +1,225 @@
+#!/bin/bash
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Main entry point for all the Dockerfile for jpegxl-builder. This centralized
+# file helps sharing code and configuration between Dockerfiles.
+
+set -eux
+
+MYDIR=$(dirname $(realpath "$0"))
+
+# List of Ubuntu arch names supported by the builder (such as "i386").
+LIST_ARCHS=(
+  amd64
+  i386
+  arm64
+  armhf
+)
+
+# List of target triplets supported by the builder.
+LIST_TARGETS=(
+  x86_64-linux-gnu
+  i686-linux-gnu
+  arm-linux-gnueabihf
+  aarch64-linux-gnu
+)
+
+# Setup the apt repositories and supported architectures.
+setup_apt() {
+  apt-get update -y
+  apt-get install -y gnupg
+
+  # gcc ppa sources.
+  cat >/etc/apt/sources.list.d/gcc.list <<EOF
+deb [arch=$(echo ${LIST_ARCHS[@]} | tr ' ' ,)] http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu bionic main
+EOF
+  apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 1E9377A2BA9EF27F
+
+  local porl_list=()
+  local main_list=()
+  local ubarch
+  for ubarch in "${LIST_ARCHS[@]}"; do
+    if [[ "${ubarch}" != "amd64" && "${ubarch}" != "i386" ]]; then
+      # other archs are not part of the main mirrors, but available in
+      # ports.ubuntu.com.
+      port_list+=("${ubarch}")
+    else
+      main_list+=("${ubarch}")
+    fi
+    # Add the arch to the system.
+    if [[ "${ubarch}" != "amd64" ]]; then
+      dpkg --add-architecture "${ubarch}"
+    fi
+  done
+
+  # Update the sources.list with the split of supported architectures.
+  local bkplist="/etc/apt/sources.list.bkp"
+  [[ -e "${bkplist}" ]] || \
+    mv /etc/apt/sources.list "${bkplist}"
+
+  local newlist="/etc/apt/sources.list.tmp"
+  rm -f "${newlist}"
+  port_list=$(echo "${port_list[@]}" | tr ' ' ,)
+  if [[ -n "${port_list}" ]]; then
+    local port_url="http://ports.ubuntu.com/ubuntu-ports/"
+    grep -v -E '^#' "${bkplist}" |
+      sed -E "s;^deb (http[^ ]+) (.*)\$;deb [arch=${port_list}] ${port_url} \\2;" \
+      >>"${newlist}"
+  fi
+
+  main_list=$(echo "${main_list[@]}" | tr ' ' ,)
+  grep -v -E '^#' "${bkplist}" |
+    sed -E "s;^deb (http[^ ]+) (.*)\$;deb [arch=${main_list}] \\1 \\2;" \
+    >>"${newlist}"
+  mv "${newlist}" /etc/apt/sources.list
+}
+
+install_pkgs() {
+  packages=(
+    # Native compilers
+    clang-6.0 clang-format-6.0 clang-tidy-6.0
+
+    # TODO: Consider adding clang-7 and clang-8 to every builder:
+    #   clang-7 clang-format-7 clang-tidy-7
+    #   clang-8 clang-format-8 clang-tidy-8
+
+    # Native tools.
+    bsdmainutils
+    cmake
+    curl
+    extra-cmake-modules
+    git
+    llvm
+    nasm
+    ninja-build
+    parallel
+    pkg-config
+
+    # These are used by the ./ci.sh lint in the native builder.
+    clang-format-7
+    clang-format-8
+
+    # For coverage builds
+    gcovr
+
+    # Common libraries.
+    libstdc++-8-dev
+    libgif-dev  # See libgif-dev comment below.
+  )
+
+  # Install packages that are arch-dependent.
+  local ubarch
+  for ubarch in "${LIST_ARCHS[@]}"; do
+    packages+=(
+      # Library dependencies. These normally depend on the target architecture
+      # we are compiling for and can't usually be installed for multiple
+      # architectures at the same time.
+      libjpeg-dev:"${ubarch}"
+      libpng-dev:"${ubarch}"
+      libqt5x11extras5-dev:"${ubarch}"
+
+      libstdc++-8-dev:"${ubarch}"
+      qtbase5-dev:"${ubarch}"
+
+      # Cross-compiling tools per arch.
+      libc6-dev-"${ubarch}"-cross
+      libstdc++-8-dev-"${ubarch}"-cross
+
+      # libgif-dev package can't be installed for multiple architectures at
+      # the same time. Instead we unpack it manually later.
+      # libgif-dev:"${ubarch}"
+    )
+  done
+
+  local target
+  for target in "${LIST_TARGETS[@]}"; do
+    # Per target cross-compiling tools.
+    if [[ "${target}" != "x86_64-linux-gnu" ]]; then
+      packages+=(
+        binutils-"${target}"
+        gcc-"${target}"
+      )
+    fi
+  done
+
+  apt install -y "${packages[@]}"
+}
+
+# Packages that are manually unpacked for each architecture.
+UNPACK_PKGS=(
+  libgif-dev
+  libclang-common-6.0-dev
+)
+
+# Main script entry point.
+main() {
+  cd "${MYDIR}"
+
+  # Configure the repositories with the sources for multi-arch cross
+  # compilation.
+  setup_apt
+  apt-get update -y
+  apt-get dist-upgrade -y
+
+  install_pkgs
+  apt clean
+
+  # Manually extract packages for the target arch that can't install it directly
+  # at the same time as the native ones.
+  local ubarch
+  for ubarch in "${LIST_ARCHS[@]}"; do
+    if [[ "${ubarch}" != "amd64" ]]; then
+      local pkg
+      for pkg in "${UNPACK_PKGS[@]}"; do
+        apt download "${pkg}":"${ubarch}"
+        dpkg -x "${pkg}"_*_"${ubarch}".deb /
+      done
+    fi
+  done
+  # TODO: Add clang from the llvm repos. This is problematic since we are
+  # installing libclang-common-6.0-dev:"${ubarch}" from the ubuntu ports repos
+  # which is not available in the llvm repos so it might have a different
+  # version than the ubuntu ones.
+
+  # Install webp manually since the version in Ubuntu is rather old.
+  local target
+  for target in "${LIST_TARGETS[@]}"; do
+    if [[ "${target}" == "x86_64-linux-gnu" ]]; then
+      CC=clang-6.0 CXX=clang++-6.0 ./webp_install.sh
+    else
+      CC=clang-6.0 CXX=clang++-6.0 ./webp_install.sh \
+        -DCMAKE_INSTALL_PREFIX=/usr/"${target}" \
+        -DCMAKE_C_COMPILER_TARGET="${target}" \
+        -DCMAKE_CXX_COMPILER_TARGET="${target}" \
+        -DCMAKE_SYSTEM_PROCESSOR="${target%%-*}" \
+        -DCMAKE_SYSTEM_NAME=Linux
+    fi
+  done
+
+  # TODO: Add msan for the target when cross-compiling. This only installs it
+  # for amd64.
+  ./msan_install.sh
+
+  # Build and install qemu user-linux targets.
+  ./qemu_install.sh
+
+  # Install emscripten SDK.
+  ./emsdk_install.sh
+
+  # Cleanup.
+  rm -rf /var/lib/apt/lists/*
+}
+
+main "$@"
diff --git a/docker/scripts/msan_install.sh b/docker/scripts/msan_install.sh
new file mode 100755
index 000000000..23d458aab
--- /dev/null
+++ b/docker/scripts/msan_install.sh
@@ -0,0 +1,140 @@
+#!/bin/bash
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eu
+
+MYDIR=$(dirname $(realpath "$0"))
+
+# Convenience flag to pass both CMAKE_C_FLAGS and CMAKE_CXX_FLAGS
+CMAKE_FLAGS=${CMAKE_FLAGS:-}
+CMAKE_C_FLAGS=${CMAKE_C_FLAGS:-${CMAKE_FLAGS}}
+CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS:-${CMAKE_FLAGS}}
+CMAKE_EXE_LINKER_FLAGS=${CMAKE_EXE_LINKER_FLAGS:-}
+
+CLANG_VERSION="${CLANG_VERSION:-}"
+# Detect the clang version suffix and store it in CLANG_VERSION. For example,
+# "6.0" for clang 6 or "7" for clang 7.
+detect_clang_version() {
+  if [[ -n "${CLANG_VERSION}" ]]; then
+    return 0
+  fi
+  local clang_version=$("${CC:-clang}" --version | head -n1)
+  local llvm_tag
+  case "${clang_version}" in
+    "clang version 6."*)
+      CLANG_VERSION="6.0"
+      ;;
+    "clang version 7."*)
+      CLANG_VERSION="7"
+      ;;
+    "clang version 8."*)
+      CLANG_VERSION="8"
+      ;;
+    "clang version 9."*)
+      CLANG_VERSION="9"
+      ;;
+    *)
+      echo "Unknown clang version: ${clang_version}" >&2
+      return 1
+  esac
+}
+
+# Temporary files cleanup hooks.
+CLEANUP_FILES=()
+cleanup() {
+  if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
+    rm -fr "${CLEANUP_FILES[@]}"
+  fi
+}
+trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT
+
+# Install libc++ libraries compiled with msan in the msan_prefix for the current
+# compiler version.
+cmd_msan_install() {
+  local tmpdir=$(mktemp -d)
+  CLEANUP_FILES+=("${tmpdir}")
+  # Detect the llvm to install:
+  export CC="${CC:-clang}"
+  export CXX="${CXX:-clang++}"
+  detect_clang_version
+  local llvm_tag
+  case "${CLANG_VERSION}" in
+    "6.0")
+      llvm_tag="llvmorg-6.0.1"
+      ;;
+    "7")
+      llvm_tag="llvmorg-7.0.1"
+      ;;
+    "8")
+      llvm_tag="llvmorg-8.0.0"
+      ;;
+    *)
+      echo "Unknown clang version: ${clang_version}" >&2
+      return 1
+  esac
+  local llvm_targz="${tmpdir}/${llvm_tag}.tar.gz"
+  curl -L --show-error -o "${llvm_targz}" \
+    "https://github.com/llvm/llvm-project/archive/${llvm_tag}.tar.gz"
+  tar -C "${tmpdir}" -zxf "${llvm_targz}"
+  local llvm_root="${tmpdir}/llvm-project-${llvm_tag}"
+
+  local msan_prefix="${HOME}/.msan/${CLANG_VERSION}"
+  rm -rf "${msan_prefix}"
+
+  declare -A CMAKE_EXTRAS
+  CMAKE_EXTRAS[libcxx]="\
+    -DLIBCXX_CXX_ABI=libstdc++ \
+    -DLIBCXX_INSTALL_EXPERIMENTAL_LIBRARY=ON"
+
+  for project in libcxx; do
+    local proj_build="${tmpdir}/build-${project}"
+    local proj_dir="${llvm_root}/${project}"
+    mkdir -p "${proj_build}"
+    cmake -B"${proj_build}" -H"${proj_dir}" \
+      -G Ninja \
+      -DCMAKE_BUILD_TYPE=Release \
+      -DLLVM_USE_SANITIZER=Memory \
+      -DLLVM_PATH="${llvm_root}/llvm" \
+      -DLLVM_CONFIG_PATH="$(which llvm-config llvm-config-7 llvm-config-6.0 | \
+                            head -n1)" \
+      -DCMAKE_CXX_FLAGS="${CMAKE_CXX_FLAGS}" \
+      -DCMAKE_C_FLAGS="${CMAKE_C_FLAGS}" \
+      -DCMAKE_EXE_LINKER_FLAGS="${CMAKE_EXE_LINKER_FLAGS}" \
+      -DCMAKE_INSTALL_PREFIX="${msan_prefix}" \
+      ${CMAKE_EXTRAS[${project}]}
+    cmake --build "${proj_build}"
+    ninja -C "${proj_build}" install
+  done
+}
+
+main() {
+  set -x
+  for version in 6.0 7 8; do
+    if ! which "clang-${version}" >/dev/null; then
+      echo "Skipping msan install for clang version ${version}"
+      continue
+    fi
+    (
+     trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT
+     export CLANG_VERSION=${version}
+     export CC=clang-${version}
+     export CXX=clang++-${version}
+     cmd_msan_install
+    ) &
+  done
+  wait
+}
+
+main "$@"
diff --git a/docker/scripts/qemu_install.sh b/docker/scripts/qemu_install.sh
new file mode 100755
index 000000000..af9d69f20
--- /dev/null
+++ b/docker/scripts/qemu_install.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+QEMU_RELEASE="4.1.0"
+QEMU_URL="https://download.qemu.org/qemu-${QEMU_RELEASE}.tar.xz"
+QEMU_ARCHS=(
+  aarch64
+  arm
+  i386
+  # TODO: Consider adding these:
+  # aarch64_be
+  # mips64el
+  # mips64
+  # mips
+  # ppc64
+  # ppc
+)
+
+# Ubuntu packages not installed that are needed to build qemu.
+QEMU_BUILD_DEPS=(
+  libglib2.0-dev
+  libpixman-1-dev
+  flex
+  bison
+)
+
+set -eu -x
+
+# Temporary files cleanup hooks.
+CLEANUP_FILES=()
+cleanup() {
+  if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
+    rm -fr "${CLEANUP_FILES[@]}"
+  fi
+}
+trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT
+
+main() {
+  local workdir=$(mktemp -d --suffix=qemu)
+  CLEANUP_FILES+=("${workdir}")
+
+  apt install -y "${QEMU_BUILD_DEPS[@]}"
+
+  local qemutar="${workdir}/qemu.tar.gz"
+  curl --output "${qemutar}" "${QEMU_URL}"
+  tar -Jxf "${qemutar}" -C "${workdir}"
+  local srcdir="${workdir}/qemu-${QEMU_RELEASE}"
+
+  local builddir="${workdir}/build"
+  local prefixdir="${workdir}/prefix"
+  mkdir -p "${builddir}"
+
+  # List of targets to build.
+  local targets=""
+  local make_targets=()
+  local target
+  for target in "${QEMU_ARCHS[@]}"; do
+    targets="${targets} ${target}-linux-user"
+    # Build just the linux-user targets.
+    make_targets+=("${target}-linux-user/all")
+  done
+
+  cd "${builddir}"
+  "${srcdir}/configure" \
+    --prefix="${prefixdir}" \
+    --static --disable-system --enable-linux-user \
+    --target-list="${targets}"
+
+  make -j $(nproc --all || echo 1) "${make_targets[@]}"
+
+  # Manually install these into the non-standard location. This script runs as
+  # root anyway.
+  for target in "${QEMU_ARCHS[@]}"; do
+    cp "${target}-linux-user/qemu-${target}" "/usr/bin/qemu-${target}-static"
+  done
+
+  apt autoremove -y --purge "${QEMU_BUILD_DEPS[@]}"
+}
+
+main "$@"
diff --git a/docker/scripts/webp_install.sh b/docker/scripts/webp_install.sh
new file mode 100755
index 000000000..46b76d4fc
--- /dev/null
+++ b/docker/scripts/webp_install.sh
@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+WEBP_RELEASE="1.0.2"
+WEBP_URL="https://codeload.github.com/webmproject/libwebp/tar.gz/v${WEBP_RELEASE}"
+
+set -eu -x
+
+# Temporary files cleanup hooks.
+CLEANUP_FILES=()
+cleanup() {
+  if [[ ${#CLEANUP_FILES[@]} -ne 0 ]]; then
+    rm -fr "${CLEANUP_FILES[@]}"
+  fi
+}
+trap "{ set +x; } 2>/dev/null; cleanup" INT TERM EXIT
+
+main() {
+  local workdir=$(mktemp -d --suffix=webp)
+  CLEANUP_FILES+=("${workdir}")
+
+  local webptar="${workdir}/webp.tar.gz"
+  curl --output "${webptar}" "${WEBP_URL}"
+  tar -zxvf "${webptar}" -C "${workdir}"
+  local srcdir="${workdir}/libwebp-${WEBP_RELEASE}"
+  local builddir="${srcdir}/build"
+  cmake -B"${builddir}" -H"${srcdir}" -G Ninja \
+    -DCMAKE_INSTALL_PREFIX=/usr \
+    "$@"
+  cmake --build "${builddir}"
+  ninja -C "${builddir}" install
+}
+
+main "$@"
diff --git a/include/jpegxl/decode.h b/include/jpegxl/decode.h
new file mode 100644
index 000000000..2f845699f
--- /dev/null
+++ b/include/jpegxl/decode.h
@@ -0,0 +1,109 @@
+/* Copyright (c) the JPEG XL Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef JPEGXL_DECODE_H_
+#define JPEGXL_DECODE_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jpegxl/jpegxl_export.h"
+#include "jpegxl/memory_manager.h"
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+/**
+ * Decoder library version.
+ *
+ * @returns the decoder library version as an integer:
+ * MAJOR_VERSION * 1000000 + MINOR_VERSION * 1000 + PATCH_VERSION. For example,
+ * version 1.2.3 would return 1002003.
+ */
+JPEGXL_EXPORT uint32_t JpegxlDecoderVersion(void);
+
+enum JpegxlSignature {
+  /** Not enough bytes were passed to determine if a valid signature was found.
+   */
+  JPEGXL_SIG_NOT_ENOUGH_BYTES = 0,
+
+  /** No valid JPEGXL header was found. */
+  JPEGXL_SIG_INVALID = 1,
+
+  /** A valid transcoded JPEG XL image signature was found. */
+  JPEGXL_SIG_BRUNSLI = 2,
+
+  /** A valid JPEG XL image signature was found. */
+  JPEGXL_SIG_JPEGXL = 4,
+
+  /* Mask representing any valid JPEGXL signature. */
+  JPEGXL_SIG_ANY = JPEGXL_SIG_BRUNSLI | JPEGXL_SIG_JPEGXL,
+};
+
+/**
+ * JPEG XL signature identification.
+ *
+ * Checks if the passed buffer contains a valid JPEG XL signature header and
+ * returns the type of signature header if found. The passed @p buf of size
+ * @p size doesn't need to be a full image, only the beginning of the file.
+ *
+ * @returns a flag indicating if a JPEG XL signature was found and what type.
+ *   - JPEGXL_SIG_INVALID no valid signature found.
+ *   - JPEGXL_SIG_BRUNSLI a valid Brunsli signature was found.
+ *   - JPEGXL_SIG_JPEGXL a valid JPEG XL signature was found.
+ *   - JPEGXL_SIG_NOT_ENOUGH_BYTES not enough bytes were passed to determine
+ *       if a valid signature is there.
+ * You can mask the result of this function with JPEGXL_SIG_ANY to determine
+ * if a valid signature (of any kind) was found.
+ */
+JPEGXL_EXPORT enum JpegxlSignature JpegxlSignatureCheck(const uint8_t* buf,
+                                                        size_t len);
+
+/**
+ * Opaque structure that holds the JPEGXL decoder.
+ *
+ * Allocated and initialized with JpegxlDecoderCreate().
+ * Cleaned up and deallocated with JpegxlDecoderDestroy().
+ */
+typedef struct JpegxlDecoderStruct JpegxlDecoder;
+
+/**
+ * Creates an instance of JpegxlDecoder and initializes it.
+ *
+ * @p memory_manager will be used for all the library dynamic allocations made
+ * from this instance. The parameter may be NULL, in which case the default
+ * allocator will be used. See jpegxl/memory_manager.h for details.
+ *
+ * @param memory_manager custom allocator function. It may be NULL. The memory
+ *        manager will be copied internally.
+ * @returns @c NULL if the instance can not be allocated or initialized
+ * @returns pointer to initialized JpegxlDecoder otherwise
+ */
+JPEGXL_EXPORT JpegxlDecoder* JpegxlDecoderCreate(
+    const JpegxlMemoryManager* memory_manager);
+
+/**
+ * Deinitializes and frees JpegxlDecoder instance.
+ *
+ * @param dec instance to be cleaned up and deallocated.
+ */
+JPEGXL_EXPORT void JpegxlDecoderDestroy(JpegxlDecoder* dec);
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif /* JPEGXL_DECODE_H_ */
diff --git a/include/jpegxl/memory_manager.h b/include/jpegxl/memory_manager.h
new file mode 100644
index 000000000..13ee2019c
--- /dev/null
+++ b/include/jpegxl/memory_manager.h
@@ -0,0 +1,71 @@
+/* Copyright (c) the JPEG XL Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef JPEGXL_MEMORY_MANAGER_H_
+#define JPEGXL_MEMORY_MANAGER_H_
+
+#include <stddef.h>
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+/**
+ * Allocating function for a memory region of a given size.
+ *
+ * Allocates a contiguous memory region of size @p size bytes. The returned
+ * memory may not be aligned to a specific size or initialized at all.
+ *
+ * @param opaque custom memory manager handle provided by the caller.
+ * @param size in bytes of the requested memory region.
+ * @returns @c 0 if the memory can not be allocated,
+ * @returns pointer to the memory otherwise.
+ */
+typedef void* (*jpegxl_alloc_func)(void* opaque, size_t size);
+
+/**
+ * Deallocating function pointer type.
+ *
+ * This function @b MUST do nothing if @p address is @c 0.
+ *
+ * @param opaque custom memory manager handle provided by the caller.
+ * @param address memory region pointer returned by ::brotli_alloc_func, or @c 0
+ */
+typedef void (*jpegxl_free_func)(void* opaque, void* address);
+
+/**
+ * Memory Manager struct.
+ * These functions, when provided by the caller, will be used to handle memory
+ * allocations.
+ */
+typedef struct JpegxlMemoryManagerStruct {
+  /* The opaque pointer that will be passed as the first parameter to all the
+   * functions in this struct. */
+  void* opaque;
+
+  /* Alloc/free functions. These can either be both NULL or none of them NULL.
+   * All dynamic memory will be allocated and freed with these functions if
+   * not NULL. */
+  jpegxl_alloc_func alloc;
+  jpegxl_free_func free;
+
+  /* TODO(deymo): Add cache-aligned alloc/free functions here. */
+} JpegxlMemoryManager;
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif  /* JPEGXL_MEMORY_MANAGER_H_ */
diff --git a/include/jpegxl/parallel_runner.h b/include/jpegxl/parallel_runner.h
new file mode 100644
index 000000000..835bbe195
--- /dev/null
+++ b/include/jpegxl/parallel_runner.h
@@ -0,0 +1,152 @@
+/* Copyright (c) the JPEG XL Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** API for running data operations in parallel in a multi-threaded environment.
+ * This module allows the JPEG XL caller to define their own way of creating and
+ * assigning threads.
+ *
+ * The JpegxlParallelRunner function type defines a parallel data processing
+ * runner that may be implemented by the caller to allow the library to process
+ * in multiple threads. The multi-threaded processing in this library only
+ * requires to run the same function over each number of a range, possibly
+ * running each call in a different thread. The JPEG XL caller is responsible
+ * for implementing this logic using the thread APIs available in their system.
+ * For convenience, a C++ implementation based on std::thread is provided in
+ * jpegxl/parallel_runner_thread.h (part of the jpegxl_threads library).
+ *
+ * Thread pools usually store small numbers of heterogeneous tasks in a queue.
+ * When tasks are identical or differ only by an integer input parameter, it is
+ * much faster to store just one function of an integer parameter and call it
+ * for each value. Conventional vector-of-tasks can be run in parallel using a
+ * lambda function adapter that simply calls task_funcs[task].
+ *
+ * If no multi-threading is desired, a @c NULL value of JpegxlParallelRunner
+ * will use an internal implementation without multi-threading.
+ */
+
+#ifndef JPEGXL_PARALLEL_RUNNER_H_
+#define JPEGXL_PARALLEL_RUNNER_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(__cplusplus) || defined(c_plusplus)
+extern "C" {
+#endif
+
+/** Return code used in the JpegxlParallel* functions as return value. A value
+ * of 0 means success and any other value means error. The special value
+ * JPEGXL_PARALLEL_RET_RUNNER_ERROR can be used by the runner to indicate any
+ * other error.
+ */
+typedef int JpegxlParallelRetCode;
+
+#define JPEGXL_PARALLEL_RET_RUNNER_ERROR (-1)
+
+/**
+ * Parallel run initialization callback. See JpegxlParallelRunner for details.
+ *
+ * This function MUST be called by the JpegxlParallelRunner only once, on the
+ * same thread that called JpegxlParallelRunner, before any parallel execution.
+ * The purpose of this call is to provide the maximum number of threads that the
+ * JpegxlParallelRunner will use, which can be used by JPEG XL to allocate
+ * per-thread storage if needed.
+ *
+ * @param jpegxl_opaque the @p jpegxl_opaque handle provided to
+ * JpegxlParallelRunner() must be passed here.
+ * @param num_threads the maximum number of threads. This value must be
+ * positive.
+ * @returns 0 if the initialization process was successful.
+ * @returns an error code if there was an error, which should be returned by
+ * JpegxlParallelRunner().
+ */
+typedef JpegxlParallelRetCode (*JpegxlParallelRunInit)(void* jpegxl_opaque,
+                                                       size_t num_threads);
+
+/**
+ * Parallel run data processing callback. See JpegxlParallelRunner for details.
+ *
+ * This function MUST be called once for every number in the range [start_range,
+ * end_range) (including start_range but not including end_range) passing this
+ * number as the @p value. Calls for different value may be executed from
+ * different threads in parallel.
+ *
+ * @param jpegxl_opaque the @p jpegxl_opaque handle provided to
+ * JpegxlParallelRunner() must be passed here.
+ * @param value the number in the range [start_range, end_range) of the call.
+ * @param thread_id the thread number where this function is being called from.
+ * This must be lower than the @p num_threads value passed to
+ * JpegxlParallelRunInit.
+ */
+typedef void (*JpegxlParallelRunFunction)(void* jpegxl_opaque, uint32_t value,
+                                          size_t thread_id);
+
+/**
+ * JpegxlParallelRunner function type. A parallel runner implementation can be
+ * provided by a JPEG XL caller to allow running computations in multiple
+ * threads. This function must call the initialization function @p init in the
+ * same thread that called it and then call the passed @p func once for every
+ * number in the range [start_range, end_range) (including start_range but not
+ * including end_range) possibly from different multiple threads in parallel.
+ *
+ * The JpegxlParallelRunner function does not need to be re-entrant. This means
+ * that the same JpegxlParallelRunner function with the same runner_opaque
+ * provided parameter will not be called from the library from either @p init or
+ * @p func in the same decoder or encoder instance. However, a single decoding
+ * or encoding instance may call the provided JpegxlParallelRunner multiple
+ * times for different parts of the decoding or encoding process.
+ *
+ * @returns 0 if the @p init call succeeded (returned 0) and no other error
+ * occurred in the runner code.
+ * @returns JPEGXL_PARALLEL_RET_RUNNER_ERROR if an error occurred in the runner
+ * code, for example, setting up the threads.
+ * @return the return value of @p init() if non-zero.
+ */
+typedef JpegxlParallelRetCode (*JpegxlParallelRunner)(
+    void* runner_opaque, void* jpegxl_opaque, JpegxlParallelRunInit init,
+    JpegxlParallelRunFunction func, uint32_t start_range, uint32_t end_range);
+
+/* The following is an example of a JpegxlParallelRunner that doesn't use any
+ * multi-threading. Note that this implementation doesn't store any state
+ * between multiple calls of the ExampleSequentialRunner function, so the
+ * runner_opaque value is not used.
+
+  JpegxlParallelRetCode ExampleSequentialRunner(void* runner_opaque,
+                                                void* jpegxl_opaque,
+                                                JpegxlParallelRunInit init,
+                                                JpegxlParallelRunFunction func,
+                                                uint32_t start_range,
+                                                uint32_t end_range) {
+    // We only use one thread (the currently running thread).
+    JpegxlParallelRetCode init_ret = (*init)(jpegxl_opaque, 1);
+    if (init_ret != 0) return init_ret;
+
+    // In case of other initialization error (for example when initializing the
+    // threads) one can return JPEGXL_PARALLEL_RET_RUNNER_ERROR.
+
+    for (uint32_t i = start_range; i < end_range; i++) {
+      // Every call is in the thread number 0. These don't need to be in any
+      // order.
+      (*func)(jpegxl_opaque, i, 0);
+    }
+    return 0;
+  }
+ */
+
+#if defined(__cplusplus) || defined(c_plusplus)
+}
+#endif
+
+#endif /* JPEGXL_PARALLEL_RUNNER_H_ */
diff --git a/include/jpegxl/thread_parallel_runner.h b/include/jpegxl/thread_parallel_runner.h
new file mode 100644
index 000000000..fee840a22
--- /dev/null
+++ b/include/jpegxl/thread_parallel_runner.h
@@ -0,0 +1,179 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+
+// Implementation of a JpegxlParallelRunner using std::threads.
+//
+// The main class in this module, ThreadParallelRunner, implements a static
+// method ThreadParallelRunner::Runner than can be passed as a
+// JpegxlParallelRunner when using the JPEG XL library. This uses std::thread
+// internally and related synchronization functions. The number of threads
+// created is fixed at construction time and the threads are re-used for every
+// ThreadParallelRunner::Runner call. Only one concurrent Runner() call per
+// instance is allowed at a time.
+//
+// This is a scalable, lower-overhead thread pool runner, especially suitable
+// for data-parallel computations in the fork-join model, where clients need to
+// know when all tasks have completed.
+//
+// This thread pool can efficiently load-balance millions of tasks using an
+// atomic counter, thus avoiding per-task virtual or system calls. With 48
+// hyperthreads and 1M tasks that add to an atomic counter, overall runtime is
+// 10-20x higher when using std::async, and ~200x for a queue-based thread pool.
+//
+// Usage:
+//   ThreadParallelRunner runner;
+//   JpegxlDecode(
+//       ... , &ThreadParallelRunner::Runner, static_cast<void*>(&runner));
+
+#ifndef JPEGXL_THREAD_PARALLEL_RUNNER_H_
+#define JPEGXL_THREAD_PARALLEL_RUNNER_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>  // max
+#include <atomic>
+#include <condition_variable>  //NOLINT
+#include <mutex>               //NOLINT
+#include <thread>              //NOLINT
+#include <vector>
+
+#include "jpegxl/parallel_runner.h"
+
+namespace jpegxl {
+
+class ThreadParallelRunner {
+ public:
+  // JpegxlParallelRunner interface.
+  static JpegxlParallelRetCode Runner(void* runner_opaque, void* jpegxl_opaque,
+                                      JpegxlParallelRunInit init,
+                                      JpegxlParallelRunFunction func,
+                                      uint32_t start_range, uint32_t end_range);
+
+  // Starts the given number of worker threads and blocks until they are ready.
+  // "num_worker_threads" defaults to one per hyperthread. If zero, all tasks
+  // run on the main thread.
+  explicit ThreadParallelRunner(
+      int num_worker_threads = std::thread::hardware_concurrency());
+
+  // Waits for all threads to exit.
+  ~ThreadParallelRunner();
+
+  // Returns number of worker threads created (some may be sleeping and never
+  // wake up in time to participate in Run). Useful for characterizing
+  // performance; 0 means "run on main thread".
+  size_t NumWorkerThreads() const { return num_worker_threads_; }
+
+  // Returns maximum number of main/worker threads that may call Func. Useful
+  // for allocating per-thread storage.
+  size_t NumThreads() const { return num_threads_; }
+
+  // Runs func(thread, thread) on all thread(s) that may participate in Run.
+  // If NumThreads() == 0, runs on the main thread with thread == 0, otherwise
+  // concurrently called by each worker thread in [0, NumThreads()).
+  template <class Func>
+  void RunOnEachThread(const Func& func) {
+    if (num_worker_threads_ == 0) {
+      const int thread = 0;
+      func(thread, thread);
+      return;
+    }
+
+    data_func_ =
+        reinterpret_cast<JpegxlParallelRunFunction>(&CallClosure<Func>);
+    jpegxl_opaque_ = const_cast<void*>(static_cast<const void*>(&func));
+    StartWorkers(kWorkerOnce);
+    WorkersReadyBarrier();
+  }
+
+ private:
+  // After construction and between calls to Run, workers are "ready", i.e.
+  // waiting on worker_start_cv_. They are "started" by sending a "command"
+  // and notifying all worker_start_cv_ waiters. (That is why all workers
+  // must be ready/waiting - otherwise, the notification will not reach all of
+  // them and the main thread waits in vain for them to report readiness.)
+  using WorkerCommand = uint64_t;
+
+  // Special values; all others encode the begin/end parameters. Note that all
+  // these are no-op ranges (begin >= end) and therefore never used to encode
+  // ranges.
+  static constexpr WorkerCommand kWorkerWait = ~1ULL;
+  static constexpr WorkerCommand kWorkerOnce = ~2ULL;
+  static constexpr WorkerCommand kWorkerExit = ~3ULL;
+
+  // Calls f(task, thread). Used for type erasure of Func arguments. The
+  // signature must match JpegxlParallelRunFunction, hence a void* argument.
+  template <class Closure>
+  static void CallClosure(void* f, const uint32_t task, const size_t thread) {
+    (*reinterpret_cast<const Closure*>(f))(task, thread);
+  }
+
+  void WorkersReadyBarrier() {
+    std::unique_lock<std::mutex> lock(mutex_);
+    // Typically only a single iteration.
+    while (workers_ready_ != threads_.size()) {
+      workers_ready_cv_.wait(lock);
+    }
+    workers_ready_ = 0;
+
+    // Safely handle spurious worker wakeups.
+    worker_start_command_ = kWorkerWait;
+  }
+
+  // Precondition: all workers are ready.
+  void StartWorkers(const WorkerCommand worker_command) {
+    mutex_.lock();
+    worker_start_command_ = worker_command;
+    // Workers will need this lock, so release it before they wake up.
+    mutex_.unlock();
+    worker_start_cv_.notify_all();
+  }
+
+  // Attempts to reserve and perform some work from the global range of tasks,
+  // which is encoded within "command". Returns after all tasks are reserved.
+  static void RunRange(ThreadParallelRunner* self, const WorkerCommand command,
+                       const int thread);
+
+  static void ThreadFunc(ThreadParallelRunner* self, int thread);
+
+  // Unmodified after ctor, but cannot be const because we call thread::join().
+  std::vector<std::thread> threads_;
+
+  const uint32_t num_worker_threads_;  // == threads_.size()
+  const uint32_t num_threads_;
+
+  std::atomic<int> depth_{0};  // detects if Run is re-entered (not supported).
+
+  std::mutex mutex_;  // guards both cv and their variables.
+  std::condition_variable workers_ready_cv_;
+  uint32_t workers_ready_ = 0;
+  std::condition_variable worker_start_cv_;
+  WorkerCommand worker_start_command_;
+
+  // Written by main thread, read by workers (after mutex lock/unlock).
+  JpegxlParallelRunFunction data_func_;
+  void* jpegxl_opaque_;
+
+  // Updated by workers; padding avoids false sharing.
+  uint8_t padding1[64];
+  std::atomic<uint32_t> num_reserved_{0};
+  uint8_t padding2[64];
+};
+
+}  // namespace jpegxl
+
+#endif /* JPEGXL_THREAD_PARALLEL_RUNNER_H_ */
diff --git a/jpegxl.cmake b/jpegxl.cmake
new file mode 100644
index 000000000..3b3ded4b4
--- /dev/null
+++ b/jpegxl.cmake
@@ -0,0 +1,438 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(JPEGXL_MAJOR_VERSION 0)
+set(JPEGXL_MINOR_VERSION 0)
+set(JPEGXL_PATCH_VERSION 1)
+set(JPEGXL_LIBRARY_VERSION
+    "${JPEGXL_MAJOR_VERSION}.${JPEGXL_MINOR_VERSION}.${JPEGXL_PATCH_VERSION}")
+
+# This the library API compatibility version.
+set(JPEGXL_LIBRARY_SOVERSION "${JPEGXL_MAJOR_VERSION}")
+
+# TODO(deymo): Split this into encoder and decoder targets
+set(JPEGXL_INTERNAL_SOURCES
+  jxl/modular/config.h
+  jxl/modular/encoding/context_predict.h
+  jxl/modular/encoding/encoding.cpp
+  jxl/modular/encoding/encoding.h
+  jxl/modular/encoding/weighted_predict.h
+  jxl/modular/image/image.cpp
+  jxl/modular/image/image.h
+  jxl/modular/ma/bit.cpp
+  jxl/modular/ma/bit.h
+  jxl/modular/ma/chance.cpp
+  jxl/modular/ma/chance.h
+  jxl/modular/ma/compound.h
+  jxl/modular/ma/compound_enc.h
+  jxl/modular/ma/rac.h
+  jxl/modular/ma/rac_enc.h
+  jxl/modular/ma/symbol.h
+  jxl/modular/ma/symbol_enc.h
+  jxl/modular/ma/util.h
+  jxl/modular/memio.h
+  jxl/modular/transform/near-lossless.h
+  jxl/modular/transform/palette.h
+  jxl/modular/transform/quantize.h
+  jxl/modular/transform/squeeze.h
+  jxl/modular/transform/subsample.h
+  jxl/modular/transform/subtractgreen.h
+  jxl/modular/transform/transform.cpp
+  jxl/modular/transform/transform.h
+  jxl/modular/transform/ycocg.h
+  jxl/modular/util.h
+  jxl/ac_context.h
+  jxl/ac_strategy.cc
+  jxl/ac_strategy.h
+  jxl/ans_common.cc
+  jxl/ans_common.h
+  jxl/ans_params.h
+  jxl/ar_control_field.cc
+  jxl/ar_control_field.h
+  jxl/aux_out.cc
+  jxl/aux_out.h
+  jxl/aux_out.h
+  jxl/aux_out_fwd.h
+  jxl/base/arch_specific.cc
+  jxl/base/arch_specific.h
+  jxl/base/bits.h
+  jxl/base/byte_order.h
+  jxl/base/cache_aligned.cc
+  jxl/base/cache_aligned.h
+  jxl/base/compiler_specific.h
+  jxl/base/data_parallel.cc
+  jxl/base/data_parallel.h
+  jxl/base/descriptive_statistics.cc
+  jxl/base/descriptive_statistics.h
+  jxl/base/fast_log.h
+  jxl/base/file_io.h
+  jxl/base/iaca.h
+  jxl/base/os_specific.cc
+  jxl/base/os_specific.h
+  jxl/base/override.h
+  jxl/base/padded_bytes.cc
+  jxl/base/padded_bytes.h
+  jxl/base/profiler.h
+  jxl/base/robust_statistics.h
+  jxl/base/span.h
+  jxl/base/status.cc
+  jxl/base/status.h
+  jxl/base/tsc_timer.h
+  jxl/block.h
+  jxl/brotli.cc
+  jxl/brotli.h
+  jxl/brunsli.cc
+  jxl/brunsli.h
+  jxl/butteraugli/butteraugli.cc
+  jxl/butteraugli/butteraugli.h
+  jxl/chroma_from_luma.cc
+  jxl/chroma_from_luma.h
+  jxl/coeff_order.cc
+  jxl/coeff_order.h
+  jxl/coeff_order_fwd.h
+  jxl/color_encoding.cc
+  jxl/color_encoding.h
+  jxl/color_management.cc
+  jxl/color_management.h
+  jxl/common.h
+  jxl/compressed_dc.cc
+  jxl/compressed_dc.h
+  jxl/convolve.h
+  jxl/dct.cc
+  jxl/dct.h
+  jxl/dct_simd_4.h
+  jxl/dct_simd_8.h
+  jxl/dct_simd_any.h
+  jxl/dct_util.cc
+  jxl/dct_util.h
+  jxl/dec_ans.cc
+  jxl/dec_ans.h
+  jxl/dec_bit_reader.h
+  jxl/dec_cache.h
+  jxl/dec_context_map.cc
+  jxl/dec_context_map.h
+  jxl/dec_file.cc
+  jxl/dec_file.h
+  jxl/dec_frame.cc
+  jxl/dec_frame.h
+  jxl/dec_group.cc
+  jxl/dec_group.h
+  jxl/dec_modular.cc
+  jxl/dec_modular.h
+  jxl/dec_params.h
+  jxl/dec_reconstruct.cc
+  jxl/dec_reconstruct.h
+  jxl/dec_xyb.cc
+  jxl/dec_xyb.h
+  jxl/detect_dots.cc
+  jxl/detect_dots.h
+  jxl/dot_dictionary.cc
+  jxl/dot_dictionary.h
+  jxl/enc_ac_strategy.cc
+  jxl/enc_ac_strategy.h
+  jxl/enc_adaptive_quantization.cc
+  jxl/enc_adaptive_quantization.h
+  jxl/enc_ans.cc
+  jxl/enc_ans.h
+  jxl/enc_bit_writer.cc
+  jxl/enc_bit_writer.h
+  jxl/enc_butteraugli_comparator.cc
+  jxl/enc_butteraugli_comparator.h
+  jxl/enc_cache.cc
+  jxl/enc_cache.h
+  jxl/enc_cluster.cc
+  jxl/enc_cluster.h
+  jxl/enc_comparator.cc
+  jxl/enc_comparator.h
+  jxl/enc_context_map.cc
+  jxl/enc_context_map.h
+  jxl/enc_file.cc
+  jxl/enc_file.h
+  jxl/enc_frame.cc
+  jxl/enc_frame.h
+  jxl/enc_gamma_correct.h
+  jxl/enc_group.cc
+  jxl/enc_group.h
+  jxl/enc_modular.cc
+  jxl/enc_modular.h
+  jxl/enc_params.h
+  jxl/enc_xyb.cc
+  jxl/enc_xyb.h
+  jxl/entropy_coder.cc
+  jxl/entropy_coder.h
+  jxl/epf.cc
+  jxl/epf.h
+  jxl/external_image.cc
+  jxl/external_image.h
+  jxl/field_encodings.h
+  jxl/fields.cc
+  jxl/fields.h
+  jxl/frame_header.cc
+  jxl/frame_header.h
+  jxl/gaborish.cc
+  jxl/gaborish.h
+  jxl/gauss_blur.cc
+  jxl/gauss_blur.h
+  jxl/headers.cc
+  jxl/headers.h
+  jxl/icc_codec.cc
+  jxl/icc_codec.h
+  jxl/image.cc
+  jxl/image.h
+  jxl/image_bundle.cc
+  jxl/image_bundle.h
+  jxl/image_ops.h
+  jxl/jpegxl/decode.cc
+  jxl/jxl_inspection.h
+  jxl/lehmer_code.h
+  jxl/linalg.cc
+  jxl/linalg.h
+  jxl/loop_filter.cc
+  jxl/loop_filter.h
+  jxl/memory_manager_internal.cc
+  jxl/memory_manager_internal.h
+  jxl/multiframe.cc
+  jxl/multiframe.h
+  jxl/noise.cc
+  jxl/noise.h
+  jxl/noise_distributions.h
+  jxl/opsin_params.cc
+  jxl/opsin_params.h
+  jxl/optimize.cc
+  jxl/optimize.h
+  jxl/passes_state.cc
+  jxl/passes_state.h
+  jxl/patch_dictionary.cc
+  jxl/patch_dictionary.h
+  jxl/predictor.cc
+  jxl/predictor.h
+  jxl/quant_weights.cc
+  jxl/quant_weights.h
+  jxl/quantizer.cc
+  jxl/quantizer.h
+  jxl/rational_polynomial.h
+  jxl/splines.cc
+  jxl/splines.h
+  jxl/splines_fastmath.h
+  jxl/toc.cc
+  jxl/toc.h
+  jxl/xorshift128plus.h
+)
+
+set(JPEGXL_INTERNAL_FLAGS
+  # F_FLAGS
+  -fmerge-all-constants
+  -fno-builtin-fwrite
+  -fno-builtin-fread
+
+  # WARN_FLAGS
+  -Wall
+  -Wextra
+  -Wc++11-compat
+  -Wc++2a-extensions
+  -Wdeprecated-increment-bool
+  -Wfloat-overflow-conversion
+  -Wfloat-zero-conversion
+  -Wfor-loop-analysis
+  -Wformat-security
+  -Wgnu-redeclared-enum
+  -Wimplicit-fallthrough
+  -Winfinite-recursion
+  -Wliteral-conversion
+  -Wno-c++98-compat
+  -Wno-register  # Needed by public headers in lcms
+  -Wno-unused-command-line-argument
+  -Wno-sign-compare
+  -Wno-unused-function
+  -Wno-unused-parameter
+  -Wnon-virtual-dtor
+  -Woverloaded-virtual
+  -Wprivate-header
+  -Wself-assign
+  -Wstring-conversion
+  -Wtautological-overlap-compare
+  -Wthread-safety-analysis
+  -Wundefined-func-template
+  -Wunused-comparison
+  -Wvla
+)
+
+if (WIN32)
+list(APPEND JPEGXL_INTERNAL_FLAGS
+  -Wno-c++98-compat-pedantic
+  -Wno-cast-align
+  -Wno-double-promotion
+  -Wno-float-equal
+  -Wno-format-nonliteral
+  -Wno-global-constructors
+  -Wno-language-extension-token
+  -Wno-missing-prototypes
+  -Wno-shadow
+  -Wno-shadow-field-in-constructor
+  -Wno-sign-conversion
+  -Wno-unused-member-function
+  -Wno-unused-template
+  -Wno-used-but-marked-unused
+  -Wno-zero-as-null-pointer-constant
+)
+else()
+list(APPEND JPEGXL_INTERNAL_FLAGS
+  -fno-signed-char
+  -fsized-deallocation
+  -fnew-alignment=8
+  -fno-cxx-exceptions
+  -fno-exceptions
+  -fno-slp-vectorize
+  -fno-vectorize
+
+  # Language flags
+  -disable-free
+  -disable-llvm-verifier
+  -fmath-errno
+)
+endif ()
+
+set(JPEGXL_INTERNAL_LIBS
+  brotlicommon-static
+  brotlienc-static
+  brotlidec-static
+  brunslicommon-static
+  brunslidec-static
+  brunslienc-static
+  hwy
+  Threads::Threads
+  ${CMAKE_DL_LIBS}
+)
+set(JPEGXL_LIBRARY_REQUIRES
+    "libbrotlicommon libbrotlienc libbrotlidec")
+
+if (JPEGXL_ENABLE_SKCMS)
+  list(APPEND JPEGXL_INTERNAL_FLAGS -DJPEGXL_ENABLE_SKCMS=1)
+  list(APPEND JPEGXL_INTERNAL_LIBS skcms)
+else ()
+  list(APPEND JPEGXL_INTERNAL_LIBS lcms2)
+endif ()
+
+if(JPEGXL_ENABLE_COVERAGE)
+set(JPEGXL_COVERAGE_FLAGS
+    -g -O0 -fprofile-arcs -ftest-coverage -DJXL_DISABLE_SLOW_TESTS
+    -DJXL_ENABLE_ASSERT=0
+)
+endif()
+
+# Object library. This is used to hold the set of objects and properties.
+add_library(jpegxl-obj OBJECT ${JPEGXL_INTERNAL_SOURCES})
+target_compile_options(jpegxl-obj PRIVATE ${JPEGXL_INTERNAL_FLAGS})
+target_compile_options(jpegxl-obj PUBLIC ${JPEGXL_COVERAGE_FLAGS})
+if (NOT "${JPEGXL_EMSCRIPTEN}")
+set_property(TARGET jpegxl-obj PROPERTY POSITION_INDEPENDENT_CODE ON)
+endif()
+target_include_directories(jpegxl-obj PUBLIC
+  ${CMAKE_CURRENT_SOURCE_DIR}
+  ${CMAKE_CURRENT_SOURCE_DIR}/include
+  $<TARGET_PROPERTY:hwy,INCLUDE_DIRECTORIES>
+  $<TARGET_PROPERTY:brotlicommon-static,INCLUDE_DIRECTORIES>
+  $<TARGET_PROPERTY:brunslicommon-static,INCLUDE_DIRECTORIES>
+)
+target_compile_definitions(jpegxl-obj PUBLIC
+  JPEGXL_MAJOR_VERSION=${JPEGXL_MAJOR_VERSION}
+  JPEGXL_MINOR_VERSION=${JPEGXL_MINOR_VERSION}
+  JPEGXL_PATCH_VERSION=${JPEGXL_PATCH_VERSION}
+  # Used to determine if we are building the library when defined or just
+  # including the library when not defined. This is public so libjpegxl shared
+  # library gets this define too.
+  JPEGXL_INTERNAL_LIBRARY_BUILD
+)
+
+if (JPEGXL_ENABLE_SKCMS)
+  target_include_directories(jpegxl-obj PRIVATE
+    $<TARGET_PROPERTY:skcms,INCLUDE_DIRECTORIES>
+  )
+else ()
+  target_include_directories(jpegxl-obj PRIVATE
+    $<TARGET_PROPERTY:lcms2,INCLUDE_DIRECTORIES>
+  )
+endif ()
+
+# Headers for exporting/importing public headers
+include(GenerateExportHeader)
+# TODO(deymo): Add these visibility properties to the static dependencies of
+# jpegxl-obj since those are currently compiled with the default visibility.
+set_target_properties(jpegxl-obj PROPERTIES
+  CXX_VISIBILITY_PRESET hidden
+  VISIBILITY_INLINES_HIDDEN 1
+  DEFINE_SYMBOL JPEGXL_INTERNAL_LIBRARY_BUILD
+)
+generate_export_header(jpegxl-obj
+  BASE_NAME JPEGXL
+  EXPORT_FILE_NAME include/jpegxl/jpegxl_export.h)
+target_include_directories(jpegxl-obj PUBLIC
+    ${CMAKE_BINARY_DIR}/include)
+
+# Private static library. This exposes all the internal functions and is used
+# for tests.
+add_library(jpegxl-static STATIC $<TARGET_OBJECTS:jpegxl-obj>)
+target_link_libraries(jpegxl-static
+  PUBLIC ${JPEGXL_COVERAGE_FLAGS} ${JPEGXL_INTERNAL_LIBS} hwy)
+target_include_directories(jpegxl-static PUBLIC
+  "${CMAKE_CURRENT_SOURCE_DIR}"
+  "${CMAKE_CURRENT_SOURCE_DIR}/include"
+  "${CMAKE_BINARY_DIR}/include")
+
+# gperftools (TCMalloc) does not compile on OSX.
+# TODO(deymo): Move TCMalloc linkage to the tools/ directory since the library
+# shouldn't do any allocs anyway.
+if(${JPEGXL_ENABLE_TCMALLOC})
+  target_link_libraries(jpegxl-static PUBLIC gperftools)
+endif()  # JPEGXL_ENABLE_TCMALLOC
+
+if (NOT "${JPEGXL_EMSCRIPTEN}")
+
+# Public shared library.
+add_library(jpegxl SHARED $<TARGET_OBJECTS:jpegxl-obj>)
+target_link_libraries(jpegxl PUBLIC ${JPEGXL_COVERAGE_FLAGS})
+target_link_libraries(jpegxl PRIVATE ${JPEGXL_INTERNAL_LIBS})
+target_include_directories(jpegxl PUBLIC
+  "${CMAKE_CURRENT_SOURCE_DIR}/include"
+  "${CMAKE_BINARY_DIR}/include")
+set_target_properties(jpegxl PROPERTIES
+  VERSION ${JPEGXL_LIBRARY_VERSION}
+  SOVERSION ${JPEGXL_LIBRARY_SOVERSION})
+
+# Add a jpegxl.version file as a version script to tag symbols with the
+# appropriate version number. This script is also used to limit what's exposed
+# in the shared library from the static dependencies bundled here.
+set_target_properties(jpegxl PROPERTIES LINK_DEPENDS ${CMAKE_SOURCE_DIR}/jxl/jpegxl.version)
+if(APPLE)
+set_property(TARGET jpegxl APPEND_STRING PROPERTY
+             LINK_FLAGS "-Wl,-exported_symbols_list,${CMAKE_SOURCE_DIR}/jxl/jpegxl_osx.syms")
+elseif(WIN32)
+  # Nothing needed here, we use __declspec(dllexport) (jpegxl_export.h)
+else()
+set_property(TARGET jpegxl APPEND_STRING PROPERTY
+             LINK_FLAGS " -Wl,--version-script=${CMAKE_SOURCE_DIR}/jxl/jpegxl.version")
+endif()  # APPLE
+install(TARGETS jpegxl
+  DESTINATION ${CMAKE_INSTALL_LIBDIR})
+install(DIRECTORY ${CMAKE_SOURCE_DIR}/include/jpegxl
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/jpegxl")
+install(DIRECTORY ${CMAKE_BINARY_DIR}/include/jpegxl
+  DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/jpegxl")
+
+# Add a pkg-config file for libjpegxl.
+configure_file("${CMAKE_SOURCE_DIR}/jxl/libjpegxl.pc.in" "libjpegxl.pc" @ONLY)
+install(FILES "${CMAKE_CURRENT_BINARY_DIR}/libjpegxl.pc"
+  DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig")
+
+endif()  # JPEGXL_EMSCRIPTEN
diff --git a/jpegxl_extras.cmake b/jpegxl_extras.cmake
new file mode 100644
index 000000000..c2e6b1786
--- /dev/null
+++ b/jpegxl_extras.cmake
@@ -0,0 +1,86 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(JPEGXL_EXTRAS_SOURCES
+  jxl/extras/codec.cc
+  jxl/extras/codec.h
+  jxl/extras/codec_apng.cc
+  jxl/extras/codec_apng.h
+  jxl/extras/codec_gif.cc
+  jxl/extras/codec_gif.h
+  jxl/extras/codec_jpg.cc
+  jxl/extras/codec_jpg.h
+  jxl/extras/codec_pgx.cc
+  jxl/extras/codec_pgx.h
+  jxl/extras/codec_png.cc
+  jxl/extras/codec_png.h
+  jxl/extras/codec_pnm.cc
+  jxl/extras/codec_pnm.h
+)
+
+find_package(GIF REQUIRED 5)
+find_package(JPEG REQUIRED)
+find_package(ZLIB REQUIRED)  # dependency of PNG
+find_package(PNG REQUIRED)
+find_package(PkgConfig)
+pkg_check_modules(OpenEXR IMPORTED_TARGET OpenEXR)
+
+foreach (lib gif jpeg png zlib)
+  string(TOUPPER "${lib}" LIB)
+  find_library(${LIB}_STATIC_LIBRARY
+    NAMES lib${lib}.a
+    HINTS
+      ENV ${LIB}_DIR
+    PATH_SUFFIXES lib
+  )
+  if(NOT "${${LIB}_STATIC_LIBRARY}" STREQUAL "${LIB}_STATIC_LIBRARY-NOTFOUND")
+    list(APPEND JPEGXL_STATIC_EXTRAS_LIBS "${${LIB}_STATIC_LIBRARY}")
+  else()
+    message("Warning: lib${lib}.a not found, using ${${LIB}_LIBRARIES}")
+    list(APPEND JPEGXL_STATIC_EXTRAS_LIBS "${${LIB}_LIBRARIES}")
+  endif()
+endforeach()
+
+# We only define a static library for jpegxl_extras since it uses internal parts
+# of jpegxl library which are not accessible from outside the library in the
+# shared library case.
+add_library(jpegxl_extras-static STATIC "${JPEGXL_EXTRAS_SOURCES}")
+target_compile_options(jpegxl_extras-static PRIVATE "${JPEGXL_INTERNAL_FLAGS}")
+set_property(TARGET jpegxl_extras-static PROPERTY POSITION_INDEPENDENT_CODE ON)
+target_include_directories(jpegxl_extras-static PUBLIC
+  "${CMAKE_CURRENT_SOURCE_DIR}"
+  "${GIF_INCLUDE_DIRS}"
+  "${JPEG_INCLUDE_DIRS}"
+  "${PNG_INCLUDE_DIRS}"
+)
+
+target_link_libraries(jpegxl_extras-static PUBLIC
+  jpegxl-static
+  lodepng
+  ${JPEGXL_STATIC_EXTRAS_LIBS}
+)
+
+if (JPEGXL_ENABLE_SJPEG)
+  target_compile_definitions(jpegxl_extras-static PUBLIC -DJPEGXL_ENABLE_SJPEG=1)
+  target_link_libraries(jpegxl_extras-static PUBLIC sjpeg)
+endif ()
+
+if (OpenEXR_FOUND)
+  target_sources(jpegxl_extras-static PRIVATE
+    jxl/extras/codec_exr.cc
+    jxl/extras/codec_exr.h
+  )
+  target_compile_definitions(jpegxl_extras-static PRIVATE -DJPEGXL_ENABLE_EXR=1)
+  target_link_libraries(jpegxl_extras-static PUBLIC PkgConfig::OpenEXR)
+endif ()
diff --git a/jpegxl_threads.cmake b/jpegxl_threads.cmake
new file mode 100644
index 000000000..ff13cbc34
--- /dev/null
+++ b/jpegxl_threads.cmake
@@ -0,0 +1,36 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(JPEGXL_THREADS_SOURCES
+  jxl/threads/thread_parallel_runner.cc
+)
+
+find_package(Threads REQUIRED)
+
+# We only provide a static jpegxl_threads library since it is a non-stable C++
+# API and very small. It is likely better to bundle it in the programs that
+# need it.
+add_library(jpegxl_threads STATIC "${JPEGXL_THREADS_SOURCES}")
+target_compile_options(jpegxl_threads PRIVATE "${JPEGXL_INTERNAL_FLAGS}")
+set_property(TARGET jpegxl_threads PROPERTY POSITION_INDEPENDENT_CODE ON)
+target_include_directories(jpegxl_threads PUBLIC ${CMAKE_CURRENT_SOURCE_DIR})
+
+target_link_libraries(jpegxl_threads
+  PUBLIC
+    Threads::Threads
+  PRIVATE
+    #TODO(deymo): Change this to use the shared library. Right now it causes
+    # problems when including both the shared and static library.
+    jpegxl-static
+)
diff --git a/jpegxltests.cmake b/jpegxltests.cmake
new file mode 100644
index 000000000..9496e7199
--- /dev/null
+++ b/jpegxltests.cmake
@@ -0,0 +1,105 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set(TEST_FILES
+  jxl/ac_strategy_test.cc
+  jxl/adaptive_reconstruction_test.cc
+  jxl/ans_common_test.cc
+  jxl/ans_test.cc
+  jxl/bit_reader_test.cc
+  jxl/bits_test.cc
+  jxl/brotli_test.cc
+  jxl/brunsli_test.cc
+  jxl/byte_order_test.cc
+  jxl/coeff_order_test.cc
+  jxl/color_encoding_test.cc
+  jxl/color_management_test.cc
+  jxl/compressed_dc_test.cc
+  jxl/compressed_image_test.cc
+  jxl/convolve_test.cc
+  jxl/data_parallel_test.cc
+  jxl/dct_test.cc
+  jxl/dct_util_test.cc
+  jxl/descriptive_statistics_test.cc
+  jxl/entropy_coder_test.cc
+  jxl/external_image_test.cc
+  jxl/extras/codec_test.cc
+  jxl/fields_test.cc
+  jxl/gaborish_test.cc
+  jxl/gamma_correct_test.cc
+  jxl/gradient_test.cc
+  jxl/iaca_test.cc
+  jxl/icc_codec_test.cc
+  jxl/image_test.cc
+  jxl/jpeg_kpasses_test.cc
+  jxl/jpegxl/decode_test.cc
+  jxl/jxl_test.cc
+  jxl/lehmer_code_test.cc
+  jxl/linalg_test.cc
+  jxl/opsin_image_test.cc
+  jxl/opsin_inverse_test.cc
+  jxl/optimize_test.cc
+  jxl/padded_bytes_test.cc
+  jxl/passes_test.cc
+  jxl/predictor_test.cc
+  jxl/preview_test.cc
+  jxl/quant_weights_test.cc
+  jxl/quantizer_test.cc
+  jxl/rational_polynomial_test.cc
+  jxl/robust_statistics_test.cc
+  jxl/speed_tier_test.cc
+  jxl/splines_test.cc
+  jxl/threads/thread_parallel_runner_test.cc
+  jxl/toc_test.cc
+  jxl/xorshift128plus_test.cc
+)
+
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests)
+foreach (TESTFILE IN LISTS TEST_FILES)
+  # The TESTNAME is the name without the extension or directory.
+  get_filename_component(TESTNAME ${TESTFILE} NAME_WE)
+  add_executable(${TESTNAME} ${TESTFILE})
+  target_compile_definitions(${TESTNAME} PRIVATE
+      -DTEST_DATA_PATH="${CMAKE_CURRENT_SOURCE_DIR}/third_party/testdata")
+  target_compile_options(${TESTNAME} PRIVATE
+    ${JPEGXL_INTERNAL_FLAGS}
+    # Add coverage flags to the test binary so code in the private headers of
+    # the library is also instrumented when running tests that execute it.
+    ${JPEGXL_COVERAGE_FLAGS}
+  )
+  target_link_libraries(${TESTNAME}
+    jpegxl-static
+    jpegxl_threads
+    jpegxl_extras-static
+    gmock
+    gtest
+    gtest_main
+  )
+  # Output test targets in the test directory.
+  set_target_properties(${TESTNAME} PROPERTIES PREFIX "tests/")
+  if (WIN32 AND ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
+    set_target_properties(${TESTNAME} PROPERTIES COMPILE_FLAGS "-Wno-error")
+  endif ()
+  if(${CMAKE_VERSION} VERSION_LESS "3.10.3")
+    gtest_discover_tests(${TESTNAME} TIMEOUT 30)
+  else ()
+    gtest_discover_tests(${TESTNAME} DISCOVERY_TIMEOUT 30)
+  endif ()
+endforeach ()
+
+# googletest doesn't compile clean with clang-cl (-Wundef)
+if (WIN32 AND ${CMAKE_CXX_COMPILER_ID} STREQUAL "Clang")
+  set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-Wno-error")
+  set_target_properties(gtest_main PROPERTIES COMPILE_FLAGS "-Wno-error")
+endif ()
diff --git a/jxl/ac_context.h b/jxl/ac_context.h
new file mode 100644
index 000000000..d9ace3dcf
--- /dev/null
+++ b/jxl/ac_context.h
@@ -0,0 +1,113 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_AC_CONTEXT_H_
+#define JXL_AC_CONTEXT_H_
+
+#include "jxl/base/bits.h"
+#include "jxl/base/status.h"
+#include "jxl/coeff_order_fwd.h"
+
+namespace jxl {
+
+// Block context used for scanning order, number of non-zeros, AC coefficients.
+// Equal to the channel.
+constexpr uint32_t kDCTOrderContextStart = 0;
+constexpr uint32_t kOrderContexts = 10;
+
+// The number of predicted nonzeros goes from 0 to 1008. We use
+// ceil(log2(predicted+1)) as a context for the number of nonzeros, so from 0 to
+// 10, inclusive.
+constexpr uint32_t kNonZeroBuckets = 37;
+
+static const uint16_t kCoeffFreqContext[64] = {
+    0xBAD, 0,  1,  2,  3,  4,  5,  6,  7,  8,  9,  10, 11, 12, 13, 14,
+    15,    15, 16, 16, 17, 17, 18, 18, 19, 19, 20, 20, 21, 21, 22, 22,
+    23,    23, 23, 23, 24, 24, 24, 24, 25, 25, 25, 25, 26, 26, 26, 26,
+    27,    27, 27, 27, 28, 28, 28, 28, 29, 29, 29, 29, 30, 30, 30, 30,
+};
+
+static const uint16_t kCoeffNumNonzeroContext[64] = {
+    0xBAD, 0,   31,  62,  62,  93,  93,  93,  93,  123, 123, 123, 123,
+    152,   152, 152, 152, 152, 152, 152, 152, 180, 180, 180, 180, 180,
+    180,   180, 180, 180, 180, 180, 180, 206, 206, 206, 206, 206, 206,
+    206,   206, 206, 206, 206, 206, 206, 206, 206, 206, 206, 206, 206,
+    206,   206, 206, 206, 206, 206, 206, 206, 206, 206, 206, 206,
+};
+
+// Supremum of ZeroDensityContext(x, y) + 1.
+constexpr int kZeroDensityContextCount = 458;
+
+/* This function is used for entropy-sources pre-clustering.
+ *
+ * Ideally, each combination of |nonzeros_left| and |k| should go to its own
+ * bucket; but it implies (64 * 63 / 2) == 2016 buckets. If there is other
+ * dimension (e.g. block context), then number of primary clusters becomes too
+ * big.
+ *
+ * To solve this problem, |nonzeros_left| and |k| values are clustered. It is
+ * known that their sum is at most 64, consequently, the total number buckets
+ * is at most A(64) * B(64).
+ */
+// TODO(user): investigate, why disabling pre-clustering makes entropy code
+// less dense. Perhaps we would need to add HQ clustering algorithm that would
+// be able to squeeze better by spending more CPU cycles.
+static JXL_INLINE size_t ZeroDensityContext(size_t nonzeros_left, size_t k,
+                                            size_t covered_blocks,
+                                            size_t log2_covered_blocks,
+                                            size_t prev) {
+  JXL_DASSERT(1 << log2_covered_blocks == covered_blocks);
+  nonzeros_left = (nonzeros_left + covered_blocks - 1) >> log2_covered_blocks;
+  k >>= log2_covered_blocks;
+  JXL_DASSERT(k > 0);
+  JXL_DASSERT(k < 64);
+  JXL_DASSERT(nonzeros_left > 0);
+  JXL_DASSERT(nonzeros_left + k < 65);
+  return (kCoeffNumNonzeroContext[nonzeros_left] + kCoeffFreqContext[k]) * 2 +
+         prev;
+}
+
+// Context map for AC coefficients consists of 2 blocks:
+//  |kOrderContexts x          : context for number of non-zeros in the block
+//   kNonZeroBuckets|            computed from block context and predicted value
+//                               (based top and left values)
+//  |kOrderContexts x          : context for AC coefficient symbols,
+//   kZeroDensityContextCount|   computed from block context,
+//                               number of non-zeros left and
+//                               index in scan order
+constexpr uint32_t kNumContexts = (kOrderContexts * kNonZeroBuckets) +
+                                  (kOrderContexts * kZeroDensityContextCount);
+
+// Non-zero context is based on number of non-zeros and block context.
+// For better clustering, contexts with same number of non-zeros are grouped.
+static inline uint32_t NonZeroContext(uint32_t non_zeros, uint32_t block_ctx) {
+  uint32_t ctx;
+  if (non_zeros >= 64) non_zeros = 64;
+  if (non_zeros < 8)
+    ctx = non_zeros;
+  else
+    ctx = 4 + non_zeros / 2;
+  return ctx * kOrderContexts + block_ctx;
+}
+
+// Non-zero context is based on number of non-zeros and block context.
+// For better clustering, contexts with same number of non-zeros are grouped.
+constexpr uint32_t ZeroDensityContextsOffset(uint32_t block_ctx) {
+  return kOrderContexts * kNonZeroBuckets +
+         kZeroDensityContextCount * block_ctx;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_AC_CONTEXT_H_
diff --git a/jxl/ac_strategy.cc b/jxl/ac_strategy.cc
new file mode 100644
index 000000000..71c751cfc
--- /dev/null
+++ b/jxl/ac_strategy.cc
@@ -0,0 +1,1540 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/ac_strategy.h"
+
+#include <string.h>
+
+#include <algorithm>
+#include <hwy/static_targets.h>
+#include <numeric>  // iota
+#include <type_traits>
+#include <utility>
+
+#include "jxl/base/profiler.h"
+#include "jxl/block.h"
+#include "jxl/common.h"
+#include "jxl/dct.h"
+#include "jxl/image_ops.h"
+
+namespace jxl {
+namespace {
+
+template <size_t ROWS, size_t COLS>
+struct DoDCT {
+  template <typename From, typename To>
+  HWY_ATTR void operator()(const From& from, const To& to) {
+    ComputeScaledDCT<ROWS, COLS>()(from, to);
+  }
+};
+
+template <size_t N>
+struct DoDCT<N, N> {
+  template <typename From, typename To>
+  HWY_ATTR void operator()(const From& from, const To& to) {
+    ComputeTransposedScaledDCT<N>()(from, to);
+  }
+};
+
+template <size_t ROWS, size_t COLS>
+struct DoIDCT {
+  template <typename From, typename To>
+  HWY_ATTR void operator()(const From& from, const To& to) {
+    ComputeScaledIDCT<ROWS, COLS>()(from, to);
+  }
+};
+
+template <size_t N>
+struct DoIDCT<N, N> {
+  template <typename From, typename To>
+  HWY_ATTR void operator()(const From& from, const To& to) const {
+    ComputeTransposedScaledIDCT<N>()(from, to);
+  }
+};
+
+// Computes the lowest-frequency LF_ROWSxLF_COLS-sized square in output, which
+// is a DCT_ROWS*DCT_COLS-sized DCT block, by doing a ROWS*COLS DCT on the
+// input block.
+template <size_t DCT_ROWS, size_t DCT_COLS, size_t LF_ROWS, size_t LF_COLS,
+          size_t ROWS, size_t COLS>
+HWY_ATTR JXL_INLINE void ReinterpretingDCT(const float* input,
+                                           const size_t input_stride,
+                                           float* output,
+                                           const size_t output_stride) {
+  static_assert(LF_ROWS == ROWS,
+                "ReinterpretingDCT should only be called with LF == N");
+  static_assert(LF_COLS == COLS,
+                "ReinterpretingDCT should only be called with LF == N");
+  HWY_ALIGN float block[ROWS * COLS] = {};
+  for (size_t y = 0; y < ROWS; y++) {
+    for (size_t x = 0; x < COLS; x++) {
+      block[y * COLS + x] = input[y * input_stride + x];
+    }
+  }
+
+  constexpr size_t OUT_ROWS = CoefficientRows(ROWS, COLS);
+  constexpr size_t OUT_COLS = CoefficientColumns(ROWS, COLS);
+  DoDCT<ROWS, COLS>()(FromBlock<ROWS, COLS>(block),
+                      ScaleToBlock<OUT_ROWS, OUT_COLS>(block));
+  if (ROWS < COLS) {
+    for (size_t y = 0; y < LF_ROWS; y++) {
+      for (size_t x = 0; x < LF_COLS; x++) {
+        output[y * output_stride + x] =
+            block[y * COLS + x] * DCTTotalResampleScale<ROWS, DCT_ROWS>(y) *
+            DCTTotalResampleScale<COLS, DCT_COLS>(x);
+      }
+    }
+  } else {
+    for (size_t y = 0; y < LF_COLS; y++) {
+      for (size_t x = 0; x < LF_ROWS; x++) {
+        output[y * output_stride + x] =
+            block[y * ROWS + x] * DCTTotalResampleScale<COLS, DCT_COLS>(y) *
+            DCTTotalResampleScale<ROWS, DCT_ROWS>(x);
+      }
+    }
+  }
+}
+
+// Inverse of ReinterpretingDCT.
+template <size_t DCT_ROWS, size_t DCT_COLS, size_t LF_ROWS, size_t LF_COLS,
+          size_t ROWS, size_t COLS>
+HWY_ATTR JXL_INLINE void ReinterpretingIDCT(const float* input,
+                                            const size_t input_stride,
+                                            float* output,
+                                            const size_t output_stride) {
+  HWY_ALIGN float block[ROWS * COLS] = {};
+  if (ROWS < COLS) {
+    for (size_t y = 0; y < LF_ROWS; y++) {
+      for (size_t x = 0; x < LF_COLS; x++) {
+        block[y * COLS + x] = input[y * input_stride + x] *
+                              DCTTotalResampleScale<DCT_ROWS, ROWS>(y) *
+                              DCTTotalResampleScale<DCT_COLS, COLS>(x);
+      }
+    }
+  } else {
+    for (size_t y = 0; y < LF_COLS; y++) {
+      for (size_t x = 0; x < LF_ROWS; x++) {
+        block[y * ROWS + x] = input[y * input_stride + x] *
+                              DCTTotalResampleScale<DCT_COLS, COLS>(y) *
+                              DCTTotalResampleScale<DCT_ROWS, ROWS>(x);
+      }
+    }
+  }
+
+  constexpr size_t IN_ROWS = CoefficientRows(ROWS, COLS);
+  constexpr size_t IN_COLS = CoefficientColumns(ROWS, COLS);
+  DoIDCT<ROWS, COLS>()(FromBlock<IN_ROWS, IN_COLS>(block),
+                       ToBlock<ROWS, COLS>(block));
+
+  for (size_t y = 0; y < ROWS; y++) {
+    for (size_t x = 0; x < COLS; x++) {
+      output[y * output_stride + x] = block[y * COLS + x];
+    }
+  }
+}
+
+template <size_t S>
+void DCT2TopBlock(const float* block, size_t stride, float* out) {
+  static_assert(kBlockDim % S == 0, "S should be a divisor of kBlockDim");
+  static_assert(S % 2 == 0, "S should be even");
+  float temp[kDCTBlockSize];
+  constexpr size_t num_2x2 = S / 2;
+  for (size_t y = 0; y < num_2x2; y++) {
+    for (size_t x = 0; x < num_2x2; x++) {
+      float c00 = block[y * 2 * stride + x * 2];
+      float c01 = block[y * 2 * stride + x * 2 + 1];
+      float c10 = block[(y * 2 + 1) * stride + x * 2];
+      float c11 = block[(y * 2 + 1) * stride + x * 2 + 1];
+      float r00 = c00 + c01 + c10 + c11;
+      float r01 = c00 + c01 - c10 - c11;
+      float r10 = c00 - c01 + c10 - c11;
+      float r11 = c00 - c01 - c10 + c11;
+      r00 *= 0.25f;
+      r01 *= 0.25f;
+      r10 *= 0.25f;
+      r11 *= 0.25f;
+      temp[y * kBlockDim + x] = r00;
+      temp[y * kBlockDim + num_2x2 + x] = r01;
+      temp[(y + num_2x2) * kBlockDim + x] = r10;
+      temp[(y + num_2x2) * kBlockDim + num_2x2 + x] = r11;
+    }
+  }
+  for (size_t y = 0; y < S; y++) {
+    for (size_t x = 0; x < S; x++) {
+      out[y * kBlockDim + x] = temp[y * kBlockDim + x];
+    }
+  }
+}
+
+template <size_t S>
+void IDCT2TopBlock(const float* block, size_t stride_out, float* out) {
+  static_assert(kBlockDim % S == 0, "S should be a divisor of kBlockDim");
+  static_assert(S % 2 == 0, "S should be even");
+  float temp[kDCTBlockSize];
+  constexpr size_t num_2x2 = S / 2;
+  for (size_t y = 0; y < num_2x2; y++) {
+    for (size_t x = 0; x < num_2x2; x++) {
+      float c00 = block[y * kBlockDim + x];
+      float c01 = block[y * kBlockDim + num_2x2 + x];
+      float c10 = block[(y + num_2x2) * kBlockDim + x];
+      float c11 = block[(y + num_2x2) * kBlockDim + num_2x2 + x];
+      float r00 = c00 + c01 + c10 + c11;
+      float r01 = c00 + c01 - c10 - c11;
+      float r10 = c00 - c01 + c10 - c11;
+      float r11 = c00 - c01 - c10 + c11;
+      temp[y * 2 * kBlockDim + x * 2] = r00;
+      temp[y * 2 * kBlockDim + x * 2 + 1] = r01;
+      temp[(y * 2 + 1) * kBlockDim + x * 2] = r10;
+      temp[(y * 2 + 1) * kBlockDim + x * 2 + 1] = r11;
+    }
+  }
+  for (size_t y = 0; y < S; y++) {
+    for (size_t x = 0; x < S; x++) {
+      out[y * stride_out + x] = temp[y * kBlockDim + x];
+    }
+  }
+}
+
+// Tries to generalize zig-zag order to non-square blocks. Surprisingly, in
+// square block frequency along the (i + j == const) diagonals is roughly the
+// same. For historical reasons, consecutive diagonals are traversed
+// in alternating directions - so called "zig-zag" (or "snake") order.
+static AcStrategy::CoeffOrderAndLut ComputeNaturalCoeffOrder() {
+  AcStrategy::CoeffOrderAndLut coeff;
+  for (size_t s = 0; s < AcStrategy::kNumValidStrategies; s++) {
+    const AcStrategy acs = AcStrategy::FromRawStrategy(s);
+    size_t cx = acs.covered_blocks_x();
+    size_t cy = acs.covered_blocks_y();
+    CoefficientLayout(&cy, &cx);
+    const size_t num_coeffs = kDCTBlockSize * cx * cy;
+    coeff_order_t* JXL_RESTRICT order_start =
+        coeff.order + s * AcStrategy::kMaxCoeffArea;
+    coeff_order_t* JXL_RESTRICT lut_start =
+        coeff.lut + s * AcStrategy::kMaxCoeffArea;
+    std::iota(order_start, order_start + num_coeffs, 0);
+
+    auto compute_key = [cx, cy](int32_t pos) {
+      JXL_DASSERT(cx != 0 && cy != 0);
+      size_t y = pos / (cx * kBlockDim);
+      size_t x = pos % (cx * kBlockDim);
+      // Ensure that LLFs are first in the order.
+      if (x < cx && y < cy) {
+        return std::make_pair(-1, static_cast<int>(y * cx + x));
+      }
+      int max_dim = std::max(cx, cy);
+      int scaled_y = y * max_dim / cy;
+      int scaled_x = x * max_dim / cx;
+      return std::make_pair(scaled_x + scaled_y, (scaled_x + scaled_y) % 2 == 0
+                                                     ? scaled_x - scaled_y
+                                                     : scaled_y - scaled_x);
+    };
+
+    std::sort(order_start, order_start + num_coeffs,
+              [compute_key](int32_t pos, int32_t other_pos) {
+                return compute_key(pos) < compute_key(other_pos);
+              });
+
+    for (size_t i = 0; i < num_coeffs; i++) {
+      lut_start[order_start[i]] = i;
+    }
+  }
+  return coeff;
+};
+
+HWY_ALIGN constexpr float k4x4AFVBasisTranspose[16][16] = {
+    {
+        0.2500000000000000,
+        0.8769029297991420,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        -0.4105377591765233,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+    },
+    {
+        0.2500000000000000,
+        0.2206518106944235,
+        0.0000000000000000,
+        0.0000000000000000,
+        -0.7071067811865474,
+        0.6235485373547691,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375376,
+        0.4067007583026075,
+        -0.2125574805828875,
+        0.0000000000000000,
+        -0.0643507165794627,
+        -0.4517556589999482,
+        -0.3046847507248690,
+        0.3017929516615495,
+        0.4082482904638627,
+        0.1747866975480809,
+        -0.2110560104933578,
+        -0.1426608480880726,
+        -0.1381354035075859,
+        -0.1743760259965107,
+        0.1135498731499434,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375375,
+        0.4444481661973445,
+        0.3085497062849767,
+        0.0000000000000000,
+        -0.0643507165794627,
+        0.1585450355184006,
+        0.5112616136591823,
+        0.2579236279634118,
+        0.0000000000000000,
+        0.0812611176717539,
+        0.1856718091610980,
+        -0.3416446842253372,
+        0.3302282550303788,
+        0.0702790691196284,
+        -0.0741750459581035,
+    },
+    {
+        0.2500000000000000,
+        0.2206518106944236,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.7071067811865476,
+        0.6235485373547694,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375378,
+        0.0000000000000000,
+        0.4706702258572536,
+        0.0000000000000000,
+        -0.0643507165794628,
+        -0.0403851516082220,
+        0.0000000000000000,
+        0.1627234014286620,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.7367497537172237,
+        0.0875511500058708,
+        -0.2921026642334881,
+        0.1940289303259434,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375377,
+        0.1957439937204294,
+        -0.1621205195722993,
+        0.0000000000000000,
+        -0.0643507165794628,
+        0.0074182263792424,
+        -0.2904801297289980,
+        0.0952002265347504,
+        0.0000000000000000,
+        -0.3675398009862027,
+        0.4921585901373873,
+        0.2462710772207515,
+        -0.0794670660590957,
+        0.3623817333531167,
+        -0.4351904965232280,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375376,
+        0.2929100136981264,
+        0.0000000000000000,
+        0.0000000000000000,
+        -0.0643507165794627,
+        0.3935103426921017,
+        -0.0657870154914280,
+        0.0000000000000000,
+        -0.4082482904638628,
+        -0.3078822139579090,
+        -0.3852501370925192,
+        -0.0857401903551931,
+        -0.4613374887461511,
+        0.0000000000000000,
+        0.2191868483885747,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375376,
+        -0.4067007583026072,
+        -0.2125574805828705,
+        0.0000000000000000,
+        -0.0643507165794627,
+        -0.4517556589999464,
+        0.3046847507248840,
+        0.3017929516615503,
+        -0.4082482904638635,
+        -0.1747866975480813,
+        0.2110560104933581,
+        -0.1426608480880734,
+        -0.1381354035075829,
+        -0.1743760259965108,
+        0.1135498731499426,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375377,
+        -0.1957439937204287,
+        -0.1621205195722833,
+        0.0000000000000000,
+        -0.0643507165794628,
+        0.0074182263792444,
+        0.2904801297290076,
+        0.0952002265347505,
+        0.0000000000000000,
+        0.3675398009862011,
+        -0.4921585901373891,
+        0.2462710772207514,
+        -0.0794670660591026,
+        0.3623817333531165,
+        -0.4351904965232251,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375375,
+        0.0000000000000000,
+        -0.4706702258572528,
+        0.0000000000000000,
+        -0.0643507165794627,
+        0.1107416575309343,
+        0.0000000000000000,
+        -0.1627234014286617,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.1488339922711357,
+        0.4972464710953509,
+        0.2921026642334879,
+        0.5550443808910661,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375377,
+        0.1137907446044809,
+        -0.1464291867126764,
+        0.0000000000000000,
+        -0.0643507165794628,
+        0.0829816309488205,
+        -0.2388977352334460,
+        -0.3531238544981630,
+        -0.4082482904638630,
+        0.4826689115059883,
+        0.1741941265991622,
+        -0.0476868035022925,
+        0.1253805944856366,
+        -0.4326608024727445,
+        -0.2546827712406646,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375377,
+        -0.4444481661973438,
+        0.3085497062849487,
+        0.0000000000000000,
+        -0.0643507165794628,
+        0.1585450355183970,
+        -0.5112616136592012,
+        0.2579236279634129,
+        0.0000000000000000,
+        -0.0812611176717504,
+        -0.1856718091610990,
+        -0.3416446842253373,
+        0.3302282550303805,
+        0.0702790691196282,
+        -0.0741750459581023,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375376,
+        -0.2929100136981264,
+        0.0000000000000000,
+        0.0000000000000000,
+        -0.0643507165794627,
+        0.3935103426921022,
+        0.0657870154914254,
+        0.0000000000000000,
+        0.4082482904638634,
+        0.3078822139579031,
+        0.3852501370925211,
+        -0.0857401903551927,
+        -0.4613374887461554,
+        0.0000000000000000,
+        0.2191868483885728,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375376,
+        -0.1137907446044814,
+        -0.1464291867126654,
+        0.0000000000000000,
+        -0.0643507165794627,
+        0.0829816309488214,
+        0.2388977352334547,
+        -0.3531238544981624,
+        0.4082482904638630,
+        -0.4826689115059858,
+        -0.1741941265991621,
+        -0.0476868035022928,
+        0.1253805944856431,
+        -0.4326608024727457,
+        -0.2546827712406641,
+    },
+    {
+        0.2500000000000000,
+        -0.1014005039375374,
+        0.0000000000000000,
+        0.4251149611657548,
+        0.0000000000000000,
+        -0.0643507165794626,
+        -0.4517556589999480,
+        0.0000000000000000,
+        -0.6035859033230976,
+        0.0000000000000000,
+        0.0000000000000000,
+        0.0000000000000000,
+        -0.1426608480880724,
+        -0.1381354035075845,
+        0.3487520519930227,
+        0.1135498731499429,
+    },
+};
+
+HWY_ALIGN constexpr float k4x4AFVBasis[16][16] = {
+    {
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+        0.25,
+    },
+    {
+        0.876902929799142,
+        0.2206518106944235,
+        -0.10140050393753763,
+        -0.1014005039375375,
+        0.2206518106944236,
+        -0.10140050393753777,
+        -0.10140050393753772,
+        -0.10140050393753763,
+        -0.10140050393753758,
+        -0.10140050393753769,
+        -0.1014005039375375,
+        -0.10140050393753768,
+        -0.10140050393753768,
+        -0.10140050393753759,
+        -0.10140050393753763,
+        -0.10140050393753741,
+    },
+    {
+        0.0,
+        0.0,
+        0.40670075830260755,
+        0.44444816619734445,
+        0.0,
+        0.0,
+        0.19574399372042936,
+        0.2929100136981264,
+        -0.40670075830260716,
+        -0.19574399372042872,
+        0.0,
+        0.11379074460448091,
+        -0.44444816619734384,
+        -0.29291001369812636,
+        -0.1137907446044814,
+        0.0,
+    },
+    {
+        0.0,
+        0.0,
+        -0.21255748058288748,
+        0.3085497062849767,
+        0.0,
+        0.4706702258572536,
+        -0.1621205195722993,
+        0.0,
+        -0.21255748058287047,
+        -0.16212051957228327,
+        -0.47067022585725277,
+        -0.1464291867126764,
+        0.3085497062849487,
+        0.0,
+        -0.14642918671266536,
+        0.4251149611657548,
+    },
+    {
+        0.0,
+        -0.7071067811865474,
+        0.0,
+        0.0,
+        0.7071067811865476,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+    },
+    {
+        -0.4105377591765233,
+        0.6235485373547691,
+        -0.06435071657946274,
+        -0.06435071657946266,
+        0.6235485373547694,
+        -0.06435071657946284,
+        -0.0643507165794628,
+        -0.06435071657946274,
+        -0.06435071657946272,
+        -0.06435071657946279,
+        -0.06435071657946266,
+        -0.06435071657946277,
+        -0.06435071657946277,
+        -0.06435071657946273,
+        -0.06435071657946274,
+        -0.0643507165794626,
+    },
+    {
+        0.0,
+        0.0,
+        -0.4517556589999482,
+        0.15854503551840063,
+        0.0,
+        -0.04038515160822202,
+        0.0074182263792423875,
+        0.39351034269210167,
+        -0.45175565899994635,
+        0.007418226379244351,
+        0.1107416575309343,
+        0.08298163094882051,
+        0.15854503551839705,
+        0.3935103426921022,
+        0.0829816309488214,
+        -0.45175565899994796,
+    },
+    {
+        0.0,
+        0.0,
+        -0.304684750724869,
+        0.5112616136591823,
+        0.0,
+        0.0,
+        -0.290480129728998,
+        -0.06578701549142804,
+        0.304684750724884,
+        0.2904801297290076,
+        0.0,
+        -0.23889773523344604,
+        -0.5112616136592012,
+        0.06578701549142545,
+        0.23889773523345467,
+        0.0,
+    },
+    {
+        0.0,
+        0.0,
+        0.3017929516615495,
+        0.25792362796341184,
+        0.0,
+        0.16272340142866204,
+        0.09520022653475037,
+        0.0,
+        0.3017929516615503,
+        0.09520022653475055,
+        -0.16272340142866173,
+        -0.35312385449816297,
+        0.25792362796341295,
+        0.0,
+        -0.3531238544981624,
+        -0.6035859033230976,
+    },
+    {
+        0.0,
+        0.0,
+        0.40824829046386274,
+        0.0,
+        0.0,
+        0.0,
+        0.0,
+        -0.4082482904638628,
+        -0.4082482904638635,
+        0.0,
+        0.0,
+        -0.40824829046386296,
+        0.0,
+        0.4082482904638634,
+        0.408248290463863,
+        0.0,
+    },
+    {
+        0.0,
+        0.0,
+        0.1747866975480809,
+        0.0812611176717539,
+        0.0,
+        0.0,
+        -0.3675398009862027,
+        -0.307882213957909,
+        -0.17478669754808135,
+        0.3675398009862011,
+        0.0,
+        0.4826689115059883,
+        -0.08126111767175039,
+        0.30788221395790305,
+        -0.48266891150598584,
+        0.0,
+    },
+    {
+        0.0,
+        0.0,
+        -0.21105601049335784,
+        0.18567180916109802,
+        0.0,
+        0.0,
+        0.49215859013738733,
+        -0.38525013709251915,
+        0.21105601049335806,
+        -0.49215859013738905,
+        0.0,
+        0.17419412659916217,
+        -0.18567180916109904,
+        0.3852501370925211,
+        -0.1741941265991621,
+        0.0,
+    },
+    {
+        0.0,
+        0.0,
+        -0.14266084808807264,
+        -0.3416446842253372,
+        0.0,
+        0.7367497537172237,
+        0.24627107722075148,
+        -0.08574019035519306,
+        -0.14266084808807344,
+        0.24627107722075137,
+        0.14883399227113567,
+        -0.04768680350229251,
+        -0.3416446842253373,
+        -0.08574019035519267,
+        -0.047686803502292804,
+        -0.14266084808807242,
+    },
+    {
+        0.0,
+        0.0,
+        -0.13813540350758585,
+        0.3302282550303788,
+        0.0,
+        0.08755115000587084,
+        -0.07946706605909573,
+        -0.4613374887461511,
+        -0.13813540350758294,
+        -0.07946706605910261,
+        0.49724647109535086,
+        0.12538059448563663,
+        0.3302282550303805,
+        -0.4613374887461554,
+        0.12538059448564315,
+        -0.13813540350758452,
+    },
+    {
+        0.0,
+        0.0,
+        -0.17437602599651067,
+        0.0702790691196284,
+        0.0,
+        -0.2921026642334881,
+        0.3623817333531167,
+        0.0,
+        -0.1743760259965108,
+        0.36238173335311646,
+        0.29210266423348785,
+        -0.4326608024727445,
+        0.07027906911962818,
+        0.0,
+        -0.4326608024727457,
+        0.34875205199302267,
+    },
+    {
+        0.0,
+        0.0,
+        0.11354987314994337,
+        -0.07417504595810355,
+        0.0,
+        0.19402893032594343,
+        -0.435190496523228,
+        0.21918684838857466,
+        0.11354987314994257,
+        -0.4351904965232251,
+        0.5550443808910661,
+        -0.25468277124066463,
+        -0.07417504595810233,
+        0.2191868483885728,
+        -0.25468277124066413,
+        0.1135498731499429,
+    },
+};
+
+using D = HWY_CAPPED(float, 16);
+
+// Coefficient layout:
+//  - (even, even) positions hold AFV coefficients
+//  - (odd, even) positions hold DCT4x4 coefficients
+//  - (any, odd) positions hold DCT4x8 coefficients
+template <size_t afv_kind>
+HWY_ATTR void AFVTransformFromPixels(const float* JXL_RESTRICT pixels,
+                                     size_t pixels_stride,
+                                     float* JXL_RESTRICT coefficients) {
+  size_t afv_x = afv_kind & 1;
+  size_t afv_y = afv_kind / 2;
+  HWY_ALIGN float block[4 * 8];
+  for (size_t iy = 0; iy < 4; iy++) {
+    for (size_t ix = 0; ix < 4; ix++) {
+      block[(afv_y == 1 ? 3 - iy : iy) * 4 + (afv_x == 1 ? 3 - ix : ix)] =
+          pixels[(iy + 4 * afv_y) * pixels_stride + ix + 4 * afv_x];
+    }
+  }
+  // AFV coefficients in (even, even) positions.
+  HWY_ALIGN float coeff[4 * 4];
+  AFVDCT4x4(block, coeff);
+  for (size_t iy = 0; iy < 4; iy++) {
+    for (size_t ix = 0; ix < 4; ix++) {
+      coefficients[iy * 2 * 8 + ix * 2] = coeff[iy * 4 + ix];
+    }
+  }
+  // 4x4 DCT of the block with same y and different x.
+  ComputeTransposedScaledDCT<4>()(
+      FromLines<4>(pixels + afv_y * 4 * pixels_stride + (afv_x == 1 ? 0 : 4),
+                   pixels_stride),
+      ScaleToBlock<4, 4>(block));
+  // ... in (odd, even) positions.
+  for (size_t iy = 0; iy < 4; iy++) {
+    for (size_t ix = 0; ix < 8; ix++) {
+      coefficients[iy * 2 * 8 + ix * 2 + 1] = block[iy * 4 + ix];
+    }
+  }
+  // 4x8 DCT of the other half of the block.
+  ComputeScaledDCT<4, 8>()(
+      FromLines<8>(pixels + (afv_y == 1 ? 0 : 4) * pixels_stride,
+                   pixels_stride),
+      ScaleToBlock<4, 8>(block));
+  for (size_t iy = 0; iy < 4; iy++) {
+    for (size_t ix = 0; ix < 8; ix++) {
+      coefficients[(1 + iy * 2) * 8 + ix] = block[iy * 8 + ix];
+    }
+  }
+  float block00 = coefficients[0] * 0.25f;
+  float block01 = coefficients[1];
+  float block10 = coefficients[8];
+  coefficients[0] = (block00 + block01 + 2 * block10) * 0.25f;
+  coefficients[1] = (block00 - block01) * 0.5f;
+  coefficients[8] = (block00 + block01 - 2 * block10) * 0.25f;
+}
+
+template <size_t afv_kind>
+HWY_ATTR void AFVTransformToPixels(const float* JXL_RESTRICT coefficients,
+                                   float* JXL_RESTRICT pixels,
+                                   size_t pixels_stride) {
+  size_t afv_x = afv_kind & 1;
+  size_t afv_y = afv_kind / 2;
+  float dcs[3] = {};
+  float block00 = coefficients[0];
+  float block01 = coefficients[1];
+  float block10 = coefficients[8];
+  dcs[0] = (block00 + block10 + block01) * 4.0f;
+  dcs[1] = (block00 + block10 - block01);
+  dcs[2] = block00 - block10;
+  // IAFV: (even, even) positions.
+  HWY_ALIGN float coeff[4 * 4];
+  coeff[0] = dcs[0];
+  for (size_t iy = 0; iy < 4; iy++) {
+    for (size_t ix = 0; ix < 4; ix++) {
+      if (ix == 0 && iy == 0) continue;
+      coeff[iy * 4 + ix] = coefficients[iy * 2 * 8 + ix * 2];
+    }
+  }
+  HWY_ALIGN float block[4 * 8];
+  AFVIDCT4x4(coeff, block);
+  for (size_t iy = 0; iy < 4; iy++) {
+    for (size_t ix = 0; ix < 4; ix++) {
+      pixels[(iy + afv_y * 4) * pixels_stride + afv_x * 4 + ix] =
+          block[(afv_y == 1 ? 3 - iy : iy) * 4 + (afv_x == 1 ? 3 - ix : ix)];
+    }
+  }
+  // IDCT4x4 in (odd, even) positions.
+  block[0] = dcs[1];
+  for (size_t iy = 0; iy < 4; iy++) {
+    for (size_t ix = 0; ix < 4; ix++) {
+      if (ix == 0 && iy == 0) continue;
+      block[iy * 4 + ix] = coefficients[iy * 2 * 8 + ix * 2 + 1];
+    }
+  }
+  ComputeTransposedScaledIDCT<4>()(
+      FromBlock<4, 4>(block),
+      ToLines<4>(pixels + afv_y * 4 * pixels_stride + (afv_x == 1 ? 0 : 4),
+                 pixels_stride));
+  // IDCT4x8.
+  block[0] = dcs[2];
+  for (size_t iy = 0; iy < 4; iy++) {
+    for (size_t ix = 0; ix < 8; ix++) {
+      if (ix == 0 && iy == 0) continue;
+      block[iy * 8 + ix] = coefficients[(1 + iy * 2) * 8 + ix];
+    }
+  }
+  ComputeScaledIDCT<4, 8>()(
+      FromBlock<4, 8>(block),
+      ToLines<8>(pixels + (afv_y == 1 ? 0 : 4) * pixels_stride, pixels_stride));
+}
+
+}  // namespace
+
+HWY_ATTR void AFVDCT4x4(const float* JXL_RESTRICT pixels,
+                        float* JXL_RESTRICT coeffs) {
+  for (size_t i = 0; i < 16; i += D::N) {
+    auto scalar = Zero(D());
+    for (size_t j = 0; j < 16; j++) {
+      auto px = Set(D(), pixels[j]);
+      auto basis = Load(D(), k4x4AFVBasisTranspose[j] + i);
+      scalar = MulAdd(px, basis, scalar);
+    }
+    Store(scalar, D(), coeffs + i);
+  }
+}
+
+HWY_ATTR void AFVIDCT4x4(const float* JXL_RESTRICT coeffs,
+                         float* JXL_RESTRICT pixels) {
+  for (size_t i = 0; i < 16; i += D::N) {
+    auto pixel = Zero(D());
+    for (size_t j = 0; j < 16; j++) {
+      auto cf = Set(D(), coeffs[j]);
+      auto basis = Load(D(), k4x4AFVBasis[j] + i);
+      pixel = MulAdd(cf, basis, pixel);
+    }
+    Store(pixel, D(), pixels + i);
+  }
+}
+
+const AcStrategy::CoeffOrderAndLut* AcStrategy::CoeffOrder() {
+  static AcStrategy::CoeffOrderAndLut order = ComputeNaturalCoeffOrder();
+  return &order;
+}
+
+// These definitions are needed before C++17.
+constexpr size_t AcStrategy::kMaxCoeffBlocks;
+constexpr size_t AcStrategy::kMaxBlockDim;
+constexpr size_t AcStrategy::kMaxCoeffArea;
+
+HWY_ATTR void AcStrategy::TransformFromPixels(
+    const float* JXL_RESTRICT pixels, size_t pixels_stride,
+    float* JXL_RESTRICT coefficients) const {
+  switch (strategy_) {
+    case Type::IDENTITY: {
+      PROFILER_ZONE("DCT Identity");
+      for (size_t y = 0; y < 2; y++) {
+        for (size_t x = 0; x < 2; x++) {
+          float block_dc = 0;
+          for (size_t iy = 0; iy < 4; iy++) {
+            for (size_t ix = 0; ix < 4; ix++) {
+              block_dc += pixels[(y * 4 + iy) * pixels_stride + x * 4 + ix];
+            }
+          }
+          block_dc *= 1.0f / 16;
+          for (size_t iy = 0; iy < 4; iy++) {
+            for (size_t ix = 0; ix < 4; ix++) {
+              if (ix == 1 && iy == 1) continue;
+              coefficients[(y + iy * 2) * 8 + x + ix * 2] =
+                  pixels[(y * 4 + iy) * pixels_stride + x * 4 + ix] -
+                  pixels[(y * 4 + 1) * pixels_stride + x * 4 + 1];
+            }
+          }
+          coefficients[(y + 2) * 8 + x + 2] = coefficients[y * 8 + x];
+          coefficients[y * 8 + x] = block_dc;
+        }
+      }
+      float block00 = coefficients[0];
+      float block01 = coefficients[1];
+      float block10 = coefficients[8];
+      float block11 = coefficients[9];
+      coefficients[0] = (block00 + block01 + block10 + block11) * 0.25f;
+      coefficients[1] = (block00 + block01 - block10 - block11) * 0.25f;
+      coefficients[8] = (block00 - block01 + block10 - block11) * 0.25f;
+      coefficients[9] = (block00 - block01 - block10 + block11) * 0.25f;
+      break;
+    }
+    case Type::DCT8X4: {
+      PROFILER_ZONE("DCT 8x4");
+      for (size_t x = 0; x < 2; x++) {
+        HWY_ALIGN float block[4 * 8];
+        ComputeScaledDCT<8, 4>()(FromLines<4>(pixels + x * 4, pixels_stride),
+                                 ScaleToBlock<4, 8>(block));
+        for (size_t iy = 0; iy < 4; iy++) {
+          for (size_t ix = 0; ix < 8; ix++) {
+            // Store transposed.
+            coefficients[(x + iy * 2) * 8 + ix] = block[iy * 8 + ix];
+          }
+        }
+      }
+      float block0 = coefficients[0];
+      float block1 = coefficients[8];
+      coefficients[0] = (block0 + block1) * 0.5f;
+      coefficients[8] = (block0 - block1) * 0.5f;
+      break;
+    }
+    case Type::DCT4X8: {
+      PROFILER_ZONE("DCT 4x8");
+      for (size_t y = 0; y < 2; y++) {
+        HWY_ALIGN float block[4 * 8];
+        ComputeScaledDCT<4, 8>()(
+            FromLines<8>(pixels + y * 4 * pixels_stride, pixels_stride),
+            ScaleToBlock<4, 8>(block));
+        for (size_t iy = 0; iy < 4; iy++) {
+          for (size_t ix = 0; ix < 8; ix++) {
+            coefficients[(y + iy * 2) * 8 + ix] = block[iy * 8 + ix];
+          }
+        }
+      }
+      float block0 = coefficients[0];
+      float block1 = coefficients[8];
+      coefficients[0] = (block0 + block1) * 0.5f;
+      coefficients[8] = (block0 - block1) * 0.5f;
+      break;
+    }
+    case Type::DCT4X4: {
+      PROFILER_ZONE("DCT 4");
+      for (size_t y = 0; y < 2; y++) {
+        for (size_t x = 0; x < 2; x++) {
+          HWY_ALIGN float block[4 * 4];
+          ComputeTransposedScaledDCT<4>()(
+              FromLines<4>(pixels + y * 4 * pixels_stride + x * 4,
+                           pixels_stride),
+              ScaleToBlock<4>(block));
+          for (size_t iy = 0; iy < 4; iy++) {
+            for (size_t ix = 0; ix < 4; ix++) {
+              coefficients[(y + iy * 2) * 8 + x + ix * 2] = block[iy * 4 + ix];
+            }
+          }
+        }
+      }
+      float block00 = coefficients[0];
+      float block01 = coefficients[1];
+      float block10 = coefficients[8];
+      float block11 = coefficients[9];
+      coefficients[0] = (block00 + block01 + block10 + block11) * 0.25f;
+      coefficients[1] = (block00 + block01 - block10 - block11) * 0.25f;
+      coefficients[8] = (block00 - block01 + block10 - block11) * 0.25f;
+      coefficients[9] = (block00 - block01 - block10 + block11) * 0.25f;
+      break;
+    }
+    case Type::DCT2X2: {
+      PROFILER_ZONE("DCT 2");
+      DCT2TopBlock<8>(pixels, pixels_stride, coefficients);
+      DCT2TopBlock<4>(coefficients, kBlockDim, coefficients);
+      DCT2TopBlock<2>(coefficients, kBlockDim, coefficients);
+      break;
+    }
+    case Type::DCT16X16: {
+      PROFILER_ZONE("DCT 16");
+      ComputeTransposedScaledDCT<2 * kBlockDim>()(
+          FromLines<2 * kBlockDim>(pixels, pixels_stride),
+          ScaleToBlock<2 * kBlockDim>(coefficients));
+      break;
+    }
+    case Type::DCT16X8: {
+      PROFILER_ZONE("DCT 16x8");
+      ComputeScaledDCT<16, 8>()(FromLines<8>(pixels, pixels_stride),
+                                ScaleToBlock<8, 16>(coefficients));
+      break;
+    }
+    case Type::DCT8X16: {
+      PROFILER_ZONE("DCT 8x16");
+      ComputeScaledDCT<8, 16>()(FromLines<16>(pixels, pixels_stride),
+                                ScaleToBlock<8, 16>(coefficients));
+      break;
+    }
+    case Type::DCT32X8: {
+      PROFILER_ZONE("DCT 32x8");
+      ComputeScaledDCT<32, 8>()(FromLines<8>(pixels, pixels_stride),
+                                ScaleToBlock<8, 32>(coefficients));
+      break;
+    }
+    case Type::DCT8X32: {
+      PROFILER_ZONE("DCT 8x32");
+      ComputeScaledDCT<8, 32>()(FromLines<32>(pixels, pixels_stride),
+                                ScaleToBlock<8, 32>(coefficients));
+      break;
+    }
+    case Type::DCT32X16: {
+      PROFILER_ZONE("DCT 32x16");
+      ComputeScaledDCT<32, 16>()(FromLines<16>(pixels, pixels_stride),
+                                 ScaleToBlock<16, 32>(coefficients));
+      break;
+    }
+    case Type::DCT16X32: {
+      PROFILER_ZONE("DCT 16x32");
+      ComputeScaledDCT<16, 32>()(FromLines<32>(pixels, pixels_stride),
+                                 ScaleToBlock<16, 32>(coefficients));
+      break;
+    }
+    case Type::DCT32X32: {
+      PROFILER_ZONE("DCT 32");
+      ComputeTransposedScaledDCT<4 * kBlockDim>()(
+          FromLines<4 * kBlockDim>(pixels, pixels_stride),
+          ScaleToBlock<4 * kBlockDim>(coefficients));
+      break;
+    }
+    case Type::DCT: {
+      PROFILER_ZONE("DCT 8");
+      ComputeTransposedScaledDCT<kBlockDim>()(
+          FromLines<kBlockDim>(pixels, pixels_stride),
+          ScaleToBlock<kBlockDim>(coefficients));
+      break;
+    }
+    case Type::AFV0: {
+      PROFILER_ZONE("AFV0");
+      AFVTransformFromPixels<0>(pixels, pixels_stride, coefficients);
+      break;
+    }
+    case Type::AFV1: {
+      PROFILER_ZONE("AFV1");
+      AFVTransformFromPixels<1>(pixels, pixels_stride, coefficients);
+      break;
+    }
+    case Type::AFV2: {
+      PROFILER_ZONE("AFV2");
+      AFVTransformFromPixels<2>(pixels, pixels_stride, coefficients);
+      break;
+    }
+    case Type::AFV3: {
+      PROFILER_ZONE("AFV3");
+      AFVTransformFromPixels<3>(pixels, pixels_stride, coefficients);
+      break;
+    }
+    case Type::kNumValidStrategies:
+      JXL_ABORT("Invalid strategy");
+  }
+}
+
+HWY_ATTR void AcStrategy::TransformToPixels(
+    const float* JXL_RESTRICT coefficients, float* JXL_RESTRICT pixels,
+    size_t pixels_stride) const {
+  switch (strategy_) {
+    case Type::IDENTITY: {
+      PROFILER_ZONE("IDCT Identity");
+      float dcs[4] = {};
+      float block00 = coefficients[0];
+      float block01 = coefficients[1];
+      float block10 = coefficients[8];
+      float block11 = coefficients[9];
+      dcs[0] = block00 + block01 + block10 + block11;
+      dcs[1] = block00 + block01 - block10 - block11;
+      dcs[2] = block00 - block01 + block10 - block11;
+      dcs[3] = block00 - block01 - block10 + block11;
+      for (size_t y = 0; y < 2; y++) {
+        for (size_t x = 0; x < 2; x++) {
+          float block_dc = dcs[y * 2 + x];
+          float residual_sum = 0;
+          for (size_t iy = 0; iy < 4; iy++) {
+            for (size_t ix = 0; ix < 4; ix++) {
+              if (ix == 0 && iy == 0) continue;
+              residual_sum += coefficients[(y + iy * 2) * 8 + x + ix * 2];
+            }
+          }
+          pixels[(4 * y + 1) * pixels_stride + 4 * x + 1] =
+              block_dc - residual_sum * (1.0f / 16);
+          for (size_t iy = 0; iy < 4; iy++) {
+            for (size_t ix = 0; ix < 4; ix++) {
+              if (ix == 1 && iy == 1) continue;
+              pixels[(y * 4 + iy) * pixels_stride + x * 4 + ix] =
+                  coefficients[(y + iy * 2) * 8 + x + ix * 2] +
+                  pixels[(4 * y + 1) * pixels_stride + 4 * x + 1];
+            }
+          }
+          pixels[y * 4 * pixels_stride + x * 4] =
+              coefficients[(y + 2) * 8 + x + 2] +
+              pixels[(4 * y + 1) * pixels_stride + 4 * x + 1];
+        }
+      }
+      break;
+    }
+    case Type::DCT8X4: {
+      PROFILER_ZONE("IDCT 8x4");
+      float dcs[2] = {};
+      float block0 = coefficients[0];
+      float block1 = coefficients[8];
+      dcs[0] = block0 + block1;
+      dcs[1] = block0 - block1;
+      for (size_t x = 0; x < 2; x++) {
+        HWY_ALIGN float block[4 * 8];
+        block[0] = dcs[x];
+        for (size_t iy = 0; iy < 4; iy++) {
+          for (size_t ix = 0; ix < 8; ix++) {
+            if (ix == 0 && iy == 0) continue;
+            block[iy * 8 + ix] = coefficients[(x + iy * 2) * 8 + ix];
+          }
+        }
+        ComputeScaledIDCT<8, 4>()(FromBlock<4, 8>(block),
+                                  ToLines<4>(pixels + x * 4, pixels_stride));
+      }
+      break;
+    }
+    case Type::DCT4X8: {
+      PROFILER_ZONE("IDCT 4x8");
+      float dcs[2] = {};
+      float block0 = coefficients[0];
+      float block1 = coefficients[8];
+      dcs[0] = block0 + block1;
+      dcs[1] = block0 - block1;
+      for (size_t y = 0; y < 2; y++) {
+        HWY_ALIGN float block[4 * 8];
+        block[0] = dcs[y];
+        for (size_t iy = 0; iy < 4; iy++) {
+          for (size_t ix = 0; ix < 8; ix++) {
+            if (ix == 0 && iy == 0) continue;
+            block[iy * 8 + ix] = coefficients[(y + iy * 2) * 8 + ix];
+          }
+        }
+        ComputeScaledIDCT<4, 8>()(
+            FromBlock<4, 8>(block),
+            ToLines<8>(pixels + y * 4 * pixels_stride, pixels_stride));
+      }
+      break;
+    }
+    case Type::DCT4X4: {
+      PROFILER_ZONE("IDCT 4");
+      float dcs[4] = {};
+      float block00 = coefficients[0];
+      float block01 = coefficients[1];
+      float block10 = coefficients[8];
+      float block11 = coefficients[9];
+      dcs[0] = block00 + block01 + block10 + block11;
+      dcs[1] = block00 + block01 - block10 - block11;
+      dcs[2] = block00 - block01 + block10 - block11;
+      dcs[3] = block00 - block01 - block10 + block11;
+      for (size_t y = 0; y < 2; y++) {
+        for (size_t x = 0; x < 2; x++) {
+          HWY_ALIGN float block[4 * 4];
+          block[0] = dcs[y * 2 + x];
+          for (size_t iy = 0; iy < 4; iy++) {
+            for (size_t ix = 0; ix < 4; ix++) {
+              if (ix == 0 && iy == 0) continue;
+              block[iy * 4 + ix] = coefficients[(y + iy * 2) * 8 + x + ix * 2];
+            }
+          }
+          ComputeTransposedScaledIDCT<4>()(
+              FromBlock<4>(block),
+              ToLines<4>(pixels + y * 4 * pixels_stride + x * 4,
+                         pixels_stride));
+        }
+      }
+      break;
+    }
+    case Type::DCT2X2: {
+      PROFILER_ZONE("IDCT 2");
+      HWY_ALIGN float coeffs[kDCTBlockSize];
+      memcpy(coeffs, coefficients, sizeof(float) * kDCTBlockSize);
+      IDCT2TopBlock<2>(coeffs, kBlockDim, coeffs);
+      IDCT2TopBlock<4>(coeffs, kBlockDim, coeffs);
+      IDCT2TopBlock<8>(coeffs, kBlockDim, coeffs);
+      for (size_t y = 0; y < kBlockDim; y++) {
+        for (size_t x = 0; x < kBlockDim; x++) {
+          pixels[y * pixels_stride + x] = coeffs[y * kBlockDim + x];
+        }
+      }
+      break;
+    }
+    case Type::DCT16X16: {
+      PROFILER_ZONE("IDCT 16");
+      ComputeTransposedScaledIDCT<2 * kBlockDim>()(
+          FromBlock<2 * kBlockDim>(coefficients),
+          ToLines<2 * kBlockDim>(pixels, pixels_stride));
+      break;
+    }
+    case Type::DCT16X8: {
+      PROFILER_ZONE("IDCT 16x8");
+      ComputeScaledIDCT<16, 8>()(FromBlock<8, 16>(coefficients),
+                                 ToLines<8>(pixels, pixels_stride));
+      break;
+    }
+    case Type::DCT8X16: {
+      PROFILER_ZONE("IDCT 8x16");
+      ComputeScaledIDCT<8, 16>()(FromBlock<8, 16>(coefficients),
+                                 ToLines<16>(pixels, pixels_stride));
+      break;
+    }
+    case Type::DCT32X8: {
+      PROFILER_ZONE("IDCT 32x8");
+      ComputeScaledIDCT<32, 8>()(FromBlock<8, 32>(coefficients),
+                                 ToLines<8>(pixels, pixels_stride));
+      break;
+    }
+    case Type::DCT8X32: {
+      PROFILER_ZONE("IDCT 8x32");
+      ComputeScaledIDCT<8, 32>()(FromBlock<8, 32>(coefficients),
+                                 ToLines<32>(pixels, pixels_stride));
+      break;
+    }
+    case Type::DCT32X16: {
+      PROFILER_ZONE("IDCT 32x16");
+      ComputeScaledIDCT<32, 16>()(FromBlock<16, 32>(coefficients),
+                                  ToLines<16>(pixels, pixels_stride));
+      break;
+    }
+    case Type::DCT16X32: {
+      PROFILER_ZONE("IDCT 16x32");
+      ComputeScaledIDCT<16, 32>()(FromBlock<16, 32>(coefficients),
+                                  ToLines<32>(pixels, pixels_stride));
+      break;
+    }
+    case Type::DCT32X32: {
+      PROFILER_ZONE("IDCT 32");
+      ComputeTransposedScaledIDCT<4 * kBlockDim>()(
+          FromBlock<4 * kBlockDim>(coefficients),
+          ToLines<4 * kBlockDim>(pixels, pixels_stride));
+      break;
+    }
+    case Type::DCT: {
+      PROFILER_ZONE("IDCT 8");
+      ComputeTransposedScaledIDCT<kBlockDim>()(
+          FromBlock<kBlockDim>(coefficients),
+          ToLines<kBlockDim>(pixels, pixels_stride));
+      break;
+    }
+    case Type::AFV0: {
+      PROFILER_ZONE("IAFV0");
+      AFVTransformToPixels<0>(coefficients, pixels, pixels_stride);
+      break;
+    }
+    case Type::AFV1: {
+      PROFILER_ZONE("IAFV1");
+      AFVTransformToPixels<1>(coefficients, pixels, pixels_stride);
+      break;
+    }
+    case Type::AFV2: {
+      PROFILER_ZONE("IAFV2");
+      AFVTransformToPixels<2>(coefficients, pixels, pixels_stride);
+      break;
+    }
+    case Type::AFV3: {
+      PROFILER_ZONE("IAFV3");
+      AFVTransformToPixels<3>(coefficients, pixels, pixels_stride);
+      break;
+    }
+    case Type::kNumValidStrategies:
+      JXL_ABORT("Invalid strategy");
+  }
+}
+
+HWY_ATTR void AcStrategy::LowestFrequenciesFromDC(const float* dc,
+                                                  size_t dc_stride,
+                                                  float* llf) const {
+  switch (strategy_) {
+    case Type::DCT16X8: {
+      ReinterpretingDCT</*DCT_ROWS=*/2 * kBlockDim, /*DCT_COLS=*/kBlockDim,
+                        /*LF_ROWS=*/2, /*LF_COLS=*/1, /*ROWS=*/2, /*COLS=*/1>(
+          dc, dc_stride, llf, 2 * kBlockDim);
+      break;
+    }
+    case Type::DCT8X16: {
+      ReinterpretingDCT</*DCT_ROWS=*/kBlockDim, /*DCT_COLS=*/2 * kBlockDim,
+                        /*LF_ROWS=*/1, /*LF_COLS=*/2, /*ROWS=*/1, /*COLS=*/2>(
+          dc, dc_stride, llf, 2 * kBlockDim);
+      break;
+    }
+    case Type::DCT16X16: {
+      ReinterpretingDCT</*DCT_ROWS=*/2 * kBlockDim, /*DCT_COLS=*/2 * kBlockDim,
+                        /*LF_ROWS=*/2, /*LF_COLS=*/2, /*ROWS=*/2, /*COLS=*/2>(
+          dc, dc_stride, llf, 2 * kBlockDim);
+      break;
+    }
+    case Type::DCT32X8: {
+      ReinterpretingDCT</*DCT_ROWS=*/4 * kBlockDim, /*DCT_COLS=*/kBlockDim,
+                        /*LF_ROWS=*/4, /*LF_COLS=*/1, /*ROWS=*/4, /*COLS=*/1>(
+          dc, dc_stride, llf, 4 * kBlockDim);
+      break;
+    }
+    case Type::DCT8X32: {
+      ReinterpretingDCT</*DCT_ROWS=*/kBlockDim, /*DCT_COLS=*/4 * kBlockDim,
+                        /*LF_ROWS=*/1, /*LF_COLS=*/4, /*ROWS=*/1, /*COLS=*/4>(
+          dc, dc_stride, llf, 4 * kBlockDim);
+      break;
+    }
+    case Type::DCT32X16: {
+      ReinterpretingDCT</*DCT_ROWS=*/4 * kBlockDim, /*DCT_COLS=*/2 * kBlockDim,
+                        /*LF_ROWS=*/4, /*LF_COLS=*/2, /*ROWS=*/4, /*COLS=*/2>(
+          dc, dc_stride, llf, 4 * kBlockDim);
+      break;
+    }
+    case Type::DCT16X32: {
+      ReinterpretingDCT</*DCT_ROWS=*/2 * kBlockDim, /*DCT_COLS=*/4 * kBlockDim,
+                        /*LF_ROWS=*/2, /*LF_COLS=*/4, /*ROWS=*/2, /*COLS=*/4>(
+          dc, dc_stride, llf, 4 * kBlockDim);
+      break;
+    }
+    case Type::DCT32X32: {
+      ReinterpretingDCT</*DCT_ROWS=*/4 * kBlockDim, /*DCT_COLS=*/4 * kBlockDim,
+                        /*LF_ROWS=*/4, /*LF_COLS=*/4, /*ROWS=*/4, /*COLS=*/4>(
+          dc, dc_stride, llf, 4 * kBlockDim);
+      break;
+    }
+    case Type::DCT:
+    case Type::DCT2X2:
+    case Type::DCT4X4:
+    case Type::DCT4X8:
+    case Type::DCT8X4:
+    case Type::AFV0:
+    case Type::AFV1:
+    case Type::AFV2:
+    case Type::AFV3:
+    case Type::IDENTITY:
+      llf[0] = dc[0];
+      break;
+    case Type::kNumValidStrategies:
+      JXL_ABORT("Invalid strategy");
+  };
+}
+
+HWY_ATTR void AcStrategy::DCFromLowestFrequencies(const float* block, float* dc,
+                                                  size_t dc_stride) const {
+  switch (strategy_) {
+    case Type::DCT16X8: {
+      ReinterpretingIDCT</*DCT_ROWS=*/2 * kBlockDim, /*DCT_COLS=*/kBlockDim,
+                         /*LF_ROWS=*/2, /*LF_COLS=*/1, /*ROWS=*/2, /*COLS=*/1>(
+          block, 2 * kBlockDim, dc, dc_stride);
+      break;
+    }
+    case Type::DCT8X16: {
+      ReinterpretingIDCT</*DCT_ROWS=*/kBlockDim, /*DCT_COLS=*/2 * kBlockDim,
+                         /*LF_ROWS=*/1, /*LF_COLS=*/2, /*ROWS=*/1, /*COLS=*/2>(
+          block, 2 * kBlockDim, dc, dc_stride);
+      break;
+    }
+    case Type::DCT16X16: {
+      ReinterpretingIDCT</*DCT_ROWS=*/2 * kBlockDim, /*DCT_COLS=*/2 * kBlockDim,
+                         /*LF_ROWS=*/2, /*LF_COLS=*/2, /*ROWS=*/2, /*COLS=*/2>(
+          block, 2 * kBlockDim, dc, dc_stride);
+      break;
+    }
+    case Type::DCT32X8: {
+      ReinterpretingIDCT</*DCT_ROWS=*/4 * kBlockDim, /*DCT_COLS=*/kBlockDim,
+                         /*LF_ROWS=*/4, /*LF_COLS=*/1, /*ROWS=*/4, /*COLS=*/1>(
+          block, 4 * kBlockDim, dc, dc_stride);
+      break;
+    }
+    case Type::DCT8X32: {
+      ReinterpretingIDCT</*DCT_ROWS=*/kBlockDim, /*DCT_COLS=*/4 * kBlockDim,
+                         /*LF_ROWS=*/1, /*LF_COLS=*/4, /*ROWS=*/1, /*COLS=*/4>(
+          block, 4 * kBlockDim, dc, dc_stride);
+      break;
+    }
+    case Type::DCT32X16: {
+      ReinterpretingIDCT</*DCT_ROWS=*/4 * kBlockDim, /*DCT_COLS=*/2 * kBlockDim,
+                         /*LF_ROWS=*/4, /*LF_COLS=*/2, /*ROWS=*/4, /*COLS=*/2>(
+          block, 4 * kBlockDim, dc, dc_stride);
+      break;
+    }
+    case Type::DCT16X32: {
+      ReinterpretingIDCT</*DCT_ROWS=*/2 * kBlockDim, /*DCT_COLS=*/4 * kBlockDim,
+                         /*LF_ROWS=*/2, /*LF_COLS=*/4, /*ROWS=*/2, /*COLS=*/4>(
+          block, 4 * kBlockDim, dc, dc_stride);
+      break;
+    }
+    case Type::DCT32X32: {
+      ReinterpretingIDCT</*DCT_ROWS=*/4 * kBlockDim, /*DCT_COLS=*/4 * kBlockDim,
+                         /*LF_ROWS=*/4, /*LF_COLS=*/4, /*ROWS=*/4, /*COLS=*/4>(
+          block, 4 * kBlockDim, dc, dc_stride);
+      break;
+    }
+    case Type::DCT:
+    case Type::DCT2X2:
+    case Type::DCT4X4:
+    case Type::DCT4X8:
+    case Type::DCT8X4:
+    case Type::AFV0:
+    case Type::AFV1:
+    case Type::AFV2:
+    case Type::AFV3:
+    case Type::IDENTITY:
+      dc[0] = block[0];
+      break;
+    case Type::kNumValidStrategies:
+      JXL_ABORT("Invalid strategy");
+  }
+}
+
+AcStrategyImage::AcStrategyImage(size_t xsize, size_t ysize)
+    : layers_(xsize, ysize) {
+  row_ = layers_.Row(0);
+  stride_ = layers_.PixelsPerRow();
+}
+
+size_t AcStrategyImage::CountBlocks(AcStrategy::Type type) const {
+  size_t ret = 0;
+  for (size_t y = 0; y < layers_.ysize(); y++) {
+    const uint8_t* JXL_RESTRICT row = layers_.ConstRow(y);
+    for (size_t x = 0; x < layers_.xsize(); x++) {
+      if (row[x] == ((static_cast<uint8_t>(type) << 1) | 1)) ret++;
+    }
+  }
+  return ret;
+}
+
+}  // namespace jxl
diff --git a/jxl/ac_strategy.h b/jxl/ac_strategy.h
new file mode 100644
index 000000000..03faa0548
--- /dev/null
+++ b/jxl/ac_strategy.h
@@ -0,0 +1,324 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_AC_STRATEGY_H_
+#define JXL_AC_STRATEGY_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/static_targets.h>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/common.h"
+#include "jxl/dct.h"
+#include "jxl/dct_util.h"
+#include "jxl/image.h"
+#include "jxl/image_ops.h"
+
+// Defines the different kinds of transforms, and heuristics to choose between
+// them.
+// `AcStrategy` represents what transform should be used, and which sub-block of
+// that transform we are currently in. Note that DCT4x4 is applied on all four
+// 4x4 sub-blocks of an 8x8 block.
+// `AcStrategyImage` defines which strategy should be used for each 8x8 block
+// of the image. The highest 4 bits represent the strategy to be used, the
+// lowest 4 represent the index of the block inside that strategy. Blocks should
+// be aligned, i.e. 32x32 blocks should only start in positions that are
+// multiples of 32.
+
+namespace jxl {
+
+template <size_t V>
+struct square_root {
+  static constexpr float value =
+      square_root<V / 2>::value * 1.4142135623730951f;
+};
+
+template <>
+struct square_root<1> {
+  static constexpr float value = 1.0f;
+};
+
+// Apply the DCT algorithm-intrinsic constants to DCTResampleScale.
+// Note that the DCTScales constants give results that are scaled with a factor
+// proportional to 1/sqrt(N), so we counteract that here.
+// We also use the fact that 1/(sqrt(N) DCTScales(N)) == sqrt(N) IDCTScales(N)
+// to avoid a division.
+template <size_t FROM, size_t TO>
+constexpr float DCTTotalResampleScale(size_t x) {
+  return square_root<FROM>::value * DCTScales<FROM>()[x] *
+         square_root<TO>::value * IDCTScales<TO>()[x] *
+         DCTResampleScales<FROM, TO>::kScales[x];
+}
+
+void AFVDCT4x4(const float* JXL_RESTRICT pixels, float* JXL_RESTRICT coeffs);
+void AFVIDCT4x4(const float* JXL_RESTRICT coeffs, float* JXL_RESTRICT pixels);
+
+class AcStrategy {
+ public:
+  // Extremal values for the number of blocks/coefficients of a single strategy.
+  static constexpr size_t kMaxCoeffBlocks = 4;
+  static constexpr size_t kMaxBlockDim = kBlockDim * kMaxCoeffBlocks;
+  static constexpr size_t kMaxCoeffArea = kMaxBlockDim * kMaxBlockDim;
+
+  // Raw strategy types.
+  enum Type : uint32_t {
+    // Regular block size DCT (value matches kQuantKind)
+    DCT = 0,
+    // Encode pixels without transforming (value matches kQuantKind)
+    IDENTITY = 1,
+    // Use 2-by-2 DCT (value matches kQuantKind)
+    DCT2X2 = 2,
+    // Use 4-by-4 DCT (value matches kQuantKind)
+    DCT4X4 = 3,
+    // Use 16-by-16 DCT
+    DCT16X16 = 4,
+    // Use 32-by-32 DCT
+    DCT32X32 = 5,
+    // Use 16-by-8 DCT
+    DCT16X8 = 6,
+    // Use 8-by-16 DCT
+    DCT8X16 = 7,
+    // Use 32-by-8 DCT
+    DCT32X8 = 8,
+    // Use 8-by-32 DCT
+    DCT8X32 = 9,
+    // Use 32-by-16 DCT
+    DCT32X16 = 10,
+    // Use 16-by-32 DCT
+    DCT16X32 = 11,
+    // 4x8 and 8x4 DCT
+    DCT4X8 = 12,
+    DCT8X4 = 13,
+    // Corner-DCT.
+    AFV0 = 14,
+    AFV1 = 15,
+    AFV2 = 16,
+    AFV3 = 17,
+    // Marker for num of valid strategies.
+    kNumValidStrategies
+  };
+
+  static constexpr uint32_t TypeBit(const Type type) {
+    return 1u << static_cast<uint32_t>(type);
+  }
+
+  // Returns true if this block is the first 8x8 block (i.e. top-left) of a
+  // possibly multi-block strategy.
+  JXL_INLINE bool IsFirstBlock() const { return is_first_; }
+
+  JXL_INLINE bool IsMultiblock() const {
+    constexpr uint32_t bits = TypeBit(Type::DCT16X16) |
+                              TypeBit(Type::DCT32X32) | TypeBit(Type::DCT16X8) |
+                              TypeBit(Type::DCT8X16) | TypeBit(Type::DCT32X8) |
+                              TypeBit(Type::DCT8X32) | TypeBit(Type::DCT16X32) |
+                              TypeBit(Type::DCT32X16);
+    return ((1u << static_cast<uint32_t>(Strategy())) & bits) != 0;
+  }
+
+  // Returns the raw strategy value. Should only be used for tokenization.
+  JXL_INLINE uint8_t RawStrategy() const {
+    return static_cast<uint8_t>(strategy_);
+  }
+
+  JXL_INLINE Type Strategy() const { return strategy_; }
+
+  // Inverse check
+  static JXL_INLINE constexpr bool IsRawStrategyValid(uint8_t raw_strategy) {
+    return raw_strategy < kNumValidStrategies;
+  }
+  static JXL_INLINE AcStrategy FromRawStrategy(uint8_t raw_strategy) {
+    return FromRawStrategy(static_cast<Type>(raw_strategy));
+  }
+  static JXL_INLINE AcStrategy FromRawStrategy(Type raw_strategy) {
+    return AcStrategy(raw_strategy, /*is_first_block=*/true);
+  }
+
+  // "Natural order" means the order of increasing of "anisotropic" frequency of
+  // continuous version of DCT basis.
+  // Round-trip, for any given strategy s:
+  //  X = NaturalCoeffOrder(s)[NaturalCoeffOrderLutN(s)[X]]
+  //  X = NaturalCoeffOrderLut(s)[NaturalCoeffOrderN(s)[X]]
+  JXL_INLINE const coeff_order_t* NaturalCoeffOrder() const {
+    return CoeffOrder()->order + RawStrategy() * kMaxCoeffArea;
+  }
+
+  JXL_INLINE const coeff_order_t* NaturalCoeffOrderLut() const {
+    return CoeffOrder()->lut + RawStrategy() * kMaxCoeffArea;
+  }
+
+  // Number of 8x8 blocks that this strategy will cover. 0 for non-top-left
+  // blocks inside a multi-block transform.
+  JXL_INLINE size_t covered_blocks_x() const {
+    static constexpr const uint8_t kLut[] = {1, 1, 1, 1, 2, 4, 1, 2, 1,
+                                             4, 2, 4, 1, 1, 1, 1, 1, 1};
+    static_assert(sizeof(kLut) / sizeof(*kLut) == kNumValidStrategies,
+                  "Update LUT");
+    return kLut[size_t(strategy_)];
+  }
+
+  JXL_INLINE size_t covered_blocks_y() const {
+    static constexpr const uint8_t kLut[] = {1, 1, 1, 1, 2, 4, 2, 1, 4,
+                                             1, 4, 2, 1, 1, 1, 1, 1, 1};
+    static_assert(sizeof(kLut) / sizeof(*kLut) == kNumValidStrategies,
+                  "Update LUT");
+    return kLut[size_t(strategy_)];
+  }
+
+  JXL_INLINE size_t log2_covered_blocks() const {
+    static constexpr const uint8_t kLut[] = {0, 0, 0, 0, 2, 4, 1, 1, 2,
+                                             2, 3, 3, 0, 0, 0, 0, 0, 0};
+    static_assert(sizeof(kLut) / sizeof(*kLut) == kNumValidStrategies,
+                  "Update LUT");
+    return kLut[size_t(strategy_)];
+  }
+
+  // 1 / covered_block_x() / covered_block_y(), for fast division.
+  // Should only be called with block_ == 0.
+  JXL_INLINE float inverse_covered_blocks() const {
+    if (strategy_ == Type::DCT32X32) return 1.0f / 16;
+    if (strategy_ == Type::DCT16X16) return 0.25f;
+    if (strategy_ == Type::DCT8X16 || strategy_ == Type::DCT16X8) return 0.5f;
+    if (strategy_ == Type::DCT8X32 || strategy_ == Type::DCT32X8) return 0.25f;
+    if (strategy_ == Type::DCT32X16 || strategy_ == Type::DCT16X32)
+      return 1.0f / 8;
+    return 1.0f;
+  }
+
+  JXL_INLINE float InverseNumACCoefficients() const {
+    JXL_DASSERT(IsFirstBlock());
+    if (strategy_ == Type::DCT32X32) return 1.0f / (32 * 32 - 16);
+    if (strategy_ == Type::DCT16X16) return 1.0f / (16 * 16 - 4);
+    if (strategy_ == Type::DCT8X16 || strategy_ == Type::DCT16X8)
+      return 1.0f / (8 * 16 - 2);
+    if (strategy_ == Type::DCT8X32 || strategy_ == Type::DCT32X8)
+      return 1.0f / (8 * 32 - 4);
+    if (strategy_ == Type::DCT32X16 || strategy_ == Type::DCT16X32)
+      return 1.0f / (32 * 16 - 8);
+    return 1.0f / (8 * 8 - 1);
+  }
+
+  // Pixel to coefficients and vice-versa
+  HWY_ATTR void TransformFromPixels(const float* JXL_RESTRICT pixels,
+                                    size_t pixels_stride,
+                                    float* JXL_RESTRICT coefficients) const;
+  HWY_ATTR void TransformToPixels(const float* JXL_RESTRICT coefficients,
+                                  float* JXL_RESTRICT pixels,
+                                  size_t pixels_stride) const;
+
+  // Same as above, but for DC image.
+  HWY_ATTR void LowestFrequenciesFromDC(const float* dc, size_t dc_stride,
+                                        float* llf) const;
+  HWY_ATTR void DCFromLowestFrequencies(const float* block, float* dc,
+                                        size_t dc_stride) const;
+
+  struct CoeffOrderAndLut {
+    coeff_order_t order[kNumValidStrategies * kMaxCoeffArea];
+    coeff_order_t lut[kNumValidStrategies * kMaxCoeffArea];
+  };
+
+ private:
+  friend class AcStrategyRow;
+  JXL_INLINE AcStrategy(Type strategy, bool is_first)
+      : strategy_(strategy), is_first_(is_first) {
+    JXL_DASSERT(IsMultiblock() || is_first == true);
+  }
+
+  Type strategy_;
+  bool is_first_;
+
+  static const CoeffOrderAndLut* CoeffOrder();
+};
+
+// Class to use a certain row of the AC strategy.
+class AcStrategyRow {
+ public:
+  explicit AcStrategyRow(const uint8_t* row) : row_(row) {}
+  AcStrategy operator[](size_t x) const {
+    return AcStrategy(static_cast<AcStrategy::Type>(row_[x] >> 1), row_[x] & 1);
+  }
+
+ private:
+  const uint8_t* JXL_RESTRICT row_;
+};
+
+class AcStrategyImage {
+ public:
+  AcStrategyImage() = default;
+  AcStrategyImage(size_t xsize, size_t ysize);
+  AcStrategyImage(AcStrategyImage&&) = default;
+  AcStrategyImage& operator=(AcStrategyImage&&) = default;
+
+  void FillDCT8() {
+    FillImage<uint8_t>((static_cast<uint8_t>(AcStrategy::Type::DCT) << 1) | 1,
+                       &layers_);
+  }
+
+  void FillInvalid() { FillImage(INVALID, &layers_); }
+
+  void Set(size_t x, size_t y, AcStrategy::Type type) {
+#if JXL_ENABLE_ASSERT
+    AcStrategy acs = AcStrategy::FromRawStrategy(type);
+#endif  // JXL_ENABLE_ASSERT
+    JXL_ASSERT(y + acs.covered_blocks_y() <= layers_.ysize());
+    JXL_ASSERT(x + acs.covered_blocks_x() <= layers_.xsize());
+    SetNoBoundsCheck(x, y, type);
+  }
+
+  void SetNoBoundsCheck(size_t x, size_t y, AcStrategy::Type type) {
+    AcStrategy acs = AcStrategy::FromRawStrategy(type);
+    for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+      for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+        row_[(y + iy) * stride_ + x + ix] =
+            (static_cast<uint8_t>(type) << 1) | ((iy | ix) == 0 ? 1 : 0);
+      }
+    }
+  }
+
+  bool IsValid(size_t x, size_t y) { return row_[y * stride_ + x] != INVALID; }
+
+  AcStrategyRow ConstRow(size_t y, size_t x_prefix = 0) const {
+    return AcStrategyRow(layers_.ConstRow(y) + x_prefix);
+  }
+
+  AcStrategyRow ConstRow(const Rect& rect, size_t y) const {
+    return ConstRow(rect.y0() + y, rect.x0());
+  }
+
+  size_t PixelsPerRow() const { return layers_.PixelsPerRow(); }
+
+  size_t xsize() const { return layers_.xsize(); }
+  size_t ysize() const { return layers_.ysize(); }
+
+  // Count the number of blocks of a given type.
+  size_t CountBlocks(AcStrategy::Type type) const;
+
+ private:
+  ImageB layers_;
+  uint8_t* JXL_RESTRICT row_;
+  size_t stride_;
+
+  // A value that does not represent a valid combined AC strategy
+  // value. Used as a sentinel.
+  static constexpr uint8_t INVALID = 0xFF;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_AC_STRATEGY_H_
diff --git a/jxl/ac_strategy_test.cc b/jxl/ac_strategy_test.cc
new file mode 100644
index 000000000..bb52028dd
--- /dev/null
+++ b/jxl/ac_strategy_test.cc
@@ -0,0 +1,405 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/ac_strategy.h"
+
+#include <string.h>
+
+#include <cmath>
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "jxl/common.h"
+#include "jxl/dct.h"
+
+namespace jxl {
+namespace {
+
+// Test that DCT -> IDCT is a noop.
+void TestAcStrategyRoundtrip(AcStrategy::Type type) {
+  AcStrategy acs = AcStrategy::FromRawStrategy(type);
+  HWY_ALIGN float coeffs[AcStrategy::kMaxCoeffArea] = {};
+  HWY_ALIGN float idct[AcStrategy::kMaxCoeffArea];
+  for (size_t i = 0; i < 64 << acs.log2_covered_blocks(); i++) {
+    HWY_ALIGN float input[AcStrategy::kMaxCoeffArea] = {};
+    input[i] = 0.2;
+    acs.TransformFromPixels(input, acs.covered_blocks_x() * 8, coeffs);
+    ASSERT_NEAR(coeffs[0], 0.2 / (64 << acs.log2_covered_blocks()), 1e-6)
+        << " i = " << i;
+    acs.TransformToPixels(coeffs, idct, acs.covered_blocks_x() * 8);
+    for (size_t j = 0; j < 64 << acs.log2_covered_blocks(); j++) {
+      EXPECT_NEAR(idct[j], input[j], 1e-6) << "j = " << j << " i = " << i;
+    }
+  }
+  // Test DC.
+  std::fill(std::begin(idct), std::end(idct), 0);
+  for (size_t y = 0; y < acs.covered_blocks_y(); y++) {
+    for (size_t x = 0; x < acs.covered_blocks_x(); x++) {
+      HWY_ALIGN float dc[AcStrategy::kMaxCoeffArea] = {};
+      dc[y * acs.covered_blocks_x() * 8 + x] = 0.2;
+      acs.LowestFrequenciesFromDC(dc, acs.covered_blocks_x() * 8, coeffs);
+      acs.DCFromLowestFrequencies(coeffs, idct, acs.covered_blocks_x() * 8);
+      for (size_t j = 0; j < 64 << acs.log2_covered_blocks(); j++) {
+        EXPECT_NEAR(idct[j], dc[j], 1e-6)
+            << "j = " << j << " x = " << x << " y = " << y;
+      }
+    }
+  }
+}
+
+// Test that DC(2x2) -> DCT coefficients -> IDCT -> downsampled IDCT is a noop.
+void TestAcStrategyRoundtripDownsample(AcStrategy::Type type) {
+  AcStrategy acs = AcStrategy::FromRawStrategy(type);
+  HWY_ALIGN float coeffs[AcStrategy::kMaxCoeffArea] = {};
+  HWY_ALIGN float idct[AcStrategy::kMaxCoeffArea];
+  for (size_t y = 0; y < acs.covered_blocks_y(); y++) {
+    for (size_t x = 0; x < acs.covered_blocks_x(); x++) {
+      HWY_ALIGN float dc[AcStrategy::kMaxCoeffArea] = {};
+      dc[y * acs.covered_blocks_x() * 8 + x] = 0.2;
+      acs.LowestFrequenciesFromDC(dc, acs.covered_blocks_x() * 8, coeffs);
+      acs.TransformToPixels(coeffs, idct, acs.covered_blocks_x() * 8);
+      // Downsample
+      for (size_t dy = 0; dy < acs.covered_blocks_y(); dy++) {
+        for (size_t dx = 0; dx < acs.covered_blocks_x(); dx++) {
+          float sum = 0;
+          for (size_t iy = 0; iy < 8; iy++) {
+            for (size_t ix = 0; ix < 8; ix++) {
+              sum += idct[(dy * 8 + iy) * 8 * acs.covered_blocks_x() + dx * 8 +
+                          ix];
+            }
+          }
+          sum /= 64.0f;
+          EXPECT_NEAR(sum, dc[dy * 8 * acs.covered_blocks_x() + dx], 1e-6);
+        }
+      }
+    }
+  }
+}
+
+// Test that IDCT(block with zeros in the non-topleft corner) -> downsampled
+// IDCT is the same as IDCT -> DC(2x2) of the same block.
+void TestAcStrategyDownsample(AcStrategy::Type type) {
+  AcStrategy acs = AcStrategy::FromRawStrategy(type);
+  HWY_ALIGN float idct[AcStrategy::kMaxCoeffArea];
+  HWY_ALIGN float idct_acs_downsampled[AcStrategy::kMaxCoeffArea] = {};
+  size_t cx = acs.covered_blocks_y();
+  size_t cy = acs.covered_blocks_x();
+  CoefficientLayout(&cy, &cx);
+  for (size_t y = 0; y < cy; y++) {
+    for (size_t x = 0; x < cx; x++) {
+      HWY_ALIGN float coeffs[AcStrategy::kMaxCoeffArea] = {};
+      coeffs[y * cx * 8 + x] = 0.2;
+      acs.TransformToPixels(coeffs, idct, acs.covered_blocks_x() * 8);
+      acs.DCFromLowestFrequencies(coeffs, idct_acs_downsampled,
+                                  acs.covered_blocks_x() * 8);
+      // Downsample
+      for (size_t dy = 0; dy < acs.covered_blocks_y(); dy++) {
+        for (size_t dx = 0; dx < acs.covered_blocks_x(); dx++) {
+          float sum = 0;
+          for (size_t iy = 0; iy < 8; iy++) {
+            for (size_t ix = 0; ix < 8; ix++) {
+              sum += idct[(dy * 8 + iy) * 8 * acs.covered_blocks_x() + dx * 8 +
+                          ix];
+            }
+          }
+          sum /= 64;
+          EXPECT_NEAR(
+              sum, idct_acs_downsampled[dy * 8 * acs.covered_blocks_x() + dx],
+              1e-6);
+        }
+      }
+    }
+  }
+}
+
+float I8(int N, int u) {
+  float eps = u == 0 ? std::sqrt(0.5) : 1.0;
+  return std::sqrt(2.0 / N) * eps * cos(u * kPi / (2.0 * N));
+}
+
+float D8(int N, int u) { return 1.0f / (N * I8(N, u)); }
+
+float I(int N, int u) {
+  if (N == 8)
+    return I8(N, u);
+  else
+    return D8(N, u);
+}
+
+float D(int N, int u) {
+  if (N == 8)
+    return D8(N, u);
+  else
+    return I8(N, u);
+}
+
+float C(int N, int n, int x) {
+  if (n > N) return 1.0 / C(n, N, x);
+  if (n == N)
+    return 1.0;
+  else
+    return cos(x * kPi / (2 * N)) * C(N / 2, n, x);
+}
+
+float ScaleF(int N, int n, int x) {
+  return std::sqrt(n * N) * D(N, x) * I(n, x) * C(N, n, x);
+}
+
+TEST(AcStrategyTest, TestConstant32) {
+  for (size_t i = 0; i < 32; i++) {
+    EXPECT_NEAR(D(32, i), DCTScales<32>()[i], 1e-4);
+    EXPECT_NEAR(I(32, i), IDCTScales<32>()[i], 1e-4);
+  }
+  for (int i = 0; i < 8; i++) {
+    float e1 = C(32, 8, i) - DCTResampleScales<32, 8>::kScales[i];
+    float e2 = C(8, 32, i) - DCTResampleScales<8, 32>::kScales[i];
+    EXPECT_LT(std::fabs(e1), 1e-4);
+    EXPECT_LT(std::fabs(e2), 1e-4);
+  }
+  EXPECT_NEAR(std::sqrt(8), square_root<8>::value, 1e-4);
+  EXPECT_NEAR(std::sqrt(16), square_root<16>::value, 1e-4);
+  for (int i = 0; i < 8; i++) {
+    float e1 = ScaleF(32, 8, i) - DCTTotalResampleScale<32, 8>(i);
+    EXPECT_LT(std::fabs(e1), 1e-4);
+  }
+  for (int i = 0; i < 4; i++) {
+    float e1 = ScaleF(32, 4, i) - DCTTotalResampleScale<32, 4>(i);
+    EXPECT_LT(std::fabs(e1), 1e-4);
+  }
+}
+
+TEST(AcStrategyTest, TestConstant16) {
+  for (size_t i = 0; i < 16; i++) {
+    EXPECT_NEAR(D(16, i), DCTScales<16>()[i], 1e-4);
+    EXPECT_NEAR(I(16, i), IDCTScales<16>()[i], 1e-4);
+  }
+  for (int i = 0; i < 4; i++) {
+    float e1 = ScaleF(16, 4, i) - DCTTotalResampleScale<16, 4>(i);
+    EXPECT_LT(std::fabs(e1), 1e-4);
+  }
+  for (int i = 0; i < 2; i++) {
+    float e1 = ScaleF(16, 2, i) - DCTTotalResampleScale<16, 2>(i);
+    EXPECT_LT(std::fabs(e1), 1e-4);
+  }
+}
+
+TEST(AcStrategyTest, TestConstant8) {
+  for (size_t i = 0; i < 8; i++) {
+    EXPECT_NEAR(D(8, i), DCTScales<8>()[i], 1e-4);
+    EXPECT_NEAR(I(8, i), IDCTScales<8>()[i], 1e-4);
+  }
+  for (int i = 0; i < 2; i++) {
+    float e1 = ScaleF(8, 2, i) - DCTTotalResampleScale<8, 2>(i);
+    EXPECT_LT(std::fabs(e1), 1e-4);
+  }
+  for (int i = 0; i < 1; i++) {
+    float e1 = ScaleF(8, 1, i) - DCTTotalResampleScale<8, 1>(i);
+    EXPECT_LT(std::fabs(e1), 1e-4);
+  }
+}
+
+TEST(AcStrategyTest, DownsampleDCT) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT);
+}
+TEST(AcStrategyTest, DownsampleIDENTITY) {
+  TestAcStrategyDownsample(AcStrategy::Type::IDENTITY);
+}
+TEST(AcStrategyTest, DownsampleDCT2X2) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT2X2);
+}
+TEST(AcStrategyTest, DownsampleDCT4X4) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT4X4);
+}
+TEST(AcStrategyTest, DownsampleDCT16X16) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT16X16);
+}
+TEST(AcStrategyTest, DownsampleDCT32X32) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT32X32);
+}
+TEST(AcStrategyTest, DownsampleDCT16X8) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT16X8);
+}
+TEST(AcStrategyTest, DownsampleDCT8X16) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT8X16);
+}
+TEST(AcStrategyTest, DownsampleDCT32X8) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT32X8);
+}
+TEST(AcStrategyTest, DownsampleDCT8X32) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT8X32);
+}
+TEST(AcStrategyTest, DownsampleDCT32X16) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT32X16);
+}
+TEST(AcStrategyTest, DownsampleDCT16X32) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT16X32);
+}
+TEST(AcStrategyTest, DownsampleDCT4X8) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT4X8);
+}
+TEST(AcStrategyTest, DownsampleDCT8X4) {
+  TestAcStrategyDownsample(AcStrategy::Type::DCT8X4);
+}
+TEST(AcStrategyTest, DownsampleAFV0) {
+  TestAcStrategyDownsample(AcStrategy::Type::AFV0);
+}
+TEST(AcStrategyTest, DownsampleAFV1) {
+  TestAcStrategyDownsample(AcStrategy::Type::AFV1);
+}
+
+TEST(AcStrategyTest, RoundtripDownsampleDCT) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT);
+}
+TEST(AcStrategyTest, RoundtripDownsampleIDENTITY) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::IDENTITY);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT2X2) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT2X2);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT4X4) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT4X4);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT16X16) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT16X16);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT32X32) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT32X32);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT16X8) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT16X8);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT8X16) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT8X16);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT32X8) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT32X8);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT8X32) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT8X32);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT32X16) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT32X16);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT16X32) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT16X32);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT4X8) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT4X8);
+}
+TEST(AcStrategyTest, RoundtripDownsampleDCT8X4) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::DCT8X4);
+}
+TEST(AcStrategyTest, RoundtripDownsampleAFV0) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::AFV0);
+}
+TEST(AcStrategyTest, RoundtripDownsampleAFV1) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::AFV1);
+}
+TEST(AcStrategyTest, RoundtripDownsampleAFV2) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::AFV2);
+}
+TEST(AcStrategyTest, RoundtripDownsampleAFV3) {
+  TestAcStrategyRoundtripDownsample(AcStrategy::Type::AFV3);
+}
+
+TEST(AcStrategyTest, RoundtripDCT) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT);
+}
+TEST(AcStrategyTest, RoundtripIDENTITY) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::IDENTITY);
+}
+TEST(AcStrategyTest, RoundtripDCT2X2) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT2X2);
+}
+TEST(AcStrategyTest, RoundtripDCT4X4) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT4X4);
+}
+TEST(AcStrategyTest, RoundtripDCT16X16) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT16X16);
+}
+TEST(AcStrategyTest, RoundtripDCT32X32) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT32X32);
+}
+TEST(AcStrategyTest, RoundtripDCT16X8) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT16X8);
+}
+TEST(AcStrategyTest, RoundtripDCT8X16) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT8X16);
+}
+TEST(AcStrategyTest, RoundtripDCT32X8) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT32X8);
+}
+TEST(AcStrategyTest, RoundtripDCT8X32) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT8X32);
+}
+TEST(AcStrategyTest, RoundtripDCT32X16) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT32X16);
+}
+TEST(AcStrategyTest, RoundtripDCT16X32) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT16X32);
+}
+TEST(AcStrategyTest, RoundtripDCT4X8) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT4X8);
+}
+TEST(AcStrategyTest, RoundtripDCT8X4) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::DCT8X4);
+}
+TEST(AcStrategyTest, RoundtripAFV0) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::AFV0);
+}
+TEST(AcStrategyTest, RoundtripAFV1) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::AFV1);
+}
+TEST(AcStrategyTest, RoundtripAFV2) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::AFV2);
+}
+TEST(AcStrategyTest, RoundtripAFV3) {
+  TestAcStrategyRoundtrip(AcStrategy::Type::AFV3);
+}
+
+TEST(AcStrategyTest, RoundtripAFVDCT) {
+  HWY_ALIGN float idct[16];
+  for (size_t i = 0; i < 16; i++) {
+    HWY_ALIGN float pixels[16] = {};
+    pixels[i] = 1;
+    HWY_ALIGN float coeffs[16] = {};
+
+    AFVDCT4x4(pixels, coeffs);
+    AFVIDCT4x4(coeffs, idct);
+    for (size_t j = 0; j < 16; j++) {
+      EXPECT_NEAR(idct[j], pixels[j], 1e-6);
+    }
+  }
+}
+
+TEST(AcStrategyTest, BenchmarkAFV) {
+  AcStrategy acs = AcStrategy::FromRawStrategy(AcStrategy::Type::AFV0);
+  HWY_ALIGN float pixels[64] = {1};
+  HWY_ALIGN float coeffs[64] = {};
+  for (size_t i = 0; i < 1 << 22; i++) {
+    acs.TransformToPixels(coeffs, pixels, 8);
+    acs.TransformFromPixels(pixels, 8, coeffs);
+  }
+}
+
+TEST(AcStrategyTest, BenchmarkAFVDCT) {
+  HWY_ALIGN float pixels[64] = {1};
+  HWY_ALIGN float coeffs[64] = {};
+  for (size_t i = 0; i < 1 << 22; i++) {
+    AFVDCT4x4(pixels, coeffs);
+    AFVIDCT4x4(coeffs, pixels);
+  }
+}
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/adaptive_reconstruction_test.cc b/jxl/adaptive_reconstruction_test.cc
new file mode 100644
index 000000000..7743c8ef5
--- /dev/null
+++ b/jxl/adaptive_reconstruction_test.cc
@@ -0,0 +1,171 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdio.h>
+
+#include <algorithm>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/ac_strategy.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/common.h"
+#include "jxl/epf.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/image_test_utils.h"
+#include "jxl/loop_filter.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+namespace {
+
+constexpr bool kPrint = false;
+
+const size_t xsize = 8;
+const size_t ysize = 8;
+
+void GenerateFlat(const float background, const float foreground,
+                  std::vector<Image3F>* images) {
+  for (size_t c = 0; c < Image3F::kNumPlanes; ++c) {
+    Image3F in(xsize, ysize);
+    // Plane c = foreground, all others = background.
+    for (size_t y = 0; y < ysize; ++y) {
+      float* rows[3] = {in.PlaneRow(0, y), in.PlaneRow(1, y),
+                        in.PlaneRow(2, y)};
+      for (size_t x = 0; x < xsize; ++x) {
+        rows[0][x] = rows[1][x] = rows[2][x] = background;
+        rows[c][x] = foreground;
+      }
+    }
+    images->push_back(std::move(in));
+  }
+}
+
+// Single foreground point at any position in any channel
+void GeneratePoints(const float background, const float foreground,
+                    std::vector<Image3F>* images) {
+  for (size_t c = 0; c < Image3F::kNumPlanes; ++c) {
+    for (size_t y = 0; y < ysize; ++y) {
+      for (size_t x = 0; x < xsize; ++x) {
+        Image3F in(xsize, ysize);
+        FillImage(background, &in);
+        in.PlaneRow(c, y)[x] = foreground;
+        images->push_back(std::move(in));
+      }
+    }
+  }
+}
+
+void GenerateHorzEdges(const float background, const float foreground,
+                       std::vector<Image3F>* images) {
+  for (size_t c = 0; c < Image3F::kNumPlanes; ++c) {
+    // Begin of foreground rows
+    for (size_t y = 1; y < ysize; ++y) {
+      Image3F in(xsize, ysize);
+      FillImage(background, &in);
+      for (size_t iy = y; iy < ysize; ++iy) {
+        std::fill(in.PlaneRow(c, iy), in.PlaneRow(c, iy) + xsize, foreground);
+      }
+      images->push_back(std::move(in));
+    }
+  }
+}
+
+void GenerateVertEdges(const float background, const float foreground,
+                       std::vector<Image3F>* images) {
+  for (size_t c = 0; c < Image3F::kNumPlanes; ++c) {
+    // Begin of foreground columns
+    for (size_t x = 1; x < xsize; ++x) {
+      Image3F in(xsize, ysize);
+      FillImage(background, &in);
+      for (size_t iy = 0; iy < ysize; ++iy) {
+        float* JXL_RESTRICT row = in.PlaneRow(c, iy);
+        for (size_t ix = x; ix < xsize; ++ix) {
+          row[ix] = foreground;
+        }
+      }
+      images->push_back(std::move(in));
+    }
+  }
+}
+
+// Ensures input remains unchanged by filter - verifies the edge-preserving
+// nature of the filter because inputs are piecewise constant.
+void EnsureUnchanged(const float background, const float foreground) {
+  std::vector<Image3F> images;
+  GenerateFlat(background, foreground, &images);
+  GeneratePoints(background, foreground, &images);
+  GenerateHorzEdges(background, foreground, &images);
+  GenerateVertEdges(background, foreground, &images);
+
+  DequantMatrices dequant;
+  size_t xsize_blocks = DivCeil(xsize, kBlockDim);
+  size_t ysize_blocks = DivCeil(ysize, kBlockDim);
+  LoopFilter lf;
+  ImageF sigma(xsize_blocks + 4, ysize_blocks + 4);
+  FillImage(-0.5f, &sigma);
+
+  Image3F storage1(xsize + 4 * kBlockDim, kEpf1InputRows);
+  Image3F storage2(xsize + 2 * kBlockDim, kEpf2InputRows);
+  lf.gab = false;
+
+  for (size_t idx_image = 0; idx_image < images.size(); ++idx_image) {
+    const Image3F& in = images[idx_image];
+
+    if (kPrint) {
+      printf("--In\n");
+      for (size_t y = 0; y < ysize; ++y) {
+        const float* row_x = in.ConstPlaneRow(0, y);
+        const float* row_y = in.ConstPlaneRow(1, y);
+        const float* row_b = in.ConstPlaneRow(2, y);
+        for (size_t x = 0; x < ysize; ++x) {
+          printf("%6.2f|%6.2f|%6.2f ", row_x[x], row_y[x], row_b[x]);
+        }
+        printf("\n");
+      }
+    }
+
+    Image3F out = CopyImage(in);  // = in_out
+    Image3F padded = PadImageSymmetric(in, 2 * kBlockDim);
+    EdgePreservingFilter(lf, Rect(0, 0, xsize, ysize), padded,
+                         Rect(0, 0, xsize_blocks, ysize_blocks), sigma,
+                         Rect(0, 0, xsize, ysize), &out, &storage1, &storage2);
+
+    if (kPrint) {
+      printf("--Out\n");
+      for (size_t y = 0; y < ysize; ++y) {
+        const float* row_x = out.ConstPlaneRow(0, y);
+        const float* row_y = out.ConstPlaneRow(1, y);
+        const float* row_b = out.ConstPlaneRow(2, y);
+        for (size_t x = 0; x < ysize; ++x) {
+          printf("%6.2f|%6.2f|%6.2f ", row_x[x], row_y[x], row_b[x]);
+        }
+        printf("\n");
+      }
+    }
+
+    VerifyRelativeError(in, out, 1E-3, 1E-4);
+  }
+}
+
+TEST(AdaptiveReconstructionTest, TestBright) { EnsureUnchanged(1.0f, 128.0f); }
+TEST(AdaptiveReconstructionTest, TestDark) { EnsureUnchanged(128.0f, 1.0f); }
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/ans_common.cc b/jxl/ans_common.cc
new file mode 100644
index 000000000..ed965de5a
--- /dev/null
+++ b/jxl/ans_common.cc
@@ -0,0 +1,143 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/ans_common.h"
+
+#include <numeric>
+
+#include "jxl/ans_params.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+std::vector<int> CreateFlatHistogram(int length, int total_count) {
+  JXL_ASSERT(length > 0);
+  JXL_ASSERT(length <= total_count);
+  const int count = total_count / length;
+  std::vector<int> result(length, count);
+  const int rem_counts = total_count % length;
+  for (int i = 0; i < rem_counts; ++i) {
+    ++result[i];
+  }
+  return result;
+}
+
+// First, all trailing non-occuring symbols are removed from the distribution;
+// if this leaves the distribution empty, a dummy symbol with max weight is
+// added. This ensures that the resulting distribution sums to total table size.
+// Then, `entry_size` is chosen to be the largest power of two so that
+// `table_size` = ANS_TAB_SIZE/`entry_size` is at least as big as the
+// distribution size.
+// Note that each entry will only ever contain two different symbols, and
+// consecutive ranges of offsets, which allows us to use a compact
+// representation.
+// Each entry is initialized with only the (symbol=i, offset) pairs; then
+// positions for which the entry overflows (i.e. distribution[i] > entry_size)
+// or is not full are computed, and put into a stack in increasing order.
+// Missing symbols in the distribution are padded with 0 (because `table_size`
+// >= number of symbols). The `cutoff` value for each entry is initialized to
+// the number of occupied slots in that entry (i.e. `distributions[i]`). While
+// the overflowing-symbol stack is not empty (which implies that the
+// underflowing-symbol stack also is not), the top overfull and underfull
+// positions are popped from the stack; the empty slots in the underfull entry
+// are then filled with as many slots as needed from the overfull entry; such
+// slots are placed after the slots in the overfull entry, and `offsets[1]` is
+// computed accordingly. The formerly underfull entry is thus now neither
+// underfull nor overfull, and represents exactly two symbols. The overfull
+// entry might be either overfull or underfull, and is pushed into the
+// corresponding stack.
+void InitAliasTable(std::vector<int> distribution, int range,
+                    AliasTable::Entry* JXL_RESTRICT a) {
+  while (!distribution.empty() && distribution.back() == 0) {
+    distribution.pop_back();
+  }
+  // Ensure that a valid table is always returned, even for an empty
+  // alphabet. Otherwise, a specially-crafted stream might crash the
+  // decoder.
+  if (distribution.empty()) {
+    distribution.emplace_back(range);
+  }
+  const int kTableSizeLog = ANS_LOG_MAX_ALPHABET_SIZE;
+  const int kTableSize = ANS_MAX_ALPHA_SIZE;
+#if JXL_ENABLE_ASSERT
+  int sum = std::accumulate(distribution.begin(), distribution.end(), 0);
+#endif  // JXL_ENABLE_ASSERT
+  JXL_ASSERT(sum == range);
+  JXL_ASSERT(distribution.size() <= (1 << kTableSizeLog));
+  // range must be a power of two
+  JXL_ASSERT((range & (range - 1)) == 0);
+  JXL_ASSERT(kTableSize >= distribution.size());
+  JXL_ASSERT(kTableSize <= range);
+  const int entry_size = range >> kTableSizeLog;  // this is exact
+  std::vector<int> underfull_posn;
+  std::vector<int> overfull_posn;
+  int cutoffs[kTableSize];
+  // Initialize entries.
+  for (size_t i = 0; i < distribution.size(); i++) {
+    cutoffs[i] = distribution[i];
+    if (cutoffs[i] > entry_size) {
+      overfull_posn.push_back(i);
+    } else if (cutoffs[i] < entry_size) {
+      underfull_posn.push_back(i);
+    }
+  }
+  for (int i = distribution.size(); i < kTableSize; i++) {
+    cutoffs[i] = 0;
+    underfull_posn.push_back(i);
+  }
+  // Reassign overflow/underflow values.
+  while (!overfull_posn.empty()) {
+    int overfull_i = overfull_posn.back();
+    overfull_posn.pop_back();
+    JXL_ASSERT(!underfull_posn.empty());
+    int underfull_i = underfull_posn.back();
+    underfull_posn.pop_back();
+    int underfull_by = entry_size - cutoffs[underfull_i];
+    cutoffs[overfull_i] -= underfull_by;
+    // overfull positions have their original symbols
+    a[underfull_i].right_value = overfull_i;
+    a[underfull_i].offsets1 = cutoffs[overfull_i];
+    // Slots in the right part of entry underfull_i were taken from the end
+    // of the symbols in entry overfull_i.
+    if (cutoffs[overfull_i] < entry_size) {
+      underfull_posn.push_back(overfull_i);
+    } else if (cutoffs[overfull_i] > entry_size) {
+      overfull_posn.push_back(overfull_i);
+    }
+  }
+  for (int i = 0; i < kTableSize; i++) {
+    // cutoffs[i] is properly initialized but the clang-analyzer doesn't infer
+    // it since it is partially initialized across two for-loops.
+    // NOLINTNEXTLINE(clang-analyzer-core.UndefinedBinaryOperatorResult)
+    if (cutoffs[i] == entry_size) {
+      a[i].right_value = i;
+      a[i].offsets1 = 0;
+      a[i].cutoff = 0;
+    } else {
+      // Note that, if cutoff is not equal to entry_size,
+      // a[i].offsets1 was initialized with (overfull cutoff) -
+      // (entry_size - a[i].cutoff). Thus, subtracting
+      // a[i].cutoff cannot make it negative.
+      a[i].offsets1 -= cutoffs[i];
+      a[i].cutoff = cutoffs[i];
+    }
+    const size_t freq0 = i < distribution.size() ? distribution[i] : 0;
+    const size_t i1 = a[i].right_value;
+    const size_t freq1 = i1 < distribution.size() ? distribution[i1] : 0;
+    a[i].freq0 = static_cast<uint16_t>(freq0);
+    a[i].freq1_xor_freq0 = static_cast<uint16_t>(freq1 ^ freq0);
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/ans_common.h b/jxl/ans_common.h
new file mode 100644
index 000000000..7f142200b
--- /dev/null
+++ b/jxl/ans_common.h
@@ -0,0 +1,152 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ANS_COMMON_H_
+#define JXL_ANS_COMMON_H_
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <hwy/cache_control.h>
+#include <hwy/static_targets.h>
+#include <vector>
+
+#include "jxl/ans_params.h"
+#include "jxl/base/byte_order.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Returns the precision (number of bits) that should be used to store
+// a histogram count such that Log2Floor(count) == logcount.
+static JXL_INLINE uint32_t GetPopulationCountPrecision(uint32_t logcount,
+                                                       uint32_t shift) {
+  return std::min(logcount, shift - ((ANS_LOG_TAB_SIZE - logcount) >> 1));
+}
+
+// Returns a histogram where the counts are positive, differ by at most 1,
+// and add up to total_count. The bigger counts (if any) are at the beginning
+// of the histogram.
+std::vector<int> CreateFlatHistogram(int length, int total_count);
+
+// An alias table implements a mapping from the [0, ANS_TAB_SIZE) range into
+// the [0, ANS_MAX_ALPHABET_SIZE) range, satisfying the following conditions:
+// - each symbol occurs as many times as specified by any valid distribution
+//   of frequencies of the symbols. A valid distribution here is an array of
+//   ANS_MAX_ALPHABET_SIZE that contains numbers in the range [0, ANS_TAB_SIZE],
+//   and whose sum is ANS_TAB_SIZE.
+// - lookups can be done in constant time, and also return how many smaller
+//   input values map into the same symbol, according to some well-defined order
+//   of input values.
+// - the space used by the alias table is given by a small constant times the
+//   index of the largest symbol with nonzero probability in the distribution.
+// Each of the entries in the table covers a range of `entry_size` values in the
+// [0, ANS_TAB_SIZE) range; consecutive entries represent consecutive
+// sub-ranges. In the range covered by entry `i`, the first `cutoff` values map
+// to symbol `i`, while the others map to symbol `right_value`.
+//
+// TODO(veluca): consider making the order used for computing offsets easier to
+// define - it is currently defined by the algorithm to compute the alias table.
+// Beware of breaking the implicit assumption that symbols that come after the
+// cutoff value should have an offset at least as big as the cutoff.
+static constexpr uint32_t kLogEntrySize =
+    ANS_LOG_TAB_SIZE - ANS_LOG_MAX_ALPHABET_SIZE;
+static constexpr uint32_t kEntrySizeMinus1 = (1 << kLogEntrySize) - 1;
+
+struct AliasTable {
+  struct Symbol {
+    size_t value;
+    size_t offset;
+    size_t freq;
+  };
+
+// Working set size matters here (~64 tables x 256 entries).
+// offsets0 is always zero (beginning of [0] side among the same symbol).
+// offsets1 is an offset of (pos >= cutoff) side decremented by cutoff.
+#pragma pack(push, 1)
+  struct Entry {
+    uint8_t cutoff;       // < kEntrySizeMinus1 when used by ANS.
+    uint8_t right_value;  // < alphabet size.
+    uint16_t freq0;
+
+    // Only used if `greater` (see Lookup)
+    uint16_t offsets1;         // <= ANS_TAB_SIZE
+    uint16_t freq1_xor_freq0;  // for branchless ternary in Lookup
+  };
+#pragma pack(pop)
+
+  // Dividing `value` by `entry_size` determines `i`, the entry which is
+  // responsible for the input. If the remainder is below `cutoff`, then the
+  // mapped symbol is `i`; since `offsets[0]` stores the number of occurences of
+  // `i` "before" the start of this entry, the offset of the input will be
+  // `offsets[0] + remainder`. If the remainder is above cutoff, the mapped
+  // symbol is `right_value`; since `offsets[1]` stores the number of occurences
+  // of `right_value` "before" this entry, minus the `cutoff` value, the input
+  // offset is then `remainder + offsets[1]`.
+  static HWY_ATTR JXL_INLINE Symbol Lookup(const Entry* JXL_RESTRICT table,
+                                           size_t value) {
+    const size_t i = value >> kLogEntrySize;
+    const size_t pos = value & kEntrySizeMinus1;
+
+#if JXL_BYTE_ORDER_LITTLE
+    uint64_t entry;
+    memcpy(&entry, &table[i].cutoff, sizeof(entry));
+    const size_t cutoff = entry & 0xFF;              // = MOVZX
+    const size_t right_value = (entry >> 8) & 0xFF;  // = MOVZX
+    const size_t freq0 = (entry >> 16) & 0xFFFF;
+#else
+    // Generates multiple loads with complex addressing.
+    const size_t cutoff = table[i].cutoff;
+    const size_t right_value = table[i].right_value;
+    const size_t freq0 = table[i].freq0;
+#endif
+
+    const bool greater = pos >= cutoff;
+
+#if JXL_BYTE_ORDER_LITTLE
+    const uint64_t conditional = greater ? entry : 0;  // = CMOV
+    const size_t offsets1_or_0 = (conditional >> 32) & 0xFFFF;
+    const size_t freq1_xor_freq0_or_0 = conditional >> 48;
+#else
+    const size_t offsets1_or_0 = greater ? table[i].offsets1 : 0;
+    const size_t freq1_xor_freq0_or_0 = greater ? table[i].freq1_xor_freq0 : 0;
+#endif
+
+    // WARNING: moving this code may interfere with CMOV heuristics.
+    Symbol s;
+    s.value = greater ? right_value : i;
+    s.offset = offsets1_or_0 + pos;
+    s.freq = freq0 ^ freq1_xor_freq0_or_0;  // = greater ? freq1 : freq0
+    // XOR avoids implementation-defined conversion from unsigned to signed.
+    // Alternatives considered: BEXTR is 2 cycles on HSW, SET+shift causes
+    // spills, simple ternary has a long dependency chain.
+
+    return s;
+  }
+
+  static HWY_ATTR HWY_INLINE void Prefetch(const Entry* JXL_RESTRICT table,
+                                           size_t value) {
+    const size_t i = value >> kLogEntrySize;
+    hwy::Prefetch(table + i);
+  }
+};
+
+// Computes an alias table for a given distribution.
+void InitAliasTable(std::vector<int> distribution, int range,
+                    AliasTable::Entry* JXL_RESTRICT a);
+
+}  // namespace jxl
+
+#endif  // JXL_ANS_COMMON_H_
diff --git a/jxl/ans_common_test.cc b/jxl/ans_common_test.cc
new file mode 100644
index 000000000..e832ed4cf
--- /dev/null
+++ b/jxl/ans_common_test.cc
@@ -0,0 +1,48 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/ans_common.h"
+
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/ans_params.h"
+
+namespace jxl {
+namespace {
+
+HWY_ATTR void VerifyAliasDistribution(const std::vector<int>& distribution,
+                                      int range) {
+  AliasTable::Entry table[ANS_MAX_ALPHA_SIZE];
+  InitAliasTable(distribution, range, table);
+  std::vector<std::vector<int>> offsets(distribution.size());
+  for (int i = 0; i < range; i++) {
+    AliasTable::Symbol s = AliasTable::Lookup(table, i);
+    offsets[s.value].push_back(s.offset);
+  }
+  for (int i = 0; i < distribution.size(); i++) {
+    ASSERT_EQ(distribution[i], offsets[i].size());
+    std::sort(offsets[i].begin(), offsets[i].end());
+    for (int j = 0; j < offsets[i].size(); j++) ASSERT_EQ(offsets[i][j], j);
+  }
+}
+
+TEST(ANSCommonTest, AliasDistributionSmoke) {
+  VerifyAliasDistribution({ANS_TAB_SIZE / 2, ANS_TAB_SIZE / 2}, ANS_TAB_SIZE);
+  VerifyAliasDistribution({ANS_TAB_SIZE}, ANS_TAB_SIZE);
+  VerifyAliasDistribution({0, 0, 0, ANS_TAB_SIZE, 0}, ANS_TAB_SIZE);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/ans_params.h b/jxl/ans_params.h
new file mode 100644
index 000000000..24a993675
--- /dev/null
+++ b/jxl/ans_params.h
@@ -0,0 +1,68 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ANS_PARAMS_H_
+#define JXL_ANS_PARAMS_H_
+
+// Common parameters that are needed for both the ANS entropy encoding and
+// decoding methods.
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include "jxl/enc_params.h"
+
+namespace jxl {
+
+// TODO(veluca): decide if 12 is the best constant here (valid range is up to
+// 16). This requires recomputing the Huffman tables in {enc,dec}_ans.cc
+// 14 gives a 0.2% improvement at d1 and makes d8 slightly worse. This is
+// likely not worth the increase in encoder complexity.
+#define ANS_LOG_TAB_SIZE 12u
+#define ANS_TAB_SIZE (1 << ANS_LOG_TAB_SIZE)
+#define ANS_TAB_MASK (ANS_TAB_SIZE - 1)
+
+// 7 is sufficient for 30 bits using ReadHybridUint. Working set is proportional
+// to ANS_MAX_ALPHA_SIZE, and does not fit in L2 cache for larger values.
+#define ANS_LOG_MAX_ALPHABET_SIZE 7
+#define ANS_MAX_ALPHA_SIZE (1 << ANS_LOG_MAX_ALPHABET_SIZE)
+#define ANS_MAX_ALPHA_MASK (ANS_MAX_ALPHA_SIZE - 1)
+
+#define ANS_SIGNATURE 0x13  // Initial state, used as CRC.
+
+struct HistogramParams {
+  enum class ClusteringType {
+    kFastest,  // Cluster everything together
+    kFast,
+    kBest,
+  };
+
+  HistogramParams() = default;
+
+  HistogramParams(SpeedTier tier, size_t num_ctx) {
+    if (tier == SpeedTier::kFalcon) {
+      clustering = ClusteringType::kFastest;
+    } else if (tier > SpeedTier::kTortoise) {
+      clustering = ClusteringType::kFast;
+    } else {
+      clustering = ClusteringType::kBest;
+    }
+  }
+
+  ClusteringType clustering = ClusteringType::kBest;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_ANS_PARAMS_H_
diff --git a/jxl/ans_test.cc b/jxl/ans_test.cc
new file mode 100644
index 000000000..db7b06935
--- /dev/null
+++ b/jxl/ans_test.cc
@@ -0,0 +1,177 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/static_targets.h>
+#include <random>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/ans_params.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/span.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_bit_writer.h"
+
+namespace jxl {
+namespace {
+
+HWY_ATTR void RoundtripTestcase(int n_histograms, int alphabet_size,
+                                const std::vector<Token>& input_values) {
+  constexpr uint16_t kMagic1 = 0x9e33;
+  constexpr uint16_t kMagic2 = 0x8b04;
+
+  BitWriter writer;
+  // Space for magic bytes.
+  BitWriter::Allotment allotment_magic1(&writer, 16);
+  writer.Write(16, kMagic1);
+  ReclaimAndCharge(&writer, &allotment_magic1, 0, nullptr);
+
+  std::vector<uint8_t> context_map;
+  EntropyEncodingData codes;
+  std::vector<std::vector<Token>> input_values_vec;
+  input_values_vec.push_back(input_values);
+
+  BuildAndEncodeHistograms(HistogramParams(), n_histograms, input_values_vec,
+                           &codes, &context_map, &writer, 0, nullptr);
+  WriteTokens(input_values_vec[0], codes, context_map, &writer, 0, nullptr);
+
+  // Magic bytes + padding
+  BitWriter::Allotment allotment_magic2(&writer, 24);
+  writer.Write(16, kMagic2);
+  writer.ZeroPadToByte();
+  ReclaimAndCharge(&writer, &allotment_magic2, 0, nullptr);
+
+  // We do not truncate the output. Reading past the end reads out zeroes
+  // anyway.
+  BitReader br(writer.GetSpan());
+
+  ASSERT_EQ(br.ReadBits(16), kMagic1);
+
+  std::vector<uint8_t> dec_context_map;
+  ANSCode decoded_codes;
+  ASSERT_TRUE(DecodeHistograms(&br, n_histograms, ANS_MAX_ALPHA_SIZE,
+                               &decoded_codes, &dec_context_map));
+  ASSERT_EQ(dec_context_map, context_map);
+  ANSSymbolReader reader(&decoded_codes, &br);
+
+  for (const Token& symbol : input_values) {
+    uint32_t read_symbol =
+        reader.ReadSymbol(dec_context_map[symbol.context], &br);
+    ASSERT_EQ(read_symbol, symbol.symbol);
+    ASSERT_EQ(br.ReadBits(symbol.nbits), symbol.bits);
+  }
+  ASSERT_TRUE(reader.CheckANSFinalState());
+
+  ASSERT_EQ(br.ReadBits(16), kMagic2);
+  EXPECT_TRUE(br.Close());
+}
+
+TEST(ANSTest, EmptyRoundtrip) {
+  RoundtripTestcase(2, ANS_MAX_ALPHA_SIZE, std::vector<Token>());
+}
+
+TEST(ANSTest, SingleSymbolRoundtrip) {
+  for (uint32_t i = 0; i < ANS_MAX_ALPHA_SIZE; i++) {
+    RoundtripTestcase(2, ANS_MAX_ALPHA_SIZE, {{0, i, 0, 0}});
+  }
+  for (uint32_t i = 0; i < ANS_MAX_ALPHA_SIZE; i++) {
+    RoundtripTestcase(2, ANS_MAX_ALPHA_SIZE,
+                      std::vector<Token>(1024, {0, i, 0, 0}));
+  }
+}
+
+#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \
+    defined(THREAD_SANITIZER)
+constexpr size_t kReps = 10;
+#else
+constexpr size_t kReps = 100;
+#endif
+
+void RoundtripRandomStream(int alphabet_size, size_t reps = kReps,
+                           size_t num = 1 << 18) {
+  constexpr int kNumHistograms = 3;
+  std::mt19937_64 rng;
+  for (size_t i = 0; i < reps; i++) {
+    std::vector<Token> symbols;
+    for (size_t j = 0; j < num; j++) {
+      int context = std::uniform_int_distribution<>(0, kNumHistograms - 1)(rng);
+      int value = std::uniform_int_distribution<>(0, alphabet_size - 1)(rng);
+      int nbits = std::uniform_int_distribution<>(0, 16)(rng);
+      int bits = std::uniform_int_distribution<>(0, (1 << nbits) - 1)(rng);
+      symbols.emplace_back(context, value, nbits, bits);
+    }
+    RoundtripTestcase(kNumHistograms, alphabet_size, symbols);
+  }
+}
+
+void RoundtripRandomUnbalancedStream(int alphabet_size) {
+  constexpr int kNumHistograms = 3;
+  constexpr int kPrecision = 1 << 10;
+  std::mt19937_64 rng;
+  for (int i = 0; i < 100; i++) {
+    std::vector<int> distributions[kNumHistograms];
+    for (int j = 0; j < kNumHistograms; j++) {
+      distributions[j].resize(kPrecision);
+      int symbol = 0;
+      int remaining = 1;
+      for (int k = 0; k < kPrecision; k++) {
+        if (remaining == 0) {
+          if (symbol < alphabet_size - 1) symbol++;
+          // There is no meaning behind this distribution: it's anything that
+          // will create a nonuniform distribution and won't have too few
+          // symbols usually. Also we want different distributions we get to be
+          // sufficiently dissimilar.
+          remaining =
+              std::uniform_int_distribution<>(0, (kPrecision - k) / 1)(rng);
+        }
+        distributions[j][k] = symbol;
+        remaining--;
+      }
+    }
+    std::vector<Token> symbols;
+    for (int j = 0; j < 1 << 18; j++) {
+      int context = std::uniform_int_distribution<>(0, kNumHistograms - 1)(rng);
+      int value = distributions[context][std::uniform_int_distribution<>(
+          0, kPrecision - 1)(rng)];
+      int nbits = std::uniform_int_distribution<>(0, 16)(rng);
+      int bits = std::uniform_int_distribution<>(0, (1 << nbits) - 1)(rng);
+      symbols.emplace_back(context, value, nbits, bits);
+    }
+    RoundtripTestcase(kNumHistograms + 1, alphabet_size, symbols);
+  }
+}
+
+TEST(ANSTest, RandomStreamRoundtrip3Small) { RoundtripRandomStream(3, 1, 16); }
+
+TEST(ANSTest, RandomStreamRoundtrip3) { RoundtripRandomStream(3); }
+
+TEST(ANSTest, RandomStreamRoundtripBig) {
+  RoundtripRandomStream(ANS_MAX_ALPHA_SIZE);
+}
+
+TEST(ANSTest, RandomUnbalancedStreamRoundtrip3) {
+  RoundtripRandomUnbalancedStream(3);
+}
+
+TEST(ANSTest, RandomUnbalancedStreamRoundtripBig) {
+  RoundtripRandomUnbalancedStream(ANS_MAX_ALPHA_SIZE);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/ar_control_field.cc b/jxl/ar_control_field.cc
new file mode 100644
index 000000000..2959520bf
--- /dev/null
+++ b/jxl/ar_control_field.cc
@@ -0,0 +1,136 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/ar_control_field.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <algorithm>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/common.h"
+#include "jxl/enc_adaptive_quantization.h"
+#include "jxl/enc_params.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+void FindBestArControlField(const Image3F& opsin, PassesEncoderState* enc_state,
+                            ThreadPool* pool) {
+  constexpr size_t N = kBlockDim;
+  const size_t xsize_blocks = enc_state->shared.frame_dim.xsize_blocks;
+  const size_t ysize_blocks = enc_state->shared.frame_dim.ysize_blocks;
+  ImageB* JXL_RESTRICT epf_sharpness = &enc_state->shared.epf_sharpness;
+  JXL_ASSERT(epf_sharpness->xsize() == xsize_blocks &&
+             epf_sharpness->ysize() == ysize_blocks);
+
+  // TODO(veluca): choose a speed tier at which the heuristics here should be
+  // enabled, once they are enabled at all.
+
+  FillImage(uint8_t(4), epf_sharpness);
+  return;
+  // TODO(veluca) - the following code ends up returning only 7.
+
+  constexpr float kChannelWeights[3] = {50.0f, 2.0f, 0.4f};
+  const size_t sharpness_stride =
+      static_cast<size_t>(epf_sharpness->PixelsPerRow());
+  const size_t opsin_stride = static_cast<size_t>(opsin.PixelsPerRow());
+
+  const float* JXL_RESTRICT in_row[3] = {opsin.ConstPlaneRow(0, 0),
+                                         opsin.ConstPlaneRow(1, 0),
+                                         opsin.ConstPlaneRow(2, 0)};
+
+  const auto process_row = [&](size_t by, int _) {
+    AcStrategyRow acs_row = enc_state->shared.ac_strategy.ConstRow(by);
+    uint8_t* JXL_RESTRICT out_row = epf_sharpness->Row(by);
+    for (size_t bx = 0; bx < xsize_blocks; bx++) {
+      AcStrategy acs = acs_row[bx];
+      if (!acs.IsFirstBlock()) continue;
+      // Find the max Laplacian in a block (for example 32x32 dct).
+      // Then find max Laplacian in two 4x4 pixel grids, offset by 2,2 pixels.
+      // The 1st max indicates amplitude of the artefacts.
+      // The 4x4 pixel grid Laplacian indicates masking of the artefacts.
+      // If masking is high or amplitude is low, then no smoothing is needed.
+      float maxval = 0;
+      float maxarray[64] = {0};
+      float maxarray2[81] = {0};
+      float total_abs = 0;
+      float total_2 = 0;
+      for (size_t iy = 0; iy < acs.covered_blocks_y() * N; iy++) {
+        const size_t cy = by * N + iy;
+        const size_t prevY = cy >= 1 ? cy - 1 : cy;
+        const size_t nextY = cy + 1 < opsin.ysize() ? cy + 1 : cy;
+        for (size_t ix = 0; ix < acs.covered_blocks_x() * N; ix++) {
+          const size_t cx = bx * N + ix;
+          const size_t prevX = cx >= 1 ? cx - 1 : cx;
+          const size_t nextX = cx + 1 < opsin.xsize() ? cx + 1 : cx;
+          float sum = 0;
+          for (size_t c = 0; c < 3; c++) {
+            float v = in_row[c][cy * opsin_stride + cx] -
+                      0.125f * (in_row[c][cy * opsin_stride + prevX] +
+                                in_row[c][cy * opsin_stride + nextX] +
+                                in_row[c][nextY * opsin_stride + prevX] +
+                                in_row[c][nextY * opsin_stride + cx] +
+                                in_row[c][nextY * opsin_stride + nextX] +
+                                in_row[c][prevY * opsin_stride + prevX] +
+                                in_row[c][prevY * opsin_stride + cx] +
+                                in_row[c][prevY * opsin_stride + nextX]);
+            v *= kChannelWeights[c];
+            sum += std::abs(v);
+          }
+          maxarray[(iy / 4) * 8 + (ix / 4)] += sum * (1.0f / 16);
+          maxarray2[((iy + 2) / 4) * 9 + ((ix + 2) / 4)] += sum * (1.0f / 16);
+          total_2 += sum * sum *
+                     (1.0f / (N * N * acs.covered_blocks_x() *
+                              acs.covered_blocks_y()));
+          total_abs += sum * (1.0f / (N * N * acs.covered_blocks_x() *
+                                      acs.covered_blocks_y()));
+        }
+      }
+      maxval = std::sqrt(total_2);
+      for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+        for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+          // Four 4x4 blocks for masking estimation.
+          const float minval = (maxarray[(2 * iy) * 8 + 2 * ix] +
+                                maxarray[(2 * iy) * 8 + 2 * ix + 1] +
+                                maxarray[(2 * iy + 1) * 8 + 2 * ix] +
+                                maxarray[(2 * iy + 1) * 8 + 2 * ix + 1] +
+                                maxarray2[(2 * iy + 1) * 9 + 2 * (ix + 1)]) *
+                               0.2f;
+          // Larger kBias, less smoothing for low intensity changes.
+          float kBias = 0.0005f;
+          float delta = (maxval + kBias) / (minval + kBias);
+          int sharpness1 = 7 + (delta - 0.5f) * 0.0;
+          int sharpness2 = 7;  // maxval * 10;
+          out_row[bx + sharpness_stride * iy + ix] =
+              std::max(0, std::min(7, std::min(sharpness1, sharpness2)));
+        }
+      }
+    }
+  };
+
+  RunOnPool(pool, 0, ysize_blocks, ThreadPool::SkipInit(), process_row,
+            "AR CF");
+}
+
+}  // namespace jxl
diff --git a/jxl/ar_control_field.h b/jxl/ar_control_field.h
new file mode 100644
index 000000000..7383c75fb
--- /dev/null
+++ b/jxl/ar_control_field.h
@@ -0,0 +1,34 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_AR_CONTROL_FIELD_H_
+#define JXL_AR_CONTROL_FIELD_H_
+
+#include "jxl/ac_strategy.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/common.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_params.h"
+#include "jxl/image.h"
+#include "jxl/quant_weights.h"
+
+namespace jxl {
+
+void FindBestArControlField(const Image3F& opsin, PassesEncoderState* enc_state,
+                            ThreadPool* pool);
+
+}  // namespace jxl
+
+#endif  // JXL_AR_CONTROL_FIELD_H_
diff --git a/jxl/aux_out.cc b/jxl/aux_out.cc
new file mode 100644
index 000000000..34818044a
--- /dev/null
+++ b/jxl/aux_out.cc
@@ -0,0 +1,128 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/aux_out.h"
+
+#include <stdint.h>
+
+#include <numeric>  // accumulate
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/enc_bit_writer.h"
+
+namespace jxl {
+
+namespace {
+
+static const char* kBlockTypes[uint8_t(BlockType::kNumBlockTypes)] = {
+    "add", "black", "white"};
+}  // namespace
+
+void AuxOut::Print(size_t num_inputs) const {
+  if (num_inputs == 0) return;
+
+  LayerTotals all_layers;
+  for (size_t i = 0; i < layers.size(); ++i) {
+    all_layers.Assimilate(layers[i]);
+  }
+
+  printf("Average butteraugli iters: %10.2f\n",
+         num_butteraugli_iters * 1.0 / num_inputs);
+
+  for (size_t i = 0; i < layers.size(); ++i) {
+    if (layers[i].total_bits != 0) {
+      printf("Total layer bits %-10s\t", LayerName(i));
+      printf("%10f%%", 100.0 * layers[i].total_bits / all_layers.total_bits);
+      layers[i].Print(num_inputs);
+    }
+  }
+  printf("Total image size           ");
+  all_layers.Print(num_inputs);
+
+  const uint32_t dc_pred_total =
+      std::accumulate(dc_pred_usage.begin(), dc_pred_usage.end(), 0u);
+  const uint32_t dc_pred_total_xb =
+      std::accumulate(dc_pred_usage_xb.begin(), dc_pred_usage_xb.end(), 0u);
+  if (dc_pred_total + dc_pred_total_xb != 0) {
+    printf("\nDC pred     Y                XB:\n");
+    for (size_t i = 0; i < dc_pred_usage.size(); ++i) {
+      printf("  %6u (%5.2f%%)    %6u (%5.2f%%)\n", dc_pred_usage[i],
+             100.0 * dc_pred_usage[i] / dc_pred_total, dc_pred_usage_xb[i],
+             100.0 * dc_pred_usage_xb[i] / dc_pred_total_xb);
+    }
+  }
+
+  size_t total_blocks = 0;
+  size_t total_positions = 0;
+  for (uint8_t block_type = 0; block_type < uint8_t(BlockType::kNumBlockTypes);
+       block_type++) {
+    total_blocks += num_block_types[block_type];
+    total_positions += num_position_types[block_type];
+  }
+  if (total_blocks != 0 && total_positions != 0) {
+    printf("\n\t\t  Blocks\t\tPositions\t\t\tBlocks/Position\n");
+    for (uint8_t block_type = 0;
+         block_type < uint8_t(BlockType::kNumBlockTypes); block_type++) {
+      printf(" %-5s:", kBlockTypes[block_type]);
+
+      printf(
+          "\t%10f%% %7zu\t %10f%% %7zu\t\t\t%10f%%\n",
+          100.0 * num_block_types[block_type] / total_blocks,
+          num_block_types[block_type],
+          100.0 * num_position_types[block_type] / total_positions,
+          num_position_types[block_type],
+          100.0 * num_block_types[block_type] / num_position_types[block_type]);
+    }
+    printf(" Total:\t\t    %7zu\t\t     %7zu \t\t\t%10f%%\n\n", total_blocks,
+           total_positions, 100.0 * total_blocks / total_positions);
+  }
+}
+
+void AuxOut::DumpCoeffImage(const char* label,
+                            const Image3S& coeff_image) const {
+  JXL_ASSERT(coeff_image.xsize() % 64 == 0);
+  Image3S reshuffled(coeff_image.xsize() / 8, coeff_image.ysize() * 8);
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t y = 0; y < coeff_image.ysize(); y++) {
+      for (size_t x = 0; x < coeff_image.xsize(); x += 64) {
+        for (size_t i = 0; i < 64; i++) {
+          reshuffled.PlaneRow(c, 8 * y + i / 8)[x / 8 + i % 8] =
+              coeff_image.PlaneRow(c, y)[x + i];
+        }
+      }
+    }
+  }
+  DumpImage(label, reshuffled);
+}
+
+void ReclaimAndCharge(BitWriter* JXL_RESTRICT writer,
+                      BitWriter::Allotment* JXL_RESTRICT allotment,
+                      size_t layer, AuxOut* JXL_RESTRICT aux_out) {
+  size_t used_bits, unused_bits;
+  allotment->PrivateReclaim(writer, &used_bits, &unused_bits);
+
+#if 0
+  printf("Layer %s bits: max %zu used %zu unused %zu\n", LayerName(layer),
+         allotment->MaxBits(), used_bits, unused_bits);
+#endif
+
+  // This may be a nested call with aux_out == null. Whenever we know that
+  // aux_out is null, we can call ReclaimUnused directly.
+  if (aux_out != nullptr) {
+    aux_out->layers[layer].total_bits += used_bits;
+    aux_out->layers[layer].histogram_bits += allotment->HistogramBits();
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/aux_out.h b/jxl/aux_out.h
new file mode 100644
index 000000000..0bb101703
--- /dev/null
+++ b/jxl/aux_out.h
@@ -0,0 +1,332 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_AUX_OUT_H_
+#define JXL_AUX_OUT_H_
+
+// Optional output information for debugging and analyzing size usage.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <array>
+#include <functional>
+#include <sstream>
+#include <string>
+#include <utility>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/dec_xyb.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/jxl_inspection.h"
+
+namespace jxl {
+
+// For LayerName and AuxOut::layers[] index. Order does not matter.
+enum {
+  kLayerHeader = 0,
+  kLayerTOC,
+  kLayerNoise,
+  kLayerQuant,
+  kLayerDequantTables,
+  kLayerOrder,
+  kLayerDownsampledDC,
+  kLayerDC,
+  kLayerAC,
+  kLayerACTokens,
+  kLayerDictionary,
+  kLayerDots,
+  kLayerSplines,
+  kLayerLossless,
+  kLayerModularGlobal,
+  kLayerModularDcGroup,
+  kLayerModularAcGroup,
+  kLayerAlpha,
+  kLayerDepth,
+  kLayerExtraChannels,
+  kNumImageLayers
+};
+
+static inline const char* LayerName(size_t layer) {
+  switch (layer) {
+    case kLayerHeader:
+      return "headers";
+    case kLayerTOC:
+      return "TOC";
+    case kLayerNoise:
+      return "noise";
+    case kLayerQuant:
+      return "quantizer";
+    case kLayerDequantTables:
+      return "quant tables";
+    case kLayerOrder:
+      return "order";
+    case kLayerDC:
+      return "DC+Group";
+    case kLayerAC:
+      return "AC";
+    case kLayerACTokens:
+      return "ACTokens";
+    case kLayerDictionary:
+      return "dictionary";
+    case kLayerDots:
+      return "dots";
+    case kLayerSplines:
+      return "splines";
+    case kLayerLossless:
+      return "lossless";
+    case kLayerModularGlobal:
+      return "modularGlobal";
+    case kLayerModularDcGroup:
+      return "modularDcGroup";
+    case kLayerModularAcGroup:
+      return "modularAcGroup";
+    case kLayerAlpha:
+      return "alpha";
+    case kLayerDepth:
+      return "depth";
+    case kLayerExtraChannels:
+      return "extra channels";
+    default:
+      JXL_ABORT("Invalid layer %zu\n", layer);
+  }
+}
+
+struct TestingAux {
+  Image3F* dc = nullptr;
+  Image3F* decoded = nullptr;
+};
+
+// Statistics gathered during compression or decompression.
+struct AuxOut {
+ private:
+  struct LayerTotals {
+    void Assimilate(const LayerTotals& victim) {
+      num_clustered_histograms += victim.num_clustered_histograms;
+      histogram_bits += victim.histogram_bits;
+      extra_bits += victim.extra_bits;
+      total_bits += victim.total_bits;
+      clustered_entropy += victim.clustered_entropy;
+    }
+    void Print(size_t num_inputs) const {
+      printf("%10zd", total_bits);
+      if (histogram_bits != 0) {
+        printf("   [%6.2f %8zd %8zd %12.3f",
+               num_clustered_histograms * 1.0 / num_inputs, histogram_bits >> 3,
+               extra_bits >> 3,
+               (histogram_bits + clustered_entropy + extra_bits) / 8.0);
+        printf("]");
+      }
+      printf("\n");
+    }
+    size_t num_clustered_histograms = 0;
+    size_t extra_bits = 0;
+
+    // Set via BitsWritten below
+    size_t histogram_bits = 0;
+    size_t total_bits = 0;
+
+    double clustered_entropy = 0.0;
+  };
+
+ public:
+  AuxOut() = default;
+  AuxOut(const AuxOut&) = default;
+
+  void Assimilate(const AuxOut& victim) {
+    for (size_t i = 0; i < layers.size(); ++i) {
+      layers[i].Assimilate(victim.layers[i]);
+    }
+    num_blocks += victim.num_blocks;
+    num_dct2_blocks += victim.num_dct2_blocks;
+    num_dct4_blocks += victim.num_dct4_blocks;
+    num_dct4x8_blocks += victim.num_dct4x8_blocks;
+    num_afv_blocks += victim.num_afv_blocks;
+    num_dct8_blocks += victim.num_dct8_blocks;
+    num_dct8x16_blocks += victim.num_dct8x16_blocks;
+    num_dct8x32_blocks += victim.num_dct8x32_blocks;
+    num_dct16_blocks += victim.num_dct16_blocks;
+    num_dct16x32_blocks += victim.num_dct16x32_blocks;
+    num_dct32_blocks += victim.num_dct32_blocks;
+    num_butteraugli_iters += victim.num_butteraugli_iters;
+    for (size_t i = 0; i < dc_pred_usage.size(); ++i) {
+      dc_pred_usage[i] += victim.dc_pred_usage[i];
+      dc_pred_usage_xb[i] += victim.dc_pred_usage_xb[i];
+    }
+    for (uint8_t block_type = 0;
+         block_type < uint8_t(BlockType::kNumBlockTypes); block_type++) {
+      num_block_types[block_type] += victim.num_block_types[block_type];
+      num_position_types[block_type] += victim.num_position_types[block_type];
+    }
+  }
+
+  void Print(size_t num_inputs) const;
+
+  template <typename T>
+  void DumpImage(const char* label, const Image3<T>& image) const {
+    if (!dump_image) return;
+    if (debug_prefix.empty()) return;
+    std::ostringstream pathname;
+    pathname << debug_prefix << label << ".png";
+    CodecInOut io;
+    io.metadata.bits_per_sample = sizeof(T) * kBitsPerByte;
+    io.metadata.color_encoding = ColorManagement::SRGB();
+    io.SetFromImage(StaticCastImage3<float>(image), io.metadata.color_encoding);
+    (void)dump_image(io, pathname.str());
+  }
+  template <typename T>
+  void DumpImage(const char* label, const Plane<T>& image) {
+    DumpImage(label,
+              Image3<T>(CopyImage(image), CopyImage(image), CopyImage(image)));
+  }
+
+  template <typename T>
+  void DumpXybImage(const char* label, const Image3<T>& image) const {
+    if (!dump_image) return;
+    if (debug_prefix.empty()) return;
+    std::ostringstream pathname;
+    pathname << debug_prefix << label << ".png";
+
+    Image3F linear(image.xsize(), image.ysize());
+    OpsinParams opsin_params;
+    opsin_params.Init();
+    OpsinToLinear(image, Rect(linear), nullptr, &linear, opsin_params);
+
+    CodecInOut io;
+    io.metadata.bits_per_sample = sizeof(T) * kBitsPerByte;
+    io.metadata.color_encoding = ColorManagement::LinearSRGB();
+    io.SetFromImage(std::move(linear), io.metadata.color_encoding);
+
+    (void)dump_image(io, pathname.str());
+  }
+
+  // Normalizes all the channels to range 0-255, creating a false-color image
+  // which allows seeing the information from non-RGB channels in an RGB debug
+  // image.
+  template <typename T>
+  void DumpImageNormalized(const char* label, const Image3<T>& image) const {
+    std::array<T, 3> min;
+    std::array<T, 3> max;
+    Image3MinMax(image, &min, &max);
+    Image3B normalized(image.xsize(), image.ysize());
+    for (size_t c = 0; c < 3; ++c) {
+      float mul = min[c] == max[c] ? 0 : (255.0f / (max[c] - min[c]));
+      for (size_t y = 0; y < image.ysize(); ++y) {
+        const T* JXL_RESTRICT row_in = image.ConstPlaneRow(c, y);
+        uint8_t* JXL_RESTRICT row_out = normalized.PlaneRow(c, y);
+        for (size_t x = 0; x < image.xsize(); ++x) {
+          row_out[x] = static_cast<uint8_t>((row_in[x] - min[c]) * mul);
+        }
+      }
+    }
+    DumpImage(label, normalized);
+  }
+
+  template <typename T>
+  void DumpPlaneNormalized(const char* label, const Plane<T>& image) const {
+    T min;
+    T max;
+    ImageMinMax(image, &min, &max);
+    Image3B normalized(image.xsize(), image.ysize());
+    for (size_t c = 0; c < 3; ++c) {
+      float mul = min == max ? 0 : (255.0f / (max - min));
+      for (size_t y = 0; y < image.ysize(); ++y) {
+        const T* JXL_RESTRICT row_in = image.ConstRow(y);
+        uint8_t* JXL_RESTRICT row_out = normalized.PlaneRow(c, y);
+        for (size_t x = 0; x < image.xsize(); ++x) {
+          row_out[x] = static_cast<uint8_t>((row_in[x] - min) * mul);
+        }
+      }
+    }
+    DumpImage(label, normalized);
+  }
+
+  // This dumps coefficients as a 16-bit PNG with coefficients of a block placed
+  // in the area that would contain that block in a normal image. To view the
+  // resulting image manually, rescale intensities by using:
+  // $ convert -auto-level IMAGE.PNG - | display -
+  void DumpCoeffImage(const char* label, const Image3S& coeff_image) const;
+
+  void SetInspectorImage3F(const jxl::InspectorImage3F& inspector) {
+    inspector_image3f_ = inspector;
+  }
+
+  // Allows hooking intermediate data inspection into various places of the
+  // processing pipeline. Returns true iff processing should proceed.
+  bool InspectImage3F(const char* label, const Image3F& image) {
+    if (inspector_image3f_ != nullptr) {
+      return inspector_image3f_(label, image);
+    }
+    return true;
+  }
+
+  std::array<LayerTotals, kNumImageLayers> layers;
+  size_t num_blocks = 0;
+
+  // Number of blocks that use larger DCT (set by ac_strategy).
+  size_t num_dct2_blocks = 0;
+  size_t num_dct4_blocks = 0;
+  size_t num_dct4x8_blocks = 0;
+  size_t num_afv_blocks = 0;
+  size_t num_dct8_blocks = 0;
+  size_t num_dct8x16_blocks = 0;
+  size_t num_dct8x32_blocks = 0;
+  size_t num_dct16_blocks = 0;
+  size_t num_dct16x32_blocks = 0;
+  size_t num_dct32_blocks = 0;
+
+  std::array<uint32_t, 8> dc_pred_usage = {0};
+  std::array<uint32_t, 8> dc_pred_usage_xb = {0};
+
+  int num_butteraugli_iters = 0;
+
+  // If not empty, additional debugging information (e.g. debug images) is
+  // saved in files with this prefix.
+  std::string debug_prefix;
+
+  // By how much the decoded image was downsampled relative to the encoded
+  // image.
+  size_t downsampling = 1;
+
+  // Number of various types of blocks
+  size_t num_block_types[uint8_t(BlockType::kNumBlockTypes)] = {0};
+  size_t num_position_types[uint8_t(BlockType::kNumBlockTypes)] = {0};
+
+  jxl::InspectorImage3F inspector_image3f_;
+
+  std::function<Status(const CodecInOut&, const std::string&)> dump_image =
+      nullptr;
+
+  // WARNING: this is actually an INPUT to some code, and must be
+  // copy-initialized from aux_out to aux_outs.
+  TestingAux testing_aux;
+};
+
+// Used to skip image creation if they won't be written to debug directory.
+static inline bool WantDebugOutput(const AuxOut* aux_out) {
+  // Need valid pointer and filename.
+  return aux_out != nullptr && !aux_out->debug_prefix.empty();
+}
+
+}  // namespace jxl
+
+#endif  // JXL_AUX_OUT_H_
diff --git a/jxl/aux_out_fwd.h b/jxl/aux_out_fwd.h
new file mode 100644
index 000000000..8d45536c1
--- /dev/null
+++ b/jxl/aux_out_fwd.h
@@ -0,0 +1,37 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_AUX_OUT_FWD_H_
+#define JXL_AUX_OUT_FWD_H_
+
+#include <stddef.h>
+
+#include "jxl/enc_bit_writer.h"
+
+namespace jxl {
+
+struct AuxOut;
+
+// Helper function that ensures the `bits_written` are charged to `layer` in
+// `aux_out`. Example usage:
+//   BitWriter::Allotment allotment(&writer, max_bits);
+//   writer.Write(..); writer.Write(..);
+//   ReclaimAndCharge(&writer, &allotment, layer, aux_out);
+void ReclaimAndCharge(BitWriter* JXL_RESTRICT writer,
+                      BitWriter::Allotment* JXL_RESTRICT allotment,
+                      size_t layer, AuxOut* JXL_RESTRICT aux_out);
+
+}  // namespace jxl
+
+#endif  // JXL_AUX_OUT_FWD_H_
diff --git a/jxl/base/arch_specific.cc b/jxl/base/arch_specific.cc
new file mode 100644
index 000000000..e12aa5203
--- /dev/null
+++ b/jxl/base/arch_specific.cc
@@ -0,0 +1,355 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/arch_specific.h"
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#if JXL_ARCH_X64
+#include <xmmintrin.h>
+#if !JXL_COMPILER_MSVC
+#include <cpuid.h>
+#endif
+#endif
+
+#if JXL_ARCH_PPC
+#include <sys/platform/ppc.h>  // __ppc_get_timebase_freq
+#endif
+
+#if JXL_ARCH_ARM
+#include <unistd.h>  // sysconf
+#endif
+
+#include <string.h>  // memcpy
+
+#include <set>
+#include <string>
+#include <vector>
+
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/os_specific.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+namespace {
+
+#if JXL_ARCH_X64
+
+// For Cpuid.
+#pragma pack(push, 1)
+struct Regs {
+  uint32_t a;
+  uint32_t b;
+  uint32_t c;
+  uint32_t d;
+};
+#pragma pack(pop)
+
+// Calls CPUID instruction with eax=level and ecx=count and fills `r`.
+// The caller must ensure `level` <= the max supported.
+void Cpuid(const uint32_t level, const uint32_t count, Regs* JXL_RESTRICT r) {
+#if JXL_COMPILER_MSVC
+  int regs[4];
+  __cpuidex(regs, level, count);
+  r->a = regs[0];
+  r->b = regs[1];
+  r->c = regs[2];
+  r->d = regs[3];
+#else
+  // WARNING: avoid using __cpuid_count, which is broken: it lacks volatile and
+  // clobber "memory", so the compiler caches CPUID results, not realizing that
+  // CPUID:1b (APIC ID) changes across calls to SetThreadAffinity.
+  __asm__ __volatile__(
+      "xchgq %%rbx,%q1\n"
+      "cpuid\n"
+      "xchgq %%rbx,%q1"
+      : "=a"(r->a), "=r"(r->b), "=c"(r->c), "=d"(r->d)
+      : "0"(level), "2"(count)
+      : "memory");
+#endif
+}
+
+class Info {
+ public:
+  Info() {
+    Regs r;
+    Cpuid(0, 0, &r);
+    max_func_ = r.a;
+
+    char vendor[13];
+    // Note unusual order, reverse of ModR/M encoding.
+    memcpy(&vendor[0], &r.b, 4);
+    memcpy(&vendor[4], &r.d, 4);
+    memcpy(&vendor[8], &r.c, 4);
+    vendor[12] = '\0';
+    intel_ = strcmp(vendor, "GenuineIntel") == 0;
+    amd_ = strcmp(vendor, "AuthenticAMD") == 0;
+
+    Cpuid(0x80000000u, 0, &r);
+    max_ext_func_ = r.a;
+  }
+
+  uint32_t MaxFunc() const { return max_func_; }
+  uint32_t MaxExtFunc() const { return max_ext_func_; }
+  bool Intel() const { return intel_; }
+  bool AMD() const { return amd_; }
+
+  std::string BrandString() const {
+    char brand_string[49];
+    Regs r;
+
+    // Check if brand string is supported (it is on all reasonable Intel/AMD)
+    if (MaxExtFunc() < 0x80000004U) return std::string();
+
+    for (uint32_t i = 0; i < 3; ++i) {
+      Cpuid(0x80000002U + i, 0, &r);
+      memcpy(brand_string + i * 16, &r, sizeof(r));
+    }
+    brand_string[48] = 0;
+    return brand_string;
+  }
+
+ private:
+  uint32_t max_func_;
+  uint32_t max_ext_func_;
+  bool intel_;
+  bool amd_;
+};
+
+// Detects number of packages/cores/logical processors (hyperthreads).
+class X64_Topology {
+ public:
+  // Enumerates all APIC IDs and partitions them into fields, or returns false
+  // if the topology cannot be detected (e.g. due to missing OS support).
+  static Status Detect(ProcessorTopology* topology) {
+    const Info info;
+    const uint32_t core_bits = CoreBits(info);
+    const uint32_t logical_bits = LogicalBits(info, core_bits);
+
+    uint32_t total_bits = 0;
+    // Order matters:
+    Field logical(logical_bits, &total_bits);
+    Field core(core_bits, &total_bits);
+    Field package(8, &total_bits);
+
+    // Query ProcessorId on each (accessible) logical processor:
+    ThreadAffinity* original_affinity = GetThreadAffinity();
+    for (int cpu : AvailableCPUs()) {
+      if (!PinThreadToCPU(cpu)) {
+        free(original_affinity);
+        return false;
+      }
+
+      const uint32_t id = ProcessorId(info);
+      // xAPIC ID of 255 is invalid. Systems with >= 255 logical processors or
+      // >= 64 cores require x2APIC (Nehalem) and CPUID:11 detection, which is a
+      // separate codepath that we have not yet implemented.
+      if (id >= 255) {
+        JXL_WARNING("x2APIC ID (%x); TODO: implement CPUID:11", id);
+      }
+
+      logical.AddValue(id);
+      core.AddValue(id);
+      package.AddValue(id);
+    }
+    JXL_CHECK(SetThreadAffinity(original_affinity));
+    free(original_affinity);
+
+    topology->logical_per_core = logical.NumValues();
+    topology->cores_per_package = core.NumValues();
+    topology->packages = package.NumValues();
+    return true;
+  }
+
+ public:
+  // How many bits in the xAPIC ID identify the core (per package).
+  // #active cores <= (1 << bits).
+  static uint32_t CoreBits(const Info& info) {
+    Regs r;
+    if (info.Intel()) {
+      if (info.MaxFunc() < 4) return 0;
+      Cpuid(4, 0, &r);
+      return static_cast<uint32_t>(CeilLog2Nonzero((r.a >> 26) + 1));
+    }
+
+    if (info.AMD()) {
+      if (info.MaxExtFunc() < 0x80000008u) return 0;
+      Cpuid(0x80000008u, 0, &r);
+      return static_cast<uint32_t>(CeilLog2Nonzero((r.c & 0xFF) + 1));
+    }
+
+    return 0;
+  }
+
+  // How many bits in the xAPIC ID identify the logical processor (per core).
+  static uint32_t LogicalBits(const Info& info, const uint32_t core_bits) {
+    Regs r;
+    Cpuid(1, 0, &r);
+
+    // No hyperthreading
+    if ((r.d & (1U << 28)) == 0) return 0;
+
+    // Early AMD falsely claim hyperthreading
+    if (info.AMD() && (r.c & 2)) return 0;
+
+    const uint32_t logical_per_package = (r.b >> 16) & 0xFF;
+    const uint32_t core_and_logical_bits =
+        static_cast<uint32_t>(CeilLog2Nonzero(logical_per_package));
+    JXL_ASSERT(core_and_logical_bits >= core_bits);
+    return core_and_logical_bits - core_bits;
+  }
+
+  // Variable-length/position field within an xAPIC ID. Counts the total
+  // number of values encountered for all given id.
+  class Field {
+   public:
+    Field(const uint32_t bits, uint32_t* JXL_RESTRICT total_bits)
+        : mask_((1U << bits) - 1), shift_(*total_bits) {
+      *total_bits += bits;
+    }
+
+    void AddValue(const uint32_t id) { values_.insert((id >> shift_) & mask_); }
+
+    size_t NumValues() const { return values_.size(); }
+
+   private:
+    const uint32_t mask_;  // zero for zero-width fields
+    const uint32_t shift_;
+    std::set<uint32_t> values_;
+  };
+
+  // Returns initial APIC ID or x2APIC ID, which uniquely identifies the current
+  // logical processor. Returns 0 on old CPUs.
+  static uint32_t ProcessorId(const Info& info) {
+    Regs r;
+
+    // Support 32-bit IDs: we will only use 8 bits, but returning the full ID
+    // allows the caller to raise a warning when x2APIC is active.
+    if (info.MaxFunc() >= 11) {
+      Cpuid(11, 0, &r);
+      // Ensure we have CPUID:11 (number of enabled logical processors != 0).
+      if (r.b != 0) {
+        // r.d is a 32-bit ID; whether or not x2APIC is actually supported and
+        // enabled, its lower 8 bits match the initial APIC ID (CPUID:1b).
+        return r.d;
+      }
+    }
+
+    // No CPUID:11 => just return initial APIC ID (8-bit).
+    Cpuid(1, 0, &r);
+    return r.b >> 24;
+  }
+};
+
+double X64_DetectNominalClockRate() {
+  const Info info;
+  const std::string& brand_string = info.BrandString();
+  // Brand strings include the maximum configured frequency. These prefixes are
+  // defined by Intel CPUID documentation.
+  const char* prefixes[3] = {"MHz", "GHz", "THz"};
+  const double multipliers[3] = {1E6, 1E9, 1E12};
+  for (size_t i = 0; i < 3; ++i) {
+    const size_t pos_prefix = brand_string.find(prefixes[i]);
+    if (pos_prefix != std::string::npos) {
+      const size_t pos_space = brand_string.rfind(' ', pos_prefix - 1);
+      if (pos_space != std::string::npos) {
+        const std::string digits =
+            brand_string.substr(pos_space + 1, pos_prefix - pos_space - 1);
+        return std::stod(digits) * multipliers[i];
+      }
+    }
+  }
+
+  return 0.0;
+}
+
+#elif JXL_ARCH_PPC
+
+double PPC_DetectNominalClockRate() {
+  double freq = -1;
+  char line[200];
+  char* s;
+  char* value;
+
+  FILE* f = fopen("/proc/cpuinfo", "r");
+  if (f != nullptr) {
+    while (fgets(line, sizeof(line), f) != nullptr) {
+      // NOTE: the ':' is the only character we can rely on
+      if (!(value = strchr(line, ':'))) continue;
+      // terminate the valuename
+      *value++ = '\0';
+      // skip any leading spaces
+      while (*value == ' ') value++;
+      if ((s = strchr(value, '\n'))) *s = '\0';
+
+      if (!strncasecmp(line, "clock", strlen("clock")) &&
+          sscanf(value, "%lf", &freq) == 1) {
+        freq *= 1E6;
+        break;
+      }
+    }
+    fclose(f);
+    return freq;
+  }
+
+  return 0.0;
+}
+
+#endif  // JXL_ARCH_*
+
+}  // namespace
+
+Status DetectProcessorTopology(ProcessorTopology* pt) {
+  if (GetProcessorTopologyFromOS(pt)) return true;
+#if JXL_ARCH_X64
+  if (X64_Topology::Detect(pt)) return true;
+#elif JXL_ARCH_ARM
+  // TODO(deymo): Actually look up the CPU topology and model the big/small
+  // core split.
+  pt->logical_per_core = 1;
+  pt->cores_per_package = sysconf(_SC_NPROCESSORS_ONLN);
+  pt->packages = 1;
+  return true;
+#endif
+  return JXL_FAILURE("Unable to detect processor topology");
+}
+
+double NominalClockRate() {
+// Thread-safe caching - this is called several times.
+#if JXL_ARCH_X64
+  static const double cycles_per_second = X64_DetectNominalClockRate();
+  return cycles_per_second;
+#elif JXL_ARCH_PPC
+  static const double cycles_per_second = PPC_DetectNominalClockRate();
+  return cycles_per_second;
+#else
+  return 0.0;
+#endif
+}
+
+double InvariantTicksPerSecond() {
+#if JXL_ARCH_PPC
+  static const double cycles_per_second = __ppc_get_timebase_freq();
+  return cycles_per_second;
+#elif JXL_ARCH_X64
+  return NominalClockRate();
+#else
+  return 1E9;  // nanoseconds - matches tsc_timer.h CLOCK_MONOTONIC fallback.
+#endif
+}
+
+}  // namespace jxl
diff --git a/jxl/base/arch_specific.h b/jxl/base/arch_specific.h
new file mode 100644
index 000000000..f480ed150
--- /dev/null
+++ b/jxl/base/arch_specific.h
@@ -0,0 +1,66 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_ARCH_SPECIFIC_H_
+#define JXL_BASE_ARCH_SPECIFIC_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/arch.h>  // kMaxVectorSize
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+#if defined(__x86_64__) || defined(_M_X64)
+#define JXL_ARCH_X64 1
+#else
+#define JXL_ARCH_X64 0
+#endif
+
+#if defined(__powerpc64__) || defined(_M_PPC)
+#define JXL_ARCH_PPC 1
+#else
+#define JXL_ARCH_PPC 0
+#endif
+
+#if defined(__aarch64__) || defined(__arm__)
+#define JXL_ARCH_ARM 1
+#else
+#define JXL_ARCH_ARM 0
+#endif
+
+struct ProcessorTopology {
+  size_t logical_per_core = 1;
+  size_t cores_per_package = 1;
+  size_t packages = 1;
+};
+
+// Relatively expensive, preferably only call once.
+Status DetectProcessorTopology(ProcessorTopology* pt);
+
+// Returns the nominal (without Turbo Boost) CPU clock rate [Hertz]. Useful for
+// (roughly) characterizing the CPU speed.
+double NominalClockRate();
+
+// Returns tsc_timer frequency, useful for converting ticks to seconds. This is
+// unaffected by CPU throttling ("invariant"). Thread-safe. Returns timebase
+// frequency on PPC, NominalClockRate on X64, otherwise 1E9.
+double InvariantTicksPerSecond();
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_ARCH_SPECIFIC_H_
diff --git a/jxl/base/bits.h b/jxl/base/bits.h
new file mode 100644
index 000000000..d7e3bd1ff
--- /dev/null
+++ b/jxl/base/bits.h
@@ -0,0 +1,149 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_BITS_H_
+#define JXL_BASE_BITS_H_
+
+// Specialized instructions for processing register-sized bit arrays.
+
+#ifdef _MSC_VER
+#include <intrin.h>
+#include <nmmintrin.h>
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/type_traits.h>
+
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+
+static JXL_INLINE size_t PopCount(hwy::SizeTag<4> /* tag */, const uint32_t x) {
+#ifdef _MSC_VER
+  return _mm_popcnt_u32(x);
+#else
+  return static_cast<size_t>(__builtin_popcount(x));
+#endif
+}
+static JXL_INLINE size_t PopCount(hwy::SizeTag<8> /* tag */, const uint64_t x) {
+#ifdef _MSC_VER
+  return _mm_popcnt_u64(x);
+#else
+  return static_cast<size_t>(__builtin_popcountll(x));
+#endif
+}
+template <typename T>
+static JXL_INLINE size_t PopCount(T x) {
+  static_assert(!hwy::IsSigned<T>(), "PopCount: use unsigned");
+  return PopCount(hwy::SizeTag<sizeof(T)>(), x);
+}
+
+// Undefined results for x == 0.
+static JXL_INLINE size_t NumZeroBitsAboveMSBNonzero(hwy::SizeTag<4> /* tag */,
+                                                    const uint32_t x) {
+#ifdef _MSC_VER
+  unsigned long index;
+  _BitScanReverse(&index, x);
+  return 31 - index;
+#else
+  return static_cast<size_t>(__builtin_clz(x));
+#endif
+}
+static JXL_INLINE size_t NumZeroBitsAboveMSBNonzero(hwy::SizeTag<8> /* tag */,
+                                                    const uint64_t x) {
+#ifdef _MSC_VER
+  unsigned long index;
+  _BitScanReverse64(&index, x);
+  return 63 - index;
+#else
+  return static_cast<size_t>(__builtin_clzll(x));
+#endif
+}
+template <typename T>
+static JXL_INLINE size_t NumZeroBitsAboveMSBNonzero(const T x) {
+  static_assert(!hwy::IsSigned<T>(),
+                "NumZeroBitsAboveMSBNonzero: use unsigned");
+  return NumZeroBitsAboveMSBNonzero(hwy::SizeTag<sizeof(T)>(), x);
+}
+
+// Undefined results for x == 0.
+static JXL_INLINE size_t NumZeroBitsBelowLSBNonzero(hwy::SizeTag<4> /* tag */,
+                                                    const uint32_t x) {
+#ifdef _MSC_VER
+  unsigned long index;
+  _BitScanForward(&index, x);
+  return index;
+#else
+  return static_cast<size_t>(__builtin_ctz(x));
+#endif
+}
+static JXL_INLINE size_t NumZeroBitsBelowLSBNonzero(hwy::SizeTag<8> /* tag */,
+                                                    const uint64_t x) {
+#ifdef _MSC_VER
+  unsigned long index;
+  _BitScanForward64(&index, x);
+  return index;
+#else
+  return static_cast<size_t>(__builtin_ctzll(x));
+#endif
+}
+template <typename T>
+static JXL_INLINE size_t NumZeroBitsBelowLSBNonzero(T x) {
+  static_assert(!hwy::IsSigned<T>(),
+                "NumZeroBitsBelowLSBNonzero: use unsigned");
+  return NumZeroBitsBelowLSBNonzero(hwy::SizeTag<sizeof(T)>(), x);
+}
+
+// Returns bit width for x == 0.
+template <typename T>
+static JXL_INLINE size_t NumZeroBitsAboveMSB(const T x) {
+  return (x == 0) ? sizeof(T) * 8 : NumZeroBitsAboveMSBNonzero(x);
+}
+
+// Returns bit width for x == 0.
+template <typename T>
+static JXL_INLINE size_t NumZeroBitsBelowLSB(const T x) {
+  return (x == 0) ? sizeof(T) * 8 : NumZeroBitsBelowLSBNonzero(x);
+}
+
+// Returns base-2 logarithm, rounded down.
+template <typename T>
+static JXL_INLINE size_t FloorLog2Nonzero(const T x) {
+  return (sizeof(T) * 8 - 1) ^ NumZeroBitsAboveMSBNonzero(x);
+}
+
+// Returns base-2 logarithm, rounded up.
+template <typename T>
+static JXL_INLINE size_t CeilLog2Nonzero(const T x) {
+  const size_t floor_log2 = FloorLog2Nonzero(x);
+  if ((x & (x - 1)) == 0) return floor_log2;  // power of two
+  return floor_log2 + 1;
+}
+
+// Reverses bit order.
+static JXL_INLINE uint8_t FlipByte(const uint8_t x) {
+  // TODO(veluca): consider trying out alternative strategies, such as a single
+  // LUT.
+  static constexpr uint8_t kNibbleLut[16] = {
+      0b0000, 0b1000, 0b0100, 0b1100, 0b0010, 0b1010, 0b0110, 0b1110,
+      0b0001, 0b1001, 0b0101, 0b1101, 0b0011, 0b1011, 0b0111, 0b1111,
+  };
+  return (kNibbleLut[x & 0xF] << 4) | kNibbleLut[x >> 4];
+}
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_BITS_H_
diff --git a/jxl/base/byte_order.h b/jxl/base/byte_order.h
new file mode 100644
index 000000000..34e7e9f79
--- /dev/null
+++ b/jxl/base/byte_order.h
@@ -0,0 +1,212 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_BYTE_ORDER_H_
+#define JXL_BASE_BYTE_ORDER_H_
+
+#include <stdint.h>
+#include <string.h>  // memcpy
+
+#include "jxl/base/compiler_specific.h"
+
+#if JXL_COMPILER_MSVC
+#include <intrin.h>  // _byteswap_*
+#endif
+
+#if (defined(__BYTE_ORDER__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__))
+#define JXL_BYTE_ORDER_LITTLE 1
+#else
+// This means that we don't know that the byte order is little endian, in
+// this case we use endian-neutral code that works for both little- and
+// big-endian.
+#define JXL_BYTE_ORDER_LITTLE 0
+#endif
+
+// Returns whether the system is little-endian (least-significant byte first).
+#if JXL_BYTE_ORDER_LITTLE
+static constexpr bool IsLittleEndian() { return true; }
+#else
+static inline bool IsLittleEndian() {
+  const uint32_t multibyte = 1;
+  uint8_t byte;
+  memcpy(&byte, &multibyte, 1);
+  return byte == 1;
+}
+#endif
+
+#if JXL_COMPILER_MSVC
+#define JXL_BSWAP32(x) _byteswap_ulong(x)
+#define JXL_BSWAP64(x) _byteswap_uint64(x)
+#else
+#define JXL_BSWAP32(x) __builtin_bswap32(x)
+#define JXL_BSWAP64(x) __builtin_bswap64(x)
+#endif
+
+static JXL_INLINE uint32_t LoadBE16(const uint8_t* p) {
+  const uint32_t byte1 = p[0];
+  const uint32_t byte0 = p[1];
+  return (byte1 << 8) | byte0;
+}
+
+static JXL_INLINE uint32_t LoadLE16(const uint8_t* p) {
+  const uint32_t byte0 = p[0];
+  const uint32_t byte1 = p[1];
+  return (byte1 << 8) | byte0;
+}
+
+static JXL_INLINE uint32_t LoadBE32(const uint8_t* p) {
+#if JXL_BYTE_ORDER_LITTLE
+  uint32_t big;
+  memcpy(&big, p, 4);
+  return JXL_BSWAP32(big);
+#else
+  // Byte-order-independent - can't assume this machine is big endian.
+  const uint32_t byte3 = p[0];
+  const uint32_t byte2 = p[1];
+  const uint32_t byte1 = p[2];
+  const uint32_t byte0 = p[3];
+  return (byte3 << 24) | (byte2 << 16) | (byte1 << 8) | byte0;
+#endif
+}
+
+static JXL_INLINE uint32_t LoadLE32(const uint8_t* p) {
+#if JXL_BYTE_ORDER_LITTLE
+  uint32_t little;
+  memcpy(&little, p, 4);
+  return little;
+#else
+  // Byte-order-independent - can't assume this machine is big endian.
+  const uint32_t byte0 = p[0];
+  const uint32_t byte1 = p[1];
+  const uint32_t byte2 = p[2];
+  const uint32_t byte3 = p[3];
+  return (byte3 << 24) | (byte2 << 16) | (byte1 << 8) | byte0;
+#endif
+}
+
+static JXL_INLINE uint64_t LoadLE64(const uint8_t* p) {
+#if JXL_BYTE_ORDER_LITTLE
+  uint64_t little;
+  memcpy(&little, p, 8);
+  return little;
+#else
+  // Byte-order-independent - can't assume this machine is big endian.
+  const uint64_t byte0 = p[0];
+  const uint64_t byte1 = p[1];
+  const uint64_t byte2 = p[2];
+  const uint64_t byte3 = p[3];
+  const uint64_t byte4 = p[4];
+  const uint64_t byte5 = p[5];
+  const uint64_t byte6 = p[6];
+  const uint64_t byte7 = p[7];
+  return (byte7 << 56) | (byte6 << 48) | (byte5 << 40) | (byte4 << 32) |
+         (byte3 << 24) | (byte2 << 16) | (byte1 << 8) | byte0;
+#endif
+}
+
+static JXL_INLINE void StoreBE16(const uint32_t native, uint8_t* p) {
+  p[0] = (native >> 8) & 0xFF;
+  p[1] = native & 0xFF;
+}
+
+static JXL_INLINE void StoreLE16(const uint32_t native, uint8_t* p) {
+  p[1] = (native >> 8) & 0xFF;
+  p[0] = native & 0xFF;
+}
+
+static JXL_INLINE void StoreBE32(const uint32_t native, uint8_t* p) {
+#if JXL_BYTE_ORDER_LITTLE
+  const uint32_t big = JXL_BSWAP32(native);
+  memcpy(p, &big, 4);
+#else
+  // Byte-order-independent - can't assume this machine is big endian.
+  p[0] = native >> 24;
+  p[1] = (native >> 16) & 0xFF;
+  p[2] = (native >> 8) & 0xFF;
+  p[3] = native & 0xFF;
+#endif
+}
+
+static JXL_INLINE void StoreLE32(const uint32_t native, uint8_t* p) {
+#if JXL_BYTE_ORDER_LITTLE
+  const uint32_t little = native;
+  memcpy(p, &little, 4);
+#else
+  // Byte-order-independent - can't assume this machine is big endian.
+  p[3] = native >> 24;
+  p[2] = (native >> 16) & 0xFF;
+  p[1] = (native >> 8) & 0xFF;
+  p[0] = native & 0xFF;
+#endif
+}
+
+static JXL_INLINE void StoreLE64(const uint64_t native, uint8_t* p) {
+#if JXL_BYTE_ORDER_LITTLE
+  const uint64_t little = native;
+  memcpy(p, &little, 8);
+#else
+  // Byte-order-independent - can't assume this machine is big endian.
+  p[7] = native >> 56;
+  p[6] = (native >> 48) & 0xFF;
+  p[5] = (native >> 40) & 0xFF;
+  p[4] = (native >> 32) & 0xFF;
+  p[3] = (native >> 24) & 0xFF;
+  p[2] = (native >> 16) & 0xFF;
+  p[1] = (native >> 8) & 0xFF;
+  p[0] = native & 0xFF;
+#endif
+}
+
+// Big/Little Endian order.
+struct OrderBE {};
+struct OrderLE {};
+
+// Wrappers for calling from generic code.
+static JXL_INLINE void Store16(OrderBE /*tag*/, const uint32_t native,
+                               uint8_t* p) {
+  return StoreBE16(native, p);
+}
+
+static JXL_INLINE void Store16(OrderLE /*tag*/, const uint32_t native,
+                               uint8_t* p) {
+  return StoreLE16(native, p);
+}
+
+static JXL_INLINE void Store32(OrderBE /*tag*/, const uint32_t native,
+                               uint8_t* p) {
+  return StoreBE32(native, p);
+}
+
+static JXL_INLINE void Store32(OrderLE /*tag*/, const uint32_t native,
+                               uint8_t* p) {
+  return StoreLE32(native, p);
+}
+
+static JXL_INLINE uint32_t Load16(OrderBE /*tag*/, const uint8_t* p) {
+  return LoadBE16(p);
+}
+
+static JXL_INLINE uint32_t Load16(OrderLE /*tag*/, const uint8_t* p) {
+  return LoadLE16(p);
+}
+
+static JXL_INLINE uint32_t Load32(OrderBE /*tag*/, const uint8_t* p) {
+  return LoadBE32(p);
+}
+
+static JXL_INLINE uint32_t Load32(OrderLE /*tag*/, const uint8_t* p) {
+  return LoadLE32(p);
+}
+
+#endif  // JXL_BASE_BYTE_ORDER_H_
diff --git a/jxl/base/cache_aligned.cc b/jxl/base/cache_aligned.cc
new file mode 100644
index 000000000..cc7a3ca3d
--- /dev/null
+++ b/jxl/base/cache_aligned.cc
@@ -0,0 +1,156 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/cache_aligned.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+// Disabled: slower than malloc + alignment.
+#define JXL_USE_MMAP 0
+
+#if JXL_USE_MMAP
+#include <sys/mman.h>
+#endif
+
+#include <algorithm>  // std::max
+#include <atomic>
+#include <hwy/arch.h>  // for kMaxVectorSize
+
+#include "jxl/base/status.h"
+
+namespace jxl {
+namespace {
+
+#pragma pack(push, 1)
+struct AllocationHeader {
+  void* allocated;
+  size_t allocated_size;
+  uint8_t left_padding[kMaxVectorSize];
+};
+#pragma pack(pop)
+
+std::atomic<uint64_t> num_allocations{0};
+std::atomic<uint64_t> bytes_in_use{0};
+std::atomic<uint64_t> max_bytes_in_use{0};
+
+}  // namespace
+
+void CacheAligned::PrintStats() {
+  printf("Allocations: %zu (max bytes in use: %E)\n",
+         size_t(num_allocations.load(std::memory_order_relaxed)),
+         double(max_bytes_in_use.load(std::memory_order_relaxed)));
+}
+
+size_t CacheAligned::NextOffset() {
+  static std::atomic<uint32_t> next{0};
+  constexpr uint32_t kGroups = CacheAligned::kAlias / CacheAligned::kAlignment;
+  const uint32_t group = next.fetch_add(1, std::memory_order_relaxed) % kGroups;
+  return CacheAligned::kAlignment * group;
+}
+
+void* CacheAligned::Allocate(const size_t payload_size, size_t offset) {
+  JXL_ASSERT(payload_size <= std::numeric_limits<size_t>::max() / 2);
+  JXL_ASSERT((offset % kAlignment == 0) && offset <= kAlias);
+
+  // What: | misalign | unused | AllocationHeader |payload
+  // Size: |<= kAlias | offset |                  |payload_size
+  //       ^allocated.^aligned.^header............^payload
+  // The header must immediately precede payload, which must remain aligned.
+  // To avoid wasting space, the header resides at the end of `unused`,
+  // which therefore cannot be empty (offset == 0).
+  if (offset == 0) {
+    offset = kAlignment;  // = round_up(sizeof(AllocationHeader), kAlignment)
+    static_assert(sizeof(AllocationHeader) <= kAlignment, "Else: round up");
+  }
+
+#if JXL_USE_MMAP
+  const size_t allocated_size = offset + payload_size;
+  const int flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_POPULATE;
+  void* allocated =
+      mmap(nullptr, allocated_size, PROT_READ | PROT_WRITE, flags, -1, 0);
+  if (allocated == MAP_FAILED) return nullptr;
+  const uintptr_t aligned = reinterpret_cast<uintptr_t>(allocated);
+#else
+  const size_t allocated_size = kAlias + offset + payload_size;
+  void* allocated = malloc(allocated_size);
+  if (allocated == nullptr) return nullptr;
+  // Always round up even if already aligned - we already asked for kAlias
+  // extra bytes and there's no way to give them back.
+  uintptr_t aligned = reinterpret_cast<uintptr_t>(allocated) + kAlias;
+  static_assert((kAlias & (kAlias - 1)) == 0, "kAlias must be a power of 2");
+  static_assert(kAlias >= kAlignment, "Cannot align to more than kAlias");
+  aligned &= ~(kAlias - 1);
+#endif
+
+#if 0
+  // No effect.
+  uintptr_t page_aligned = reinterpret_cast<uintptr_t>(allocated);
+  page_aligned &= ~(4096 - 1);
+  if (madvise(reinterpret_cast<void*>(page_aligned), allocated_size,
+              MADV_WILLNEED) != 0) {
+    JXL_NOTIFY_ERROR("madvise failed");
+  }
+#elif 0
+  // INCREASES both first and subsequent decode times.
+  if (mlock(allocated, allocated_size) != 0) {
+    JXL_NOTIFY_ERROR("mlock failed");
+  }
+#endif
+
+  // Update statistics (#allocations and max bytes in use)
+  num_allocations.fetch_add(1, std::memory_order_relaxed);
+  const uint64_t prev_bytes =
+      bytes_in_use.fetch_add(allocated_size, std::memory_order_acq_rel);
+  uint64_t expected_max = max_bytes_in_use.load(std::memory_order_acquire);
+  for (;;) {
+    const uint64_t desired =
+        std::max(expected_max, prev_bytes + allocated_size);
+    if (max_bytes_in_use.compare_exchange_strong(expected_max, desired,
+                                                 std::memory_order_acq_rel)) {
+      break;
+    }
+  }
+
+  const uintptr_t payload = aligned + offset;  // still aligned
+
+  // Stash `allocated` and payload_size inside header for use by Free().
+  AllocationHeader* header = reinterpret_cast<AllocationHeader*>(payload) - 1;
+  header->allocated = allocated;
+  header->allocated_size = allocated_size;
+
+  return JXL_ASSUME_ALIGNED(reinterpret_cast<void*>(payload), 64);
+}
+
+void CacheAligned::Free(const void* aligned_pointer) {
+  if (aligned_pointer == nullptr) {
+    return;
+  }
+  const uintptr_t payload = reinterpret_cast<uintptr_t>(aligned_pointer);
+  JXL_ASSERT(payload % kAlignment == 0);
+  const AllocationHeader* header =
+      reinterpret_cast<const AllocationHeader*>(payload) - 1;
+
+  // Subtract (2's complement negation).
+  bytes_in_use.fetch_add(~header->allocated_size + 1,
+                         std::memory_order_acq_rel);
+
+#if JXL_USE_MMAP
+  munmap(header->allocated, header->allocated_size);
+#else
+  free(header->allocated);
+#endif
+}
+
+}  // namespace jxl
diff --git a/jxl/base/cache_aligned.h b/jxl/base/cache_aligned.h
new file mode 100644
index 000000000..e448d1a00
--- /dev/null
+++ b/jxl/base/cache_aligned.h
@@ -0,0 +1,84 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_CACHE_ALIGNED_H_
+#define JXL_BASE_CACHE_ALIGNED_H_
+
+// Memory allocator with support for alignment + misalignment.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <memory>
+
+#include "jxl/base/arch_specific.h"
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+
+// Functions that depend on the cache line size.
+class CacheAligned {
+ public:
+  static void PrintStats();
+
+  static constexpr size_t kPointerSize = sizeof(void*);
+  static constexpr size_t kCacheLineSize = 64;
+  // To avoid RFOs, match L2 fill size (pairs of lines).
+  static constexpr size_t kAlignment = 2 * kCacheLineSize;
+  // Minimum multiple for which cache set conflicts and/or loads blocked by
+  // preceding stores can occur.
+  static constexpr size_t kAlias = 2048;
+
+  // Returns a 'random' (cyclical) offset suitable for Allocate.
+  static size_t NextOffset();
+
+  // Returns null or memory whose address is congruent to `offset` (mod kAlias).
+  // This reduces cache conflicts and load/store stalls, especially with large
+  // allocations that would otherwise have similar alignments. At least
+  // `payload_size` (which can be zero) bytes will be accessible.
+  static void* Allocate(size_t payload_size, size_t offset);
+
+  static void* Allocate(const size_t payload_size) {
+    return Allocate(payload_size, NextOffset());
+  }
+
+  static void Free(const void* aligned_pointer);
+};
+
+// Avoids the need for a function pointer (deleter) in CacheAlignedUniquePtr.
+struct CacheAlignedDeleter {
+  void operator()(uint8_t* aligned_pointer) const {
+    return CacheAligned::Free(aligned_pointer);
+  }
+};
+
+using CacheAlignedUniquePtr = std::unique_ptr<uint8_t[], CacheAlignedDeleter>;
+
+// Does not invoke constructors.
+static inline CacheAlignedUniquePtr AllocateArray(const size_t bytes) {
+  return CacheAlignedUniquePtr(
+      static_cast<uint8_t*>(CacheAligned::Allocate(bytes)),
+      CacheAlignedDeleter());
+}
+
+static inline CacheAlignedUniquePtr AllocateArray(const size_t bytes,
+                                                  const size_t offset) {
+  return CacheAlignedUniquePtr(
+      static_cast<uint8_t*>(CacheAligned::Allocate(bytes, offset)),
+      CacheAlignedDeleter());
+}
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_CACHE_ALIGNED_H_
diff --git a/jxl/base/compiler_specific.h b/jxl/base/compiler_specific.h
new file mode 100644
index 000000000..fac035b72
--- /dev/null
+++ b/jxl/base/compiler_specific.h
@@ -0,0 +1,144 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_COMPILER_SPECIFIC_H_
+#define JXL_BASE_COMPILER_SPECIFIC_H_
+
+// Macros for compiler version + nonstandard keywords, e.g. __builtin_expect.
+
+#include <stdint.h>
+
+// #if is shorter and safer than #ifdef. *_VERSION are zero if not detected,
+// otherwise 100 * major + minor version. Note that other packages check for
+// #ifdef COMPILER_MSVC, so we cannot use that same name.
+
+#ifdef _MSC_VER
+#define JXL_COMPILER_MSVC _MSC_VER
+#else
+#define JXL_COMPILER_MSVC 0
+#endif
+
+#ifdef __GNUC__
+#define JXL_COMPILER_GCC (__GNUC__ * 100 + __GNUC_MINOR__)
+#else
+#define JXL_COMPILER_GCC 0
+#endif
+
+#ifdef __clang__
+#define JXL_COMPILER_CLANG (__clang_major__ * 100 + __clang_minor__)
+// Clang pretends to be GCC for compatibility.
+#undef JXL_COMPILER_GCC
+#define JXL_COMPILER_GCC 0
+#else
+#define JXL_COMPILER_CLANG 0
+#endif
+
+#if JXL_COMPILER_MSVC
+#define JXL_RESTRICT __restrict
+#elif JXL_COMPILER_GCC || JXL_COMPILER_CLANG
+#define JXL_RESTRICT __restrict__
+#else
+#define JXL_RESTRICT
+#endif
+
+#if JXL_COMPILER_MSVC
+#define JXL_INLINE __forceinline
+#define JXL_NOINLINE __declspec(noinline)
+#else
+#define JXL_INLINE inline __attribute__((always_inline))
+#define JXL_NOINLINE __attribute__((noinline))
+#endif
+
+#if JXL_COMPILER_MSVC
+#define JXL_NORETURN __declspec(noreturn)
+#elif JXL_COMPILER_GCC || JXL_COMPILER_CLANG
+#define JXL_NORETURN __attribute__((noreturn))
+#endif
+
+#if JXL_COMPILER_MSVC
+#define JXL_UNREACHABLE __assume(false)
+#elif JXL_COMPILER_CLANG || JXL_COMPILER_GCC >= 405
+#define JXL_UNREACHABLE __builtin_unreachable()
+#else
+#define JXL_UNREACHABLE
+#endif
+
+#if JXL_COMPILER_MSVC
+// Unsupported, __assume is not the same.
+#define JXL_LIKELY(expr) expr
+#define JXL_UNLIKELY(expr) expr
+#else
+#define JXL_LIKELY(expr) __builtin_expect(!!(expr), 1)
+#define JXL_UNLIKELY(expr) __builtin_expect(!!(expr), 0)
+#endif
+
+#if JXL_COMPILER_MSVC
+#include <intrin.h>
+
+#pragma intrinsic(_ReadWriteBarrier)
+#define JXL_COMPILER_FENCE _ReadWriteBarrier()
+#elif JXL_COMPILER_GCC || JXL_COMPILER_CLANG
+#define JXL_COMPILER_FENCE asm volatile("" : : : "memory")
+#else
+#define JXL_COMPILER_FENCE
+#endif
+
+// Returns a void* pointer which the compiler then assumes is N-byte aligned.
+// Example: float* JXL_RESTRICT aligned = (float*)JXL_ASSUME_ALIGNED(in, 32);
+//
+// The assignment semantics are required by GCC/Clang. ICC provides an in-place
+// __assume_aligned, whereas MSVC's __assume appears unsuitable.
+#if JXL_COMPILER_CLANG
+// Early versions of Clang did not support __builtin_assume_aligned.
+#define JXL_HAS_ASSUME_ALIGNED __has_builtin(__builtin_assume_aligned)
+#elif JXL_COMPILER_GCC
+#define JXL_HAS_ASSUME_ALIGNED 1
+#else
+#define JXL_HAS_ASSUME_ALIGNED 0
+#endif
+
+#if JXL_HAS_ASSUME_ALIGNED
+#define JXL_ASSUME_ALIGNED(ptr, align) __builtin_assume_aligned((ptr), (align))
+#else
+#define JXL_ASSUME_ALIGNED(ptr, align) (ptr) /* not supported */
+#endif
+
+#ifdef __has_attribute
+#define JXL_HAVE_ATTRIBUTE(x) __has_attribute(x)
+#else
+#define JXL_HAVE_ATTRIBUTE(x) 0
+#endif
+
+// Raises warnings if the function return value is unused. Should appear as the
+// first part of a function definition/declaration.
+#if JXL_HAVE_ATTRIBUTE(nodiscard)
+#define JXL_MUST_USE_RESULT [[nodiscard]]
+#elif JXL_COMPILER_CLANG && JXL_HAVE_ATTRIBUTE(warn_unused_result)
+#define JXL_MUST_USE_RESULT __attribute__((warn_unused_result))
+#else
+#define JXL_MUST_USE_RESULT
+#endif
+
+#if JXL_HAVE_ATTRIBUTE(__format__)
+#define JXL_FORMAT(idx_fmt, idx_arg) \
+  __attribute__((__format__(__printf__, idx_fmt, idx_arg)))
+#else
+#define JXL_FORMAT(idx_fmt, idx_arg)
+#endif
+
+#if JXL_COMPILER_MSVC
+using ssize_t = intptr_t;
+#endif
+
+#endif  // JXL_BASE_COMPILER_SPECIFIC_H_
diff --git a/jxl/base/data_parallel.cc b/jxl/base/data_parallel.cc
new file mode 100644
index 000000000..3020e2b9a
--- /dev/null
+++ b/jxl/base/data_parallel.cc
@@ -0,0 +1,46 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/data_parallel.h"
+
+#define DATA_PARALLEL_TRACE 0
+
+namespace jxl {
+
+// static
+JpegxlParallelRetCode ThreadPool::SequentialRunnerStatic(
+    void* runner_opaque, void* jpegxl_opaque, JpegxlParallelRunInit init,
+    JpegxlParallelRunFunction func, uint32_t start_range, uint32_t end_range) {
+  JpegxlParallelRetCode init_ret = (*init)(jpegxl_opaque, 1);
+  if (init_ret != 0) return init_ret;
+
+  for (uint32_t i = start_range; i < end_range; i++) {
+    (*func)(jpegxl_opaque, i, 0);
+  }
+  return 0;
+}
+
+#if DATA_PARALLEL_TRACE
+void TraceRunBegin(const char* /*caller*/, double* t0) { *t0 = Now(); }
+
+void TraceRunEnd(const char* caller, double t0) {
+  const double elapsed = Now() - t0;
+  fprintf(stderr, "%27s: %5.1f ms\n", caller, elapsed * 1E3);
+}
+#else
+void TraceRunBegin(const char* /*caller*/, double* /*t0*/) {}
+void TraceRunEnd(const char* /*caller*/, double /*t0*/) {}
+#endif
+
+}  // namespace jxl
diff --git a/jxl/base/data_parallel.h b/jxl/base/data_parallel.h
new file mode 100644
index 000000000..957ee1aec
--- /dev/null
+++ b/jxl/base/data_parallel.h
@@ -0,0 +1,170 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_DATA_PARALLEL_H_
+#define JXL_BASE_DATA_PARALLEL_H_
+
+// Portable, low-overhead C++11 ThreadPool alternative to OpenMP for
+// data-parallel computations.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "include/jpegxl/parallel_runner.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+class ThreadPool {
+ public:
+  // Use this type as an InitFunc to skip the initialization step in Run().
+  // When this is used the return value of Run() is always true and does not
+  // need to be checked.
+  struct SkipInit {};
+
+  ThreadPool(JpegxlParallelRunner runner, void* runner_opaque)
+      : runner_(runner ? runner : &ThreadPool::SequentialRunnerStatic),
+        runner_opaque_(runner ? runner_opaque : static_cast<void*>(this)) {}
+
+  ThreadPool(const ThreadPool&) = delete;
+  ThreadPool& operator&(const ThreadPool&) = delete;
+
+  // Runs init_func(num_threads) followed by data_func(task, thread) on worker
+  // thread(s) for every task in [begin, end). init_func() must return a Status
+  // indicating whether the initialization succeeded.
+  // "thread" is an integer smaller than num_threads.
+  // Not thread-safe - no two calls to Run may overlap.
+  // Subsequent calls will reuse the same threads.
+  //
+  // Precondition: begin <= end.
+  template <class InitFunc, class DataFunc>
+  Status Run(uint32_t begin, uint32_t end, const InitFunc& init_func,
+             const DataFunc& data_func, const char* caller = "") {
+    JXL_ASSERT(begin <= end);
+    if (begin == end) return true;
+    RunCallState<InitFunc, DataFunc> call_state(init_func, data_func);
+    // The runner_ uses the C convention and returns 0 in case of error, so we
+    // convert it to an Status.
+    return (*runner_)(runner_opaque_, static_cast<void*>(&call_state),
+                      &call_state.CallInitFunc, &call_state.CallDataFunc, begin,
+                      end) == 0;
+  }
+
+  // Specialization that returns bool when SkipInit is used.
+  template <class DataFunc>
+  bool Run(uint32_t begin, uint32_t end, const SkipInit /* tag */,
+           const DataFunc& data_func, const char* caller = "") {
+    return Run(begin, end, ReturnTrueInit, data_func, caller);
+  }
+
+ private:
+  static Status ReturnTrueInit(size_t num_threads) { return true; }
+
+  // class holding the state of a Run() call to pass to the runner_ as an
+  // opaque_jpegxl pointer.
+  template <class InitFunc, class DataFunc>
+  class RunCallState final {
+   public:
+    RunCallState(const InitFunc& init_func, const DataFunc& data_func)
+        : init_func_(init_func), data_func_(data_func) {}
+
+    // JpegxlParallelRunInit interface.
+    static int CallInitFunc(void* jpegxl_opaque, size_t num_threads) {
+      const auto* self =
+          static_cast<RunCallState<InitFunc, DataFunc>*>(jpegxl_opaque);
+      // Returns -1 when the internal init function returns false Status to
+      // indicate an error.
+      return self->init_func_(num_threads) ? 0 : -1;
+    }
+
+    // JpegxlParallelRunFunction interface.
+    static void CallDataFunc(void* jpegxl_opaque, uint32_t value,
+                             size_t thread_id) {
+      const auto* self =
+          static_cast<RunCallState<InitFunc, DataFunc>*>(jpegxl_opaque);
+      return self->data_func_(value, thread_id);
+    }
+
+   private:
+    const InitFunc& init_func_;
+    const DataFunc& data_func_;
+  };
+
+  // Default JpegxlParallelRunner used when no runner is provided by the
+  // caller. This runner doesn't use any threading and thread_id is always 0.
+  static JpegxlParallelRetCode SequentialRunnerStatic(
+      void* runner_opaque, void* jpegxl_opaque, JpegxlParallelRunInit init,
+      JpegxlParallelRunFunction func, uint32_t start_range, uint32_t end_range);
+
+  // The caller supplied runner function and its opaque void*.
+  const JpegxlParallelRunner runner_;
+  void* const runner_opaque_;
+};
+
+void TraceRunBegin(const char* caller, double* t0);
+void TraceRunEnd(const char* caller, double t0);
+
+// TODO(deymo): Convert the return value to a Status when not using SkipInit.
+template <class InitFunc, class DataFunc>
+bool RunOnPool(ThreadPool* pool, const uint32_t begin, const uint32_t end,
+               const InitFunc& init_func, const DataFunc& data_func,
+               const char* caller) {
+  Status ret = true;
+  double t0;
+  TraceRunBegin(caller, &t0);
+  if (pool == nullptr) {
+    ThreadPool default_pool(nullptr, nullptr);
+    ret = default_pool.Run(begin, end, init_func, data_func, caller);
+  } else {
+    ret = pool->Run(begin, end, init_func, data_func, caller);
+  }
+  TraceRunEnd(caller, t0);
+  return ret;
+}
+
+// Accelerates multiple unsigned 32-bit divisions with the same divisor by
+// precomputing a multiplier. This is useful for splitting a contiguous range of
+// indices (the task index) into 2D indices. Exhaustively tested on dividends
+// up to 4M with non-power of two divisors up to 2K.
+class Divider {
+ public:
+  // "d" is the divisor (what to divide by).
+  explicit Divider(const uint32_t d) : shift_(FloorLog2Nonzero(d)) {
+    // Power of two divisors (including 1) are not supported because it is more
+    // efficient to special-case them at a higher level.
+    JXL_ASSERT((d & (d - 1)) != 0);
+
+    // ceil_log2 = floor_log2 + 1 because we ruled out powers of two above.
+    const uint64_t next_pow2 = 1ULL << (shift_ + 1);
+
+    mul_ = ((next_pow2 - d) << 32) / d + 1;
+  }
+
+  // "n" is the numerator (what is being divided).
+  inline uint32_t operator()(const uint32_t n) const {
+    // Algorithm from "Division by Invariant Integers using Multiplication".
+    // Its "sh1" is hardcoded to 1 because we don't need to handle d=1.
+    const uint32_t hi = (uint64_t(mul_) * n) >> 32;
+    return (hi + ((n - hi) >> 1)) >> shift_;
+  }
+
+ private:
+  uint32_t mul_;
+  const int shift_;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_DATA_PARALLEL_H_
diff --git a/jxl/base/descriptive_statistics.cc b/jxl/base/descriptive_statistics.cc
new file mode 100644
index 000000000..7f3a74f92
--- /dev/null
+++ b/jxl/base/descriptive_statistics.cc
@@ -0,0 +1,111 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/descriptive_statistics.h"
+
+#include <stdio.h>
+
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+void Stats::Assimilate(const Stats& other) {
+  const int64_t total_n = n_ + other.n_;
+  if (total_n == 0) return;  // Nothing to do; prevents div by zero.
+
+  min_ = std::min(min_, other.min_);
+  max_ = std::max(max_, other.max_);
+
+  product_ *= other.product_;
+
+  const double product_n = n_ * other.n_;
+  const double n2 = n_ * n_;
+  const double other_n2 = other.n_ * other.n_;
+  // Warning: multiplying int64 can overflow here.
+  const double total_n2 = static_cast<double>(total_n) * total_n;
+  const double total_n3 = static_cast<double>(total_n2) * total_n;
+  // Precompute reciprocal for speed - used at least twice.
+  const double inv_total_n = 1.0 / total_n;
+  const double inv_total_n2 = 1.0 / total_n2;
+
+  const double delta = other.m1_ - m1_;
+  const double delta2 = delta * delta;
+  const double delta3 = delta * delta2;
+  const double delta4 = delta2 * delta2;
+
+  m1_ = (n_ * m1_ + other.n_ * other.m1_) * inv_total_n;
+
+  const double new_m2 = m2_ + other.m2_ + delta2 * product_n * inv_total_n;
+
+  const double new_m3 =
+      m3_ + other.m3_ + delta3 * product_n * (n_ - other.n_) * inv_total_n2 +
+      3.0 * delta * (n_ * other.m2_ - other.n_ * m2_) * inv_total_n;
+
+  m4_ += other.m4_ +
+         delta4 * product_n * (n2 - product_n + other_n2) / total_n3 +
+         6.0 * delta2 * (n2 * other.m2_ + other_n2 * m2_) * inv_total_n2 +
+         4.0 * delta * (n_ * other.m3_ - other.n_ * m3_) * inv_total_n;
+
+  m2_ = new_m2;
+  m3_ = new_m3;
+  n_ = total_n;
+}
+
+std::string Stats::ToString(int exclude) const {
+  if (Count() == 0) return std::string("(none)");
+
+  char buf[300];
+  size_t pos = 0;
+  int ret;  // snprintf - bytes written or negative for error.
+
+  if ((exclude & kNoCount) == 0) {
+    ret = snprintf(buf + pos, sizeof(buf) - pos, "Count=%6zu ",
+                   static_cast<size_t>(Count()));
+    JXL_ASSERT(ret > 0);
+    pos += ret;
+  }
+
+  if ((exclude & kNoMeanSD) == 0) {
+    ret = snprintf(buf + pos, sizeof(buf) - pos, "Mean=%9.6f SD=%8.5f ", Mean(),
+                   StandardDeviation());
+    JXL_ASSERT(ret > 0);
+    pos += ret;
+  }
+
+  if ((exclude & kNoMinMax) == 0) {
+    ret = snprintf(buf + pos, sizeof(buf) - pos, "Min=%8.5f Max=%8.5f ", Min(),
+                   Max());
+    JXL_ASSERT(ret > 0);
+    pos += ret;
+  }
+
+  if ((exclude & kNoSkewKurt) == 0) {
+    ret = snprintf(buf + pos, sizeof(buf) - pos, "Skew=%5.2f Kurt=%7.2f ",
+                   Skewness(), Kurtosis());
+    JXL_ASSERT(ret > 0);
+    pos += ret;
+  }
+
+  if ((exclude & kNoGeomean) == 0) {
+    ret = snprintf(buf + pos, sizeof(buf) - pos, "GeoMean=%9.6f ",
+                   GeometricMean());
+    JXL_ASSERT(ret > 0);
+    pos += ret;
+  }
+
+  JXL_ASSERT(pos < sizeof(buf));
+  return buf;
+}
+
+}  // namespace jxl
diff --git a/jxl/base/descriptive_statistics.h b/jxl/base/descriptive_statistics.h
new file mode 100644
index 000000000..f116440bc
--- /dev/null
+++ b/jxl/base/descriptive_statistics.h
@@ -0,0 +1,135 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_DESCRIPTIVE_STATISTICS_H_
+#define JXL_BASE_DESCRIPTIVE_STATISTICS_H_
+
+// For analyzing the range/distribution of scalars.
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <cmath>
+#include <string>
+
+namespace jxl {
+
+// Descriptive statistics of a variable (4 moments).
+class Stats {
+ public:
+  void Notify(const float x) {
+    ++n_;
+
+    min_ = std::min(min_, x);
+    max_ = std::max(max_, x);
+
+    product_ *= x;
+
+    // Online moments. Reference: https://goo.gl/9ha694
+    const double d = x - m1_;
+    const double d_div_n = d / n_;
+    const double d2n1_div_n = d * (n_ - 1) * d_div_n;
+    const int64_t n_poly = n_ * n_ - 3 * n_ + 3;
+    m1_ += d_div_n;
+    m4_ += d_div_n * (d_div_n * (d2n1_div_n * n_poly + 6.0 * m2_) - 4.0 * m3_);
+    m3_ += d_div_n * (d2n1_div_n * (n_ - 2) - 3.0 * m2_);
+    m2_ += d2n1_div_n;
+  }
+
+  void Assimilate(const Stats& other);
+
+  int64_t Count() const { return n_; }
+
+  float Min() const { return min_; }
+  float Max() const { return max_; }
+
+  double GeometricMean() const {
+    return n_ == 0 ? 0.0 : pow(product_, 1.0 / n_);
+  }
+
+  double Mean() const { return m1_; }
+  // Same as Mu2. Assumes n_ is large.
+  double SampleVariance() const {
+    return n_ == 0 ? 0.0 : m2_ / static_cast<int>(n_);
+  }
+  // Unbiased estimator for population variance even for smaller n_.
+  double Variance() const {
+    if (n_ == 0) return 0.0;
+    if (n_ == 1) return m2_;
+    return m2_ / static_cast<int>(n_ - 1);
+  }
+  double StandardDeviation() const { return std::sqrt(Variance()); }
+  // Near zero for normal distributions; if positive on a unimodal distribution,
+  // the right tail is fatter. Assumes n_ is large.
+  double SampleSkewness() const {
+    if (std::abs(m2_) < 1E-7) return 0.0;
+    return m3_ * std::sqrt(static_cast<double>(n_)) / std::pow(m2_, 1.5);
+  }
+  // Corrected for bias (same as Wikipedia and Minitab but not Excel).
+  double Skewness() const {
+    if (n_ == 0) return 0.0;
+    const double biased = SampleSkewness();
+    const double r = (n_ - 1.0) / n_;
+    return biased * std::pow(r, 1.5);
+  }
+  // Near zero for normal distributions; smaller values indicate fewer/smaller
+  // outliers and larger indicates more/larger outliers. Assumes n_ is large.
+  double SampleKurtosis() const {
+    if (std::abs(m2_) < 1E-7) return 0.0;
+    return m4_ * n_ / (m2_ * m2_);
+  }
+  // Corrected for bias (same as Wikipedia and Minitab but not Excel).
+  double Kurtosis() const {
+    if (n_ == 0) return 0.0;
+    const double biased = SampleKurtosis();
+    const double r = (n_ - 1.0) / n_;
+    return biased * r * r;
+  }
+
+  // Central moments, useful for "method of moments"-based parameter estimation
+  // of a mixture of two Gaussians. Assumes Count() != 0.
+  double Mu1() const { return m1_; }
+  double Mu2() const { return m2_ / static_cast<int>(n_); }
+  double Mu3() const { return m3_ / static_cast<int>(n_); }
+  double Mu4() const { return m4_ / static_cast<int>(n_); }
+
+  // Which statistics to EXCLUDE in ToString
+  enum {
+    kNoCount = 1,
+    kNoMeanSD = 2,
+    kNoMinMax = 4,
+    kNoSkewKurt = 8,
+    kNoGeomean = 16
+  };
+
+  std::string ToString(int exclude = 0) const;
+
+ private:
+  int64_t n_ = 0;  // signed for faster conversion + safe subtraction
+
+  float min_ = 1E30f;
+  float max_ = -1E30f;
+
+  double product_ = 1.0;
+
+  // Moments
+  double m1_ = 0.0;
+  double m2_ = 0.0;
+  double m3_ = 0.0;
+  double m4_ = 0.0;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_DESCRIPTIVE_STATISTICS_H_
diff --git a/jxl/base/fast_log.h b/jxl/base/fast_log.h
new file mode 100644
index 000000000..2f85efc8d
--- /dev/null
+++ b/jxl/base/fast_log.h
@@ -0,0 +1,133 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_FAST_LOG_H_
+#define JXL_BASE_FAST_LOG_H_
+
+#include <stdint.h>
+
+#include <cmath>
+
+namespace jxl {
+
+// A lookup table for small values of log2(int) to be used in entropy
+// computation.
+//
+// ", ".join(["%.16ff" % x for x in [0.0]+[log2(x) for x in range(1, 256)]])
+static const float kLog2Table[] = {
+    0.0000000000000000f, 0.0000000000000000f, 1.0000000000000000f,
+    1.5849625007211563f, 2.0000000000000000f, 2.3219280948873622f,
+    2.5849625007211561f, 2.8073549220576042f, 3.0000000000000000f,
+    3.1699250014423126f, 3.3219280948873626f, 3.4594316186372978f,
+    3.5849625007211565f, 3.7004397181410922f, 3.8073549220576037f,
+    3.9068905956085187f, 4.0000000000000000f, 4.0874628412503400f,
+    4.1699250014423122f, 4.2479275134435852f, 4.3219280948873626f,
+    4.3923174227787607f, 4.4594316186372973f, 4.5235619560570131f,
+    4.5849625007211570f, 4.6438561897747244f, 4.7004397181410926f,
+    4.7548875021634691f, 4.8073549220576037f, 4.8579809951275728f,
+    4.9068905956085187f, 4.9541963103868758f, 5.0000000000000000f,
+    5.0443941193584534f, 5.0874628412503400f, 5.1292830169449664f,
+    5.1699250014423122f, 5.2094533656289501f, 5.2479275134435852f,
+    5.2854022188622487f, 5.3219280948873626f, 5.3575520046180838f,
+    5.3923174227787607f, 5.4262647547020979f, 5.4594316186372973f,
+    5.4918530963296748f, 5.5235619560570131f, 5.5545888516776376f,
+    5.5849625007211570f, 5.6147098441152083f, 5.6438561897747244f,
+    5.6724253419714961f, 5.7004397181410926f, 5.7279204545631996f,
+    5.7548875021634691f, 5.7813597135246599f, 5.8073549220576046f,
+    5.8328900141647422f, 5.8579809951275719f, 5.8826430493618416f,
+    5.9068905956085187f, 5.9307373375628867f, 5.9541963103868758f,
+    5.9772799234999168f, 6.0000000000000000f, 6.0223678130284544f,
+    6.0443941193584534f, 6.0660891904577721f, 6.0874628412503400f,
+    6.1085244567781700f, 6.1292830169449672f, 6.1497471195046822f,
+    6.1699250014423122f, 6.1898245588800176f, 6.2094533656289510f,
+    6.2288186904958804f, 6.2479275134435861f, 6.2667865406949019f,
+    6.2854022188622487f, 6.3037807481771031f, 6.3219280948873617f,
+    6.3398500028846252f, 6.3575520046180847f, 6.3750394313469254f,
+    6.3923174227787598f, 6.4093909361377026f, 6.4262647547020979f,
+    6.4429434958487288f, 6.4594316186372982f, 6.4757334309663976f,
+    6.4918530963296748f, 6.5077946401986964f, 6.5235619560570131f,
+    6.5391588111080319f, 6.5545888516776376f, 6.5698556083309478f,
+    6.5849625007211561f, 6.5999128421871278f, 6.6147098441152092f,
+    6.6293566200796095f, 6.6438561897747253f, 6.6582114827517955f,
+    6.6724253419714952f, 6.6865005271832185f, 6.7004397181410917f,
+    6.7142455176661224f, 6.7279204545631988f, 6.7414669864011465f,
+    6.7548875021634691f, 6.7681843247769260f, 6.7813597135246599f,
+    6.7944158663501062f, 6.8073549220576037f, 6.8201789624151887f,
+    6.8328900141647422f, 6.8454900509443757f, 6.8579809951275719f,
+    6.8703647195834048f, 6.8826430493618416f, 6.8948177633079437f,
+    6.9068905956085187f, 6.9188632372745955f, 6.9307373375628867f,
+    6.9425145053392399f, 6.9541963103868758f, 6.9657842846620879f,
+    6.9772799234999168f, 6.9886846867721664f, 7.0000000000000000f,
+    7.0112272554232540f, 7.0223678130284544f, 7.0334230015374501f,
+    7.0443941193584534f, 7.0552824355011898f, 7.0660891904577721f,
+    7.0768155970508317f, 7.0874628412503400f, 7.0980320829605272f,
+    7.1085244567781700f, 7.1189410727235076f, 7.1292830169449664f,
+    7.1395513523987937f, 7.1497471195046822f, 7.1598713367783891f,
+    7.1699250014423130f, 7.1799090900149345f, 7.1898245588800176f,
+    7.1996723448363644f, 7.2094533656289492f, 7.2191685204621621f,
+    7.2288186904958804f, 7.2384047393250794f, 7.2479275134435861f,
+    7.2573878426926521f, 7.2667865406949019f, 7.2761244052742384f,
+    7.2854022188622487f, 7.2946207488916270f, 7.3037807481771031f,
+    7.3128829552843557f, 7.3219280948873617f, 7.3309168781146177f,
+    7.3398500028846243f, 7.3487281542310781f, 7.3575520046180847f,
+    7.3663222142458151f, 7.3750394313469254f, 7.3837042924740528f,
+    7.3923174227787607f, 7.4008794362821844f, 7.4093909361377026f,
+    7.4178525148858991f, 7.4262647547020979f, 7.4346282276367255f,
+    7.4429434958487288f, 7.4512111118323299f, 7.4594316186372973f,
+    7.4676055500829976f, 7.4757334309663976f, 7.4838157772642564f,
+    7.4918530963296748f, 7.4998458870832057f, 7.5077946401986964f,
+    7.5156998382840436f, 7.5235619560570131f, 7.5313814605163119f,
+    7.5391588111080319f, 7.5468944598876373f, 7.5545888516776376f,
+    7.5622424242210728f, 7.5698556083309478f, 7.5774288280357487f,
+    7.5849625007211561f, 7.5924570372680806f, 7.5999128421871278f,
+    7.6073303137496113f, 7.6147098441152075f, 7.6220518194563764f,
+    7.6293566200796095f, 7.6366246205436488f, 7.6438561897747244f,
+    7.6510516911789290f, 7.6582114827517955f, 7.6653359171851765f,
+    7.6724253419714952f, 7.6794800995054464f, 7.6865005271832185f,
+    7.6934869574993252f, 7.7004397181410926f, 7.7073591320808825f,
+    7.7142455176661224f, 7.7210991887071856f, 7.7279204545631996f,
+    7.7347096202258392f, 7.7414669864011465f, 7.7481928495894596f,
+    7.7548875021634691f, 7.7615512324444795f, 7.7681843247769260f,
+    7.7747870596011737f, 7.7813597135246608f, 7.7879025593914317f,
+    7.7944158663501062f, 7.8008998999203047f, 7.8073549220576037f,
+    7.8137811912170374f, 7.8201789624151887f, 7.8265484872909159f,
+    7.8328900141647422f, 7.8392037880969445f, 7.8454900509443757f,
+    7.8517490414160571f, 7.8579809951275719f, 7.8641861446542798f,
+    7.8703647195834048f, 7.8765169465650002f, 7.8826430493618425f,
+    7.8887432488982601f, 7.8948177633079446f, 7.9008668079807496f,
+    7.9068905956085187f, 7.9128893362299619f, 7.9188632372745955f,
+    7.9248125036057813f, 7.9307373375628867f, 7.9366379390025719f,
+    7.9425145053392399f, 7.9483672315846778f, 7.9541963103868758f,
+    7.9600019320680806f, 7.9657842846620870f, 7.9715435539507720f,
+    7.9772799234999168f, 7.9829935746943104f, 7.9886846867721664f,
+    7.9943534368588578f};
+
+// Faster logarithm for small integers, with the property of log2(0) == 0.
+static inline float FastLog2(uint32_t v) {
+  if (v < (sizeof(kLog2Table) / sizeof(kLog2Table[0]))) {
+    return kLog2Table[v];
+  }
+  return static_cast<float>(log2(v));
+}
+
+static inline float FastLog2f(float f) {
+  int exp;
+  float fr = std::frexp(f, &exp);
+  // TODO(veluca): improve constants.
+  return exp + (-1.34752046f * fr + 3.98979143f) * fr - 2.64898502f;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_FAST_LOG_H_
diff --git a/jxl/base/file_io.h b/jxl/base/file_io.h
new file mode 100644
index 000000000..7a2c795d7
--- /dev/null
+++ b/jxl/base/file_io.h
@@ -0,0 +1,107 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_FILE_IO_H_
+#define JXL_BASE_FILE_IO_H_
+
+// Helper functions for reading/writing files.
+
+#include <stdio.h>
+
+#include <string>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Returns extension including the dot, or empty string if none. Assumes
+// filename is not a hidden file (e.g. ".bashrc"). May be called with a pathname
+// if the filename contains a dot and/or no other path component does.
+static inline std::string Extension(const std::string& filename) {
+  const size_t pos = filename.rfind('.');
+  if (pos == std::string::npos) return std::string();
+  return filename.substr(pos);
+}
+
+// RAII, ensures files are closed even when returning early.
+class FileWrapper {
+ public:
+  FileWrapper(const FileWrapper& other) = delete;
+  FileWrapper& operator=(const FileWrapper& other) = delete;
+
+  explicit FileWrapper(const std::string& pathname, const char* mode)
+      : file_(fopen(pathname.c_str(), mode)) {}
+
+  ~FileWrapper() {
+    if (file_ != nullptr) {
+      const int err = fclose(file_);
+      JXL_CHECK(err == 0);
+    }
+  }
+
+  // We intend to use FileWrapper as a replacement of FILE.
+  // NOLINTNEXTLINE(google-explicit-constructor)
+  operator FILE*() const { return file_; }
+
+ private:
+  FILE* const file_;
+};
+
+template <typename ContainerType>
+static inline Status ReadFile(const std::string& pathname,
+                              ContainerType* JXL_RESTRICT bytes) {
+  FileWrapper f(pathname, "rb");
+  if (f == nullptr) return JXL_FAILURE("Failed to open file for reading");
+
+  if (fseek(f, 0, SEEK_END) != 0) return JXL_FAILURE("Failed to seek end");
+  bytes->resize(ftell(f));
+  if (bytes->empty()) return JXL_FAILURE("Zero-length file");
+  if (fseek(f, 0, SEEK_SET) != 0) return JXL_FAILURE("Failed to seek set");
+
+  size_t pos = 0;
+  while (pos < bytes->size()) {
+    // Needed in case ContainerType is std::string, whose data() is const.
+    char* bytes_writable = reinterpret_cast<char*>(&(*bytes)[0]);
+    const size_t bytes_read =
+        fread(bytes_writable + pos, 1, bytes->size() - pos, f);
+    if (bytes_read == 0) return JXL_FAILURE("Failed to read");
+    pos += bytes_read;
+  }
+  JXL_ASSERT(pos == bytes->size());
+  return true;
+}
+
+template <typename ContainerType>
+static inline Status WriteFile(const ContainerType& bytes,
+                               const std::string& pathname) {
+  FileWrapper f(pathname, "wb");
+  if (f == nullptr) return JXL_FAILURE("Failed to open file for writing");
+
+  size_t pos = 0;
+  while (pos < bytes.size()) {
+    const size_t bytes_written =
+        fwrite(bytes.data() + pos, 1, bytes.size() - pos, f);
+    if (bytes_written == 0) return JXL_FAILURE("Failed to write");
+    pos += bytes_written;
+  }
+  JXL_ASSERT(pos == bytes.size());
+
+  return true;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_FILE_IO_H_
diff --git a/jxl/base/iaca.h b/jxl/base/iaca.h
new file mode 100644
index 000000000..83d0d0674
--- /dev/null
+++ b/jxl/base/iaca.h
@@ -0,0 +1,74 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_IACA_H_
+#define JXL_BASE_IACA_H_
+
+#include <hwy/compiler_specific.h>
+
+// IACA (Intel's Code Analyzer) analyzes instruction latencies, but only for
+// code between special markers. These functions embed such markers in an
+// executable, but only for reading via IACA - they deliberately trigger a
+// crash if executed to ensure they are removed in normal builds.
+
+#ifndef JXL_IACA_ENABLED
+#define JXL_IACA_ENABLED 0
+#endif
+
+namespace jxl {
+
+// Call before the region of interest.
+static HWY_INLINE void BeginIACA() {
+#if JXL_IACA_ENABLED && (JXL_COMPILER_GCC || JXL_COMPILER_CLANG)
+  asm volatile(
+      // UD2 "instruction" raises an invalid opcode exception.
+      ".byte 0x0F, 0x0B\n\t"
+      // Magic sequence recognized by IACA (MOV + addr32 fs:NOP). This actually
+      // clobbers EBX, but we don't care because the code won't be run, and we
+      // want IACA to observe the same code the compiler would have generated
+      // without this marker.
+      "movl $111, %%ebx\n\t"
+      ".byte 0x64, 0x67, 0x90\n\t"
+      :
+      :
+      // (Allegedly) clobbering memory may prevent reordering.
+      : "memory");
+#endif
+}
+
+// Call after the region of interest.
+static HWY_INLINE void EndIACA() {
+#if JXL_IACA_ENABLED && (JXL_COMPILER_GCC || JXL_COMPILER_CLANG)
+  asm volatile(
+      // See above.
+      "movl $222, %%ebx\n\t"
+      ".byte 0x64, 0x67, 0x90\n\t"
+      // UD2
+      ".byte 0x0F, 0x0B\n\t"
+      :
+      :
+      // (Allegedly) clobbering memory may prevent reordering.
+      : "memory");
+#endif
+}
+
+// Add to a scope to mark a region.
+struct ScopeIACA {
+  HWY_INLINE ScopeIACA() { BeginIACA(); }
+  HWY_INLINE ~ScopeIACA() { EndIACA(); }
+};
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_IACA_H_
diff --git a/jxl/base/os_specific.cc b/jxl/base/os_specific.cc
new file mode 100644
index 000000000..b581ee6b1
--- /dev/null
+++ b/jxl/base/os_specific.cc
@@ -0,0 +1,366 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/os_specific.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <ctime>
+#include <random>
+
+#include "jxl/base/arch_specific.h"  // ProcessorTopology
+
+#if defined(_WIN32) || defined(_WIN64)
+#define OS_WIN 1
+#define NOMINMAX
+#include <windows.h>
+#else
+#define OS_WIN 0
+#endif
+
+#ifdef __linux__
+#define OS_LINUX 1
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+#include <pthread.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#else
+#define OS_LINUX 0
+#endif
+
+#ifdef __MACH__
+#define OS_MAC 1
+#include <mach/mach.h>
+#include <mach/mach_time.h>
+#include <mach/thread_act.h>
+#include <mach/thread_policy.h>
+#include <sys/sysctl.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#else
+#define OS_MAC 0
+#endif
+
+#ifdef __FreeBSD__
+#define OS_FREEBSD 1
+#include <sys/cpuset.h>
+#include <sys/param.h>
+#include <sys/types.h>
+#include <sys/wait.h>
+#include <unistd.h>
+#else
+#define OS_FREEBSD 0
+#endif
+
+namespace jxl {
+
+double Now() {
+#if OS_WIN
+  LARGE_INTEGER counter;
+  (void)QueryPerformanceCounter(&counter);
+  LARGE_INTEGER freq;
+  (void)QueryPerformanceFrequency(&freq);
+  return double(counter.QuadPart) / freq.QuadPart;
+#elif OS_MAC
+  const auto t = mach_absolute_time();
+  // On OSX/iOS platform the elapsed time is cpu time unit
+  // We have to query the time base information to convert it back
+  // See https://developer.apple.com/library/mac/qa/qa1398/_index.html
+  static mach_timebase_info_data_t timebase;
+  if (timebase.denom == 0) {
+    (void)mach_timebase_info(&timebase);
+  }
+  return double(t) * timebase.numer / timebase.denom * 1E-9;
+#else
+  timespec t;
+  clock_gettime(CLOCK_MONOTONIC, &t);
+  return t.tv_sec + t.tv_nsec * 1E-9;
+#endif
+}
+
+// Emulate Linux type (cpu_set_t) + interface on other platforms
+
+#if OS_FREEBSD
+using cpu_set_t = cpuset_t;
+#elif OS_WIN || OS_MAC
+using cpu_set_t = uint64_t;
+
+static inline void CPU_ZERO(cpu_set_t* set) { *set = 0; }
+
+static inline int CPU_ISSET(int cpu, const cpu_set_t* set) {
+  return (*set & (1ULL << cpu)) != 0;
+}
+
+static inline void CPU_SET(int cpu, cpu_set_t* set) { *set |= (1ULL << cpu); }
+#endif
+
+struct ThreadAffinity {
+  cpu_set_t set;
+};
+
+#if OS_MAC
+namespace {
+Status GetSystemValue(const char* name, size_t* value) {
+  int64_t value_i64 = 0;
+  size_t size = sizeof(value_i64);
+  const int err = sysctlbyname(name, &value_i64, &size, nullptr, 0);
+  if (err != 0) return JXL_FAILURE("sysctl packages failed");
+  JXL_ASSERT(value_i64 >= 0);
+  *value = static_cast<size_t>(value_i64);
+  return true;
+}
+
+// Returns mask with the lowest N bits set, one per logical processor.
+cpu_set_t SetOfAllLogicalProcessors() {
+  size_t logical;
+  // On failure, assume there is at least one logical processor.
+  if (!GetSystemValue("machdep.cpu.thread_count", &logical)) return 1;
+
+  if (logical > 64) {
+    printf("Warning: more than 64 logical processors, update cpu_set_t");
+    return ~0ull;
+  }
+  if (logical == 64) return ~0ull;
+
+  return (1ull << logical) - 1;
+}
+}  // namespace
+#endif
+
+Status GetProcessorTopologyFromOS(ProcessorTopology* pt) {
+#if OS_MAC
+  size_t packages, cores, logical;  // totals, not per package/core!
+  JXL_RETURN_IF_ERROR(GetSystemValue("hw.packages", &packages));
+  JXL_RETURN_IF_ERROR(GetSystemValue("machdep.cpu.core_count", &cores));
+  JXL_RETURN_IF_ERROR(GetSystemValue("machdep.cpu.thread_count", &logical));
+
+  // All succeeded: now set `pt`
+  pt->packages = packages;
+  pt->cores_per_package = cores / packages;
+  pt->logical_per_core = logical / cores;
+
+  return true;
+#else
+  // Not needed on X64 if the affinity APIs work (DetectProcessorTopology will
+  // succeed)
+  return false;
+#endif
+}
+
+ThreadAffinity* GetThreadAffinity() {
+  ThreadAffinity* affinity =
+      static_cast<ThreadAffinity*>(malloc(sizeof(ThreadAffinity)));
+#if OS_WIN
+  DWORD_PTR process_affinity, system_affinity;
+  const BOOL ok = GetProcessAffinityMask(GetCurrentProcess(), &process_affinity,
+                                         &system_affinity);
+  JXL_CHECK(ok);
+  affinity->set = process_affinity;
+#elif OS_LINUX
+  CPU_ZERO(&affinity->set);
+  const int err =
+      pthread_getaffinity_np(pthread_self(), sizeof(cpu_set_t), &affinity->set);
+  JXL_CHECK(err == 0);
+#elif OS_FREEBSD
+  const pid_t pid = getpid();  // current thread
+  const int err = cpuset_getaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid,
+                                     sizeof(cpuset_t), &affinity->set);
+  JXL_CHECK(err == 0);
+#elif OS_MAC
+  static cpu_set_t all = SetOfAllLogicalProcessors();
+  affinity->set = all;
+#endif
+  return affinity;
+}
+
+namespace {
+
+ThreadAffinity* OriginalThreadAffinity() {
+  static ThreadAffinity* original = GetThreadAffinity();
+  return original;
+}
+
+}  // namespace
+
+Status SetThreadAffinity(ThreadAffinity* affinity) {
+  // Ensure original is initialized before changing.
+  const ThreadAffinity* const original = OriginalThreadAffinity();
+  JXL_CHECK(original != nullptr);
+
+#if OS_WIN
+  const HANDLE hThread = GetCurrentThread();
+  const DWORD_PTR prev = SetThreadAffinityMask(hThread, affinity->set);
+  if (prev == 0) return JXL_FAILURE("SetThreadAffinityMask failed");
+  return true;
+#elif OS_LINUX
+  const int err =
+      pthread_setaffinity_np(pthread_self(), sizeof(cpu_set_t), &affinity->set);
+  if (err != 0) return JXL_FAILURE("pthread_setaffinity_np failed");
+  return true;
+#elif OS_FREEBSD
+  const pid_t pid = getpid();  // current thread
+  const int err = cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_PID, pid,
+                                     sizeof(cpuset_t), &affinity->set);
+  if (err != 0) return JXL_FAILURE("cpuset_setaffinity failed");
+  return true;
+#elif OS_MAC
+  // As of 2019-03 we are unaware of a way to reliably restrict a thread to
+  // core(s); THREAD_AFFINITY_POLICY is only a hint.
+  (void)affinity;
+  return false;
+#else
+  printf("Don't know how to SetThreadAffinity on this platform.\n");
+  return false;
+#endif
+}
+
+std::vector<int> AvailableCPUs() {
+  std::vector<int> cpus;
+  cpus.reserve(128);
+#if OS_WIN || OS_LINUX || OS_FREEBSD || OS_MAC
+  const ThreadAffinity* const affinity = OriginalThreadAffinity();
+  for (int cpu = 0; cpu < static_cast<int>(sizeof(cpu_set_t)) * 8; ++cpu) {
+    if (CPU_ISSET(cpu, &affinity->set)) {
+      cpus.push_back(static_cast<int>(cpu));
+    }
+  }
+#else
+  cpus.push_back(0);
+#endif
+  return cpus;
+}
+
+Status PinThreadToCPU(const int cpu) {
+#if OS_WIN || OS_LINUX || OS_FREEBSD || OS_MAC
+  ThreadAffinity affinity;
+  CPU_ZERO(&affinity.set);
+  CPU_SET(cpu, &affinity.set);
+  return SetThreadAffinity(&affinity);
+#else
+  return false;
+#endif
+}
+
+Status PinThreadToRandomCPU() {
+  std::vector<int> cpus = AvailableCPUs();
+
+  // Remove first two CPUs because interrupts are often pinned to them.
+  JXL_CHECK(cpus.size() > 2);
+  cpus.erase(cpus.begin(), cpus.begin() + 2);
+
+  // Random choice to prevent burning up the same core.
+  std::random_device device;
+  std::ranlux48 generator(device());
+  std::shuffle(cpus.begin(), cpus.end(), generator);
+  const int cpu = cpus.front();
+
+  return PinThreadToCPU(cpu);
+}
+
+namespace {
+
+size_t DetectTotalMemoryMiB() {
+#if OS_LINUX || OS_FREEBSD || OS_MAC
+  const long page_size = sysconf(_SC_PAGESIZE);
+  const long num_pages = sysconf(_SC_PHYS_PAGES);
+  if (page_size == -1 || num_pages == -1) {
+    JXL_WARNING("Failed to detect page size (%ld) and/or num pages (%ld)",
+                page_size, num_pages);
+    return 0;
+  }
+  JXL_ASSERT(page_size > 0 && num_pages > 0);
+  const uint64_t bytes =
+      static_cast<uint64_t>(num_pages) * static_cast<uint64_t>(page_size);
+  return bytes >> 20;
+#elif OS_WIN
+  MEMORYSTATUSEX ms;
+  ms.dwLength = sizeof(ms);
+  if (!GlobalMemoryStatusEx(&ms)) {
+    JXL_WARNING("Failed to get memory status");
+    return 0;
+  }
+  const uint64_t bytes = ms.ullTotalPhys;
+  // `bytes` excludes nonpaged pool reserved during boot; round up to whole MiB
+  // to improve the estimate.
+  return (bytes + (1U << 20) - 1) >> 20;
+#else
+  JXL_WARNING("Implement DetectTotalMemoryMiB for this platform");
+  return 0;
+#endif
+}
+
+}  // namespace
+
+size_t TotalMemoryMiB() {
+  static size_t mib = DetectTotalMemoryMiB();
+  return mib;
+}
+
+/*
+Status RunCommand(const std::vector<std::string>& args) {
+#if _POSIX_VERSION >= 200112L
+  // Avoid system(), but do not try to be over-zealous about not passing along
+  // some special resources further (such as: inherited-not-marked-FD_CLOEXEC
+  // file descriptors).
+  std::vector<const char*> c_args;
+  c_args.reserve(args.size() + 1);
+  for (size_t i = 0; i < args.size(); ++i) {
+    c_args.push_back(args[i].c_str());
+  }
+  c_args.push_back(nullptr);
+  const pid_t pid = fork();
+  if (pid == -1)  // fork() failed.
+    return false;
+  if (pid != 0) {  // Parent process.
+    int ret_status;
+    if (pid != waitpid(pid, &ret_status, 0)) {
+      return false;  // waitpid() error.
+    }
+    return ret_status == 0;
+  } else {  // Child process.
+    execvp(c_args[0],
+           // Address benign-but-annoying execvp() signature weirdness.
+           const_cast<char* const*>(c_args.data()));
+    JXL_ABORT("Failed to run command.\n");
+  }
+#elif OS_WIN
+  // Synthesize a string for system(). And warn about it.
+  // TODO(user): Fix this - research the safe way to run a command on Windows.
+  // Likely, the solution is along these lines:
+  // https://docs.microsoft.com/en-us/windows/desktop/ProcThread/creating-processes
+  std::ostringstream cmd;
+  std::copy(args.begin(), args.end(),
+            std::ostream_iterator<std::string>(cmd, " "));
+  printf(stderr, "Warning: Using system() on string: %s\n", cmd.str.c_str());
+  int ret = system(cmd.str.c_str());
+  if (errno != ENOENT &&  // Windows: Command interpreter not found.
+      ret == 0) {
+    return true;
+  }
+  return false;
+#else
+#error Neither a POSIX-1.2001 nor a Windows System.
+#endif
+}
+*/
+
+}  // namespace jxl
diff --git a/jxl/base/os_specific.h b/jxl/base/os_specific.h
new file mode 100644
index 000000000..2c0b028c7
--- /dev/null
+++ b/jxl/base/os_specific.h
@@ -0,0 +1,72 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_OS_SPECIFIC_H_
+#define JXL_BASE_OS_SPECIFIC_H_
+
+// OS-specific functions (e.g. timing and thread affinity)
+
+#include <stddef.h>
+
+#include <string>
+#include <vector>
+
+#include "jxl/base/arch_specific.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Returns current time [seconds] from a monotonic clock with unspecified
+// starting point - only suitable for computing elapsed time.
+double Now();
+
+// Called by arch_specific. Returns false if `pt` remains unchanged. Only
+// implemented/needed on OSX.
+struct ProcessorTopology;
+
+Status GetProcessorTopologyFromOS(ProcessorTopology* pt);
+
+// Returns logical processor numbers in [0, N), where N is the number of bits in
+// the thread's initial affinity (unaffected by any SetThreadAffinity).
+std::vector<int> AvailableCPUs();
+
+// Opaque.
+struct ThreadAffinity;
+
+// Returns current affinity; useful for restoring the original value.
+// Caller must free() the pointer - dynamic allocation is required because
+// ThreadAffinity is an incomplete type.
+ThreadAffinity* GetThreadAffinity();
+
+// Restores a previous affinity returned by GetThreadAffinity.
+Status SetThreadAffinity(ThreadAffinity* affinity);
+
+// Ensures the thread is running on the specified cpu, and no others.
+// Useful for reducing nanobenchmark variability (fewer context switches).
+// Calls SetThreadAffinity.
+Status PinThreadToCPU(int cpu);
+
+// Random choice of CPU avoids overloading any one core. Calls PinThreadToCPU.
+Status PinThreadToRandomCPU();
+
+// Returns total physical memory size [MiB], or 0 if unknown. This function
+// returns a cached value initialized on the first call.
+size_t TotalMemoryMiB();
+
+// Executes a command in a subprocess.
+//Status RunCommand(const std::vector<std::string>& args);
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_OS_SPECIFIC_H_
diff --git a/jxl/base/override.h b/jxl/base/override.h
new file mode 100644
index 000000000..e0a59b3f2
--- /dev/null
+++ b/jxl/base/override.h
@@ -0,0 +1,38 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_OVERRIDE_H_
+#define JXL_BASE_OVERRIDE_H_
+
+// 'Trool' for command line arguments: force enable/disable, or use default.
+
+namespace jxl {
+
+// No effect if kDefault, otherwise forces a feature (typically a FrameHeader
+// flag) on or off.
+enum class Override : int { kOn = 1, kOff = 0, kDefault = -1 };
+
+static inline Override OverrideFromBool(bool flag) {
+  return flag ? Override::kOn : Override::kOff;
+}
+
+static inline bool ApplyOverride(Override o, bool default_condition) {
+  if (o == Override::kOn) return true;
+  if (o == Override::kOff) return false;
+  return default_condition;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_OVERRIDE_H_
diff --git a/jxl/base/padded_bytes.cc b/jxl/base/padded_bytes.cc
new file mode 100644
index 000000000..e907125d5
--- /dev/null
+++ b/jxl/base/padded_bytes.cc
@@ -0,0 +1,47 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/padded_bytes.h"
+
+#include <string.h>
+
+namespace jxl {
+
+void PaddedBytes::IncreaseCapacityTo(size_t capacity) {
+  JXL_ASSERT(capacity > capacity_);
+
+  // BitWriter writes up to 7 bytes past the end.
+  CacheAlignedUniquePtr new_data = AllocateArray(capacity + 8);
+  if (new_data == nullptr) {
+    // Allocation failed, discard all data to ensure this is noticed.
+    size_ = capacity_ = 0;
+    return;
+  }
+
+  if (data_ == nullptr) {
+    // First allocation: ensure first byte is initialized (won't be copied).
+    new_data[0] = 0;
+  } else {
+    // Subsequent resize: copy existing data to new location.
+    memcpy(new_data.get(), data_.get(), size_);
+    // Ensure that the first new byte is initialized, to allow write_bits to
+    // safely append to the newly-resized PaddedBytes.
+    new_data[size_] = 0;
+  }
+
+  capacity_ = capacity;
+  std::swap(new_data, data_);
+}
+
+}  // namespace jxl
diff --git a/jxl/base/padded_bytes.h b/jxl/base/padded_bytes.h
new file mode 100644
index 000000000..4dfac380f
--- /dev/null
+++ b/jxl/base/padded_bytes.h
@@ -0,0 +1,203 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_PADDED_BYTES_H_
+#define JXL_BASE_PADDED_BYTES_H_
+
+// std::vector replacement with padding to reduce bounds checks in WriteBits
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include <algorithm>  // std::max
+#include <memory>
+#include <utility>
+
+#include "jxl/base/cache_aligned.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Provides a subset of the std::vector interface with some differences:
+// - allows BitWriter to write 64 bits at a time without bounds checking;
+// - ONLY zero-initializes the first byte (required by BitWriter);
+// - ensures cache-line alignment.
+class PaddedBytes {
+ public:
+  // Required for output params.
+  PaddedBytes() : size_(0), capacity_(0) {}
+
+  explicit PaddedBytes(size_t size) : size_(size), capacity_(0) {
+    if (size != 0) IncreaseCapacityTo(size);
+  }
+
+  PaddedBytes(size_t size, uint8_t value) : size_(size), capacity_(0) {
+    if (size != 0) {
+      IncreaseCapacityTo(size);
+    }
+    if (size_ != 0) {
+      memset(data(), value, size);
+    }
+  }
+
+  PaddedBytes(const PaddedBytes& other) : size_(other.size_), capacity_(0) {
+    if (size_ != 0) IncreaseCapacityTo(size_);
+    if (data() != nullptr) memcpy(data(), other.data(), size_);
+  }
+  PaddedBytes& operator=(const PaddedBytes& other) {
+    // Self-assignment is safe.
+    resize(other.size());
+    if (data() != nullptr) memmove(data(), other.data(), size_);
+    return *this;
+  }
+
+  // default is not OK - need to set other.size_ to 0!
+  PaddedBytes(PaddedBytes&& other) noexcept
+      : size_(other.size_),
+        capacity_(other.capacity_),
+        data_(std::move(other.data_)) {
+    other.size_ = other.capacity_ = 0;
+  }
+  PaddedBytes& operator=(PaddedBytes&& other) noexcept {
+    size_ = other.size_;
+    capacity_ = other.capacity_;
+    data_ = std::move(other.data_);
+
+    if (&other != this) {
+      other.size_ = other.capacity_ = 0;
+    }
+    return *this;
+  }
+
+  void swap(PaddedBytes& other) {
+    std::swap(size_, other.size_);
+    std::swap(capacity_, other.capacity_);
+    std::swap(data_, other.data_);
+  }
+
+  void reserve(size_t capacity) {
+    if (capacity > capacity_) IncreaseCapacityTo(capacity);
+  }
+  // NOTE: unlike vector, this does not initialize the new data!
+  // However, we guarantee that write_bits can safely append after
+  // the resize, as we zero-initialize the first new byte of data.
+  void resize(size_t size) {
+    if (size > capacity_) IncreaseCapacityTo(size);
+    size_ = (data() == nullptr) ? 0 : size;
+  }
+
+  // Explicitely initializes the new data with the value.
+  void resize(size_t size, uint8_t value) {
+    size_t old_size = size_;
+    resize(size);
+    if (size_ > old_size) {
+      memset(data() + old_size, value, size_ - old_size);
+    }
+  }
+
+  // Amortized constant complexity due to exponential growth.
+  void push_back(uint8_t x) {
+    if (size_ == capacity_) {
+      IncreaseCapacityTo(std::max<size_t>(3 * capacity_ / 2, 64));
+      if (data() == nullptr) return;
+    }
+
+    data_[size_++] = x;
+  }
+
+  size_t size() const { return size_; }
+  size_t capacity() const { return capacity_; }
+
+  uint8_t* data() { return data_.get(); }
+  const uint8_t* data() const { return data_.get(); }
+
+  // std::vector operations implemented in terms of the public interface above.
+
+  void clear() { resize(0); }
+  bool empty() const { return size() == 0; }
+
+  void assign(std::initializer_list<uint8_t> il) {
+    resize(il.size());
+    memcpy(data(), il.begin(), il.size());
+  }
+
+  void assign(const uint8_t* begin, const uint8_t* end) {
+    resize(end - begin);
+    memcpy(data(), begin, end - begin);
+  }
+
+  uint8_t* begin() { return data(); }
+  const uint8_t* begin() const { return data(); }
+  uint8_t* end() { return begin() + size(); }
+  const uint8_t* end() const { return begin() + size(); }
+
+  uint8_t& operator[](const size_t i) {
+    BoundsCheck(i);
+    return data()[i];
+  }
+  const uint8_t& operator[](const size_t i) const {
+    BoundsCheck(i);
+    return data()[i];
+  }
+
+  uint8_t& back() {
+    JXL_ASSERT(size() != 0);
+    return data()[size() - 1];
+  }
+  const uint8_t& back() const {
+    JXL_ASSERT(size() != 0);
+    return data()[size() - 1];
+  }
+
+  template <typename T>
+  void append(const T& other) {
+    size_t old_size = size();
+    resize(size() + other.size());
+    memcpy(data() + old_size, other.data(), other.size());
+  }
+
+  void append(const uint8_t* begin, const uint8_t* end) {
+    size_t old_size = size();
+    resize(size() + end - begin);
+    memcpy(data() + old_size, begin, end - begin);
+  }
+
+ private:
+  void BoundsCheck(size_t i) const {
+    // <= is safe due to padding and required by BitWriter.
+    JXL_ASSERT(i <= size());
+  }
+
+  // Copies existing data to newly allocated "data_". If allocation fails,
+  // data() == nullptr and size_ = capacity_ = 0.
+  void IncreaseCapacityTo(size_t capacity);
+
+  size_t size_;
+  size_t capacity_;
+  CacheAlignedUniquePtr data_;
+};
+
+template <typename T>
+static inline void Append(const T& s, PaddedBytes* out,
+                          size_t* JXL_RESTRICT byte_pos) {
+  memcpy(out->data() + *byte_pos, s.data(), s.size());
+  *byte_pos += s.size();
+  JXL_CHECK(*byte_pos <= out->size());
+}
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_PADDED_BYTES_H_
diff --git a/jxl/base/profiler.h b/jxl/base/profiler.h
new file mode 100644
index 000000000..37418a220
--- /dev/null
+++ b/jxl/base/profiler.h
@@ -0,0 +1,753 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_PROFILER_H_
+#define JXL_BASE_PROFILER_H_
+
+// High precision, low overhead time measurements. Returns exact call counts and
+// total elapsed time for user-defined 'zones' (code regions, i.e. C++ scopes).
+//
+// Usage: add this header to BUILD srcs; instrument regions of interest:
+// { PROFILER_ZONE("name"); /*code*/ } or
+// void FuncToMeasure() { PROFILER_FUNC; /*code*/ }.
+// After all threads have exited any zones, invoke PROFILER_PRINT_RESULTS() to
+// print call counts and average durations [CPU cycles] to stdout, sorted in
+// descending order of total duration.
+
+// Configuration settings:
+
+// If zero, this file has no effect and no measurements will be recorded.
+#ifndef PROFILER_ENABLED
+#define PROFILER_ENABLED 0
+#endif
+
+// How many mebibytes to allocate (if PROFILER_ENABLED) per thread that
+// enters at least one zone. Once this buffer is full, the thread will analyze
+// and discard packets, thus temporarily adding some observer overhead.
+// Each zone occupies 16 bytes.
+#ifndef PROFILER_THREAD_STORAGE
+#define PROFILER_THREAD_STORAGE 32ULL
+#endif
+
+#if PROFILER_ENABLED
+
+#define PROFILER_PRINT_OVERHEAD 0
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>  // memcpy
+
+#include <algorithm>  // min/max
+#include <atomic>
+#include <cinttypes>  // PRIu64
+#include <hwy/static_targets.h>
+#include <new>
+
+#include "jxl/base/arch_specific.h"
+#include "jxl/base/cache_aligned.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/robust_statistics.h"
+#include "jxl/base/status.h"
+#include "jxl/base/tsc_timer.h"
+
+// Non-portable aspects:
+// - SSE2 128-bit load/store (write-combining, UpdateOrAdd)
+// - RDTSCP timestamps (serializing, high-resolution)
+// - assumes string literals are stored within an 8 MiB range
+// - compiler-specific annotations (restrict, alignment, fences)
+#if JXL_ARCH_X64
+#include <emmintrin.h>
+#if JXL_COMPILER_MSVC
+#include <intrin.h>
+#else
+#include <x86intrin.h>
+#endif
+#endif
+
+namespace jxl {
+
+// Upper bounds for various fixed-size data structures (guarded via JXL_ASSERT):
+
+// How many unique threads can enter a zone (those that don't do not count).
+// Memory use is about kMaxThreads * PROFILER_THREAD_STORAGE MiB.
+// WARNING: fiber libraries and multiple ThreadPool can spawn >100 threads.
+static constexpr size_t kMaxThreads = 256;
+
+// Maximum nesting of zones.
+static constexpr size_t kMaxDepth = 64;
+
+// Total number of zones.
+static constexpr size_t kMaxZones = 256;
+
+// Represents zone entry/exit events. Stores a full-resolution timestamp plus
+// an offset (representing zone name or identifying exit packets). POD.
+class Packet {
+ public:
+  // If offsets do not fit, UpdateOrAdd will overrun our heap allocation
+  // (governed by kMaxZones). We have seen ~100 MiB static binaries.
+  static constexpr size_t kOffsetBits = 27;
+  static constexpr uintptr_t kOffsetBias = 1ULL << (kOffsetBits - 1);
+
+  // We need full-resolution timestamps; at an effective rate of 4 GHz,
+  // this permits 34 second zone durations (for longer durations, split into
+  // multiple zones). Wraparound is handled by masking.
+  static constexpr size_t kTimestampBits = 64 - kOffsetBits;
+  static constexpr uintptr_t kTimestampMask = (1ULL << kTimestampBits) - 1;
+
+  static Packet Make(const uintptr_t biased_offset, const uint64_t timestamp) {
+    JXL_ASSERT(biased_offset < (1ULL << kOffsetBits));
+
+    Packet packet;
+    packet.bits_ =
+        (biased_offset << kTimestampBits) + (timestamp & kTimestampMask);
+    return packet;
+  }
+
+  uint64_t Timestamp() const { return bits_ & kTimestampMask; }
+
+  uintptr_t BiasedOffset() const { return (bits_ >> kTimestampBits); }
+
+ private:
+  uint64_t bits_;
+};
+static_assert(sizeof(Packet) == 8, "Wrong Packet size");
+
+// Returns the address of a string literal. Assuming zone names are also
+// literals and stored nearby, we can represent them as offsets, which are
+// faster to compute than hashes or even a static index.
+//
+// This function must not be static - each call (even from other translation
+// units) must return the same value.
+inline uintptr_t StringOrigin() {
+  // Chosen such that no zone name is a prefix nor suffix of this string
+  // to ensure they aren't merged (offset 0 identifies zone-exit packets).
+  static const char* string_origin = "__#Origin#__";
+
+  return reinterpret_cast<uintptr_t>(string_origin) - Packet::kOffsetBias;
+}
+
+// Representation of an active zone, stored in a stack. Used to deduct
+// child duration from the parent's self time. POD.
+struct ProfilerNode {
+  Packet packet;
+  uint64_t child_total;
+};
+
+// Holds statistics for all zones with the same name. POD.
+struct Accumulator {
+  static constexpr size_t kNumCallBits = 64 - Packet::kOffsetBits;
+
+  uintptr_t BiasedOffset() const { return num_calls >> kNumCallBits; }
+  uint64_t NumCalls() const { return num_calls & ((1ULL << kNumCallBits) - 1); }
+
+  // UpdateOrAdd relies upon this layout.
+  uint64_t num_calls = 0;  // upper bits = biased_offset.
+  uint64_t total_duration = 0;
+};
+#if JXL_ARCH_X64
+static_assert(sizeof(Accumulator) == 2 * sizeof(uint64_t), "Accumulator size");
+#endif
+
+template <typename T>
+inline T ClampedSubtract(const T minuend, const T subtrahend) {
+  if (subtrahend > minuend) {
+    return 0;
+  }
+  return minuend - subtrahend;
+}
+
+// Per-thread call graph (stack) and Accumulator for each zone.
+class Results {
+ public:
+  Results() {
+    // Zero-initialize first accumulator to avoid a check for num_zones_ == 0.
+    memset(zones_, 0, sizeof(Accumulator));
+  }
+
+  // Used for computing overhead when this thread encounters its first Zone.
+  // This has no observable effect apart from increasing "analyze_elapsed_".
+  uint64_t ZoneDuration(const Packet* packets) {
+    JXL_CHECK(depth_ == 0);
+    JXL_CHECK(num_zones_ == 0);
+    AnalyzePackets(packets, 2);
+    const uint64_t duration = zones_[0].total_duration;
+    zones_[0].num_calls = 0;
+    zones_[0].total_duration = 0;
+    JXL_CHECK(depth_ == 0);
+    num_zones_ = 0;
+    return duration;
+  }
+
+  void SetSelfOverhead(const uint64_t self_overhead) {
+    self_overhead_ = self_overhead;
+  }
+
+  void SetChildOverhead(const uint64_t child_overhead) {
+    child_overhead_ = child_overhead;
+  }
+
+  // Draw all required information from the packets, which can be discarded
+  // afterwards. Called whenever this thread's storage is full.
+  void AnalyzePackets(const Packet* packets, const size_t num_packets) {
+    const uint64_t t0 = TicksBefore();
+
+    for (size_t i = 0; i < num_packets; ++i) {
+      const Packet p = packets[i];
+      // Entering a zone
+      if (p.BiasedOffset() != Packet::kOffsetBias) {
+        JXL_ASSERT(depth_ < kMaxDepth);
+        nodes_[depth_].packet = p;
+        nodes_[depth_].child_total = 0;
+        ++depth_;
+        continue;
+      }
+
+      JXL_ASSERT(depth_ != 0);
+      const ProfilerNode& node = nodes_[depth_ - 1];
+      // Masking correctly handles unsigned wraparound.
+      const uint64_t duration =
+          (p.Timestamp() - node.packet.Timestamp()) & Packet::kTimestampMask;
+      const uint64_t self_duration = ClampedSubtract(
+          duration, self_overhead_ + child_overhead_ + node.child_total);
+
+      UpdateOrAdd(node.packet.BiasedOffset(), 1, self_duration);
+      --depth_;
+
+      // Deduct this nested node's time from its parent's self_duration.
+      if (depth_ != 0) {
+        nodes_[depth_ - 1].child_total += duration + child_overhead_;
+      }
+    }
+
+    const uint64_t t1 = TicksAfter();
+    analyze_elapsed_ += t1 - t0;
+  }
+
+  // Incorporates results from another thread. Call after all threads have
+  // exited any zones.
+  void Assimilate(const Results& other) {
+    const uint64_t t0 = TicksBefore();
+    JXL_ASSERT(depth_ == 0);
+    JXL_ASSERT(other.depth_ == 0);
+
+    for (size_t i = 0; i < other.num_zones_; ++i) {
+      const Accumulator& zone = other.zones_[i];
+      UpdateOrAdd(zone.BiasedOffset(), zone.NumCalls(), zone.total_duration);
+    }
+    const uint64_t t1 = TicksAfter();
+    analyze_elapsed_ += t1 - t0 + other.analyze_elapsed_;
+  }
+
+  // Single-threaded.
+  void Print() {
+    const uint64_t t0 = TicksBefore();
+    MergeDuplicates();
+
+    // Sort by decreasing total (self) cost.
+    std::sort(zones_, zones_ + num_zones_,
+              [](const Accumulator& r1, const Accumulator& r2) {
+                return r1.total_duration > r2.total_duration;
+              });
+
+    const uintptr_t string_origin = StringOrigin();
+    uint64_t total_visible_duration = 0;
+    for (size_t i = 0; i < num_zones_; ++i) {
+      const Accumulator& r = zones_[i];
+      const uint64_t num_calls = r.NumCalls();
+      const char* name =
+          reinterpret_cast<const char*>(string_origin + r.BiasedOffset());
+      if (name[0] != '@') {
+        total_visible_duration += r.total_duration;
+        printf("%-40s: %10" PRIu64 " x %15" PRIu64 "= %15" PRIu64 "\n", name,
+               num_calls, r.total_duration / num_calls, r.total_duration);
+      }
+    }
+
+    const uint64_t t1 = TicksAfter();
+    analyze_elapsed_ += t1 - t0;
+    printf("Total clocks during analysis: %" PRIu64 "\n", analyze_elapsed_);
+    printf("Total clocks measured: %" PRIu64 "\n", total_visible_duration);
+  }
+
+  // Single-threaded. Clears all results as if no zones had been recorded.
+  void Reset() {
+    analyze_elapsed_ = 0;
+    JXL_CHECK(depth_ == 0);
+    num_zones_ = 0;
+    memset(nodes_, 0, sizeof(nodes_));
+    memset(zones_, 0, sizeof(zones_));
+  }
+
+ private:
+#if JXL_ARCH_X64
+  static bool SameOffset(const __m128i zone, const uintptr_t biased_offset) {
+    const uint64_t num_calls = _mm_cvtsi128_si64(zone);
+    return (num_calls >> Accumulator::kNumCallBits) == biased_offset;
+  }
+#endif
+
+  // Updates an existing Accumulator (uniquely identified by biased_offset) or
+  // adds one if this is the first time this thread analyzed that zone.
+  // Uses a self-organizing list data structure, which avoids dynamic memory
+  // allocations and is far faster than unordered_map. Loads, updates and
+  // stores the entire Accumulator with vector instructions.
+  void UpdateOrAdd(const uintptr_t biased_offset, const uint64_t num_calls,
+                   const uint64_t duration) {
+    JXL_ASSERT(biased_offset < (1ULL << Packet::kOffsetBits));
+
+#if JXL_ARCH_X64
+    const __m128i num_calls_64 = _mm_cvtsi64_si128(num_calls);
+    const __m128i duration_64 = _mm_cvtsi64_si128(duration);
+    const __m128i add_duration_call =
+        _mm_unpacklo_epi64(num_calls_64, duration_64);
+
+    __m128i* const JXL_RESTRICT zones = reinterpret_cast<__m128i*>(zones_);
+
+    // Special case for first zone: (maybe) update, without swapping.
+    __m128i prev = _mm_load_si128(zones);
+    if (SameOffset(prev, biased_offset)) {
+      prev = _mm_add_epi64(prev, add_duration_call);
+      JXL_ASSERT(SameOffset(prev, biased_offset));
+      _mm_store_si128(zones, prev);
+      return;
+    }
+
+    // Look for a zone with the same offset.
+    for (size_t i = 1; i < num_zones_; ++i) {
+      __m128i zone = _mm_load_si128(zones + i);
+      if (SameOffset(zone, biased_offset)) {
+        zone = _mm_add_epi64(zone, add_duration_call);
+        JXL_ASSERT(SameOffset(zone, biased_offset));
+        // Swap with predecessor (more conservative than move to front,
+        // but at least as successful).
+        _mm_store_si128(zones + i - 1, zone);
+        _mm_store_si128(zones + i, prev);
+        return;
+      }
+      prev = zone;
+    }
+
+    // Not found; create a new Accumulator.
+    const __m128i biased_offset_64 = _mm_slli_epi64(
+        _mm_cvtsi64_si128(biased_offset), Accumulator::kNumCallBits);
+    const __m128i zone = _mm_add_epi64(biased_offset_64, add_duration_call);
+    JXL_ASSERT(SameOffset(zone, biased_offset));
+
+    JXL_ASSERT(num_zones_ < kMaxZones);
+    _mm_store_si128(zones + num_zones_, zone);
+    ++num_zones_;
+#else
+    // Special case for first zone: (maybe) update, without swapping.
+    if (zones_[0].BiasedOffset() == biased_offset) {
+      zones_[0].total_duration += duration;
+      zones_[0].num_calls += num_calls;
+      JXL_ASSERT(zones_[0].BiasedOffset() == biased_offset);
+      return;
+    }
+
+    // Look for a zone with the same offset.
+    for (size_t i = 1; i < num_zones_; ++i) {
+      if (zones_[i].BiasedOffset() == biased_offset) {
+        zones_[i].total_duration += duration;
+        zones_[i].num_calls += num_calls;
+        JXL_ASSERT(zones_[i].BiasedOffset() == biased_offset);
+        // Swap with predecessor (more conservative than move to front,
+        // but at least as successful).
+        const Accumulator prev = zones_[i - 1];
+        zones_[i - 1] = zones_[i];
+        zones_[i] = prev;
+        return;
+      }
+    }
+
+    // Not found; create a new Accumulator.
+    JXL_ASSERT(num_zones_ < kMaxZones);
+    Accumulator* JXL_RESTRICT zone = zones_ + num_zones_;
+    zone->num_calls = (biased_offset << Accumulator::kNumCallBits) + num_calls;
+    zone->total_duration = duration;
+    JXL_ASSERT(zone->BiasedOffset() == biased_offset);
+    ++num_zones_;
+#endif
+  }
+
+  // Each instantiation of a function template seems to get its own copy of
+  // __func__ and GCC doesn't merge them. An N^2 search for duplicates is
+  // acceptable because we only expect a few dozen zones.
+  void MergeDuplicates() {
+    const uintptr_t string_origin = StringOrigin();
+    for (size_t i = 0; i < num_zones_; ++i) {
+      const uintptr_t biased_offset = zones_[i].BiasedOffset();
+      const char* name =
+          reinterpret_cast<const char*>(string_origin + biased_offset);
+      // Separate num_calls from biased_offset so we can add them together.
+      uint64_t num_calls = zones_[i].NumCalls();
+
+      // Add any subsequent duplicates to num_calls and total_duration.
+      for (size_t j = i + 1; j < num_zones_;) {
+        if (!strcmp(name, reinterpret_cast<const char*>(
+                              string_origin + zones_[j].BiasedOffset()))) {
+          num_calls += zones_[j].NumCalls();
+          zones_[i].total_duration += zones_[j].total_duration;
+          // Fill hole with last item.
+          zones_[j] = zones_[--num_zones_];
+        } else {  // Name differed, try next Accumulator.
+          ++j;
+        }
+      }
+
+      JXL_ASSERT(num_calls < (1ULL << Accumulator::kNumCallBits));
+
+      // Re-pack regardless of whether any duplicates were found.
+      zones_[i].num_calls =
+          (biased_offset << Accumulator::kNumCallBits) + num_calls;
+    }
+  }
+
+  uint64_t analyze_elapsed_ = 0;
+  uint64_t self_overhead_ = 0;
+  uint64_t child_overhead_ = 0;
+
+  size_t depth_ = 0;      // Number of active zones.
+  size_t num_zones_ = 0;  // Number of retired zones.
+
+  alignas(64) ProfilerNode nodes_[kMaxDepth];  // Stack
+  alignas(64) Accumulator zones_[kMaxZones];   // Self-organizing list
+};
+
+// Per-thread packet storage, allocated via CacheAligned.
+class ThreadSpecific {
+  static constexpr size_t kBufferCapacity =
+      CacheAligned::kCacheLineSize / sizeof(Packet);
+
+ public:
+  // "name" is used to sanity-check offsets fit in kOffsetBits.
+  explicit ThreadSpecific(const char* name)
+      : packets_(static_cast<Packet*>(
+            CacheAligned::Allocate(PROFILER_THREAD_STORAGE << 20))),
+        num_packets_(0),
+        max_packets_(PROFILER_THREAD_STORAGE << 17),
+        string_origin_(StringOrigin()) {
+    // Even in optimized builds (with NDEBUG), verify that this zone's name
+    // offset fits within the allotted space. If not, UpdateOrAdd is likely to
+    // overrun zones_[]. We also JXL_ASSERT(), but users often do not run debug
+    // builds. Checking here on the cold path (only reached once per thread)
+    // is cheap, but it only covers one zone.
+    const uintptr_t biased_offset =
+        reinterpret_cast<uintptr_t>(name) - string_origin_;
+    JXL_CHECK(biased_offset <= (1ULL << Packet::kOffsetBits));
+  }
+
+  ~ThreadSpecific() { CacheAligned::Free(packets_); }
+
+  // Depends on Zone => defined below.
+  void ComputeOverhead();
+
+  void WriteEntry(const char* name, const uint64_t timestamp) {
+    const uintptr_t biased_offset =
+        reinterpret_cast<uintptr_t>(name) - string_origin_;
+    Write(Packet::Make(biased_offset, timestamp));
+  }
+
+  void WriteExit(const uint64_t timestamp) {
+    const uintptr_t biased_offset = Packet::kOffsetBias;
+    Write(Packet::Make(biased_offset, timestamp));
+  }
+
+  void AnalyzeRemainingPackets() {
+#if JXL_ARCH_X64
+    // Ensures prior weakly-ordered streaming stores are globally visible.
+    _mm_sfence();
+
+    // Storage full => empty it.
+    if (num_packets_ + buffer_size_ > max_packets_) {
+      results_.AnalyzePackets(packets_, num_packets_);
+      num_packets_ = 0;
+    }
+    memcpy(packets_ + num_packets_, buffer_, buffer_size_ * sizeof(Packet));
+    num_packets_ += buffer_size_;
+    buffer_size_ = 0;
+#endif
+
+    results_.AnalyzePackets(packets_, num_packets_);
+    num_packets_ = 0;
+  }
+
+  Results& GetResults() { return results_; }
+
+ private:
+#if JXL_ARCH_X64 && HWY_BITS != 0
+  // Overwrites `to` without loading it into cache (read-for-ownership).
+  // Copies kCacheLineSize bytes from/to naturally aligned addresses.
+  template <typename T>
+  static HWY_ATTR void StreamCacheLine(const T* JXL_RESTRICT from,
+                                       T* JXL_RESTRICT to) {
+    static_assert(16 % sizeof(T) == 0, "T must fit in a lane");
+    constexpr size_t kLanes = 16 / sizeof(T);
+    const hwy::Desc<T, kLanes> d;
+    JXL_COMPILER_FENCE;
+    const auto v0 = Load(d, from + 0 * kLanes);
+    const auto v1 = Load(d, from + 1 * kLanes);
+    const auto v2 = Load(d, from + 2 * kLanes);
+    const auto v3 = Load(d, from + 3 * kLanes);
+    static_assert(sizeof(v0) * 4 == 64, "Wrong #vectors");
+    // Fences prevent the compiler from reordering loads/stores, which may
+    // interfere with write-combining.
+    JXL_COMPILER_FENCE;
+    Stream(v0, d, to + 0 * kLanes);
+    Stream(v1, d, to + 1 * kLanes);
+    Stream(v2, d, to + 2 * kLanes);
+    Stream(v3, d, to + 3 * kLanes);
+    JXL_COMPILER_FENCE;
+  }
+#endif
+
+  // Write packet to buffer/storage, emptying them as needed.
+  void Write(const Packet packet) {
+#if JXL_ARCH_X64 && HWY_BITS != 0
+    // Buffer full => copy to storage.
+    if (buffer_size_ == kBufferCapacity) {
+      // Storage full => empty it.
+      if (num_packets_ + kBufferCapacity > max_packets_) {
+        results_.AnalyzePackets(packets_, num_packets_);
+        num_packets_ = 0;
+      }
+      // This buffering halves observer overhead and decreases the overall
+      // runtime by about 3%.
+      StreamCacheLine(buffer_, packets_ + num_packets_);
+      num_packets_ += kBufferCapacity;
+      buffer_size_ = 0;
+    }
+    buffer_[buffer_size_] = packet;
+    ++buffer_size_;
+#else
+    // Write directly to storage.
+    if (num_packets_ >= max_packets_) {
+      results_.AnalyzePackets(packets_, num_packets_);
+      num_packets_ = 0;
+    }
+    packets_[num_packets_] = packet;
+    ++num_packets_;
+#endif
+  }
+
+  // Write-combining buffer to avoid cache pollution. Must be the first
+  // non-static member to ensure cache-line alignment.
+#if JXL_ARCH_X64
+  Packet buffer_[kBufferCapacity];
+  size_t buffer_size_ = 0;
+#endif
+
+  // Contiguous storage for zone enter/exit packets.
+  Packet* const JXL_RESTRICT packets_;
+  size_t num_packets_;
+  const size_t max_packets_;
+  // Cached here because we already read this cache line on zone entry/exit.
+  uintptr_t string_origin_;
+  Results results_;
+};
+
+class ThreadList {
+ public:
+  // Thread-safe.
+  void Add(ThreadSpecific* const ts) {
+    const uint32_t index = num_threads_.fetch_add(1, std::memory_order_relaxed);
+    JXL_CHECK(index < kMaxThreads);
+    threads_[index] = ts;
+  }
+
+  // Single-threaded.
+  void PrintResults() {
+    const uint32_t num_threads = num_threads_.load(std::memory_order_relaxed);
+    for (uint32_t i = 0; i < num_threads; ++i) {
+      threads_[i]->AnalyzeRemainingPackets();
+    }
+
+    // Combine all threads into a single Result.
+    for (uint32_t i = 1; i < num_threads; ++i) {
+      threads_[0]->GetResults().Assimilate(threads_[i]->GetResults());
+    }
+
+    if (num_threads != 0) {
+      threads_[0]->GetResults().Print();
+
+      for (uint32_t i = 0; i < num_threads; ++i) {
+        threads_[i]->GetResults().Reset();
+      }
+    }
+  }
+
+ private:
+  // Owning pointers.
+  alignas(64) ThreadSpecific* threads_[kMaxThreads];
+  std::atomic<uint32_t> num_threads_{0};
+};
+
+// RAII zone enter/exit recorder constructed by the ZONE macro; also
+// responsible for initializing ThreadSpecific.
+class Zone {
+ public:
+  // "name" must be a string literal (see StringOrigin).
+  JXL_NOINLINE explicit Zone(const char* name) {
+    JXL_COMPILER_FENCE;
+    ThreadSpecific* JXL_RESTRICT thread_specific = StaticThreadSpecific();
+    if (JXL_UNLIKELY(thread_specific == nullptr)) {
+      void* mem = CacheAligned::Allocate(sizeof(ThreadSpecific));
+      thread_specific = new (mem) ThreadSpecific(name);
+      // Must happen before ComputeOverhead, which re-enters this ctor.
+      Threads().Add(thread_specific);
+      StaticThreadSpecific() = thread_specific;
+      thread_specific->ComputeOverhead();
+    }
+
+    // (Capture timestamp ASAP, not inside WriteEntry.)
+    JXL_COMPILER_FENCE;
+    const uint64_t timestamp = TicksBefore();
+    thread_specific->WriteEntry(name, timestamp);
+  }
+
+  JXL_NOINLINE ~Zone() {
+    JXL_COMPILER_FENCE;
+    const uint64_t timestamp = TicksAfter();
+    StaticThreadSpecific()->WriteExit(timestamp);
+    JXL_COMPILER_FENCE;
+  }
+
+  // Call exactly once after all threads have exited all zones.
+  static void PrintResults() { Threads().PrintResults(); }
+
+ private:
+  // Returns reference to the thread's ThreadSpecific pointer (initially null).
+  // Function-local static avoids needing a separate definition.
+  static ThreadSpecific*& StaticThreadSpecific() {
+    static thread_local ThreadSpecific* thread_specific;
+    return thread_specific;
+  }
+
+  // Returns the singleton ThreadList. Non time-critical.
+  static ThreadList& Threads() {
+    static ThreadList threads_;
+    return threads_;
+  }
+};
+
+// Creates a zone starting from here until the end of the current scope.
+// Timestamps will be recorded when entering and exiting the zone.
+// "name" must be a string literal, which is ensured by merging with "".
+#define PROFILER_ZONE(name)        \
+  JXL_COMPILER_FENCE;              \
+  const ::jxl::Zone zone("" name); \
+  JXL_COMPILER_FENCE
+
+// Creates a zone for an entire function (when placed at its beginning).
+// Shorter/more convenient than ZONE.
+#define PROFILER_FUNC               \
+  JXL_COMPILER_FENCE;               \
+  const ::jxl::Zone zone(__func__); \
+  JXL_COMPILER_FENCE
+
+#define PROFILER_PRINT_RESULTS ::jxl::Zone::PrintResults
+
+inline void ThreadSpecific::ComputeOverhead() {
+  // Delay after capturing timestamps before/after the actual zone runs. Even
+  // with frequency throttling disabled, this has a multimodal distribution,
+  // including 32, 34, 48, 52, 59, 62.
+  uint64_t self_overhead;
+  {
+    const size_t kNumSamples = 32;
+    uint32_t samples[kNumSamples];
+    for (size_t idx_sample = 0; idx_sample < kNumSamples; ++idx_sample) {
+      const size_t kNumDurations = 1024;
+      uint32_t durations[kNumDurations];
+
+      for (size_t idx_duration = 0; idx_duration < kNumDurations;
+           ++idx_duration) {
+        {
+          PROFILER_ZONE("Dummy Zone (never shown)");
+        }
+#if JXL_ARCH_X64
+        const uint64_t duration = results_.ZoneDuration(buffer_);
+        buffer_size_ = 0;
+#else
+        const uint64_t duration = results_.ZoneDuration(packets_);
+        num_packets_ = 0;
+#endif
+        durations[idx_duration] = static_cast<uint32_t>(duration);
+        JXL_CHECK(num_packets_ == 0);
+      }
+      CountingSort(durations, durations + kNumDurations);
+      samples[idx_sample] = HalfSampleMode()(durations, kNumDurations);
+    }
+    // Median.
+    CountingSort(samples, samples + kNumSamples);
+    self_overhead = samples[kNumSamples / 2];
+#if PROFILER_PRINT_OVERHEAD
+    printf("Overhead: %zu\n", self_overhead);
+#endif
+    results_.SetSelfOverhead(self_overhead);
+  }
+
+  // Delay before capturing start timestamp / after end timestamp.
+  const size_t kNumSamples = 32;
+  uint32_t samples[kNumSamples];
+  for (size_t idx_sample = 0; idx_sample < kNumSamples; ++idx_sample) {
+    const size_t kNumDurations = 16;
+    uint32_t durations[kNumDurations];
+    for (size_t idx_duration = 0; idx_duration < kNumDurations;
+         ++idx_duration) {
+      const size_t kReps = 10000;
+      // Analysis time should not be included => must fit within buffer.
+      JXL_CHECK(kReps * 2 < max_packets_);
+#if JXL_ARCH_X64
+      _mm_mfence();
+#endif
+      const uint64_t t0 = TicksBefore();
+      for (size_t i = 0; i < kReps; ++i) {
+        PROFILER_ZONE("Dummy");
+      }
+#if JXL_ARCH_X64
+      _mm_sfence();
+#endif
+      const uint64_t t1 = TicksAfter();
+#if JXL_ARCH_X64
+      JXL_CHECK(num_packets_ + buffer_size_ == kReps * 2);
+      buffer_size_ = 0;
+#else
+      JXL_CHECK(num_packets_ == kReps * 2);
+#endif
+      num_packets_ = 0;
+      const uint64_t avg_duration = (t1 - t0 + kReps / 2) / kReps;
+      durations[idx_duration] =
+          static_cast<uint32_t>(ClampedSubtract(avg_duration, self_overhead));
+    }
+    CountingSort(durations, durations + kNumDurations);
+    samples[idx_sample] = HalfSampleMode()(durations, kNumDurations);
+  }
+  CountingSort(samples, samples + kNumSamples);
+  const uint64_t child_overhead = samples[9 * kNumSamples / 10];
+#if PROFILER_PRINT_OVERHEAD
+  printf("Child overhead: %zu\n", child_overhead);
+#endif
+  results_.SetChildOverhead(child_overhead);
+}
+
+}  // namespace jxl
+
+#else  // !PROFILER_ENABLED
+#define PROFILER_ZONE(name)
+#define PROFILER_FUNC
+#define PROFILER_PRINT_RESULTS()
+#endif
+
+#endif  // JXL_BASE_PROFILER_H_
diff --git a/jxl/base/robust_statistics.h b/jxl/base/robust_statistics.h
new file mode 100644
index 000000000..b87b54ba0
--- /dev/null
+++ b/jxl/base/robust_statistics.h
@@ -0,0 +1,361 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_ROBUST_STATISTICS_H_
+#define JXL_BASE_ROBUST_STATISTICS_H_
+
+// Robust statistics: Mode, Median, MedianAbsoluteDeviation.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <cmath>
+#include <limits>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/arch_specific.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Round up for integers
+template <class T, typename std::enable_if<
+                       std::numeric_limits<T>::is_integer>::type* = nullptr>
+inline T Half(T x) {
+  return (x + 1) / 2;
+}
+
+// Mul is faster than div.
+template <class T, typename std::enable_if<
+                       !std::numeric_limits<T>::is_integer>::type* = nullptr>
+inline T Half(T x) {
+  return x * 0.5;
+}
+
+// Returns the median value. Side effect: values <= median will appear before,
+// values >= median after the middle index.
+// Guarantees average speed O(num_values).
+template <typename T>
+T Median(T* samples, const size_t num_samples) {
+  JXL_ASSERT(num_samples != 0);
+  std::nth_element(samples, samples + num_samples / 2, samples + num_samples);
+  T result = samples[num_samples / 2];
+  // If even size, find largest element in the partially sorted vector to
+  // use as second element to average with
+  if ((num_samples & 1) == 0) {
+    T biggest = *std::max_element(samples, samples + num_samples / 2);
+    result = Half(result + biggest);
+  }
+  return result;
+}
+
+template <typename T>
+T Median(std::vector<T>* samples) {
+  return Median(samples->data(), samples->size());
+}
+
+template <typename T>
+static inline T Median3(const T a, const T b, const T c) {
+  return std::max(std::min(a, b), std::min(c, std::max(a, b)));
+}
+
+template <typename T>
+static inline T Median5(const T a, const T b, const T c, const T d, const T e) {
+  return Median3(e, std::max(std::min(a, b), std::min(c, d)),
+                 std::min(std::max(a, b), std::max(c, d)));
+}
+
+// Returns a robust measure of variability.
+template <typename T>
+T MedianAbsoluteDeviation(const T* samples, const size_t num_samples,
+                          const T median) {
+  JXL_ASSERT(num_samples != 0);
+  std::vector<T> abs_deviations;
+  abs_deviations.reserve(num_samples);
+  for (size_t i = 0; i < num_samples; ++i) {
+    abs_deviations.push_back(std::abs(samples[i] - median));
+  }
+  return Median(&abs_deviations);
+}
+
+template <typename T>
+T MedianAbsoluteDeviation(const std::vector<T>& samples, const T median) {
+  return MedianAbsoluteDeviation(samples.data(), samples.size(), median);
+}
+
+// Half{Range/Sample}Mode are implementations of "Robust estimators of the mode
+// and skewness of continuous data". The mode is less affected by outliers in
+// highly-skewed distributions than the median.
+
+// Robust estimator of the mode for data given as sorted values.
+// O(N*logN), N=num_values.
+class HalfSampleMode {
+ public:
+  // Returns mode. "sorted" must be in ascending order.
+  template <typename T>
+  T operator()(const T* const JXL_RESTRICT sorted,
+               const size_t num_values) const {
+    int64_t center = num_values / 2;
+    int64_t width = num_values;
+
+    // Zoom in on modal intervals of decreasing width. Stop before we reach
+    // width=1, i.e. single values, for which there is no "slope".
+    while (width > 2) {
+      // Round up so we can still reach the outer edges of odd widths.
+      width = Half(width);
+
+      center = CenterOfIntervalWithMinSlope(sorted, num_values, center, width);
+    }
+
+    return sorted[center];  // mode := middle value in modal interval.
+  }
+
+ private:
+  // Returns center of the densest region [c-radius, c+radius].
+  template <typename T>
+  static JXL_INLINE int64_t CenterOfIntervalWithMinSlope(
+      const T* JXL_RESTRICT sorted, const int64_t total_values,
+      const int64_t center, const int64_t width) {
+    const int64_t radius = Half(width);
+
+    auto compute_slope = [radius, total_values, sorted](
+                             int64_t c, int64_t* actual_center = nullptr) {
+      // For symmetry, check 2*radius+1 values, i.e. [min, max].
+      const int64_t min = std::max(c - radius, int64_t(0));
+      const int64_t max = std::min(c + radius, total_values - 1);
+      JXL_ASSERT(min < max);
+      JXL_ASSERT(sorted[min] <=
+                 sorted[max] + std::numeric_limits<float>::epsilon());
+      const float dx = max - min + 1;
+      const float slope = (sorted[max] - sorted[min]) / dx;
+
+      if (actual_center != nullptr) {
+        // c may be out of bounds, so return center of the clamped bounds.
+        *actual_center = Half(min + max);
+      }
+      return slope;
+    };
+
+    // First find min_slope for all centers.
+    float min_slope = std::numeric_limits<float>::max();
+    for (int64_t c = center - radius; c <= center + radius; ++c) {
+      min_slope = std::min(min_slope, compute_slope(c));
+    }
+
+    // Candidates := centers with slope ~= min_slope.
+    std::vector<int64_t> candidates;
+    for (int64_t c = center - radius; c <= center + radius; ++c) {
+      int64_t actual_center;
+      const float slope = compute_slope(c, &actual_center);
+      if (slope <= min_slope * 1.001f) {
+        candidates.push_back(actual_center);
+      }
+    }
+
+    // Keep the median.
+    JXL_ASSERT(!candidates.empty());
+    if (candidates.size() == 1) return candidates[0];
+    return Median(&candidates);
+  }
+};
+
+// Robust estimator of the mode for data given as a CDF.
+// O(N*logN), N=num_bins.
+class HalfRangeMode {
+ public:
+  // Returns mode expressed as a histogram bin index. "cdf" must be weakly
+  // monotonically increasing, e.g. from std::partial_sum.
+  int operator()(const uint32_t* JXL_RESTRICT cdf,
+                 const size_t num_bins) const {
+    int center = num_bins / 2;
+    int width = num_bins;
+
+    // Zoom in on modal intervals of decreasing width. Stop before we reach
+    // width=1, i.e. original bins, because those are noisy.
+    while (width > 2) {
+      // Round up so we can still reach the outer edges of odd widths.
+      width = Half(width);
+
+      center = CenterOfIntervalWithMaxDensity(cdf, num_bins, center, width);
+    }
+
+    return center;  // mode := midpoint of modal interval.
+  }
+
+ private:
+  // Returns center of the densest interval [c-radius, c+radius].
+  static JXL_INLINE int CenterOfIntervalWithMaxDensity(
+      const uint32_t* JXL_RESTRICT cdf, const int total_bins, const int center,
+      const int width) {
+    const int radius = Half(width);
+
+    auto compute_density = [radius, total_bins, cdf](
+                               int c, int* actual_center = nullptr) {
+      // For symmetry, check 2*radius+1 bins, i.e. [min, max].
+      const int min = std::max(c - radius, 1);  // for -1 below
+      const int max = std::min(c + radius, total_bins - 1);
+      JXL_ASSERT(min < max);
+      JXL_ASSERT(cdf[min] <= cdf[max - 1]);
+      const int num_bins = max - min + 1;
+      // Sum over [min, max] == CDF(max) - CDF(min-1).
+      const float density = float(cdf[max] - cdf[min - 1]) / num_bins;
+
+      if (actual_center != nullptr) {
+        // c may be out of bounds, so take center of the clamped bounds.
+        *actual_center = Half(min + max);
+      }
+      return density;
+    };
+
+    // First find max_density for all centers.
+    float max_density = 0.0f;
+    for (int c = center - radius; c <= center + radius; ++c) {
+      max_density = std::max(max_density, compute_density(c));
+    }
+
+    // Candidates := centers with density ~= max_density.
+    std::vector<int> candidates;
+    for (int c = center - radius; c <= center + radius; ++c) {
+      int actual_center;
+      const float density = compute_density(c, &actual_center);
+      if (density >= max_density * 0.999f) {
+        candidates.push_back(actual_center);
+      }
+    }
+
+    // Keep the median.
+    JXL_ASSERT(!candidates.empty());
+    if (candidates.size() == 1) return candidates[0];
+    return Median(&candidates);
+  }
+};
+
+// Sorts integral values in ascending order. About 3x faster than std::sort for
+// input distributions with very few unique values.
+template <class T>
+void CountingSort(T* begin, T* end) {
+  // Unique values and their frequency (similar to flat_map).
+  using Unique = std::pair<T, int>;
+  std::vector<Unique> unique;
+  for (const T* p = begin; p != end; ++p) {
+    const T value = *p;
+    const auto pos =
+        std::find_if(unique.begin(), unique.end(),
+                     [value](const Unique& u) { return u.first == value; });
+    if (pos == unique.end()) {
+      unique.push_back(std::make_pair(*p, 1));
+    } else {
+      ++pos->second;
+    }
+  }
+
+  // Sort in ascending order of value (pair.first).
+  std::sort(unique.begin(), unique.end());
+
+  // Write that many copies of each unique value to the array.
+  T* JXL_RESTRICT p = begin;
+  for (const auto& value_count : unique) {
+    std::fill(p, p + value_count.second, value_count.first);
+    p += value_count.second;
+  }
+  JXL_ASSERT(p == end);
+}
+
+struct Bivariate {
+  Bivariate(float x, float y) : x(x), y(y) {}
+  float x;
+  float y;
+};
+
+class Line {
+ public:
+  constexpr Line(const float slope, const float intercept)
+      : slope_(slope), intercept_(intercept) {}
+
+  constexpr float slope() const { return slope_; }
+  constexpr float intercept() const { return intercept_; }
+
+  // Robust line fit using Siegel's repeated-median algorithm.
+  explicit Line(const std::vector<Bivariate>& points) {
+    const size_t N = points.size();
+    // This straightforward N^2 implementation is OK for small N.
+    JXL_ASSERT(N < 10 * 1000);
+
+    // One for every point i.
+    std::vector<float> medians;
+    medians.reserve(N);
+
+    // One for every j != i. Never cleared to avoid reallocation.
+    std::vector<float> slopes(N - 1);
+
+    for (size_t i = 0; i < N; ++i) {
+      // Index within slopes[] (avoids the hole where j == i).
+      size_t idx_slope = 0;
+
+      for (size_t j = 0; j < N; ++j) {
+        if (j == i) continue;
+
+        const float dy = points[j].y - points[i].y;
+        const float dx = points[j].x - points[i].x;
+        JXL_ASSERT(std::abs(dx) > 1E-7f);  // x must be distinct
+        slopes[idx_slope++] = dy / dx;
+      }
+      JXL_ASSERT(idx_slope == N - 1);
+
+      const float median = Median(&slopes);
+      medians.push_back(median);
+    }
+
+    slope_ = Median(&medians);
+
+    // Solve for intercept, overwriting medians[].
+    for (size_t i = 0; i < N; ++i) {
+      medians[i] = points[i].y - slope_ * points[i].x;
+    }
+    intercept_ = Median(&medians);
+  }
+
+  constexpr float operator()(float x) const { return x * slope_ + intercept_; }
+
+ private:
+  float slope_;
+  float intercept_;
+};
+
+static inline void EvaluateQuality(const Line& line,
+                                   const std::vector<Bivariate>& points,
+                                   float* JXL_RESTRICT max_l1,
+                                   float* JXL_RESTRICT median_abs_deviation) {
+  // For computing median_abs_deviation.
+  std::vector<float> abs_deviations;
+  abs_deviations.reserve(points.size());
+
+  *max_l1 = 0.0f;
+  for (const Bivariate& point : points) {
+    const float l1 = std::abs(line(point.x) - point.y);
+    *max_l1 = std::max(*max_l1, l1);
+    abs_deviations.push_back(l1);
+  }
+
+  *median_abs_deviation = Median(&abs_deviations);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_ROBUST_STATISTICS_H_
diff --git a/jxl/base/span.h b/jxl/base/span.h
new file mode 100644
index 000000000..c0b5e4f30
--- /dev/null
+++ b/jxl/base/span.h
@@ -0,0 +1,67 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_SPAN_H_
+#define JXL_BASE_SPAN_H_
+
+// Span (array view) is a non-owning container that provides cheap "cut"
+// operations and could be used as "ArrayLike" data source for PaddedBytes.
+
+#include <stddef.h>
+
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+template <typename T>
+class Span {
+ public:
+  constexpr Span() noexcept : Span(nullptr, 0) {}
+
+  constexpr Span(T* array, size_t length) noexcept
+      : ptr_(array), len_(length) {}
+
+  template <size_t N>
+  explicit constexpr Span(T (&a)[N]) noexcept : Span(a, N) {}
+
+  template <typename ArrayLike>
+  explicit constexpr Span(const ArrayLike& other) noexcept
+      : Span(reinterpret_cast<T*>(other.data()), other.size()) {
+    static_assert(sizeof(*other.data()) == sizeof(T),
+                  "Incompatible type of source.");
+  }
+
+  constexpr T* data() const noexcept { return ptr_; }
+
+  constexpr size_t size() const noexcept { return len_; }
+
+  constexpr T& operator[](size_t i) const noexcept {
+    // MSVC 2015 accepts this as constexpr, but not ptr_[i]
+    return *(data() + i);
+  }
+
+  void remove_prefix(size_t n) noexcept {
+    JXL_ASSERT(size() >= n);
+    ptr_ += n;
+    len_ -= n;
+  }
+
+ private:
+  T* ptr_;
+  size_t len_;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_SPAN_H_
diff --git a/jxl/base/status.cc b/jxl/base/status.cc
new file mode 100644
index 000000000..e8e2f9005
--- /dev/null
+++ b/jxl/base/status.cc
@@ -0,0 +1,57 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/status.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <string>
+
+#include "jxl/base/os_specific.h"
+
+namespace jxl {
+
+bool Debug(const char* format, ...) {
+// Show the debug messages in debug or opt mode, not in release mode.
+#if JXL_DEBUG_WARNING
+  va_list args;
+  va_start(args, format);
+  vfprintf(stderr, format, args);
+  va_end(args);
+#endif
+  return false;
+}
+
+bool Abort(const char* file, int line, const char* format, ...) {
+  char buf[2000];
+  va_list args;
+  va_start(args, format);
+  vsnprintf(buf, sizeof(buf), format, args);
+  va_end(args);
+
+  const std::string call_stack;
+
+  fprintf(stderr, "Abort at %s:%d: %s\n%s\n", file, line, buf,
+          call_stack.c_str());
+#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \
+    defined(THREAD_SANITIZER)
+  // If compiled with UBSAN, triggering an error gives us a call stack.
+  printf("Deliberate ubsan div zero %zu\n", 10 / (TotalMemoryMiB() >> 30));
+#endif
+  abort();
+}
+
+}  // namespace jxl
diff --git a/jxl/base/status.h b/jxl/base/status.h
new file mode 100644
index 000000000..4dc1d83cb
--- /dev/null
+++ b/jxl/base/status.h
@@ -0,0 +1,185 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_STATUS_H_
+#define JXL_BASE_STATUS_H_
+
+// Error handling: Status return type + helper macros.
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+
+// Uncomment to abort when JXL_FAILURE is reached
+// #define JXL_CRASH_ON_ERROR
+
+#ifndef JXL_ENABLE_ASSERT
+#define JXL_ENABLE_ASSERT 1
+#endif
+
+// Pass -DJXL_DEBUG_ON_ERROR at compile time to print debug messages when a
+// function returns JXL_FAILURE or calls JXL_NOTIFY_ERROR. Note that this is
+// irrelevant if you also pass -DJXL_CRASH_ON_ERROR.
+#ifdef JXL_DEBUG_ON_ERROR
+#undef JXL_DEBUG_ON_ERROR
+#define JXL_DEBUG_ON_ERROR 1
+#else  // JXL_DEBUG_ON_ERROR
+#ifdef NDEBUG
+#define JXL_DEBUG_ON_ERROR 0
+#else  // JXL_DEBUG_ON_ERROR
+#define JXL_DEBUG_ON_ERROR 1
+#endif  // NDEBUG
+#endif  // JXL_DEBUG_ON_ERROR
+
+// The Verbose level for the library
+#ifndef JXL_DEBUG_V_LEVEL
+#define JXL_DEBUG_V_LEVEL 0
+#endif  // JXL_DEBUG_V_LEVEL
+
+// Print a debug message on standard error. You should use the JXL_DEBUG macro
+// instead of calling Debug directly. This function returns false, so it can be
+// used as a return value in JXL_FAILURE.
+JXL_FORMAT(1, 2)
+bool Debug(const char* format, ...);
+
+// Print a debug message on standard error if "enabled" is true. "enabled" is
+// normally a macro that evaluates to 0 or 1 at compile time, so the Debug
+// function is never called and optimized out in release builds. Note that the
+// arguments are compiled but not evaluated when enabled is false. The format
+// string must be a explicit string in the call, for example:
+//   JXL_DEBUG(JXL_DEBUG_MYMODULE, "my module message: %d", some_var);
+// Add a header at the top of your module's .cc or .h file (depending on whether
+// you have JXL_DEBUG calls from the .h as well) like this:
+//   #ifndef JXL_DEBUG_MYMODULE
+//   #define JXL_DEBUG_MYMODULE 0
+//   #endif JXL_DEBUG_MYMODULE
+#define JXL_DEBUG(enabled, format, ...)                         \
+  do {                                                          \
+    if (enabled) {                                              \
+      ::jxl::Debug(("%s:%d: " format "\n"), __FILE__, __LINE__, \
+                   ##__VA_ARGS__);                              \
+    }                                                           \
+  } while (0)
+
+// JXL_DEBUG version that prints the debug message if the global verbose level
+// defined at compile time by JXL_DEBUG_V_LEVEL is greater or equal than the
+// passed level.
+#define JXL_DEBUG_V(level, ...) \
+  JXL_DEBUG(level <= JXL_DEBUG_V_LEVEL, __VA_ARGS__)
+
+// Warnings (via JXL_WARNING) are enabled by default in debug builds (opt and
+// debug).
+#ifdef JXL_DEBUG_WARNING
+#undef JXL_DEBUG_WARNING
+#define JXL_DEBUG_WARNING 1
+#else  // JXL_DEBUG_WARNING
+#ifdef NDEBUG
+#define JXL_DEBUG_WARNING 0
+#else  // JXL_DEBUG_WARNING
+#define JXL_DEBUG_WARNING 1
+#endif  // NDEBUG
+#endif  // JXL_DEBUG_WARNING
+#define JXL_WARNING(...) JXL_DEBUG(JXL_DEBUG_WARNING, __VA_ARGS__)
+
+// Exits the program after printing file/line plus a formatted string.
+JXL_FORMAT(3, 4)
+JXL_NORETURN bool Abort(const char* file, int line, const char* format, ...);
+
+// Exits the program after printing file/line plus a formatted string.
+#define JXL_ABORT(...) Abort(__FILE__, __LINE__, __VA_ARGS__)
+
+// Does not guarantee running the code, use only for debug mode checks.
+#if JXL_ENABLE_ASSERT
+#define JXL_ASSERT(condition)                                    \
+  do {                                                           \
+    if (!(condition)) {                                          \
+      ::jxl::Abort(__FILE__, __LINE__, "Assert %s", #condition); \
+    }                                                            \
+  } while (0)
+#else
+#define JXL_ASSERT(condition) \
+  do {                        \
+  } while (0)
+#endif
+
+// Same as above, but only runs in debug builds (builds where NDEBUG is not
+// defined). This is useful for slower asserts that we want to run more rarely
+// than usual. These will run on asan, msan and other debug builds, but not in
+// opt or release.
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER) || \
+    defined(MEMORY_SANITIZER) || defined(THREAD_SANITIZER)
+#define JXL_DASSERT(condition)                                         \
+  do {                                                                 \
+    if (!(condition)) {                                                \
+      ::jxl::Abort(__FILE__, __LINE__, "Debug Assert %s", #condition); \
+    }                                                                  \
+  } while (0)
+#else
+#define JXL_DASSERT(condition) \
+  do {                         \
+  } while (0)
+#endif
+
+// Always runs the condition, so can be used for non-debug calls.
+#define JXL_CHECK(condition)                                    \
+  do {                                                          \
+    if (!(condition)) {                                         \
+      ::jxl::Abort(__FILE__, __LINE__, "Check %s", #condition); \
+    }                                                           \
+  } while (0)
+
+// Always runs the condition, so can be used for non-debug calls.
+#define JXL_RETURN_IF_ERROR(condition) \
+  do {                                 \
+    if (!(condition)) return false;    \
+  } while (0)
+
+// Annotation for the location where an error condition is first noticed.
+// Error codes are too unspecific to pinpoint the exact location, so we
+// add a build flag that crashes and dumps stack at the actual error source.
+#ifdef JXL_CRASH_ON_ERROR
+#define JXL_NOTIFY_ERROR(...) \
+  (void)::jxl::Abort(__FILE__, __LINE__, __VA_ARGS__)
+#define JXL_FAILURE(...) ::jxl::Abort(__FILE__, __LINE__, __VA_ARGS__)
+#else  // JXL_CRASH_ON_ERROR
+#define JXL_NOTIFY_ERROR(...) JXL_DEBUG(JXL_DEBUG_ON_ERROR, __VA_ARGS__)
+#define JXL_FAILURE(format, ...)                                               \
+  ((JXL_DEBUG_ON_ERROR) &&                                                     \
+   ::jxl::Debug(("%s:%d: " format "\n"), __FILE__, __LINE__, ##__VA_ARGS__) && \
+   false)
+#endif  // JXL_CRASH_ON_ERROR
+
+// Drop-in replacement for bool that raises compiler warnings if not used
+// after being returned from a function. Example:
+// Status LoadFile(...) { return true; } is more compact than
+// bool JXL_MUST_USE_RESULT LoadFile(...) { return true; }
+class JXL_MUST_USE_RESULT Status {
+ public:
+  // We want implicit constructor from bool to allow returning "true" or "false"
+  // on a function when using Status.
+  Status(bool ok) : ok_(ok) {}  // NOLINT(google-explicit-constructor)
+
+  // We also want implicit cast to bool to check for return values of functions.
+  operator bool() const { return ok_; }  // NOLINT(google-explicit-constructor)
+
+ private:
+  bool ok_;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_STATUS_H_
diff --git a/jxl/base/thread_pool_internal.h b/jxl/base/thread_pool_internal.h
new file mode 100644
index 000000000..6d37b063f
--- /dev/null
+++ b/jxl/base/thread_pool_internal.h
@@ -0,0 +1,61 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_THREAD_TEST_UTILS_H_
+#define JXL_THREAD_TEST_UTILS_H_
+
+#include <stddef.h>
+
+#include <cmath>
+
+#include "include/jpegxl/parallel_runner.h"
+#include "include/jpegxl/thread_parallel_runner.h"
+#include "jxl/base/data_parallel.h"
+
+namespace jxl {
+
+// Helper class to pass an internal ThreadPool-like object using threads. This
+// is only suitable for tests or tools that access the internal API of JPEG XL.
+// In other cases the caller will provide a JpegxlParallelRunner() for handling
+// this. This class uses jpegxl::ThreadParallelRunner (from jpegxl_threads
+// library). For interface details check jpegxl::ThreadParallelRunner.
+class ThreadPoolInternal : public ThreadPool {
+ public:
+  // Starts the given number of worker threads and blocks until they are ready.
+  // "num_worker_threads" defaults to one per hyperthread. If zero, all tasks
+  // run on the main thread.
+  explicit ThreadPoolInternal(
+      int num_worker_threads = std::thread::hardware_concurrency())
+      : ThreadPool(&jpegxl::ThreadParallelRunner::Runner,
+                   static_cast<void*>(&runner_)),
+        runner_(num_worker_threads) {}
+
+  ThreadPoolInternal(const ThreadPoolInternal&) = delete;
+  ThreadPoolInternal& operator&(const ThreadPoolInternal&) = delete;
+
+  size_t NumThreads() const { return runner_.NumThreads(); }
+  size_t NumWorkerThreads() const { return runner_.NumWorkerThreads(); }
+
+  template <class Func>
+  void RunOnEachThread(const Func& func) {
+    runner_.RunOnEachThread(func);
+  }
+
+ private:
+  jpegxl::ThreadParallelRunner runner_;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_THREAD_TEST_UTILS_H_
diff --git a/jxl/base/tsc_timer.h b/jxl/base/tsc_timer.h
new file mode 100644
index 000000000..41908777a
--- /dev/null
+++ b/jxl/base/tsc_timer.h
@@ -0,0 +1,142 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BASE_TSC_TIMER_H_
+#define JXL_BASE_TSC_TIMER_H_
+
+// High-resolution (~10 ns) timestamps, using fences to prevent reordering and
+// ensure exactly the desired regions are measured.
+
+#include <stdint.h>
+
+#include <ctime>
+#include <hwy/cache_control.h>  // LoadFence
+
+#include "jxl/base/arch_specific.h"
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+
+// TicksBefore/After return absolute timestamps and must be placed immediately
+// before and after the region to measure. The functions are distinct because
+// they use different fences.
+//
+// Background: RDTSC is not 'serializing'; earlier instructions may complete
+// after it, and/or later instructions may complete before it. 'Fences' ensure
+// regions' elapsed times are independent of such reordering. The only
+// documented unprivileged serializing instruction is CPUID, which acts as a
+// full fence (no reordering across it in either direction). Unfortunately
+// the latency of CPUID varies wildly (perhaps made worse by not initializing
+// its EAX input). Because it cannot reliably be deducted from the region's
+// elapsed time, it must not be included in the region to measure (i.e.
+// between the two RDTSC).
+//
+// The newer RDTSCP is sometimes described as serializing, but it actually
+// only serves as a half-fence with release semantics. Although all
+// instructions in the region will complete before the final timestamp is
+// captured, subsequent instructions may leak into the region and increase the
+// elapsed time. Inserting another fence after the final RDTSCP would prevent
+// such reordering without affecting the measured region.
+//
+// Fortunately, such a fence exists. The LFENCE instruction is only documented
+// to delay later loads until earlier loads are visible. However, Intel's
+// reference manual says it acts as a full fence (waiting until all earlier
+// instructions have completed, and delaying later instructions until it
+// completes). AMD assigns the same behavior to MFENCE.
+//
+// We need a fence before the initial RDTSC to prevent earlier instructions
+// from leaking into the region, and arguably another after RDTSC to avoid
+// region instructions from completing before the timestamp is recorded.
+// When surrounded by fences, the additional RDTSCP half-fence provides no
+// benefit, so the initial timestamp can be recorded via RDTSC, which has
+// lower overhead than RDTSCP because it does not read TSC_AUX. In summary,
+// we define Before = LFENCE/RDTSC/LFENCE; After = RDTSCP/LFENCE.
+//
+// Using Before+Before leads to higher variance and overhead than After+After.
+// However, After+After includes an LFENCE in the region measurements, which
+// adds a delay dependent on earlier loads. The combination of Before+After
+// is faster than Before+Before and more consistent than Stop+Stop because
+// the first LFENCE already delayed subsequent loads before the measured
+// region. This combination seems not to have been considered in prior work:
+// http://akaros.cs.berkeley.edu/lxr/akaros/kern/arch/x86/rdtsc_test.c
+//
+// Note: performance counters can measure 'exact' instructions-retired or
+// (unhalted) cycle counts. The RDPMC instruction is not serializing and also
+// requires fences. Unfortunately, it is not accessible on all OSes and we
+// prefer to avoid kernel-mode drivers. Performance counters are also affected
+// by several under/over-count errata, so we use the TSC instead.
+
+// Returns a 64-bit timestamp in unit of 'ticks'; to convert to seconds,
+// divide by InvariantTicksPerSecond. Although 32-bit ticks are faster to read,
+// they overflow too quickly to measure long regions.
+static inline uint64_t TicksBefore() {
+  uint64_t t;
+#if JXL_ARCH_PPC
+  asm volatile("mfspr %0, %1" : "=r"(t) : "i"(268));
+#elif JXL_ARCH_X64 && JXL_COMPILER_MSVC
+  LoadFence();
+  JXL_COMPILER_FENCE;
+  t = __rdtsc();
+  LoadFence();
+  JXL_COMPILER_FENCE;
+#elif JXL_ARCH_X64 && (JXL_COMPILER_CLANG || JXL_COMPILER_GCC)
+  asm volatile(
+      "lfence\n\t"
+      "rdtsc\n\t"
+      "shl $32, %%rdx\n\t"
+      "or %%rdx, %0\n\t"
+      "lfence"
+      : "=a"(t)
+      :
+      // "memory" avoids reordering. rdx = TSC >> 32.
+      // "cc" = flags modified by SHL.
+      : "rdx", "memory", "cc");
+#else
+  // Fall back to OS - unsure how to reliably query cntvct_el0 frequency.
+  timespec ts;
+  clock_gettime(CLOCK_MONOTONIC, &ts);
+  t = ts.tv_sec * 1000000000LL + ts.tv_nsec;
+#endif
+  return t;
+}
+
+static inline uint64_t TicksAfter() {
+  uint64_t t;
+#if JXL_ARCH_X64 && JXL_COMPILER_MSVC
+  JXL_COMPILER_FENCE;
+  unsigned aux;
+  t = __rdtscp(&aux);
+  LoadFence();
+  JXL_COMPILER_FENCE;
+#elif JXL_ARCH_X64 && (JXL_COMPILER_CLANG || JXL_COMPILER_GCC)
+  // Use inline asm because __rdtscp generates code to store TSC_AUX (ecx).
+  asm volatile(
+      "rdtscp\n\t"
+      "shl $32, %%rdx\n\t"
+      "or %%rdx, %0\n\t"
+      "lfence"
+      : "=a"(t)
+      :
+      // "memory" avoids reordering. rcx = TSC_AUX. rdx = TSC >> 32.
+      // "cc" = flags modified by SHL.
+      : "rcx", "rdx", "memory", "cc");
+#else
+  t = TicksBefore();  // no difference on other platforms.
+#endif
+  return t;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_BASE_TSC_TIMER_H_
diff --git a/jxl/bit_reader_test.cc b/jxl/bit_reader_test.cc
new file mode 100644
index 000000000..3ab2fb5f2
--- /dev/null
+++ b/jxl/bit_reader_test.cc
@@ -0,0 +1,276 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <array>
+#include <random>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/span.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/common.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+
+namespace jxl {
+namespace {
+
+HWY_ATTR void BitReaderTest_ExtendsWithZeroes() {
+  for (size_t size = 4; size < 32; ++size) {
+    std::vector<uint8_t> data(size, 0xff);
+
+    for (size_t n_bytes = 0; n_bytes < size; n_bytes++) {
+      BitReader br(Span<const uint8_t>(data.data(), n_bytes));
+      // Read all the bits
+      for (size_t i = 0; i < n_bytes * kBitsPerByte; i++) {
+        ASSERT_EQ(br.ReadBits(1), 1) << "n_bytes=" << n_bytes << " i=" << i;
+      }
+
+      // PEEK more than the declared size - all will be zero. Cannot consume.
+      for (size_t i = 0; i < BitReader::kMaxBitsPerCall; i++) {
+        ASSERT_EQ(br.PeekBits(i), 0)
+            << "size=" << size << "n_bytes=" << n_bytes << " i=" << i;
+      }
+
+      EXPECT_TRUE(br.Close());
+    }
+  }
+}
+TEST(BitReaderTest, ExtendsWithZeroes) { BitReaderTest_ExtendsWithZeroes(); }
+
+struct Symbol {
+  uint32_t num_bits;
+  uint32_t value;
+};
+
+// Reading from output gives the same values.
+TEST(BitReaderTest, TestRoundTrip) {
+  ThreadPoolInternal pool(8);
+  pool.Run(0, 1000, ThreadPool::SkipInit(),
+           [](const int task, const int /* thread */) HWY_ATTR {
+             constexpr size_t kMaxBits = 8000;
+             BitWriter writer;
+             BitWriter::Allotment allotment(&writer, kMaxBits);
+
+             std::vector<Symbol> symbols;
+             symbols.reserve(1000);
+
+             std::mt19937 rng(55537 + 129 * task);
+             std::uniform_int_distribution<> dist(1, 32);  // closed interval
+
+             for (;;) {
+               const uint32_t num_bits = dist(rng);
+               if (writer.BitsWritten() + num_bits > kMaxBits) break;
+               const uint32_t value = rng() >> (32 - num_bits);
+               symbols.push_back({num_bits, value});
+               writer.Write(num_bits, value);
+             }
+
+             writer.ZeroPadToByte();
+             ReclaimAndCharge(&writer, &allotment, 0, nullptr);
+             BitReader reader(writer.GetSpan());
+             for (const Symbol& s : symbols) {
+               EXPECT_EQ(s.value, reader.ReadBits(s.num_bits));
+             }
+             EXPECT_TRUE(reader.Close());
+           });
+}
+
+// SkipBits is the same as reading that many bits.
+TEST(BitReaderTest, TestSkip) {
+  ThreadPoolInternal pool(8);
+  pool.Run(
+      0, 96, ThreadPool::SkipInit(),
+      [](const int task, const int /* thread */) HWY_ATTR {
+        constexpr size_t kSize = 100;
+
+        for (size_t skip = 0; skip < 128; ++skip) {
+          BitWriter writer;
+          BitWriter::Allotment allotment(&writer, kSize * kBitsPerByte);
+          // Start with "task" 1-bits.
+          for (int i = 0; i < task; ++i) {
+            writer.Write(1, 1);
+          }
+
+          // Write 0-bits that we will skip over
+          for (size_t i = 0; i < skip; ++i) {
+            writer.Write(1, 0);
+          }
+
+          // Write terminator bits '101'
+          writer.Write(3, 5);
+          EXPECT_EQ(task + skip + 3, writer.BitsWritten());
+          writer.ZeroPadToByte();
+          AuxOut aux_out;
+          ReclaimAndCharge(&writer, &allotment, 0, &aux_out);
+          EXPECT_LT(aux_out.layers[0].total_bits, kSize * 8);
+
+          BitReader reader1(writer.GetSpan());
+          BitReader reader2(writer.GetSpan());
+          // Verify initial 1-bits
+          for (int i = 0; i < task; ++i) {
+            EXPECT_EQ(1, reader1.ReadBits(1));
+            EXPECT_EQ(1, reader2.ReadBits(1));
+          }
+
+          // SkipBits or manually read "skip" bits
+          reader1.SkipBits(skip);
+          for (size_t i = 0; i < skip; ++i) {
+            EXPECT_EQ(0, reader2.ReadBits(1)) << " skip=" << skip << " i=" << i;
+          }
+          EXPECT_EQ(reader1.TotalBitsConsumed(), reader2.TotalBitsConsumed());
+
+          // Ensure both readers see the terminator bits.
+          EXPECT_EQ(5, reader1.ReadBits(3));
+          EXPECT_EQ(5, reader2.ReadBits(3));
+
+          EXPECT_TRUE(reader1.Close());
+          EXPECT_TRUE(reader2.Close());
+        }
+      });
+}
+
+// Verifies byte order and different groupings of bits.
+HWY_ATTR void BitReaderTest_TestOrder() {
+  constexpr size_t kMaxBits = 16;
+
+  // u(1) - bits written into LSBs of first byte
+  {
+    BitWriter writer;
+    BitWriter::Allotment allotment(&writer, kMaxBits);
+    for (size_t i = 0; i < 5; ++i) {
+      writer.Write(1, 1);
+    }
+    for (size_t i = 0; i < 5; ++i) {
+      writer.Write(1, 0);
+    }
+    for (size_t i = 0; i < 6; ++i) {
+      writer.Write(1, 1);
+    }
+
+    writer.ZeroPadToByte();
+    ReclaimAndCharge(&writer, &allotment, 0, nullptr);
+    BitReader reader(writer.GetSpan());
+    EXPECT_EQ(0x1F, reader.ReadFixedBits<8>());
+    EXPECT_EQ(0xFC, reader.ReadFixedBits<8>());
+    EXPECT_TRUE(reader.Close());
+  }
+
+  // u(8) - get bytes in the same order
+  {
+    BitWriter writer;
+    BitWriter::Allotment allotment(&writer, kMaxBits);
+    writer.Write(8, 0xF8);
+    writer.Write(8, 0x3F);
+
+    writer.ZeroPadToByte();
+    ReclaimAndCharge(&writer, &allotment, 0, nullptr);
+    BitReader reader(writer.GetSpan());
+    EXPECT_EQ(0xF8, reader.ReadFixedBits<8>());
+    EXPECT_EQ(0x3F, reader.ReadFixedBits<8>());
+    EXPECT_TRUE(reader.Close());
+  }
+
+  // u(16) - little-endian bytes
+  {
+    BitWriter writer;
+    BitWriter::Allotment allotment(&writer, kMaxBits);
+    writer.Write(16, 0xF83F);
+
+    writer.ZeroPadToByte();
+    ReclaimAndCharge(&writer, &allotment, 0, nullptr);
+    BitReader reader(writer.GetSpan());
+    EXPECT_EQ(0x3F, reader.ReadFixedBits<8>());
+    EXPECT_EQ(0xF8, reader.ReadFixedBits<8>());
+    EXPECT_TRUE(reader.Close());
+  }
+
+  // Non-byte-aligned, mixed sizes
+  {
+    BitWriter writer;
+    BitWriter::Allotment allotment(&writer, kMaxBits);
+    writer.Write(1, 1);
+    writer.Write(3, 6);
+    writer.Write(8, 0xDB);
+    writer.Write(4, 8);
+
+    writer.ZeroPadToByte();
+    ReclaimAndCharge(&writer, &allotment, 0, nullptr);
+    BitReader reader(writer.GetSpan());
+    EXPECT_EQ(0xBD, reader.ReadFixedBits<8>());
+    EXPECT_EQ(0x8D, reader.ReadFixedBits<8>());
+    EXPECT_TRUE(reader.Close());
+  }
+}
+
+TEST(BitReaderTest, TestOrder) { BitReaderTest_TestOrder(); }
+
+HWY_ATTR void BitReaderTest_TotalCountersTest() {
+  uint8_t buf[4] = {1, 2, 3, 4};
+  BitReader reader(Span<const uint8_t>(buf, sizeof(buf)));
+
+  EXPECT_EQ(sizeof(buf), reader.TotalBytes());
+  EXPECT_EQ(0, reader.TotalBitsConsumed());
+  reader.ReadFixedBits<1>();
+  EXPECT_EQ(1, reader.TotalBitsConsumed());
+
+  reader.ReadFixedBits<10>();
+  EXPECT_EQ(11, reader.TotalBitsConsumed());
+
+  reader.ReadFixedBits<4>();
+  EXPECT_EQ(15, reader.TotalBitsConsumed());
+
+  reader.ReadFixedBits<1>();
+  EXPECT_EQ(16, reader.TotalBitsConsumed());
+
+  reader.ReadFixedBits<16>();
+  EXPECT_EQ(32, reader.TotalBitsConsumed());
+
+  EXPECT_TRUE(reader.Close());
+}
+
+TEST(BitReaderTest, TotalCountersTest) { BitReaderTest_TotalCountersTest(); }
+
+HWY_ATTR void BitReaderTest_MoveTest() {
+  uint8_t buf[4] = {1, 2, 3, 4};
+  BitReader reader2;
+  {
+    BitReader reader1(Span<const uint8_t>(buf, sizeof(buf)));
+
+    EXPECT_EQ(0, reader1.TotalBitsConsumed());
+    reader1.ReadFixedBits<16>();
+    EXPECT_EQ(16, reader1.TotalBitsConsumed());
+
+    reader2 = std::move(reader1);
+    // From this point reader1 is invalid, but can continue to access reader2
+    // and we don't need to call Close() on reader1.
+  }
+
+  EXPECT_EQ(16, reader2.TotalBitsConsumed());
+  EXPECT_EQ(3U, reader2.ReadFixedBits<8>());
+  EXPECT_EQ(24, reader2.TotalBitsConsumed());
+
+  EXPECT_TRUE(reader2.Close());
+}
+
+TEST(BitReaderTest, MoveTest) { BitReaderTest_MoveTest(); }
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/bits_test.cc b/jxl/bits_test.cc
new file mode 100644
index 000000000..9df0c2571
--- /dev/null
+++ b/jxl/bits_test.cc
@@ -0,0 +1,105 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/bits.h"
+
+#include "gtest/gtest.h"
+
+namespace jxl {
+namespace {
+
+TEST(BitsTest, TestPopCount) {
+  EXPECT_EQ(0, PopCount(0u));
+  EXPECT_EQ(1, PopCount(1u));
+  EXPECT_EQ(1, PopCount(2u));
+  EXPECT_EQ(2, PopCount(3u));
+  EXPECT_EQ(1, PopCount(0x80000000u));
+  EXPECT_EQ(31, PopCount(0x7FFFFFFFu));
+  EXPECT_EQ(32, PopCount(0xFFFFFFFFu));
+
+  EXPECT_EQ(1, PopCount(0x80000000ull));
+  EXPECT_EQ(31, PopCount(0x7FFFFFFFull));
+  EXPECT_EQ(32, PopCount(0xFFFFFFFFull));
+  EXPECT_EQ(33, PopCount(0x10FFFFFFFFull));
+  EXPECT_EQ(63, PopCount(0xFFFEFFFFFFFFFFFFull));
+  EXPECT_EQ(64, PopCount(0xFFFFFFFFFFFFFFFFull));
+}
+
+TEST(BitsTest, TestNumZeroBits) {
+  // Zero input is well-defined.
+  EXPECT_EQ(32, NumZeroBitsAboveMSB(0u));
+  EXPECT_EQ(64, NumZeroBitsAboveMSB(0ull));
+  EXPECT_EQ(32, NumZeroBitsBelowLSB(0u));
+  EXPECT_EQ(64, NumZeroBitsBelowLSB(0ull));
+
+  EXPECT_EQ(31, NumZeroBitsAboveMSB(1u));
+  EXPECT_EQ(30, NumZeroBitsAboveMSB(2u));
+  EXPECT_EQ(63, NumZeroBitsAboveMSB(1ull));
+  EXPECT_EQ(62, NumZeroBitsAboveMSB(2ull));
+
+  EXPECT_EQ(0, NumZeroBitsBelowLSB(1u));
+  EXPECT_EQ(0, NumZeroBitsBelowLSB(1ull));
+  EXPECT_EQ(1, NumZeroBitsBelowLSB(2u));
+  EXPECT_EQ(1, NumZeroBitsBelowLSB(2ull));
+
+  EXPECT_EQ(0, NumZeroBitsAboveMSB(0x80000000u));
+  EXPECT_EQ(0, NumZeroBitsAboveMSB(0x8000000000000000ull));
+  EXPECT_EQ(31, NumZeroBitsBelowLSB(0x80000000u));
+  EXPECT_EQ(63, NumZeroBitsBelowLSB(0x8000000000000000ull));
+}
+
+TEST(BitsTest, TestFloorLog2) {
+  // for input = [1, 7]
+  const int expected[7] = {0, 1, 1, 2, 2, 2, 2};
+  for (uint32_t i = 1; i <= 7; ++i) {
+    EXPECT_EQ(expected[i - 1], FloorLog2Nonzero(i)) << " " << i;
+    EXPECT_EQ(expected[i - 1], FloorLog2Nonzero(uint64_t(i))) << " " << i;
+  }
+
+  EXPECT_EQ(31, FloorLog2Nonzero(0x80000000u));
+  EXPECT_EQ(31, FloorLog2Nonzero(0x80000001u));
+  EXPECT_EQ(31, FloorLog2Nonzero(0xFFFFFFFFu));
+
+  EXPECT_EQ(31, FloorLog2Nonzero(0x80000000ull));
+  EXPECT_EQ(31, FloorLog2Nonzero(0x80000001ull));
+  EXPECT_EQ(31, FloorLog2Nonzero(0xFFFFFFFFull));
+
+  EXPECT_EQ(63, FloorLog2Nonzero(0x8000000000000000ull));
+  EXPECT_EQ(63, FloorLog2Nonzero(0x8000000000000001ull));
+  EXPECT_EQ(63, FloorLog2Nonzero(0xFFFFFFFFFFFFFFFFull));
+}
+
+TEST(BitsTest, TestCeilLog2) {
+  // for input = [1, 7]
+  const int expected[7] = {0, 1, 2, 2, 3, 3, 3};
+  for (uint32_t i = 1; i <= 7; ++i) {
+    EXPECT_EQ(expected[i - 1], CeilLog2Nonzero(i)) << " " << i;
+    EXPECT_EQ(expected[i - 1], CeilLog2Nonzero(uint64_t(i))) << " " << i;
+  }
+
+  EXPECT_EQ(31, CeilLog2Nonzero(0x80000000u));
+  EXPECT_EQ(32, CeilLog2Nonzero(0x80000001u));
+  EXPECT_EQ(32, CeilLog2Nonzero(0xFFFFFFFFu));
+
+  EXPECT_EQ(31, CeilLog2Nonzero(0x80000000ull));
+  EXPECT_EQ(32, CeilLog2Nonzero(0x80000001ull));
+  EXPECT_EQ(32, CeilLog2Nonzero(0xFFFFFFFFull));
+
+  EXPECT_EQ(63, CeilLog2Nonzero(0x8000000000000000ull));
+  EXPECT_EQ(64, CeilLog2Nonzero(0x8000000000000001ull));
+  EXPECT_EQ(64, CeilLog2Nonzero(0xFFFFFFFFFFFFFFFFull));
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/block.h b/jxl/block.h
new file mode 100644
index 000000000..c7da3cd02
--- /dev/null
+++ b/jxl/block.h
@@ -0,0 +1,221 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BLOCK_H_
+#define JXL_BLOCK_H_
+
+// Adapters for DCT input/output: from/to contiguous blocks or image rows.
+
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/common.h"
+
+namespace jxl {
+
+// Adapters for source/destination.
+//
+// Block: (x, y) <-> (N * y + x)
+// Lines: (x, y) <-> (stride * y + x)
+//
+// I.e. Block is a specialization of Lines with fixed stride.
+//
+// FromXXX should implement Read and Load (Read vector).
+// ToXXX should implement Write and Store (Write vector).
+
+template <size_t N>
+using BlockDesc = HWY_CAPPED(float, N);
+
+// Here and in the following, the SZ template parameter specifies the number of
+// values to load/store. Needed because we want to handle 4x4 sub-blocks of
+// 16x16 blocks.
+template <size_t ROWS, size_t COLS = ROWS>
+class FromBlock {
+ public:
+  explicit FromBlock(const float* block) : block_(block) {}
+
+  FromBlock View(size_t dx, size_t dy) const {
+    return FromBlock<ROWS, COLS>(Address(dx, dy));
+  }
+
+  template <size_t SZ>
+  HWY_ATTR HWY_INLINE hwy::VT<BlockDesc<SZ>> LoadPart(const size_t row,
+                                                      size_t i) const {
+    static_assert(SZ <= COLS, "Requesting more values that are present!");
+    return Load(BlockDesc<SZ>(), block_ + row * COLS + i);
+  }
+
+  HWY_ATTR HWY_INLINE hwy::VT<BlockDesc<COLS>> LoadVec(const size_t row,
+                                                       size_t i) const {
+    return LoadPart<COLS>(row, i);
+  }
+
+  HWY_ATTR HWY_INLINE float Read(const size_t row, const size_t i) const {
+    return *Address(row, i);
+  }
+
+  constexpr HWY_INLINE const float* Address(const size_t row,
+                                            const size_t i) const {
+    return block_ + row * COLS + i;
+  }
+
+ private:
+  const float* block_;
+};
+
+template <size_t ROWS, size_t COLS = ROWS>
+class ToBlock {
+ public:
+  explicit ToBlock(float* block) : block_(block) {}
+
+  ToBlock View(size_t dx, size_t dy) const {
+    return ToBlock<ROWS, COLS>(Address(dx, dy));
+  }
+
+  template <size_t SZ>
+  HWY_ATTR HWY_INLINE void StorePart(const hwy::VT<BlockDesc<SZ>>& v,
+                                     const size_t row, const size_t i) const {
+    static_assert(SZ <= COLS, "Requesting more values that are present!");
+    Store(v, BlockDesc<SZ>(), Address(row, i));
+  }
+
+  HWY_ATTR HWY_INLINE void StoreVec(const hwy::VT<BlockDesc<COLS>>& v,
+                                    const size_t row, size_t i) const {
+    return StorePart<COLS>(v, row, i);
+  }
+
+  HWY_ATTR HWY_INLINE void Write(float v, const size_t row,
+                                 const size_t i) const {
+    *Address(row, i) = v;
+  }
+
+  constexpr HWY_INLINE float* Address(const size_t row, const size_t i) const {
+    return block_ + row * COLS + i;
+  }
+
+ private:
+  float* block_;
+};
+
+// Same as ToBlock, but multiplies result by (N * N)
+template <size_t ROWS, size_t COLS = ROWS>
+class ScaleToBlock {
+ public:
+  explicit HWY_ATTR ScaleToBlock(float* block) : block_(block) {}
+
+  ScaleToBlock View(size_t dx, size_t dy) const {
+    return ScaleToBlock<ROWS, COLS>(Address(dx, dy));
+  }
+
+  template <size_t SZ>
+  HWY_ATTR HWY_INLINE void StorePart(const hwy::VT<BlockDesc<SZ>>& v,
+                                     const size_t row, const size_t i) const {
+    using BlockDesc = jxl::BlockDesc<SZ>;
+    const auto mul_ = Set(BlockDesc(), 1.0f / (COLS * ROWS));
+    Store(v * mul_, BlockDesc(), Address(row, i));
+  }
+
+  HWY_ATTR HWY_INLINE void StoreVec(const hwy::VT<BlockDesc<COLS>>& v,
+                                    const size_t row, size_t i) const {
+    return StorePart<COLS>(v, row, i);
+  }
+
+  HWY_ATTR HWY_INLINE void Write(float v, const size_t row,
+                                 const size_t i) const {
+    constexpr float mul_ = 1.0f / (COLS * ROWS);
+    *Address(row, i) = v * mul_;
+  }
+
+  constexpr HWY_INLINE float* Address(const size_t row, const size_t i) const {
+    return block_ + row * COLS + i;
+  }
+
+ private:
+  float* block_;
+};
+
+template <size_t N>
+class FromLines {
+ public:
+  FromLines(const float* top_left, size_t stride)
+      : top_left_(top_left), stride_(stride) {}
+
+  FromLines View(size_t dx, size_t dy) const {
+    return FromLines(Address(dx, dy), stride_);
+  }
+
+  template <size_t SZ>
+  HWY_ATTR HWY_INLINE hwy::VT<BlockDesc<SZ>> LoadPart(const size_t row,
+                                                      const size_t i) const {
+    return Load(BlockDesc<SZ>(), Address(row, i));
+  }
+
+  HWY_ATTR HWY_INLINE hwy::VT<BlockDesc<N>> LoadVec(const size_t row,
+                                                    size_t i) const {
+    return LoadPart<N>(row, i);
+  }
+
+  HWY_ATTR HWY_INLINE float Read(const size_t row, const size_t i) const {
+    return *Address(row, i);
+  }
+
+  HWY_INLINE const float* Address(const size_t row, const size_t i) const {
+    return top_left_ + row * stride_ + i;
+  }
+
+ private:
+  const float* HWY_RESTRICT top_left_;
+  size_t stride_;  // move to next line by adding this to pointer
+};
+
+// Pointers are restrict-qualified: assumes we don't use both FromLines and
+// ToLines in the same DCT. NOTE: Transpose uses From/ToBlock, not *Lines.
+template <size_t N>
+class ToLines {
+ public:
+  ToLines(float* top_left, size_t stride)
+      : top_left_(top_left), stride_(stride) {}
+
+  ToLines View(const ToLines& other, size_t dx, size_t dy) const {
+    return ToLines(Address(dx, dy), stride_);
+  }
+
+  template <size_t SZ>
+  HWY_ATTR HWY_INLINE void StorePart(const hwy::VT<BlockDesc<SZ>>& v,
+                                     const size_t row, const size_t i) const {
+    Store(v, BlockDesc<SZ>(), Address(row, i));
+  }
+
+  HWY_ATTR HWY_INLINE void StoreVec(const hwy::VT<BlockDesc<N>>& v,
+                                    const size_t row, size_t i) const {
+    return StorePart<N>(v, row, i);
+  }
+
+  HWY_ATTR HWY_INLINE void Write(float v, const size_t row,
+                                 const size_t i) const {
+    *Address(row, i) = v;
+  }
+
+  HWY_INLINE float* Address(const size_t row, const size_t i) const {
+    return top_left_ + row * stride_ + i;
+  }
+
+ private:
+  float* HWY_RESTRICT top_left_;
+  size_t stride_;  // move to next line by adding this to pointer
+};
+
+}  // namespace jxl
+
+#endif  // JXL_BLOCK_H_
diff --git a/jxl/brotli.cc b/jxl/brotli.cc
new file mode 100644
index 000000000..f444e02de
--- /dev/null
+++ b/jxl/brotli.cc
@@ -0,0 +1,117 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/brotli.h"
+
+#include <string.h>  // memcpy
+
+#include <memory>
+
+#include "brotli/decode.h"
+#include "brotli/encode.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+Status BrotliCompress(int quality, const uint8_t* in, const size_t in_size,
+                      PaddedBytes* JXL_RESTRICT out) {
+  std::unique_ptr<BrotliEncoderState, decltype(BrotliEncoderDestroyInstance)*>
+      enc(BrotliEncoderCreateInstance(nullptr, nullptr, nullptr),
+          BrotliEncoderDestroyInstance);
+  if (!enc) return JXL_FAILURE("BrotliEncoderCreateInstance failed");
+
+  BrotliEncoderSetParameter(enc.get(), BROTLI_PARAM_QUALITY, quality);
+  BrotliEncoderSetParameter(enc.get(), BROTLI_PARAM_LGWIN, 24);
+  BrotliEncoderSetParameter(enc.get(), BROTLI_PARAM_LGBLOCK, 0);
+  BrotliEncoderSetParameter(enc.get(), BROTLI_PARAM_MODE, BROTLI_MODE_FONT);
+
+  const size_t kBufferSize = 128 * 1024;
+  PaddedBytes temp_buffer(kBufferSize);
+
+  size_t avail_in = in_size;
+  const uint8_t* next_in = in;
+
+  size_t total_out = 0;
+
+  while (true) {
+    size_t out_size;
+    size_t avail_out = kBufferSize;
+    uint8_t* next_out = temp_buffer.data();
+    if (!BrotliEncoderCompressStream(enc.get(), BROTLI_OPERATION_FINISH,
+                                     &avail_in, &next_in, &avail_out, &next_out,
+                                     &total_out)) {
+      return JXL_FAILURE("Brotli compression failed");
+    }
+    out_size = next_out - temp_buffer.data();
+    out->resize(out->size() + out_size);
+    memcpy(out->data() + out->size() - out_size, temp_buffer.data(), out_size);
+    if (BrotliEncoderIsFinished(enc.get())) break;
+  }
+
+  return true;
+}
+
+Status BrotliCompress(int quality, const uint8_t* in, const size_t in_size,
+                      uint8_t* JXL_RESTRICT out,
+                      size_t* JXL_RESTRICT total_out_size) {
+  PaddedBytes temp_buffer;
+  if (!BrotliCompress(quality, in, in_size, &temp_buffer)) {
+    return false;
+  }
+  *total_out_size = temp_buffer.size();
+  memcpy(out, temp_buffer.data(), temp_buffer.size());
+  return true;
+}
+
+Status BrotliCompress(int quality, const PaddedBytes& in,
+                      PaddedBytes* JXL_RESTRICT out) {
+  return BrotliCompress(quality, in.data(), in.size(), out);
+}
+
+Status BrotliDecompress(const Span<const uint8_t> in, size_t max_output_size,
+                        size_t* JXL_RESTRICT bytes_read,
+                        PaddedBytes* JXL_RESTRICT out) {
+  std::unique_ptr<BrotliDecoderState, decltype(BrotliDecoderDestroyInstance)*>
+      s(BrotliDecoderCreateInstance(nullptr, nullptr, nullptr),
+        BrotliDecoderDestroyInstance);
+  if (!s) return JXL_FAILURE("BrotliDecoderCreateInstance failed");
+
+  const size_t kBufferSize = 128 * 1024;
+  PaddedBytes temp_buffer(kBufferSize);
+
+  size_t avail_in = in.size();
+  if (in.size() == 0) return false;
+  const uint8_t* next_in = in.data();
+  BrotliDecoderResult code;
+
+  while (true) {
+    size_t out_size;
+    size_t avail_out = kBufferSize;
+    uint8_t* next_out = temp_buffer.data();
+    code = BrotliDecoderDecompressStream(s.get(), &avail_in, &next_in,
+                                         &avail_out, &next_out, nullptr);
+    out_size = next_out - temp_buffer.data();
+    out->resize(out->size() + out_size);
+    if (out->size() > max_output_size)
+      return JXL_FAILURE("Brotli output too large");
+    memcpy(out->data() + out->size() - out_size, temp_buffer.data(), out_size);
+    if (code != BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT) break;
+  }
+  if (code != BROTLI_DECODER_RESULT_SUCCESS)
+    return JXL_FAILURE("Brotli decompression failed");
+  *bytes_read += (in.size() - avail_in);
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/brotli.h b/jxl/brotli.h
new file mode 100644
index 000000000..b1730d595
--- /dev/null
+++ b/jxl/brotli.h
@@ -0,0 +1,46 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BROTLI_H_
+#define JXL_BROTLI_H_
+
+// Convenience functions for Brotli compression/decompression.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Appends to out.
+Status BrotliCompress(int quality, const PaddedBytes& in,
+                      PaddedBytes* JXL_RESTRICT out);
+
+// Appends to out + *total_out_size.
+Status BrotliCompress(int quality, const uint8_t* in, const size_t in_size,
+                      uint8_t* JXL_RESTRICT out,
+                      size_t* JXL_RESTRICT total_out_size);
+
+// Appends to out and ADDS to "bytes_read", which must be pre-initialized.
+Status BrotliDecompress(const Span<const uint8_t> in, size_t max_output_size,
+                        size_t* JXL_RESTRICT bytes_read,
+                        PaddedBytes* JXL_RESTRICT out);
+
+}  // namespace jxl
+
+#endif  // JXL_BROTLI_H_
diff --git a/jxl/brotli_test.cc b/jxl/brotli_test.cc
new file mode 100644
index 000000000..eac6fb22c
--- /dev/null
+++ b/jxl/brotli_test.cc
@@ -0,0 +1,78 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/brotli.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <random>
+
+#include "gtest/gtest.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/thread_pool_internal.h"
+
+namespace jxl {
+namespace {
+
+TEST(BrotliTest, TestCompressEmpty) {
+  PaddedBytes in;
+  PaddedBytes out;
+  EXPECT_TRUE(BrotliCompress(6, in, &out));
+  EXPECT_TRUE(in.empty());
+}
+
+TEST(BrotliTest, TestDecompressEmpty) {
+  size_t bytes_read = 0;
+  PaddedBytes in;
+  PaddedBytes out;
+  EXPECT_FALSE(BrotliDecompress(Span<const uint8_t>(in), 1, &bytes_read, &out));
+  EXPECT_EQ(0, bytes_read);
+  EXPECT_TRUE(in.empty());
+  EXPECT_TRUE(out.empty());
+}
+
+TEST(BrotliTest, TestRoundTrip) {
+  ThreadPoolInternal pool(0);
+  pool.Run(1, 65, ThreadPool::SkipInit(), [](const int task, const int thread) {
+    const size_t size = task;
+
+    PaddedBytes in(size);
+    std::mt19937_64 rng(thread * 65537 + task * 129);
+    std::generate(in.begin(), in.end(), rng);
+    PaddedBytes compressed;
+    PaddedBytes out;
+
+    for (int quality = 1; quality < 7; ++quality) {
+      compressed.clear();
+      EXPECT_TRUE(BrotliCompress(quality, in, &compressed));
+      size_t bytes_read = 0;
+      out.clear();
+      EXPECT_TRUE(BrotliDecompress(Span<const uint8_t>(compressed), size,
+                                   &bytes_read, &out));
+      EXPECT_EQ(compressed.size(), bytes_read);
+      EXPECT_EQ(in.size(), out.size());
+      for (size_t i = 0; i < in.size(); ++i) {
+        if (in[i] != out[i]) {
+          printf("Mismatch at %zu (%zu)\n", i, size);
+          exit(1);
+        }
+      }
+    }
+  });
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/brunsli.cc b/jxl/brunsli.cc
new file mode 100644
index 000000000..803cc5698
--- /dev/null
+++ b/jxl/brunsli.cc
@@ -0,0 +1,1702 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/brunsli.h"
+
+#include <brunsli/brunsli_decode.h>
+#include <brunsli/brunsli_encode.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <cstring>
+#include <hwy/static_targets.h>
+#include <utility>
+#include <vector>
+
+#include "jxl/block.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/dct.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_xyb.h"
+#include "jxl/gaborish.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "third_party/brunsli/c/common/constants.h"
+#include "third_party/brunsli/c/common/context.h"
+#include "third_party/brunsli/c/dec/state.h"
+#include "third_party/brunsli/c/enc/state.h"
+
+namespace jxl {
+
+namespace {
+
+const uint8_t kBrunsliMagic[] = {0x0A, 0x04, 'B', 0xd2, 0xd5, 'N', 0x12};
+constexpr const size_t kBrunsliMagicSize = sizeof(kBrunsliMagic);
+
+const uint8_t kBrunsliXHdr[] = {'B', 'R', 'N', ':', 'H', 'D', 'R'};
+
+const uint8_t kIccProfileTag[] = {'I', 'C', 'C', '_', 'P', 'R',
+                                  'O', 'F', 'I', 'L', 'E', 0x00};
+
+// JPEG markers.
+constexpr const uint8_t kSof0 = 0xC0;
+constexpr const uint8_t kDht = 0xC4;
+constexpr const uint8_t kEoi = 0xD9;
+constexpr const uint8_t kSos = 0xDA;
+constexpr const uint8_t kDqt = 0xDB;
+constexpr const uint8_t kApp0 = 0xE0;
+constexpr const uint8_t kApp2 = 0xE2;
+constexpr const uint8_t kApp9 = 0xE9;
+
+constexpr const uint32_t kHuffmanCodeDcSlotOffset = 0x00;
+constexpr const uint32_t kHuffmanCodeAcSlotOffset = 0x10;
+
+// JFIF / v1.1 / no units / square pixel / no thumbnail.
+const uint8_t kApp0Template[] = {kApp0, 0x00, 0x10, 0x4A, 0x46, 0x49,
+                                 0x46,  0x00, 0x01, 0x01, 0x00, 0x00,
+                                 0x01,  0x00, 0x01, 0x00, 0x00};
+constexpr const size_t kApp0TemplateSize = sizeof(kApp0Template);
+
+/*struct EntropySource {
+  PaddedBytes payload;
+  bool active = false;
+  std::unique_ptr<BitReader> reader;
+  ANSCode code;
+  std::vector<uint8_t> context_map;
+  std::unique_ptr<ANSSymbolReader> decoder;
+};*/
+
+struct BrunsliExtensions {
+  std::string hdr_orig_colorspace;
+  std::string hdr_colorspace;
+};
+
+using ByteSpan = Span<const uint8_t>;
+
+// Transform RGB to YCbCr.
+// Could be performed in-place (i.e. Y, Cb and Cr could alias R, B and B).
+HWY_ATTR void RgbToYcbcr(const ImageF& r_plane, const ImageF& g_plane,
+                         const ImageF& b_plane, ImageF* y_plane,
+                         ImageF* cb_plane, ImageF* cr_plane) {
+  const HWY_FULL(float) df;
+  constexpr size_t S = df.N;  // Step.
+
+  const size_t xsize = r_plane.xsize();
+  const size_t ysize = r_plane.ysize();
+
+  const auto cm128 = Set(df, -128.0f);
+  const auto c05 = Set(df, 0.5f);
+  const auto cyr = Set(df, 0.299f);
+  const auto cyg = Set(df, 0.587f);
+  const auto cyb = Set(df, 0.114f);
+  const auto ccbr = Set(df, -0.1687f);
+  const auto ccbg = Set(df, -0.3313f);
+  const auto& ccbb = c05;
+  const auto& ccrr = c05;
+  const auto ccrg = Set(df, -0.4187f);
+  const auto ccrb = Set(df, -0.0813f);
+  for (size_t y = 0; y < ysize; ++y) {
+    const float* JXL_RESTRICT r_row = r_plane.ConstRow(y);
+    const float* JXL_RESTRICT g_row = g_plane.ConstRow(y);
+    const float* JXL_RESTRICT b_row = b_plane.ConstRow(y);
+    float* JXL_RESTRICT y_row = y_plane->Row(y);
+    float* JXL_RESTRICT cb_row = cb_plane->Row(y);
+    float* JXL_RESTRICT cr_row = cr_plane->Row(y);
+    for (size_t x = 0; x < xsize; x += S) {
+      const auto r_vec = Load(df, r_row + x);
+      const auto g_vec = Load(df, g_row + x);
+      const auto b_vec = Load(df, b_row + x);
+      const auto y_vec =
+          MulAdd(cyr, r_vec, MulAdd(cyg, g_vec, MulAdd(cyb, b_vec, cm128)));
+      const auto cb_vec =
+          MulAdd(ccbr, r_vec, MulAdd(ccbg, g_vec, ccbb * b_vec));
+      const auto cr_vec =
+          MulAdd(ccrr, r_vec, MulAdd(ccrg, g_vec, ccrb * b_vec));
+      Store(y_vec, df, y_row + x);
+      Store(cb_vec, df, cb_row + x);
+      Store(cr_vec, df, cr_row + x);
+    }
+  }
+}
+
+struct GetDcCoeff {
+  const ::brunsli::coeff_t* JXL_RESTRICT src;
+  size_t xsize_blocks;
+
+  ::brunsli::coeff_t operator()(size_t u, size_t v) const {
+    return src[(u + v * xsize_blocks) * kDCTBlockSize];
+  }
+};
+
+struct GetAcCoeff {
+  const ::brunsli::coeff_t* JXL_RESTRICT src;
+  size_t xsize_blocks;
+
+  ::brunsli::coeff_t operator()(size_t u, size_t v) const {
+    return src[(u + v * xsize_blocks) * kDCTBlockSize + 1];
+  }
+};
+
+struct GetDcCoeffTransposed {
+  const ::brunsli::coeff_t* JXL_RESTRICT src;
+  size_t xsize_blocks;
+
+  ::brunsli::coeff_t operator()(size_t u, size_t v) const {
+    return src[(v + u * xsize_blocks) * kDCTBlockSize];
+  }
+};
+
+struct GetAcCoeffTransposed {
+  const ::brunsli::coeff_t* JXL_RESTRICT src;
+  size_t xsize_blocks;
+
+  ::brunsli::coeff_t operator()(size_t u, size_t v) const {
+    return src[(v + u * xsize_blocks) * kDCTBlockSize + 8];
+  }
+};
+
+struct MakeSlope {
+  template <typename GetDcCoeffFn, typename GetAcCoeffFn>
+  bool operator()(const GetDcCoeffFn& get_dc_coeff,
+                  const GetAcCoeffFn& get_ac_coeff, float q_ac_dc, size_t y,
+                  size_t x0, size_t x1, size_t xsize, float* JXL_RESTRICT from,
+                  float* JXL_RESTRICT to) {
+    const float current = static_cast<float>(get_dc_coeff(x0, y));
+    float ll = current;
+    const float lr = current;
+    if (x0 != 0) {
+      ll = get_dc_coeff(x0 - 1, y) - q_ac_dc * get_ac_coeff(x0 - 1, y);
+    }
+    const float l = (ll + lr) * 0.5f;
+
+    const float rl = current;
+    float rr = current;
+    if (x1 != xsize - 1) {
+      rr = get_dc_coeff(x1 + 1, y) + q_ac_dc * get_ac_coeff(x1 + 1, y);
+    }
+    const float r = (rl + rr) * 0.5f;
+
+    const float l_delta = l - current;
+    const float r_delta = r - current;
+    const float l_delta_abs = std::abs(l_delta);
+    const float r_delta_abs = std::abs(r_delta);
+    static constexpr const float kThreshold = 0.5f;
+    const float step_threshold = 0.5f;
+    bool is_sane = (std::abs(l - r) > step_threshold) &&
+                   (l_delta_abs <= kThreshold) && (r_delta_abs <= kThreshold);
+    *from = l;
+    *to = r;
+    return is_sane;
+  }
+};
+
+// First and last indices (inclusive) of the range of repeated DC values.
+struct Range {
+  size_t x0;
+  size_t x1;
+  Range(size_t left, size_t right) : x0(left), x1(right){};
+};
+typedef std::vector<Range> RowRanges;
+typedef std::vector<RowRanges> Ranges;
+
+template <typename GetDcCoeffFn, typename GetAcCoeffFn>
+Ranges FindPlateaux(const GetDcCoeffFn& get_dc_coeff,
+                    const GetAcCoeffFn& get_ac_coeff, size_t xsize,
+                    size_t ysize) {
+  Ranges result(ysize);
+  for (size_t y = 0; y < ysize; ++y) {
+    RowRanges& row = result[y];
+    ::brunsli::coeff_t span_dc = get_dc_coeff(0, y);
+    size_t span_start = 0;
+    for (size_t x = 0; x <= xsize; ++x) {
+      bool end = (x == xsize);
+      ::brunsli::coeff_t dc = end ? (span_dc + 8192) : get_dc_coeff(x, y);
+      bool step = span_dc != dc;
+      bool bump = end || (get_ac_coeff(x, y) != 0);
+      if (step || bump) {
+        if (span_start < x) row.emplace_back(span_start, x - 1);
+        span_start = x + (bump ? 1 : 0);
+        span_dc = dc;
+      }
+    }
+  }
+  return result;
+}
+
+template <typename GetDcCoeffFn, typename GetAcCoeffFn, typename SlopeFn>
+void BuildSlopes(const GetDcCoeffFn& get_dc_coeff,
+                 const GetAcCoeffFn& get_ac_coeff, SlopeFn* make_slope,
+                 const float q_ac_dc, const Ranges& ranges, ImageF* approx,
+                 ImageB* presence, ImageF* slope) {
+  const size_t xsize = approx->xsize();
+  const size_t ysize = approx->ysize();
+
+  for (size_t y = 0; y < ysize; ++y) {
+    float* JXL_RESTRICT approx_row = approx->Row(y);
+    float* JXL_RESTRICT slope_row = slope->Row(y);
+    uint8_t* JXL_RESTRICT presence_row = presence->Row(y);
+    std::fill_n(presence_row, xsize, 0);
+    for (auto& range : ranges[y]) {
+      const size_t x0 = range.x0;
+      const size_t x1 = range.x1;
+      float from;
+      float to;
+      bool has_slope = (*make_slope)(get_dc_coeff, get_ac_coeff, q_ac_dc, y, x0,
+                                     x1, xsize, &from, &to);
+      if (!has_slope) continue;
+      const size_t span = x1 - x0 + 1;
+      const float delta = to - from;
+      const float step = delta / span;
+      const float base = from + 0.5f * step;
+      for (size_t x = x0; x <= x1; ++x) {
+        presence_row[x] = 1;
+        const float diff = static_cast<float>(x - x0);
+        approx_row[x] = base + diff * step;
+        // Base function f(u, k) = k * cos(u / Pi) -> f(0, k) = k, f(1, k) = -k.
+        // To stitch consecutive "steps" we want
+        // f(0) = -0.5 * step, f(1) = 0.5 * step
+        slope_row[x] = -0.5f * step;
+      }
+    }
+  }
+}
+
+HWY_ATTR void FixDc(const ::brunsli::coeff_t* JXL_RESTRICT coeffs, int q_dc,
+                    int q_ach, int q_acv, size_t xsize_blocks,
+                    size_t ysize_blocks, float* JXL_RESTRICT dst,
+                    size_t dst_stride) {
+  constexpr size_t N = kBlockDim;
+  static_assert(N == 8, "JPEG block dim must be 8");
+  static_assert(kDCTBlockSize == N * N, "JPEG block size must be 64");
+  const float dequant_mult =
+      static_cast<float>(q_dc) * IDCTScales<N>()[0] * IDCTScales<N>()[0];
+  const float q_ach_dc = static_cast<float>(q_ach) / static_cast<float>(q_dc);
+  const float q_acv_dc = static_cast<float>(q_acv) / static_cast<float>(q_dc);
+
+  MakeSlope make_slope;
+
+  const GetDcCoeff get_dc_coeff = {coeffs, xsize_blocks};
+  const GetAcCoeff get_ac_coeff = {coeffs, xsize_blocks};
+  Ranges ranges_h =
+      FindPlateaux(get_dc_coeff, get_ac_coeff, xsize_blocks, ysize_blocks);
+  ImageF approx_h(xsize_blocks, ysize_blocks);
+  ImageF slope_h(xsize_blocks, ysize_blocks);
+  ImageB has_h(xsize_blocks, ysize_blocks);
+  BuildSlopes(get_dc_coeff, get_ac_coeff, &make_slope, q_ach_dc, ranges_h,
+              &approx_h, &has_h, &slope_h);
+
+  const GetDcCoeffTransposed get_dc_t_coeff = {coeffs, xsize_blocks};
+  const GetAcCoeffTransposed get_ac_t_coeff = {coeffs, xsize_blocks};
+  Ranges ranges_v =
+      FindPlateaux(get_dc_t_coeff, get_ac_t_coeff, ysize_blocks, xsize_blocks);
+  // NB: transposed.
+  ImageF approx_v(ysize_blocks, xsize_blocks);
+  ImageF slope_v(ysize_blocks, xsize_blocks);
+  ImageB has_v(ysize_blocks, xsize_blocks);
+  BuildSlopes(get_dc_t_coeff, get_ac_t_coeff, &make_slope, q_acv_dc, ranges_v,
+              &approx_v, &has_v, &slope_v);
+
+  for (size_t by = 0; by < ysize_blocks; ++by) {
+    const ::brunsli::coeff_t* JXL_RESTRICT coeffs_row =
+        &coeffs[by * xsize_blocks * kDCTBlockSize];
+    const uint8_t* JXL_RESTRICT has_h_row = has_h.ConstRow(by);
+    const float* JXL_RESTRICT approx_h_row = approx_h.ConstRow(by);
+    for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+      const bool has_h_val = (has_h_row[bx] != 0);
+      const bool has_v_val = (has_v.ConstRow(bx)[by] != 0);
+      float fixed_val;
+      if (has_h_val && has_v_val) {
+        fixed_val = 0.5f * (approx_h_row[bx] + approx_v.ConstRow(bx)[by]);
+      } else if (has_h_val) {
+        fixed_val = approx_h_row[bx];
+      } else if (has_v_val) {
+        fixed_val = approx_v.ConstRow(bx)[by];
+      } else {
+        fixed_val = coeffs_row[bx * kDCTBlockSize];
+      }
+      dst[by * dst_stride + bx * kDCTBlockSize] = fixed_val * dequant_mult;
+    }
+  }
+
+  for (size_t by = 0; by < ysize_blocks; ++by) {
+    for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+      const int16_t* JXL_RESTRICT src_coeffs =
+          &coeffs[(by * xsize_blocks + bx) * kDCTBlockSize];
+      const bool has_v_val = (has_v.ConstRow(bx)[by] != 0);
+      const float slope_v_val = slope_v.ConstRow(bx)[by];
+      if (has_v_val && (src_coeffs[8] == 0)) {
+        dst[by * dst_stride + bx * kDCTBlockSize + 8] =
+            slope_v_val * dequant_mult;
+      }
+    }
+  }
+  for (size_t by = 0; by < ysize_blocks; ++by) {
+    const uint8_t* JXL_RESTRICT has_h_row = has_h.ConstRow(by);
+    const float* JXL_RESTRICT slope_h_row = slope_h.ConstRow(by);
+    for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+      const bool has_h_val = (has_h_row[bx] != 0);
+      const float slope_h_val = slope_h_row[bx];
+      const int16_t* JXL_RESTRICT src_coeffs =
+          &coeffs[(by * xsize_blocks + bx) * kDCTBlockSize];
+      if (has_h_val && (src_coeffs[1] == 0)) {
+        dst[by * dst_stride + bx * kDCTBlockSize + 1] =
+            slope_h_val * dequant_mult;
+      }
+    }
+  }
+}
+
+HWY_ATTR Status JpegDataToPixels(const brunsli::JPEGData& src,
+                                 const BrunsliDecoderOptions& options,
+                                 const BrunsliExtensions& extensions,
+                                 Image3F* out, bool* is_gray,
+                                 ThreadPool* pool) {
+  constexpr size_t N = kBlockDim;
+  static_assert(N == 8, "JPEG block dim must be 8");
+  static_assert(kDCTBlockSize == N * N, "JPEG block size must be 64");
+  const size_t xsize = src.width;
+  const size_t ysize = src.height;
+
+  const HWY_FULL(float) df;
+  constexpr size_t S = df.N;  // Step.
+  static_assert(kDCTBlockSize % S == 0,
+                "Block size should be divisible by SIMD length");
+  const HWY_CAPPED(int32_t, S) di32;
+  const HWY_CAPPED(int16_t, S) di16;
+  JXL_RETURN_IF_ERROR(src.components.size() == 1 || src.components.size() == 3);
+  *is_gray = src.components.size() == 1;
+
+  ImageF planes[3];
+
+  const size_t xmcu = N * src.max_h_samp_factor;
+  const size_t ymcu = N * src.max_v_samp_factor;
+
+  for (size_t c = 0; c < src.components.size(); ++c) {
+    const brunsli::JPEGComponent& component = src.components[c];
+    const size_t factor_x = src.max_h_samp_factor / component.h_samp_factor;
+    const size_t factor_y = src.max_v_samp_factor / component.v_samp_factor;
+    JXL_RETURN_IF_ERROR(src.max_h_samp_factor % component.h_samp_factor == 0);
+    JXL_RETURN_IF_ERROR(src.max_v_samp_factor % component.v_samp_factor == 0);
+    JXL_RETURN_IF_ERROR(component.quant_idx <
+                        static_cast<int>(src.quant.size()));
+    const size_t comp_xsize = component.width_in_blocks * N * factor_x;
+    const size_t comp_ysize = component.height_in_blocks * N * factor_y;
+    JXL_RETURN_IF_ERROR(xsize <= comp_xsize);
+    JXL_RETURN_IF_ERROR(xsize > comp_xsize - xmcu);
+    JXL_RETURN_IF_ERROR(ysize <= comp_ysize);
+    JXL_RETURN_IF_ERROR(ysize > comp_ysize - ymcu);
+    if (c == 0) {
+      JXL_RETURN_IF_ERROR((factor_x == 1) && (factor_y == 1));
+    } else {
+      JXL_RETURN_IF_ERROR((factor_x <= 2) && (factor_y <= 2));
+    }
+  }
+
+  for (size_t c = 0; c < src.components.size(); ++c) {
+    const brunsli::JPEGComponent& component = src.components[c];
+    const brunsli::JPEGQuantTable& quant_table = src.quant[component.quant_idx];
+    const size_t xsize_blocks = component.width_in_blocks;
+    const size_t ysize_blocks = component.height_in_blocks;
+    ImageF pixels(xsize_blocks * N, ysize_blocks * N);
+    const size_t pixels_stride = pixels.PixelsPerRow();
+    const size_t factor_x = src.max_h_samp_factor / component.h_samp_factor;
+    const size_t factor_y = src.max_v_samp_factor / component.v_samp_factor;
+
+    HWY_ALIGN float quant[kDCTBlockSize];
+
+    for (size_t y = 0; y < N; ++y) {
+      for (size_t x = 0; x < N; ++x) {
+        const size_t i = y * N + x;
+        quant[i] = static_cast<float>(quant_table.values[i]) *
+                   IDCTScales<N>()[x] * IDCTScales<N>()[y];
+      }
+    }
+
+    // Dequantize.
+    constexpr size_t group_dim = kGroupDimInBlocks;
+    ImageF dequantized(xsize_blocks * kDCTBlockSize, ysize_blocks);
+    const size_t xsize_groups = DivCeil(xsize_blocks, group_dim);
+    const size_t ysize_groups = DivCeil(ysize_blocks, group_dim);
+    const auto dequantize = [&](int idx, int /* thread */) HWY_ATTR {
+      HWY_ALIGN int16_t coeffs[kDCTBlockSize];
+      const size_t gx = idx % xsize_groups;
+      const size_t gy = idx / xsize_groups;
+      const Rect group_rect_blocks(gx * group_dim, gy * group_dim, group_dim,
+                                   group_dim, xsize_blocks, ysize_blocks);
+      const size_t bx0 = group_rect_blocks.x0();
+      const size_t bx1 = bx0 + group_rect_blocks.xsize();
+      const size_t by0 = group_rect_blocks.y0();
+      const size_t by1 = by0 + group_rect_blocks.ysize();
+      for (size_t by = by0; by < by1; ++by) {
+        float* JXL_RESTRICT dequantized_row = dequantized.Row(by);
+        for (size_t bx = bx0; bx < bx1; ++bx) {
+          float* JXL_RESTRICT dequantized_block =
+              dequantized_row + bx * kDCTBlockSize;
+          const int16_t* JXL_RESTRICT src_coeffs =
+              &component.coeffs[(by * xsize_blocks + bx) * kDCTBlockSize];
+          memcpy(coeffs, src_coeffs, sizeof(int16_t) * kDCTBlockSize);
+
+          for (size_t i = 0; i < kDCTBlockSize; i += S) {
+            const auto coeff =
+                ConvertTo(df, ConvertTo(di32, Load(di16, coeffs + i)));
+            const auto mult = Load(df, quant + i);
+            Store(coeff * mult, df, dequantized_block + i);
+          }
+        }
+      }
+    };
+    RunOnPool(pool, 0, static_cast<int>(xsize_groups * ysize_groups),
+              ThreadPool::SkipInit(), dequantize, "Brunsli:Dequantize");
+
+    // TODO(eustas): turn to pipeline and parallelize.
+    if (options.fix_dc_staircase) {
+      const int q_dc = src.quant[component.quant_idx].values[0];
+      const int q_ach = src.quant[component.quant_idx].values[1];
+      const int q_acv = src.quant[component.quant_idx].values[8];
+      FixDc(component.coeffs.data(), q_dc, q_ach, q_acv, xsize_blocks,
+            ysize_blocks, dequantized.Row(0), dequantized.PixelsPerRow());
+    }
+
+    const auto idct = [&](int idx, int /* thread */) HWY_ATTR {
+      const size_t gx = idx % xsize_groups;
+      const size_t gy = idx / xsize_groups;
+      const Rect group_rect_blocks(gx * group_dim, gy * group_dim, group_dim,
+                                   group_dim, xsize_blocks, ysize_blocks);
+      const size_t bx0 = group_rect_blocks.x0();
+      const size_t bx1 = bx0 + group_rect_blocks.xsize();
+      const size_t by0 = group_rect_blocks.y0();
+      const size_t by1 = by0 + group_rect_blocks.ysize();
+      for (size_t by = by0; by < by1; ++by) {
+        const float* JXL_RESTRICT dequantized_row = dequantized.Row(by);
+        float* JXL_RESTRICT pixels_row = pixels.Row(by * N);
+        for (size_t bx = bx0; bx < bx1; ++bx) {
+          ComputeTransposedScaledIDCT<N>()(
+              FromBlock<N>(dequantized_row + bx * kDCTBlockSize),
+              ToLines<N>(pixels_row + bx * N, pixels_stride));
+          GenericTransposeBlockInplace<N>(
+              FromLines<N>(pixels_row + bx * N, pixels_stride),
+              ToLines<N>(pixels_row + bx * N, pixels_stride));
+        }
+      }
+    };
+    RunOnPool(pool, 0, static_cast<int>(xsize_groups * ysize_groups),
+              ThreadPool::SkipInit(), idct, "Brunsli:IDCT");
+
+    // TODO: before or after upsampling?
+    if (options.gaborish) {
+      ImageF gab_pixels(pixels.xsize(), pixels.ysize());
+      ConvolveGaborish(pixels, 0.2, 0.2, pool, &gab_pixels);
+      for (size_t by = 0; by < ysize_blocks; ++by) {
+        for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+          const float* JXL_RESTRICT gab_pixels_row =
+              gab_pixels.Row(by * N) + bx * N;
+          float* JXL_RESTRICT pixels_row = pixels.Row(by * N) + bx * N;
+          size_t count = 0;
+          for (size_t u = 1; u < N - 1; ++u) {
+            for (size_t v = 1; v < N - 1; ++v) {
+              float a = gab_pixels_row[u + pixels_stride * v];
+              float b = pixels_row[u + pixels_stride * v];
+              if (std::abs(a - b) > 5) count++;
+            }
+          }
+          if (count >= 9) continue;
+          // TODO: vectorize
+          for (size_t u = 0; u < N; ++u) {
+            for (size_t v = 0; v < N; ++v) {
+              pixels_row[u + pixels_stride * v] =
+                  gab_pixels_row[u + pixels_stride * v];
+            }
+          }
+        }
+      }
+    }
+
+    if (factor_x == 1) {
+      if (factor_y == 1) {
+        planes[c] = std::move(pixels);
+      } else {
+        planes[c] = UpsampleV2(pixels, pool);
+      }
+    } else {
+      if (factor_y == 1) {
+        planes[c] = UpsampleH2(pixels, pool);
+      } else {
+        planes[c] = UpsampleV2(UpsampleH2(pixels, pool), pool);
+      }
+    }
+  }
+
+  if (*is_gray) {
+    const auto c128 = Set(df, 128.0f);
+    for (size_t y = 0; y < ysize; ++y) {
+      float* JXL_RESTRICT y_row = planes[0].Row(y);
+      for (size_t x = 0; x < xsize; x += S) {
+        Store(Load(df, y_row + x) + c128, df, y_row + x);
+      }
+    }
+  } else {
+    YcbcrToRgb(planes[0], planes[1], planes[2], &planes[0], &planes[1],
+               &planes[2], pool);
+  }
+
+  if (*is_gray) {
+    planes[1] = CopyImage(planes[0]);
+    planes[2] = CopyImage(planes[0]);
+  }
+  Image3F rgb(std::move(planes[0]), std::move(planes[1]), std::move(planes[2]));
+  rgb.ShrinkTo(xsize, ysize);
+  *out = std::move(rgb);
+
+  return true;
+}
+
+bool GetMarkerPayload(const std::string& marker, ByteSpan* payload) {
+  if (marker.size() < 3) {
+    return false;
+  }
+  size_t hi = marker[1];
+  size_t lo = marker[2];
+  size_t size = (hi << 8u) | lo;
+  // Second byte of marker is not counted towards size.
+  if (size != marker.size() - 1) {
+    return false;
+  }
+  // cut second marker byte and "length" from payload.
+  *payload = ByteSpan(marker);
+  payload->remove_prefix(3);
+  return true;
+}
+
+Status ParseChunkedMarker(const brunsli::JPEGData& src, uint8_t marker_type,
+                          const ByteSpan& tag, PaddedBytes* output) {
+  output->clear();
+
+  std::vector<ByteSpan> chunks;
+  std::vector<bool> presence;
+  size_t expected_number_of_parts = 0;
+  bool is_first_chunk = true;
+  for (const std::string& marker : src.app_data) {
+    if (marker.empty() || marker[0] != marker_type) {
+      continue;
+    }
+    ByteSpan payload;
+    if (!GetMarkerPayload(marker, &payload)) {
+      // Something is wrong with this marker; does not care.
+      continue;
+    }
+    if ((payload.size() < tag.size()) ||
+        memcmp(payload.data(), tag.data(), tag.size()) != 0) {
+      continue;
+    }
+    payload.remove_prefix(tag.size());
+    if (payload.size() < 2) {
+      return JXL_FAILURE("Chunk is too small.");
+    }
+    uint8_t index = payload[0];
+    uint8_t total = payload[1];
+    payload.remove_prefix(2);
+
+    JXL_RETURN_IF_ERROR(total != 0);
+    if (is_first_chunk) {
+      is_first_chunk = false;
+      expected_number_of_parts = total;
+      // 1-based indices; 0-th element is added for convenience.
+      chunks.resize(total + 1);
+      presence.resize(total + 1);
+    } else {
+      JXL_RETURN_IF_ERROR(expected_number_of_parts == total);
+    }
+
+    if (index == 0 || index > total) {
+      return JXL_FAILURE("Invalid chunk index.");
+    }
+
+    if (presence[index]) {
+      return JXL_FAILURE("Duplicate chunk.");
+    }
+    presence[index] = true;
+    chunks[index] = payload;
+  }
+
+  for (size_t i = 0; i < expected_number_of_parts; ++i) {
+    // 0-th element is not used.
+    size_t index = i + 1;
+    if (!presence[index]) {
+      return JXL_FAILURE("Missing chunk.");
+    }
+    output->append(chunks[index]);
+  }
+
+  return true;
+}
+
+Status AddChunkedMarker(brunsli::JPEGData* out, uint8_t marker_type,
+                        const ByteSpan& tag, const PaddedBytes& payload) {
+  // 2 bytes encode size itself, 2 bytes are index / total.
+  const size_t kChunkOverhead = 4 + tag.size();
+  // Chunk size is an uint16_t.
+  constexpr const size_t kChunkSizeBits = 16;
+  const size_t kMaxChunkSize = (1u << kChunkSizeBits) - kChunkOverhead;
+
+  const size_t full_size = payload.size();
+  const size_t num_chunks = DivCeil(full_size, kMaxChunkSize);
+  if (num_chunks == 0 || num_chunks > 255) {
+    return false;
+  }
+  for (size_t i = 0; i < num_chunks; ++i) {
+    const size_t start = i * kMaxChunkSize;
+    const size_t end = std::min(start + kMaxChunkSize, full_size);
+    const size_t part_length = end - start;
+    const size_t chunk_length = part_length + kChunkOverhead;
+
+    JXL_ASSERT((chunk_length >> kChunkSizeBits) == 0);
+    const uint8_t hi = chunk_length >> 8u;
+    const uint8_t lo = chunk_length & 0xFFu;
+
+    std::string extension_chunk;
+    extension_chunk.push_back(marker_type);
+    extension_chunk.push_back(hi);
+    extension_chunk.push_back(lo);
+    extension_chunk.append(reinterpret_cast<const char*>(tag.data()),
+                           tag.size());
+    extension_chunk.push_back(i + 1);
+    extension_chunk.push_back(num_chunks);
+    extension_chunk.append(
+        reinterpret_cast<const char*>(payload.data() + start), part_length);
+
+    // 1 extra for marker byte.
+    JXL_ASSERT(extension_chunk.size() == (chunk_length + 1));
+    out->app_data.push_back(extension_chunk);
+    out->marker_order.push_back(marker_type);
+  }
+  return true;
+}
+
+/*void ParseEntropySource(const brunsli::JPEGData& src, EntropySource* dst,
+                        ByteSpan tag) {
+  if (!ParseChunkedMarker(src, kApp9, tag, &dst->payload)) {
+    dst->payload.clear();
+    JXL_WARNING("ReJPEG: corrupted extension payload\n");
+    return;
+  }
+  if (dst->payload.empty()) {
+    return;
+  }
+
+  dst->reader = make_unique<BitReader>(dst->payload);
+  if (!DecodeHistograms(dst->reader.get(), 1, 32, &dst->code,
+                        &dst->context_map)) {
+    JXL_WARNING("ReJPEG: failed to decode extension histograms\n");
+    return;
+  }
+
+  dst->decoder = make_unique<ANSSymbolReader>(&dst->code, dst->reader.get());
+  dst->active = true;
+}*/
+
+BrunsliExtensions ParseBrunsliExtensions(const brunsli::JPEGData& src) {
+  BrunsliExtensions result{};
+  /*for (const std::string& marker : src.app_data) {
+    // Filter out non-APP9 markers.
+    if (marker.empty() || marker[0] != kApp9) {
+      continue;
+    }
+    ByteSpan payload;
+    if (!GetMarkerPayload(marker, &payload)) {
+      // Something is wrong with this marker; does not care.
+      continue;
+    }
+  }*/
+
+  PaddedBytes hdr_payload;
+  if (!ParseChunkedMarker(src, kApp9, ByteSpan(kBrunsliXHdr), &hdr_payload)) {
+    hdr_payload.clear();
+    JXL_WARNING("ReJPEG: corrupted hdr extension payload\n");
+  }
+  if (!hdr_payload.empty()) {
+    std::string colorspaces = std::string(
+        reinterpret_cast<const char*>(hdr_payload.data()), hdr_payload.size());
+    const size_t pos = colorspaces.find('>', 0);
+    if (pos != std::string::npos) {
+      result.hdr_orig_colorspace = colorspaces.substr(0, pos);
+      result.hdr_colorspace = colorspaces.substr(pos + 1);
+    }
+  }
+
+  return result;
+}
+
+// Update |data| to point to the start of the next section.
+bool SkipSection(const uint8_t** data, size_t len) {
+  size_t section_len = 0;
+  uint64_t b = 0x80;
+  size_t off = 1;
+  for (size_t i = 0; (i < 9) && (b & 0x80u); ++i) {
+    if (off >= len) return false;
+    b = (*data)[off++];
+    section_len |= (b & 0x7Fu) << (i * 7);
+  }
+  if ((b & 0x80u) != 0) return false;
+  off += section_len;
+  if (off > len) return false;
+  *data += off;
+  return true;
+}
+
+}  // namespace
+
+Status BrunsliToPixels(jxl::Span<const uint8_t> compressed,
+                       jxl::CodecInOut* JXL_RESTRICT io,
+                       const BrunsliDecoderOptions& options,
+                       BrunsliDecoderMeta* metadata, jxl::ThreadPool* pool) {
+  io->enc_size = compressed.size();
+
+  ::brunsli::JPEGData jpg;
+  ::brunsli::BrunsliStatus status =
+      ::brunsli::BrunsliDecodeJpeg(compressed.data(), compressed.size(), &jpg);
+  if (status != ::brunsli::BRUNSLI_OK) {
+    return JXL_FAILURE("Failed to parse Brunsli input.");
+  }
+
+  BrunsliExtensions extensions = ParseBrunsliExtensions(jpg);
+
+  PaddedBytes icc_profile;
+  if (!ParseChunkedMarker(jpg, kApp2, ByteSpan(kIccProfileTag), &icc_profile)) {
+    JXL_WARNING("ReJPEG: corrupted ICC profile\n");
+    icc_profile.clear();
+  }
+
+  Image3F rgb;
+  bool is_gray;
+  JXL_RETURN_IF_ERROR(
+      JpegDataToPixels(jpg, options, extensions, &rgb, &is_gray, pool));
+
+  ColorEncoding color_encoding;
+  if (!extensions.hdr_colorspace.empty()) {
+    JXL_RETURN_IF_ERROR(
+        ParseDescription(extensions.hdr_colorspace, &color_encoding));
+    JXL_RETURN_IF_ERROR(ColorManagement::CreateProfile(&color_encoding));
+  } else {
+    if (!ColorManagement::SetProfile(std::move(icc_profile), &color_encoding)) {
+      color_encoding = ColorManagement::SRGB(is_gray);
+    }
+  }
+
+  // TODO(eustas): also import EXIF, etc.
+  metadata->hdr_orig_colorspace = extensions.hdr_orig_colorspace;
+
+  io->metadata.bits_per_sample =
+      extensions.hdr_orig_colorspace.empty() ? 8 : 32;
+  io->metadata.color_encoding = color_encoding;
+  io->SetFromImage(std::move(rgb), color_encoding);
+
+  io->dec_pixels += jpg.width * jpg.height;
+
+  return true;
+}
+
+// TODO(eustas): use VerifySignature from Brunsli library.
+BrunsliFileSignature IsBrunsliFile(jxl::Span<const uint8_t> compressed) {
+  if (memcmp(compressed.data(), kBrunsliMagic,
+             std::min(kBrunsliMagicSize, compressed.size())) != 0) {
+    return BrunsliFileSignature::kInvalid;
+  }
+  if (compressed.size() < kBrunsliMagicSize) {
+    return BrunsliFileSignature::kNotEnoughData;
+  }
+  return BrunsliFileSignature::kBrunsli;
+}
+
+HWY_ATTR ImageF Dct(const ImageF& image) {
+  constexpr size_t N = kBlockDim;
+  static_assert(N == 8, "JPEG block dim must be 8");
+  static_assert(kDCTBlockSize == N * N, "JPEG block size must be 64");
+
+  JXL_ASSERT(image.xsize() % N == 0);
+  JXL_ASSERT(image.ysize() % N == 0);
+  const size_t xsize_blocks = image.xsize() / N;
+  const size_t ysize_blocks = image.ysize() / N;
+  ImageF dct(xsize_blocks * kDCTBlockSize, ysize_blocks);
+
+  for (size_t by = 0; by < ysize_blocks; ++by) {
+    const float* JXL_RESTRICT row_in = image.ConstRow(by * kBlockDim);
+    float* JXL_RESTRICT row_dct = dct.Row(by);
+    for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+      ComputeTransposedScaledDCT<kBlockDim>()(
+          FromLines<kBlockDim>(row_in + bx * kBlockDim, image.PixelsPerRow()),
+          ScaleToBlock<kBlockDim>(row_dct + bx * kDCTBlockSize));
+    }
+  }
+  return dct;
+}
+
+namespace {
+
+const uint8_t kDefaultQuantMatrix[2][64] = {
+    {16, 11, 10, 16, 24,  40,  51,  61,  12, 12, 14, 19, 26,  58,  60,  55,
+     14, 13, 16, 24, 40,  57,  69,  56,  14, 17, 22, 29, 51,  87,  80,  62,
+     18, 22, 37, 56, 68,  109, 103, 77,  24, 35, 55, 64, 81,  104, 113, 92,
+     49, 64, 78, 87, 103, 121, 120, 101, 72, 92, 95, 98, 112, 100, 103, 99},
+    {17, 18, 24, 47, 99, 99, 99, 99, 18, 21, 26, 66, 99, 99, 99, 99,
+     24, 26, 56, 99, 99, 99, 99, 99, 47, 66, 99, 99, 99, 99, 99, 99,
+     99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99,
+     99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99, 99}};
+
+void FillQuantMatrix(bool is_chroma, float scale, std::vector<int>* dst) {
+  const uint8_t* const in = kDefaultQuantMatrix[is_chroma];
+  for (uint32_t i = 0; i < kDCTBlockSize; ++i) {
+    const uint32_t v = static_cast<float>(in[i]) * scale;
+    dst->at(i) = (v < 1) ? 1 : (v > 255) ? 255u : v;
+  }
+}
+
+// Convert YCbCr pixels to JPEGData components.
+void ConvertPixels(const Image3F& from, brunsli::JPEGData* to,
+                   const int quant_map[3], jxl::ThreadPool* pool) {
+  constexpr size_t N = kBlockDim;
+  // TODO(eustas): consider grayscale / CMYK.
+  const size_t num_c = 3;
+
+  for (size_t c = 0; c < num_c; ++c) {
+    // TODO(eustas): use pool.
+    const ImageF& plane = Dct(from.Plane(c));
+    ::brunsli::JPEGComponent component;
+    component.id = static_cast<int>(c) + 1;  // YCbCr
+    component.h_samp_factor = 1;
+    component.v_samp_factor = 1;
+    component.quant_idx = quant_map[c];
+    const int* JXL_RESTRICT quant_table =
+        to->quant[component.quant_idx].values.data();
+    const size_t xsize_blocks = component.h_samp_factor * to->MCU_cols;
+    component.width_in_blocks = xsize_blocks;
+    const size_t ysize_blocks = component.v_samp_factor * to->MCU_rows;
+    component.height_in_blocks = ysize_blocks;
+    const size_t num_blocks = xsize_blocks * ysize_blocks;
+    component.num_blocks = num_blocks;
+    std::vector<::brunsli::coeff_t> coeffs(num_blocks * kDCTBlockSize);
+    // TODO(eustas): use pool.
+    for (size_t by = 0; by < ysize_blocks; ++by) {
+      const float* JXL_RESTRICT from_row = plane.ConstRow(by);
+      ::brunsli::coeff_t* JXL_RESTRICT to_row =
+          &coeffs[by * xsize_blocks * kDCTBlockSize];
+      for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+        const float* JXL_RESTRICT from_block = &from_row[bx * kDCTBlockSize];
+        ::brunsli::coeff_t* JXL_RESTRICT to_block = &to_row[bx * kDCTBlockSize];
+        // TODO(eustas): SIMDify
+        for (size_t u = 0; u < N; ++u) {
+          for (size_t v = 0; v < N; ++v) {
+            size_t to_idx = u * N + v;
+            size_t from_idx = v * N + u;
+            float raw = (static_cast<float>(N * N) * DCTScales<N>()[u] *
+                         DCTScales<N>()[v]) *
+                        from_block[from_idx];
+            raw += std::copysign(0.5f * static_cast<float>(quant_table[to_idx]),
+                                 raw);
+            to_block[to_idx] = raw / static_cast<float>(quant_table[to_idx]);
+          }
+        }
+      }
+    }
+
+    component.coeffs = std::move(coeffs);
+    to->components.emplace_back(std::move(component));
+  }
+}
+
+}  // namespace
+
+Status PixelsToBrunsli(const jxl::CodecInOut* JXL_RESTRICT io,
+                       jxl::PaddedBytes* compressed,
+                       const BrunsliEncoderOptions& options,
+                       jxl::ThreadPool* pool) {
+  constexpr size_t N = kBlockDim;
+  static_assert(N == 8, "JPEG block dim must be 8");
+  static_assert(kDCTBlockSize == N * N, "JPEG block size must be 64");
+
+  const jxl::ImageBundle& ib = io->Main();
+  if (!ib.HasColor()) {
+    return false;
+  }
+  const size_t num_c = 3;
+
+  const size_t xsize = io->xsize();
+  const size_t ysize = io->ysize();
+
+  const Image3F& src = PadImageToMultiple(ib.color(), N);
+  ImageF planes[3];
+  for (auto& plane : planes) {
+    plane = ImageF(src.xsize(), src.ysize());
+  }
+  RgbToYcbcr(src.Plane(0), src.Plane(1), src.Plane(2), &planes[0], &planes[1],
+             &planes[2]);
+
+  ::brunsli::JPEGData out;
+  out.width = xsize;
+  out.height = ysize;
+  out.version = 0;           // regular Brunsli
+  out.restart_interval = 0;  // don't care
+  out.max_h_samp_factor = 1;
+  out.max_v_samp_factor = 1;
+  out.MCU_rows = DivCeil(out.height, out.max_v_samp_factor * N);
+  out.MCU_cols = DivCeil(out.width, out.max_h_samp_factor * N);
+  out.has_zero_padding_bit = false;
+
+  out.app_data.emplace_back(reinterpret_cast<const char*>(kApp0Template),
+                            kApp0TemplateSize);
+  out.marker_order.push_back(kApp0);
+
+  if (options.hdr_orig_colorspace.empty()) {
+    // TODO(eustas): use "predefined profile" optimization, when eligible.
+    if (!AddChunkedMarker(&out, kApp2, ByteSpan(kIccProfileTag),
+                          io->metadata.color_encoding.icc)) {
+      JXL_ABORT("Brunsli: failed to add ICC profile\n");
+    }
+  } else {
+    PaddedBytes hdr_payload;
+    std::string colorspace = Description(ib.c_current());
+    std::string payload = options.hdr_orig_colorspace + ">" + colorspace;
+    hdr_payload.append(payload);
+    if (!AddChunkedMarker(&out, kApp9, ByteSpan(kBrunsliXHdr), hdr_payload)) {
+      JXL_ABORT("Brunsli: failed to add HDR extension\n");
+    }
+  }
+
+  ::brunsli::JPEGQuantTable quant_luma;
+  FillQuantMatrix(false, options.quant_scale, &quant_luma.values);
+  quant_luma.precision = 0;
+  quant_luma.index = 0;
+  quant_luma.is_last = false;
+  out.quant.emplace_back(std::move(quant_luma));
+  ::brunsli::JPEGQuantTable quant_chroma;
+  FillQuantMatrix(true, options.quant_scale, &quant_chroma.values);
+  quant_chroma.precision = 0;
+  quant_chroma.index = 1;
+  quant_chroma.is_last = true;
+  out.quant.emplace_back(std::move(quant_chroma));
+  out.marker_order.push_back(kDqt);
+
+  {
+    Image3F pixels(std::move(planes[0]), std::move(planes[1]),
+                   std::move(planes[2]));
+    constexpr int kQuantMap[] = {0, 1, 1};
+    ConvertPixels(pixels, &out, kQuantMap, pool);
+  }
+
+  out.marker_order.push_back(kSof0);
+
+  ::brunsli::JPEGHuffmanCode dc_huff;
+  // kJpegDCAlphabetSize + 1 = 13 = 3 + 10; 3/8 + 10/16 = 1
+  dc_huff.counts[3] = 3;
+  dc_huff.counts[4] = 10;
+  for (size_t i = 0; i < ::brunsli::kJpegDCAlphabetSize; ++i) {
+    dc_huff.values[i] = i;
+  }
+  dc_huff.values[::brunsli::kJpegDCAlphabetSize] =
+      ::brunsli::kJpegHuffmanAlphabetSize;
+  dc_huff.slot_id = 0 + kHuffmanCodeDcSlotOffset;
+  dc_huff.is_last = false;
+  out.huffman_code.emplace_back(std::move(dc_huff));
+  ::brunsli::JPEGHuffmanCode ac_huff;
+  // kJpegHuffmanAlphabetSize + 1 = 257 = 255 + 2; 255 / 256 + 2 / 512 = 1
+  ac_huff.counts[8] = 255;
+  ac_huff.counts[9] = 2;
+  for (size_t i = 0; i < ::brunsli::kJpegHuffmanAlphabetSize; ++i) {
+    ac_huff.values[i] = i;
+  }
+  ac_huff.values[::brunsli::kJpegHuffmanAlphabetSize] =
+      ::brunsli::kJpegHuffmanAlphabetSize;
+  ac_huff.slot_id = 0 + kHuffmanCodeAcSlotOffset;
+  ac_huff.is_last = true;
+  out.huffman_code.emplace_back(std::move(ac_huff));
+  out.marker_order.push_back(kDht);
+
+  ::brunsli::JPEGScanInfo scan_info;
+  scan_info.Ss = 0;
+  scan_info.Se = 63;
+  scan_info.Ah = 0;
+  scan_info.Al = 0;
+  scan_info.components.resize(num_c);
+  for (size_t i = 0; i < num_c; ++i) {
+    scan_info.components[i].comp_idx = i;
+    scan_info.components[i].dc_tbl_idx = 0;
+    scan_info.components[i].ac_tbl_idx = 0;
+  }
+  out.scan_info.emplace_back(std::move(scan_info));
+  out.marker_order.push_back(kSos);
+
+  out.marker_order.push_back(kEoi);
+
+  size_t output_size = ::brunsli::GetMaximumBrunsliEncodedSize(out);
+  std::vector<uint8_t> output(output_size);
+  // TODO(eustas): introduce streaming API?
+  if (!::brunsli::BrunsliEncodeJpeg(out, output.data(), &output_size)) {
+    return false;
+  }
+  compressed->append(Span<uint8_t>(output.data(), output_size));
+
+  return true;
+}
+
+class BrunsliFrameEncoderInternal {
+ public:
+  BrunsliFrameEncoderInternal(const FrameDimensions& frame_dim,
+                              ThreadPool* pool)
+      : pool_(pool), dc_states_(nullptr), ac_states_(nullptr) {
+    jpg_.width = frame_dim.xsize;
+    jpg_.height = frame_dim.ysize;
+    jpg_.version = 0;           // regular Brunsli
+    jpg_.restart_interval = 0;  // don't care
+    jpg_.has_zero_padding_bit = false;
+  }
+
+  bool ReadSourceImage(const ImageBundle* src,
+                       const std::vector<int>& quant_table,
+                       YCbCrChromaSubsampling subsampling) {
+    if (!src->HasColor()) return false;
+
+    // TODO(eustas): consider grayscale / CMYK.
+    const size_t num_c = 3;
+
+    constexpr size_t kInvPlaneOrder[] = {1, 0, 2};
+    for (size_t c = 0; c < num_c; ++c) {
+      ::brunsli::JPEGQuantTable quant;
+      size_t src_q = kInvPlaneOrder[c] * 64;
+      for (size_t i = 0; i < 64; ++i) {
+        // TODO(eustas): add a sanity check?
+        quant.values[i] = quant_table[i + src_q];
+      }
+      quant.precision = 0;
+      quant.index = c;
+      quant.is_last = (c + 1 == num_c);
+      jpg_.quant.emplace_back(std::move(quant));
+    }
+
+    constexpr size_t N = kBlockDim;
+    constexpr int kQuantMap[] = {0, 1, 2};
+
+    constexpr size_t kPlaneOrder[] = {1, 0, 2};
+
+    for (size_t c = 0; c < num_c; ++c) {
+      ::brunsli::JPEGComponent component;
+      component.id = static_cast<int>(c) + 1;  // YCbCr
+      component.h_samp_factor = 1;
+      component.v_samp_factor = 1;
+      component.quant_idx = kQuantMap[c];
+      jpg_.components.emplace_back(std::move(component));
+    }
+    if (subsampling == YCbCrChromaSubsampling::k420) {
+      jpg_.components[0].h_samp_factor = 2;
+      jpg_.components[0].v_samp_factor = 2;
+    } else if (subsampling == YCbCrChromaSubsampling::k422) {
+      jpg_.components[0].h_samp_factor = 2;
+    } else if (subsampling == YCbCrChromaSubsampling::k411) {
+      jpg_.components[0].h_samp_factor = 4;
+    }
+    jpg_.max_h_samp_factor = jpg_.components[0].h_samp_factor;
+    jpg_.max_v_samp_factor = jpg_.components[0].v_samp_factor;
+    jpg_.MCU_rows = DivCeil(jpg_.height, jpg_.max_v_samp_factor * N);
+    jpg_.MCU_cols = DivCeil(jpg_.width, jpg_.max_h_samp_factor * N);
+
+    for (size_t c = 0; c < num_c; ++c) {
+      const ImageF& plane = src->color().Plane(kPlaneOrder[c]);
+      ::brunsli::JPEGComponent& component = jpg_.components[c];
+      component.quant_idx = kQuantMap[c];
+      const size_t xsize_blocks = component.h_samp_factor * jpg_.MCU_cols;
+      component.width_in_blocks = xsize_blocks;
+      const size_t ysize_blocks = component.v_samp_factor * jpg_.MCU_rows;
+      component.height_in_blocks = ysize_blocks;
+      const size_t num_blocks = xsize_blocks * ysize_blocks;
+      component.num_blocks = num_blocks;
+      std::vector<::brunsli::coeff_t> coeffs(num_blocks * kDCTBlockSize);
+      const size_t from_stride = plane.PixelsPerRow();
+      constexpr size_t group_dim = kGroupDimInBlocks;
+      const size_t xsize_groups = DivCeil(xsize_blocks, group_dim);
+      const size_t ysize_groups = DivCeil(ysize_blocks, group_dim);
+      const auto copy_coeffs = [&](int idx, int /*thread*/) {
+        const size_t gx = idx % xsize_groups;
+        const size_t gy = idx / xsize_groups;
+        const Rect group_rect_blocks(gx * group_dim, gy * group_dim, group_dim,
+                                     group_dim, xsize_blocks, ysize_blocks);
+        const size_t bx0 = group_rect_blocks.x0();
+        const size_t bx1 = bx0 + group_rect_blocks.xsize();
+        const size_t by0 = group_rect_blocks.y0();
+        const size_t by1 = by0 + group_rect_blocks.ysize();
+        const size_t bx_max = std::min<size_t>(DivCeil(plane.xsize(), N), bx1);
+        const size_t by_max = std::min<size_t>(DivCeil(plane.ysize(), N), by1);
+        for (size_t by = by0; by < by_max; ++by) {
+          const float* JXL_RESTRICT from_row = plane.ConstRow(by * N);
+          ::brunsli::coeff_t* JXL_RESTRICT to_row =
+              &coeffs[by * xsize_blocks * kDCTBlockSize];
+          for (size_t bx = bx0; bx < bx_max; ++bx) {
+            const float* JXL_RESTRICT from_block = &from_row[bx * N];
+            ::brunsli::coeff_t* JXL_RESTRICT to_block =
+                &to_row[bx * kDCTBlockSize];
+            // TODO(eustas): SIMDify
+            for (size_t n = 0; n < N * N; ++n) {
+              to_block[n] = from_block[(n % N) + (n / N) * from_stride];
+            }
+          }
+          memset(to_row + bx_max * kDCTBlockSize, 0,
+                 (bx1 - bx_max) * kDCTBlockSize * sizeof(::brunsli::coeff_t));
+        }
+        // Complete the MCU, if necessary.
+        for (size_t by = by_max; by < by1; ++by) {
+          ::brunsli::coeff_t* JXL_RESTRICT to_row =
+              &coeffs[by * xsize_blocks * kDCTBlockSize];
+          memset(to_row, 0,
+                 xsize_blocks * kDCTBlockSize * sizeof(::brunsli::coeff_t));
+        }
+      };
+      RunOnPool(pool_, 0, static_cast<int>(xsize_groups * ysize_groups),
+                ThreadPool::SkipInit(), copy_coeffs, "Brunsli:CopyCoeffs");
+
+      component.coeffs = std::move(coeffs);
+    }
+
+    return true;
+  }
+
+  bool DoEncode() {
+    using brunsli::internal::enc::ComponentMeta;
+    using brunsli::internal::enc::SelectContextBits;
+    using brunsli::internal::enc::State;
+
+    size_t num_components = jpg_.components.size();
+    std::vector<size_t> approx_total_nonzeros(num_components);
+
+    size_t width_in_blocks = jpg_.MCU_cols * jpg_.max_h_samp_factor;
+    size_t height_in_blocks = jpg_.MCU_rows * jpg_.max_v_samp_factor;
+
+    size_t w_ac = (width_in_blocks + kGroupDimInBlocks - 1) / kGroupDimInBlocks;
+    size_t h_ac =
+        (height_in_blocks + kGroupDimInBlocks - 1) / kGroupDimInBlocks;
+
+    size_t w_dc =
+        (width_in_blocks + kDcGroupDimInBlocks - 1) / kDcGroupDimInBlocks;
+    size_t h_dc =
+        (height_in_blocks + kDcGroupDimInBlocks - 1) / kDcGroupDimInBlocks;
+
+    std::vector<std::vector<brunsli::coeff_t>> dc_prediction_errors(
+        num_components);
+    std::vector<std::vector<uint8_t>> block_state(num_components);
+    for (size_t i = 0; i < num_components; ++i) {
+      const brunsli::JPEGComponent& c = jpg_.components[i];
+      dc_prediction_errors[i].resize(c.width_in_blocks * c.height_in_blocks);
+      block_state[i].resize(c.width_in_blocks * c.height_in_blocks);
+    }
+
+    size_t dc_state_size = w_dc * h_dc;
+    size_t ac_state_size = w_ac * h_ac;
+    states_.resize(1 + dc_state_size + ac_state_size);
+    State& state = states_[0];
+    dc_states_ = states_.data() + 1;
+    ac_states_ = dc_states_ + dc_state_size;
+
+    if (!CalculateMeta(jpg_, &state)) return false;
+    for (size_t c = 0; c < num_components; ++c) {
+      ComponentMeta& m = state.meta[c];
+      m.dc_prediction_errors = dc_prediction_errors[c].data();
+      m.block_state = block_state[c].data();
+    }
+
+    for (size_t y = 0; y < h_dc; ++y) {
+      for (size_t x = 0; x < w_dc; ++x) {
+        State& s = dc_states_[x + y * w_dc];
+        std::vector<ComponentMeta>& meta = s.meta;
+        if (!CalculateMeta(jpg_, &s)) return false;
+        for (size_t c = 0; c < num_components; ++c) {
+          ComponentMeta& m = meta[c];
+          size_t h_group_dim =
+              m.h_samp * kDcGroupDimInBlocks / jpg_.max_h_samp_factor;
+          size_t first_x = x * h_group_dim;
+          size_t last_x =
+              std::min<size_t>(first_x + h_group_dim, m.width_in_blocks);
+          size_t v_group_dim =
+              m.v_samp * kDcGroupDimInBlocks / jpg_.max_v_samp_factor;
+          size_t first_y = y * v_group_dim;
+          size_t last_y =
+              std::min<size_t>(first_y + v_group_dim, m.height_in_blocks);
+          m.ac_coeffs +=
+              first_x * brunsli::kDCTBlockSize + first_y * m.ac_stride;
+          m.width_in_blocks = last_x - first_x;
+          m.height_in_blocks = last_y - first_y;
+          m.dc_prediction_errors =
+              dc_prediction_errors[c].data() + first_x + first_y * m.dc_stride;
+          m.block_state =
+              block_state[c].data() + first_x + first_y * m.b_stride;
+        }
+      }
+    }
+
+    for (size_t y = 0; y < h_ac; ++y) {
+      for (size_t x = 0; x < w_ac; ++x) {
+        State& s = ac_states_[x + y * w_ac];
+        std::vector<ComponentMeta>& meta = s.meta;
+        if (!CalculateMeta(jpg_, &s)) return false;
+        for (size_t c = 0; c < num_components; ++c) {
+          ComponentMeta& m = meta[c];
+          size_t h_group_dim =
+              m.h_samp * kGroupDimInBlocks / jpg_.max_h_samp_factor;
+          size_t first_x = x * h_group_dim;
+          size_t last_x =
+              std::min<size_t>(first_x + h_group_dim, m.width_in_blocks);
+          size_t v_group_dim =
+              m.v_samp * kGroupDimInBlocks / jpg_.max_v_samp_factor;
+          size_t first_y = y * v_group_dim;
+          size_t last_y =
+              std::min<size_t>(first_y + v_group_dim, m.height_in_blocks);
+          m.ac_coeffs +=
+              first_x * brunsli::kDCTBlockSize + first_y * m.ac_stride;
+          m.width_in_blocks = last_x - first_x;
+          m.height_in_blocks = last_y - first_y;
+          m.dc_prediction_errors =
+              dc_prediction_errors[c].data() + first_x + first_y * m.dc_stride;
+          m.block_state =
+              block_state[c].data() + first_x + first_y * m.b_stride;
+        }
+      }
+    }
+
+    State* ac_states = ac_states_;
+    const auto sample_nonzeros = [num_components, ac_states](int idx,
+                                                             int /* thread */) {
+      for (size_t c = 0; c < num_components; ++c) {
+        ComponentMeta& m = ac_states[idx].meta[c];
+        m.approx_total_nonzeros = SampleNumNonZeros(&m);
+      }
+    };
+    RunOnPool(pool_, 0, static_cast<int>(ac_state_size), ThreadPool::SkipInit(),
+              sample_nonzeros, "Brunsli:SampleNonzeros");
+
+    // Groups workflow: reduce approx_total_nonzeros.
+    for (size_t y = 0; y < h_ac; ++y) {
+      for (size_t x = 0; x < w_ac; ++x) {
+        for (size_t c = 0; c < num_components; ++c) {
+          approx_total_nonzeros[c] +=
+              ac_states[x + y * w_ac].meta[c].approx_total_nonzeros;
+        }
+      }
+    }
+
+    int32_t num_contexts = num_components;
+    for (size_t c = 0; c < num_components; ++c) {
+      ComponentMeta& m = state.meta[c];
+      m.context_bits = SelectContextBits(approx_total_nonzeros[c] + 1);
+      m.context_offset = num_contexts;
+      num_contexts += brunsli::kNumNonzeroContextSkip[m.context_bits];
+    }
+    state.num_contexts = num_contexts;
+
+    // Groups workflow: distribute context_bits.
+    for (size_t y = 0; y < h_ac; ++y) {
+      for (size_t x = 0; x < w_ac; ++x) {
+        State& s = ac_states[x + y * w_ac];
+        for (size_t c = 0; c < num_components; ++c) {
+          ComponentMeta& m = state.meta[c];
+          s.meta[c].context_bits = m.context_bits;
+          s.meta[c].context_offset = m.context_offset;
+        }
+        s.num_contexts = state.num_contexts;
+      }
+    }
+
+    for (size_t y = 0; y < h_dc; ++y) {
+      for (size_t x = 0; x < w_dc; ++x) {
+        State& s = dc_states_[x + y * w_dc];
+        for (size_t c = 0; c < num_components; ++c) {
+          ComponentMeta& m = state.meta[c];
+          s.meta[c].context_bits = m.context_bits;
+          s.meta[c].context_offset = m.context_offset;
+        }
+        s.num_contexts = state.num_contexts;
+      }
+    }
+
+    std::atomic<int> num_errors{0};
+    State* dc_states = dc_states_;
+    const auto encode_dc = [&num_errors, dc_states](int idx, int /* thread */) {
+      if (!PredictDCCoeffs(&dc_states[idx])) {
+        num_errors.fetch_add(1, std::memory_order_relaxed);
+        return;
+      }
+      EncodeDC(&dc_states[idx]);
+    };
+    RunOnPool(pool_, 0, static_cast<int>(dc_state_size), ThreadPool::SkipInit(),
+              encode_dc, "Brunsli:EncodeDc");
+    if (num_errors.load(std::memory_order_relaxed) != 0) return false;
+
+    const auto encode_ac = [ac_states](int idx, int /* thread */) {
+      EncodeAC(&ac_states[idx]);
+    };
+    RunOnPool(pool_, 0, static_cast<int>(ac_state_size), ThreadPool::SkipInit(),
+              encode_ac, "Brunsli:EncodeAc");
+
+    // Groups workflow: merge histograms.
+    // TODO: SIMDify.
+    state.entropy_source.Resize(num_contexts);
+    for (size_t y = 0; y < h_dc; ++y) {
+      for (size_t x = 0; x < w_dc; ++x) {
+        state.entropy_source.Merge(dc_states[x + y * w_dc].entropy_source);
+      }
+    }
+    for (size_t y = 0; y < h_ac; ++y) {
+      for (size_t x = 0; x < w_ac; ++x) {
+        state.entropy_source.Merge(ac_states[x + y * w_ac].entropy_source);
+      }
+    }
+
+    entropy_codes_ = PrepareEntropyCodes(&state);
+
+    return true;
+  }
+
+  bool SerializeHeader(BitWriter* out) {
+    // TODO: pull entropy codes serialization "side effect".
+    auto& state = states_[0];
+    state.entropy_codes = entropy_codes_.get();
+    size_t part_size = 20480;
+
+    // TODO: take into account histograms.
+    uint32_t skip_flags = ~((1u << brunsli::kBrunsliQuantDataTag) |
+                            (1u << brunsli::kBrunsliHistogramDataTag));
+    PaddedBytes part(part_size);
+    if (!BrunsliSerialize(&state, jpg_, skip_flags, part.data(), &part_size)) {
+      return false;
+    }
+
+    part.resize(part_size);
+    out->AppendByteAligned(BitWriter(std::move(part)));
+
+    return true;
+  }
+
+  bool SerializeDcGroup(size_t idx, BitWriter* out, AuxOut* aux_out) {
+    auto& s = dc_states_[idx];
+    // TODO: reduce for subsampled
+    size_t part_size =
+        kDcGroupDimInBlocks * kDcGroupDimInBlocks * jpg_.components.size();
+    part_size += part_size / 8;
+    PaddedBytes part(part_size);
+    s.entropy_codes = entropy_codes_.get();
+    uint32_t skip_flags = ~(1u << brunsli::kBrunsliDCDataTag);
+    if (!BrunsliSerialize(&s, jpg_, skip_flags, part.data(), &part_size)) {
+      return false;
+    }
+
+    part.resize(part_size);
+    out->AppendByteAligned(BitWriter(std::move(part)));
+
+    return true;
+  }
+
+  bool SerializeAcGroup(size_t idx, BitWriter* out, AuxOut* aux_out) {
+    auto& s = ac_states_[idx];
+    // TODO: reduce for subsampled.
+    size_t part_size = 32 * 32 * 63 * jpg_.components.size();
+    PaddedBytes part(part_size);
+    s.entropy_codes = entropy_codes_.get();
+    uint32_t skip_flags = ~(1u << brunsli::kBrunsliACDataTag);
+    if (!BrunsliSerialize(&s, jpg_, skip_flags, part.data(), &part_size)) {
+      return false;
+    }
+
+    part.resize(part_size);
+    out->AppendByteAligned(BitWriter(std::move(part)));
+
+    return true;
+  }
+
+ private:
+  ThreadPool* pool_;
+
+  brunsli::JPEGData jpg_;
+  std::vector<brunsli::internal::enc::State> states_;
+  brunsli::internal::enc::State* dc_states_;
+  brunsli::internal::enc::State* ac_states_;
+  std::unique_ptr<brunsli::internal::enc::EntropyCodes> entropy_codes_;
+};
+
+BrunsliFrameEncoder::BrunsliFrameEncoder(const FrameDimensions& frame_dim,
+                                         ThreadPool* pool)
+    : impl_(make_unique<BrunsliFrameEncoderInternal>(frame_dim, pool)) {}
+BrunsliFrameEncoder::~BrunsliFrameEncoder() = default;
+bool BrunsliFrameEncoder::ReadSourceImage(const ImageBundle* src,
+                                          const std::vector<int>& quant_table,
+                                          YCbCrChromaSubsampling subsampling) {
+  return impl_->ReadSourceImage(src, quant_table, subsampling);
+}
+bool BrunsliFrameEncoder::DoEncode() { return impl_->DoEncode(); }
+bool BrunsliFrameEncoder::SerializeHeader(BitWriter* out) {
+  return impl_->SerializeHeader(out);
+}
+bool BrunsliFrameEncoder::SerializeDcGroup(size_t index, BitWriter* out,
+                                           AuxOut* aux_out) {
+  return impl_->SerializeDcGroup(index, out, aux_out);
+}
+bool BrunsliFrameEncoder::SerializeAcGroup(size_t index, BitWriter* out,
+                                           AuxOut* aux_out) {
+  return impl_->SerializeAcGroup(index, out, aux_out);
+}
+
+class BrunsliFrameDecoderInternal {
+ public:
+  explicit BrunsliFrameDecoderInternal(ThreadPool* pool)
+      : pool_(pool), dc_states_(nullptr), ac_states_(nullptr) {}
+
+  bool ReadHeader(const FrameDimensions* frame_dim, BitReader* src,
+                  YCbCrChromaSubsampling subsampling) {
+    using brunsli::BrunsliStatus;
+    using brunsli::JPEGQuantTable;
+    using brunsli::internal::dec::ComponentMeta;
+    using brunsli::internal::dec::PrepareMeta;
+    using brunsli::internal::dec::Stage;
+    using brunsli::internal::dec::State;
+    using brunsli::internal::dec::UpdateSubsamplingDerivatives;
+
+    // TODO(eustas): consider grayscale / CMYK.
+    const size_t num_c = 3;
+    jpg_.version = 0;
+    jpg_.width = frame_dim->xsize;
+    jpg_.height = frame_dim->ysize;
+    jpg_.components.resize(num_c);
+    for (size_t c = 0; c < num_c; ++c) {
+      jpg_.components[c].h_samp_factor = 1;
+      jpg_.components[c].v_samp_factor = 1;
+    }
+    if (subsampling == YCbCrChromaSubsampling::k420) {
+      jpg_.components[0].h_samp_factor = 2;
+      jpg_.components[0].v_samp_factor = 2;
+    } else if (subsampling == YCbCrChromaSubsampling::k422) {
+      jpg_.components[0].h_samp_factor = 2;
+    } else if (subsampling == YCbCrChromaSubsampling::k411) {
+      jpg_.components[0].h_samp_factor = 4;
+    }
+    if (!UpdateSubsamplingDerivatives(&jpg_)) return false;
+
+    // == num_c?
+    const int num_q = 3;
+    jpg_.quant.resize(num_q);
+    for (size_t i = 0; i < num_q; ++i) {
+      JPEGQuantTable& q = jpg_.quant[i];
+      q.index = i;
+      q.is_last = (i == num_q - 1);
+      q.precision = 0;
+    }
+
+    const size_t num_dc_groups =
+        frame_dim->xsize_dc_groups * frame_dim->ysize_dc_groups;
+    const size_t num_ac_groups =
+        frame_dim->xsize_groups * frame_dim->ysize_groups;
+    states_.resize(1 + num_dc_groups + num_ac_groups);
+    dc_states_ = states_.data() + 1;
+    ac_states_ = dc_states_ + num_dc_groups;
+
+    size_t pos = src->TotalBitsConsumed();
+    if ((pos & 0x7u) != 0) return false;
+    pos >>= 3u;
+
+    const uint8_t* data = src->FirstByte() + pos;
+    const uint8_t* data_end = src->FirstByte() + src->TotalBytes();
+    const uint8_t* chunk_end = data;
+    if (!SkipSection(&chunk_end, data_end - chunk_end)) return false;
+    if (!SkipSection(&chunk_end, data_end - chunk_end)) return false;
+    const size_t part_length = chunk_end - data;
+
+    State& state = states_[0];
+    state.stage = Stage::SECTION;
+    state.tags_met = ~((1u << brunsli::kBrunsliQuantDataTag) |
+                       (1u << brunsli::kBrunsliHistogramDataTag));
+    state.data = data;
+    state.len = part_length;
+    PrepareMeta(&jpg_, &state);
+
+    BrunsliStatus status = ProcessJpeg(&state, &jpg_);
+    if (status != BrunsliStatus::BRUNSLI_OK) return false;
+    WarmupMeta(&jpg_, &state);
+
+    src->SkipBits(part_length << 3u);
+
+    for (size_t idx = 0; idx < num_dc_groups; ++idx) {
+      size_t y = idx / frame_dim->xsize_dc_groups;
+      size_t x = idx % frame_dim->xsize_dc_groups;
+      State& dc_state = dc_states_[idx];
+      dc_state.stage = Stage::SECTION;
+      dc_state.tags_met = ~(1u << brunsli::kBrunsliDCDataTag);
+
+      dc_state.context_map = state.context_map;
+      dc_state.entropy_codes = state.entropy_codes;
+
+      std::vector<ComponentMeta>& meta = dc_state.meta;
+
+      PrepareMeta(&jpg_, &dc_state);
+      dc_state.is_storage_allocated = true;
+      WarmupMeta(&jpg_, &dc_state);
+      for (size_t c = 0; c < num_c; ++c) {
+        ComponentMeta& m = meta[c];
+        size_t h_group_dim =
+            m.h_samp * kDcGroupDimInBlocks / jpg_.max_h_samp_factor;
+        size_t first_x = x * h_group_dim;
+        size_t last_x =
+            std::min<size_t>(first_x + h_group_dim, m.width_in_blocks);
+        size_t v_group_dim =
+            m.v_samp * kDcGroupDimInBlocks / jpg_.max_v_samp_factor;
+        size_t first_y = y * v_group_dim;
+        size_t last_y =
+            std::min<size_t>(first_y + v_group_dim, m.height_in_blocks);
+        m.ac_coeffs += first_x * brunsli::kDCTBlockSize + first_y * m.ac_stride;
+        m.block_state =
+            state.block_state_[c].data() + first_x + first_y * m.b_stride;
+        m.width_in_blocks = last_x - first_x;
+        m.height_in_blocks = last_y - first_y;
+      }
+    }
+
+    for (size_t idx = 0; idx < num_ac_groups; ++idx) {
+      size_t y = idx / frame_dim->xsize_groups;
+      size_t x = idx % frame_dim->xsize_groups;
+      State& ac_state = ac_states_[idx];
+      ac_state.stage = Stage::SECTION;
+      ac_state.tags_met = ~(1u << brunsli::kBrunsliACDataTag);
+
+      ac_state.context_map = state.context_map;
+      ac_state.entropy_codes = state.entropy_codes;
+
+      std::vector<ComponentMeta>& meta = ac_state.meta;
+
+      PrepareMeta(&jpg_, &ac_state);
+      ac_state.is_storage_allocated = true;
+      WarmupMeta(&jpg_, &ac_state);
+      for (size_t c = 0; c < num_c; ++c) {
+        ComponentMeta& m = meta[c];
+        size_t h_group_dim =
+            m.h_samp * kGroupDimInBlocks / jpg_.max_h_samp_factor;
+        size_t first_x = x * h_group_dim;
+        size_t last_x =
+            std::min<size_t>(first_x + h_group_dim, m.width_in_blocks);
+        size_t v_group_dim =
+            m.v_samp * kGroupDimInBlocks / jpg_.max_v_samp_factor;
+        size_t first_y = y * v_group_dim;
+        size_t last_y =
+            std::min<size_t>(first_y + v_group_dim, m.height_in_blocks);
+        m.context_bits = state.meta[c].context_bits;
+        m.context_offset = state.meta[c].context_offset;
+        m.ac_coeffs += first_x * brunsli::kDCTBlockSize + first_y * m.ac_stride;
+        m.block_state =
+            state.block_state_[c].data() + first_x + first_y * m.b_stride;
+        m.width_in_blocks = last_x - first_x;
+        m.height_in_blocks = last_y - first_y;
+      }
+    }
+
+    return true;
+  }
+
+  bool DecodeDcGroup(int idx, jxl::BitReader* src) {
+    using brunsli::BrunsliStatus;
+    using brunsli::internal::dec::ProcessJpeg;
+    using brunsli::internal::dec::Stage;
+    using brunsli::internal::dec::State;
+
+    size_t pos = src->TotalBitsConsumed();
+    if ((pos & 0x7u) != 0) return false;
+    pos >>= 3u;
+
+    const uint8_t* data = src->FirstByte() + pos;
+    const uint8_t* data_end = src->FirstByte() + src->TotalBytes();
+    const uint8_t* chunk_end = data;
+    if (!SkipSection(&chunk_end, data_end - chunk_end)) return false;
+    const size_t part_length = chunk_end - data;
+
+    State& dc_state = dc_states_[idx];
+    dc_state.data = data;
+    dc_state.len = part_length;
+
+    BrunsliStatus status = ProcessJpeg(&dc_state, &jpg_);
+    if (status != brunsli::BRUNSLI_OK) return false;
+
+    src->SkipBits(part_length << 3u);
+
+    return true;
+  }
+
+  bool DecodeAcGroup(int idx, jxl::BitReader* src, Image3F* img,
+                     const Rect& rect) {
+    using brunsli::BrunsliStatus;
+    using brunsli::internal::dec::ProcessJpeg;
+    using brunsli::internal::dec::Stage;
+    using brunsli::internal::dec::State;
+
+    size_t pos = src->TotalBitsConsumed();
+    if ((pos & 0x7u) != 0) return false;
+    pos >>= 3u;
+
+    const uint8_t* data = src->FirstByte() + pos;
+    const uint8_t* data_end = src->FirstByte() + src->TotalBytes();
+    const uint8_t* chunk_end = data;
+    if (!SkipSection(&chunk_end, data_end - chunk_end)) return false;
+    const size_t part_length = chunk_end - data;
+
+    State& ac_state = ac_states_[idx];
+    ac_state.data = data;
+    ac_state.len = part_length;
+
+    BrunsliStatus status = ProcessJpeg(&ac_state, &jpg_);
+    if (status != brunsli::BRUNSLI_OK) return false;
+
+    src->SkipBits(part_length << 3u);
+
+    return true;
+  }
+
+  bool FinalizeDecoding(Image3F* opsin) {
+    BrunsliDecoderOptions options{};
+    BrunsliExtensions extensions{};
+    bool is_gray;
+    bool ok =
+        JpegDataToPixels(jpg_, options, extensions, opsin, &is_gray, pool_);
+    if (!ok) return false;
+    return true;
+  }
+
+ private:
+  ThreadPool* pool_;
+
+  brunsli::JPEGData jpg_;
+  std::vector<brunsli::internal::dec::State> states_;
+  brunsli::internal::dec::State* dc_states_;
+  brunsli::internal::dec::State* ac_states_;
+};
+
+BrunsliFrameDecoder::BrunsliFrameDecoder(ThreadPool* pool)
+    : impl_(make_unique<BrunsliFrameDecoderInternal>(pool)) {}
+BrunsliFrameDecoder::~BrunsliFrameDecoder() = default;
+bool BrunsliFrameDecoder::ReadHeader(const FrameDimensions* frame_dim,
+                                     BitReader* src,
+                                     YCbCrChromaSubsampling subsampling) {
+  return impl_->ReadHeader(frame_dim, src, subsampling);
+}
+bool BrunsliFrameDecoder::DecodeDcGroup(int idx, jxl::BitReader* src) {
+  return impl_->DecodeDcGroup(idx, src);
+}
+bool BrunsliFrameDecoder::DecodeAcGroup(int idx, jxl::BitReader* src,
+                                        Image3F* img, const Rect& rect) {
+  return impl_->DecodeAcGroup(idx, src, img, rect);
+}
+bool BrunsliFrameDecoder::FinalizeDecoding(Image3F* opsin) {
+  return impl_->FinalizeDecoding(opsin);
+}
+}  // namespace jxl
diff --git a/jxl/brunsli.h b/jxl/brunsli.h
new file mode 100644
index 000000000..5b88a34ba
--- /dev/null
+++ b/jxl/brunsli.h
@@ -0,0 +1,100 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_BRUNSLI_H_
+#define JXL_BRUNSLI_H_
+
+#include <cstdint>
+#include <functional>
+#include <string>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/dec_params.h"
+
+// Utilities for rasterization of intermediate Brunsli representation.
+
+namespace jxl {
+
+enum class BrunsliFileSignature {
+  kBrunsli,
+  kNotEnoughData,
+  kInvalid,
+};
+
+BrunsliFileSignature IsBrunsliFile(jxl::Span<const uint8_t> compressed);
+
+struct BrunsliEncoderOptions {
+  float quant_scale = 1.0f;
+  std::string hdr_orig_colorspace;
+};
+
+struct BrunsliDecoderMeta {
+  std::string hdr_orig_colorspace;
+};
+
+Status BrunsliToPixels(jxl::Span<const uint8_t> compressed,
+                       jxl::CodecInOut* JXL_RESTRICT io,
+                       const BrunsliDecoderOptions& options,
+                       BrunsliDecoderMeta* metadata, jxl::ThreadPool* pool);
+
+Status PixelsToBrunsli(const jxl::CodecInOut* JXL_RESTRICT io,
+                       jxl::PaddedBytes* compressed,
+                       const BrunsliEncoderOptions& options,
+                       jxl::ThreadPool* pool);
+
+// Actual encoder has a lot of brunsli-specific state. Let's not expose it.
+class BrunsliFrameEncoderInternal;
+
+class BrunsliFrameEncoder {
+ public:
+  BrunsliFrameEncoder(const FrameDimensions& frame_dim, ThreadPool* pool);
+  ~BrunsliFrameEncoder();
+  bool ReadSourceImage(const ImageBundle* src,
+                       const std::vector<int>& quant_table,
+                       YCbCrChromaSubsampling subsampling);
+  bool DoEncode();
+  bool SerializeHeader(BitWriter* out);
+  bool SerializeDcGroup(size_t index, BitWriter* out, AuxOut* aux_out);
+  bool SerializeAcGroup(size_t index, BitWriter* out, AuxOut* aux_out);
+
+ private:
+  std::unique_ptr<BrunsliFrameEncoderInternal> impl_;
+};
+
+// Actual decoder has a lot of brunsli-specific state. Let's not expose it.
+class BrunsliFrameDecoderInternal;
+
+class BrunsliFrameDecoder {
+ public:
+  explicit BrunsliFrameDecoder(ThreadPool* pool);
+  ~BrunsliFrameDecoder();
+
+  bool ReadHeader(const FrameDimensions* frame_dim, BitReader* src,
+                  YCbCrChromaSubsampling subsampling);
+  bool DecodeDcGroup(int idx, BitReader* src);
+  bool DecodeAcGroup(int idx, BitReader* src, Image3F* img, const Rect& rect);
+  bool FinalizeDecoding(Image3F* opsin);
+
+ private:
+  std::unique_ptr<BrunsliFrameDecoderInternal> impl_;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_BRUNSLI_H_
diff --git a/jxl/brunsli_test.cc b/jxl/brunsli_test.cc
new file mode 100644
index 000000000..3b47855a2
--- /dev/null
+++ b/jxl/brunsli_test.cc
@@ -0,0 +1,164 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/brunsli.h"
+
+#include <stddef.h>
+
+#include <string>
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/enc_butteraugli_comparator.h"
+#include "jxl/extras/codec.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+
+// Returns compressed size [bytes].
+size_t Roundtrip(CodecInOut* io, const BrunsliEncoderOptions& enc_options,
+                 const BrunsliDecoderOptions& dec_options, ThreadPool* pool,
+                 CodecInOut* JXL_RESTRICT io2, float max_butteraugli_score) {
+  PaddedBytes compressed;
+
+  EXPECT_TRUE(PixelsToBrunsli(io, &compressed, enc_options, pool));
+
+  BrunsliDecoderMeta meta;
+  EXPECT_TRUE(BrunsliToPixels(jxl::Span<const uint8_t>(compressed), io2,
+                              dec_options, &meta, pool));
+
+  EXPECT_LE(ButteraugliDistance(*io, *io2, /* hf_assymetry */ 1.0f,
+                                /*distmap=*/nullptr, pool),
+            max_butteraugli_score);
+
+  return compressed.size();
+}
+
+TEST(BrunsliTest, RoundtripSinglePixel) {
+  ThreadPool* pool = nullptr;
+  Image3F image(1, 1);
+  BrunsliEncoderOptions enc_options;
+  BrunsliDecoderOptions dec_options;
+
+  image.PlaneRow(0, 0)[0] = 0.0f;
+  image.PlaneRow(1, 0)[0] = 0.0f;
+  image.PlaneRow(2, 0)[0] = 0.0f;
+
+  CodecInOut io;
+  io.metadata.bits_per_sample = 8;
+  io.metadata.color_encoding = ColorManagement::SRGB();
+  io.SetFromImage(std::move(image), io.metadata.color_encoding);
+  CodecInOut io2;
+  Roundtrip(&io, enc_options, dec_options, pool, &io2, 0.001f);
+}
+
+TEST(BrunsliTest, RoundtripTiny) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(16, 16);
+
+  BrunsliEncoderOptions enc_options;
+  enc_options.quant_scale = 0.0f;
+  BrunsliDecoderOptions dec_options;
+
+  CodecInOut io2;
+  Roundtrip(&io, enc_options, dec_options, pool, &io2, 0.3f);
+}
+
+TEST(BrunsliTest, RoundtripSmallQ0) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+
+  BrunsliEncoderOptions enc_options;
+  enc_options.quant_scale = 0.0f;
+  BrunsliDecoderOptions dec_options;
+
+  CodecInOut io2;
+  Roundtrip(&io, enc_options, dec_options, pool, &io2, 0.5f);
+}
+
+TEST(BrunsliTest, RoundtripDecOptions) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+
+  BrunsliEncoderOptions enc_options;
+  BrunsliDecoderOptions dec_options;
+  CodecInOut io2;
+
+  // Also try different decoding options.
+  enc_options.quant_scale = 2.0f;
+
+  dec_options.fix_dc_staircase = false;
+  dec_options.gaborish = false;
+  Roundtrip(&io, enc_options, dec_options, pool, &io2, 5.4f);
+
+  dec_options.fix_dc_staircase = true;
+  dec_options.gaborish = false;
+  Roundtrip(&io, enc_options, dec_options, pool, &io2, 5.2f);
+
+  dec_options.fix_dc_staircase = false;
+  dec_options.gaborish = true;
+  Roundtrip(&io, enc_options, dec_options, pool, &io2, 4.777f);
+}
+
+TEST(BrunsliTest, RoundtripUnalignedQ0_5) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 12, io.ysize() / 7);
+
+  BrunsliEncoderOptions enc_options;
+  enc_options.quant_scale = 0.5f;
+  BrunsliDecoderOptions dec_options;
+
+  CodecInOut io2;
+  Roundtrip(&io, enc_options, dec_options, pool, &io2, 1.3579f);
+}
+
+TEST(BrunsliTest, RoundtripLarge) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+
+  BrunsliEncoderOptions enc_options;
+  enc_options.quant_scale = 2.0f;
+  BrunsliDecoderOptions dec_options;
+
+  CodecInOut io2;
+  Roundtrip(&io, enc_options, dec_options, pool, &io2, 8.88888);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/butteraugli/butteraugli.cc b/jxl/butteraugli/butteraugli.cc
new file mode 100644
index 000000000..2b90b79a7
--- /dev/null
+++ b/jxl/butteraugli/butteraugli.cc
@@ -0,0 +1,2029 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: Jyrki Alakuijala (jyrki.alakuijala@gmail.com)
+//
+// The physical architecture of butteraugli is based on the following naming
+// convention:
+//   * Opsin - dynamics of the photosensitive chemicals in the retina
+//             with their immediate electrical processing
+//   * Xyb - hybrid opponent/trichromatic color space
+//     x is roughly red-subtract-green.
+//     y is yellow.
+//     b is blue.
+//     Xyb values are computed from Opsin mixing, not directly from rgb.
+//   * Mask - for visual masking
+//   * Hf - color modeling for spatially high-frequency features
+//   * Lf - color modeling for spatially low-frequency features
+//   * Diffmap - to cluster and build an image of error between the images
+//   * Blur - to hold the smoothing code
+
+#include "jxl/butteraugli/butteraugli.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <new>
+#include <vector>
+
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/image_ops.h"
+
+#ifndef PROFILER_ENABLED
+#define PROFILER_ENABLED 0
+#endif
+#if PROFILER_ENABLED
+#else
+#define PROFILER_FUNC
+#define PROFILER_ZONE(name)
+#endif
+
+namespace jxl {
+namespace butteraugli {
+
+static inline bool IsNan(const float x) {
+  uint32_t bits;
+  memcpy(&bits, &x, sizeof(bits));
+  const uint32_t bitmask_exp = 0x7F800000;
+  return (bits & bitmask_exp) == bitmask_exp && (bits & 0x7FFFFF);
+}
+
+static inline bool IsNan(const double x) {
+  uint64_t bits;
+  memcpy(&bits, &x, sizeof(bits));
+  return (0x7ff0000000000001ULL <= bits && bits <= 0x7fffffffffffffffULL) ||
+         (0xfff0000000000001ULL <= bits && bits <= 0xffffffffffffffffULL);
+}
+
+static inline void CheckImage(const ImageF& image, const char* name) {
+  PROFILER_FUNC;
+  for (size_t y = 0; y < image.ysize(); ++y) {
+    const float* BUTTERAUGLI_RESTRICT row = image.Row(y);
+    for (size_t x = 0; x < image.xsize(); ++x) {
+      if (IsNan(row[x])) {
+        printf("NAN: Image %s @ %zu,%zu (of %zu,%zu)\n", name, x, y,
+               image.xsize(), image.ysize());
+        exit(1);
+      }
+    }
+  }
+}
+
+#if BUTTERAUGLI_ENABLE_CHECKS
+
+#define CHECK_NAN(x, str)                \
+  do {                                   \
+    if (IsNan(x)) {                      \
+      printf("%d: %s\n", __LINE__, str); \
+      abort();                           \
+    }                                    \
+  } while (0)
+
+#define CHECK_IMAGE(image, name) CheckImage(image, name)
+
+#else
+
+#define CHECK_NAN(x, str)
+#define CHECK_IMAGE(image, name)
+
+#endif
+
+// Calculate a 2x2 subsampled image for purposes of recursive butteraugli at
+// multiresolution.
+static Image3F SubSample2x(const Image3F &in) {
+  size_t xs = (in.xsize() + 1) / 2;
+  size_t ys = (in.ysize() + 1) / 2;
+  Image3F retval(xs, ys);
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < ys; ++y) {
+      for (size_t x = 0; x < xs; ++x) {
+        retval.PlaneRow(c, y)[x] = 0;
+      }
+    }
+  }
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < in.ysize(); ++y) {
+      for (size_t x = 0; x < in.xsize(); ++x) {
+        retval.PlaneRow(c, y / 2)[x / 2] += 0.25 * in.PlaneRow(c, y)[x];
+      }
+    }
+    if ((in.xsize() & 1) != 0) {
+      for (size_t y = 0; y < retval.ysize(); ++y) {
+        size_t last_column = retval.xsize() - 1;
+        retval.PlaneRow(c, y)[last_column] *= 2.0;
+      }
+    }
+    if ((in.ysize() & 1) != 0) {
+      for (size_t x = 0; x < retval.xsize(); ++x) {
+        size_t last_row = retval.ysize() - 1;
+        retval.PlaneRow(c, last_row)[x] *= 2.0;
+      }
+    }
+  }
+  return retval;
+}
+
+// Supersample src by 2x and add it to dest.
+static void AddSupersampled2x(const ImageF &src, float w, ImageF &dest) {
+  for (size_t y = 0; y < dest.ysize(); ++y) {
+    for (size_t x = 0; x < dest.xsize(); ++x) {
+      // There will be less errors from the more averaged images.
+      // We take it into account to some extent using a scaler.
+      static const double kHeuristicMixingValue = 0.3;
+      dest.Row(y)[x] *= 1.0 - kHeuristicMixingValue * w;
+      dest.Row(y)[x] += w * src.Row(y / 2)[x / 2];
+    }
+  }
+}
+
+
+// Purpose of kInternalGoodQualityThreshold:
+// Normalize 'ok' image degradation to 1.0 across different versions of
+// butteraugli.
+static const double kInternalGoodQualityThreshold = 42.5;
+static const double kGlobalScale = 1.0 / kInternalGoodQualityThreshold;
+
+inline float MaskColor(const float color[3], const float mask[2]) {
+  return color[0] * mask[0] + color[1] * mask[1] + color[2] * mask[1];
+}
+
+std::vector<float> ComputeKernel(float sigma) {
+  const float m = 2.25;  // Accuracy increases when m is increased.
+  const float scaler = -1.0 / (2 * sigma * sigma);
+  const int diff = std::max<int>(1, m * std::fabs(sigma));
+  std::vector<float> kernel(2 * diff + 1);
+  for (int i = -diff; i <= diff; ++i) {
+    kernel[i + diff] = std::exp(scaler * i * i);
+  }
+  return kernel;
+}
+
+void ConvolveBorderColumn(const ImageF& in, const std::vector<float>& kernel,
+                          const float weight_no_border,
+                          const float border_ratio, const size_t x,
+                          float* BUTTERAUGLI_RESTRICT row_out) {
+  const size_t offset = kernel.size() / 2;
+  int minx = x < offset ? 0 : x - offset;
+  int maxx = std::min<int>(in.xsize() - 1, x + offset);
+  float weight = 0.0f;
+  for (int j = minx; j <= maxx; ++j) {
+    weight += kernel[j - x + offset];
+  }
+  // Interpolate linearly between the no-border scaling and border scaling.
+  weight = (1.0f - border_ratio) * weight + border_ratio * weight_no_border;
+  float scale = 1.0f / weight;
+  for (size_t y = 0; y < in.ysize(); ++y) {
+    const float* BUTTERAUGLI_RESTRICT row_in = in.Row(y);
+    float sum = 0.0f;
+    for (int j = minx; j <= maxx; ++j) {
+      sum += row_in[j] * kernel[j - x + offset];
+    }
+    row_out[y] = sum * scale;
+  }
+}
+
+// Computes a horizontal convolution and transposes the result.
+void ConvolutionWithTranspose(const ImageF& in,
+                              const std::vector<float>& kernel,
+                              const float border_ratio,
+                              ImageF* BUTTERAUGLI_RESTRICT out) {
+  PROFILER_FUNC;
+  JXL_CHECK(out->xsize() == in.ysize());
+  JXL_CHECK(out->ysize() == in.xsize());
+  const size_t len = kernel.size();
+  const size_t offset = len / 2;
+  float weight_no_border = 0.0f;
+  for (size_t j = 0; j < len; ++j) {
+    weight_no_border += kernel[j];
+  }
+  const float scale_no_border = 1.0f / weight_no_border;
+  const size_t border1 = std::min(in.xsize(), offset);
+  const size_t border2 = in.xsize() > offset ? in.xsize() - offset : 0;
+  float* BUTTERAUGLI_RESTRICT scaled_kernel =
+      static_cast<float*>(malloc((len / 2 + 1) * sizeof(float)));
+  for (size_t i = 0; i <= len / 2; ++i) {
+    scaled_kernel[i] = kernel[i] * scale_no_border;
+  }
+  // left border
+  for (size_t x = 0; x < border1; ++x) {
+    ConvolveBorderColumn(in, kernel, weight_no_border, border_ratio, x,
+                         out->Row(x));
+  }
+  // middle
+  switch (len) {
+#if 1  // speed-optimized version
+    case 5: {
+      const float sk0 = scaled_kernel[0];
+      const float sk1 = scaled_kernel[1];
+      const float sk2 = scaled_kernel[2];
+      for (size_t y = 0; y < in.ysize(); ++y) {
+        const float* BUTTERAUGLI_RESTRICT row_in = in.Row(y) + border1 - offset;
+        for (size_t x = border1; x < border2; ++x, ++row_in) {
+          float sum = (row_in[0] + row_in[4]) * sk0;
+          sum += (row_in[1] + row_in[3]) * sk1;
+          sum += (row_in[2]) * sk2;
+          float* BUTTERAUGLI_RESTRICT row_out = out->Row(x);
+          row_out[y] = sum;
+        }
+      }
+    } break;
+    case 9: {
+      const float sk0 = scaled_kernel[0];
+      const float sk1 = scaled_kernel[1];
+      const float sk2 = scaled_kernel[2];
+      const float sk3 = scaled_kernel[3];
+      const float sk4 = scaled_kernel[4];
+      for (size_t y = 0; y < in.ysize(); ++y) {
+        const float* BUTTERAUGLI_RESTRICT row_in = in.Row(y) + border1 - offset;
+        for (size_t x = border1; x < border2; ++x, ++row_in) {
+          float sum = (row_in[0] + row_in[8]) * sk0;
+          sum += (row_in[1] + row_in[7]) * sk1;
+          sum += (row_in[2] + row_in[6]) * sk2;
+          sum += (row_in[3] + row_in[5]) * sk3;
+          sum += (row_in[4]) * sk4;
+          float* BUTTERAUGLI_RESTRICT row_out = out->Row(x);
+          row_out[y] = sum;
+        }
+      }
+    } break;
+    case 17:
+      for (size_t y = 0; y < in.ysize(); ++y) {
+        const float* BUTTERAUGLI_RESTRICT row_in = in.Row(y) + border1 - offset;
+        for (size_t x = border1; x < border2; ++x, ++row_in) {
+          float sum = (row_in[0] + row_in[16]) * scaled_kernel[0];
+          sum += (row_in[1] + row_in[15]) * scaled_kernel[1];
+          sum += (row_in[2] + row_in[14]) * scaled_kernel[2];
+          sum += (row_in[3] + row_in[13]) * scaled_kernel[3];
+          sum += (row_in[4] + row_in[12]) * scaled_kernel[4];
+          sum += (row_in[5] + row_in[11]) * scaled_kernel[5];
+          sum += (row_in[6] + row_in[10]) * scaled_kernel[6];
+          sum += (row_in[7] + row_in[9]) * scaled_kernel[7];
+          sum += (row_in[8]) * scaled_kernel[8];
+          float* BUTTERAUGLI_RESTRICT row_out = out->Row(x);
+          row_out[y] = sum;
+        }
+      }
+      break;
+    case 33:
+      for (size_t y = 0; y < in.ysize(); ++y) {
+        const float* BUTTERAUGLI_RESTRICT row_in = in.Row(y) + border1 - offset;
+        for (size_t x = border1; x < border2; ++x, ++row_in) {
+          float sum = (row_in[0] + row_in[32]) * scaled_kernel[0];
+          sum += (row_in[1] + row_in[31]) * scaled_kernel[1];
+          sum += (row_in[2] + row_in[30]) * scaled_kernel[2];
+          sum += (row_in[3] + row_in[29]) * scaled_kernel[3];
+          sum += (row_in[4] + row_in[28]) * scaled_kernel[4];
+          sum += (row_in[5] + row_in[27]) * scaled_kernel[5];
+          sum += (row_in[6] + row_in[26]) * scaled_kernel[6];
+          sum += (row_in[7] + row_in[25]) * scaled_kernel[7];
+          sum += (row_in[8] + row_in[24]) * scaled_kernel[8];
+          sum += (row_in[9] + row_in[23]) * scaled_kernel[9];
+          sum += (row_in[10] + row_in[22]) * scaled_kernel[10];
+          sum += (row_in[11] + row_in[21]) * scaled_kernel[11];
+          sum += (row_in[12] + row_in[20]) * scaled_kernel[12];
+          sum += (row_in[13] + row_in[19]) * scaled_kernel[13];
+          sum += (row_in[14] + row_in[18]) * scaled_kernel[14];
+          sum += (row_in[15] + row_in[17]) * scaled_kernel[15];
+          sum += (row_in[16]) * scaled_kernel[16];
+          float* BUTTERAUGLI_RESTRICT row_out = out->Row(x);
+          row_out[y] = sum;
+        }
+      }
+      break;
+    case 11:
+      for (size_t y = 0; y < in.ysize(); ++y) {
+        const float* BUTTERAUGLI_RESTRICT row_in = in.Row(y) + border1 - offset;
+        for (size_t x = border1; x < border2; ++x, ++row_in) {
+          float sum = (row_in[0] + row_in[10]) * scaled_kernel[0];
+          sum += (row_in[1] + row_in[9]) * scaled_kernel[1];
+          sum += (row_in[2] + row_in[8]) * scaled_kernel[2];
+          sum += (row_in[3] + row_in[7]) * scaled_kernel[3];
+          sum += (row_in[4] + row_in[6]) * scaled_kernel[4];
+          sum += (row_in[5]) * scaled_kernel[5];
+          float* BUTTERAUGLI_RESTRICT row_out = out->Row(x);
+          row_out[y] = sum;
+        }
+      }
+      break;
+    case 41:
+      for (size_t y = 0; y < in.ysize(); ++y) {
+        const float* BUTTERAUGLI_RESTRICT row_in = in.Row(y) + border1 - offset;
+        for (size_t x = border1; x < border2; ++x, ++row_in) {
+          float sum = (row_in[0] + row_in[40]) * scaled_kernel[0];
+          sum += (row_in[1] + row_in[39]) * scaled_kernel[1];
+          sum += (row_in[2] + row_in[38]) * scaled_kernel[2];
+          sum += (row_in[3] + row_in[37]) * scaled_kernel[3];
+          sum += (row_in[4] + row_in[36]) * scaled_kernel[4];
+          sum += (row_in[5] + row_in[35]) * scaled_kernel[5];
+          sum += (row_in[6] + row_in[34]) * scaled_kernel[6];
+          sum += (row_in[7] + row_in[33]) * scaled_kernel[7];
+          sum += (row_in[8] + row_in[32]) * scaled_kernel[8];
+          sum += (row_in[9] + row_in[31]) * scaled_kernel[9];
+          sum += (row_in[10] + row_in[30]) * scaled_kernel[10];
+          sum += (row_in[11] + row_in[29]) * scaled_kernel[11];
+          sum += (row_in[12] + row_in[28]) * scaled_kernel[12];
+          sum += (row_in[13] + row_in[27]) * scaled_kernel[13];
+          sum += (row_in[14] + row_in[26]) * scaled_kernel[14];
+          sum += (row_in[15] + row_in[25]) * scaled_kernel[15];
+          sum += (row_in[16] + row_in[24]) * scaled_kernel[16];
+          sum += (row_in[17] + row_in[23]) * scaled_kernel[17];
+          sum += (row_in[18] + row_in[22]) * scaled_kernel[18];
+          sum += (row_in[19] + row_in[21]) * scaled_kernel[19];
+          sum += (row_in[20]) * scaled_kernel[20];
+          float* BUTTERAUGLI_RESTRICT row_out = out->Row(x);
+          row_out[y] = sum;
+        }
+      }
+      break;
+    case 47:
+      for (size_t y = 0; y < in.ysize(); ++y) {
+        const float* BUTTERAUGLI_RESTRICT row_in = in.Row(y) + border1 - offset;
+        for (size_t x = border1; x < border2; ++x, ++row_in) {
+          float sum = (row_in[0] + row_in[46]) * scaled_kernel[0];
+          sum += (row_in[1] + row_in[45]) * scaled_kernel[1];
+          sum += (row_in[2] + row_in[44]) * scaled_kernel[2];
+          sum += (row_in[3] + row_in[43]) * scaled_kernel[3];
+          sum += (row_in[4] + row_in[42]) * scaled_kernel[4];
+          sum += (row_in[5] + row_in[41]) * scaled_kernel[5];
+          sum += (row_in[6] + row_in[40]) * scaled_kernel[6];
+          sum += (row_in[7] + row_in[39]) * scaled_kernel[7];
+          sum += (row_in[8] + row_in[38]) * scaled_kernel[8];
+          sum += (row_in[9] + row_in[37]) * scaled_kernel[9];
+          sum += (row_in[10] + row_in[36]) * scaled_kernel[10];
+          sum += (row_in[11] + row_in[35]) * scaled_kernel[11];
+          sum += (row_in[12] + row_in[34]) * scaled_kernel[12];
+          sum += (row_in[13] + row_in[33]) * scaled_kernel[13];
+          sum += (row_in[14] + row_in[32]) * scaled_kernel[14];
+          sum += (row_in[15] + row_in[31]) * scaled_kernel[15];
+          sum += (row_in[16] + row_in[30]) * scaled_kernel[16];
+          sum += (row_in[17] + row_in[29]) * scaled_kernel[17];
+          sum += (row_in[18] + row_in[28]) * scaled_kernel[18];
+          sum += (row_in[19] + row_in[27]) * scaled_kernel[19];
+          sum += (row_in[20] + row_in[26]) * scaled_kernel[20];
+          sum += (row_in[21] + row_in[25]) * scaled_kernel[21];
+          sum += (row_in[22] + row_in[24]) * scaled_kernel[22];
+          sum += (row_in[23]) * scaled_kernel[23];
+          float* BUTTERAUGLI_RESTRICT row_out = out->Row(x);
+          row_out[y] = sum;
+        }
+      }
+      break;
+    default:
+      //      printf("Warning: Unexpected kernel size! %d\n", len);
+#else
+    default:
+#endif
+      for (size_t y = 0; y < in.ysize(); ++y) {
+        const float* BUTTERAUGLI_RESTRICT row_in = in.Row(y);
+        for (size_t j, x = border1; x < border2; ++x) {
+          const int d = x - offset;
+          float* BUTTERAUGLI_RESTRICT row_out = out->Row(x);
+          float sum = 0.0f;
+          for (j = 0; j <= len / 2; ++j) {
+            sum += row_in[d + j] * scaled_kernel[j];
+          }
+          for (; j < len; ++j) {
+            sum += row_in[d + j] * scaled_kernel[len - 1 - j];
+          }
+          row_out[y] = sum;
+        }
+      }
+  }
+  // right border
+  for (size_t x = border2; x < in.xsize(); ++x) {
+    ConvolveBorderColumn(in, kernel, weight_no_border, border_ratio, x,
+                         out->Row(x));
+  }
+  free(scaled_kernel);
+}
+
+// A blur somewhat similar to a 2D Gaussian blur.
+// See: https://en.wikipedia.org/wiki/Gaussian_blur
+void Blur(const ImageF& in, float sigma, float border_ratio,
+          ImageF* BUTTERAUGLI_RESTRICT out) {
+  std::vector<float> kernel = ComputeKernel(sigma);
+  ImageF tmp(in.ysize(), in.xsize());
+  ConvolutionWithTranspose(in, kernel, border_ratio, &tmp);
+  ConvolutionWithTranspose(tmp, kernel, border_ratio, out);
+}
+
+// Clamping linear interpolator.
+inline double InterpolateClampNegative(const double* array, int size,
+                                       double ix) {
+  if (ix < 0) {
+    ix = 0;
+  }
+  int baseix = static_cast<int>(ix);
+  double res;
+  if (baseix >= size - 1) {
+    res = array[size - 1];
+  } else if (baseix < 0) {
+    res = array[0];
+  } else {
+    double mix = ix - baseix;
+    int nextix = baseix + 1;
+    res = array[baseix] + mix * (array[nextix] - array[baseix]);
+  }
+  return res;
+}
+
+double GammaMinArg() {
+  double out0, out1, out2;
+  OpsinAbsorbance(0.0, 0.0, 0.0, &out0, &out1, &out2);
+  return std::min(out0, std::min(out1, out2));
+}
+
+double GammaMaxArg() {
+  double out0, out1, out2;
+  OpsinAbsorbance(255.0, 255.0, 255.0, &out0, &out1, &out2);
+  return std::max(out0, std::max(out1, out2));
+}
+
+double SimpleGamma(double v) {
+  // A simple HDR compatible gamma function.
+  static const double kRetMul = 19.245013259874995;
+  static const double kRetAdd = -23.16046239805755;
+  static const double kVOffset = 9.9710635769299145;
+  if (v < 0) {
+    // This should happen rarely, but may lead to a NaN in log, which is
+    // undesirable. Since negative photons don't exist we solve the NaNs by
+    // clamping here.
+    v = 0;
+  }
+  return kRetMul * log(v + kVOffset) + kRetAdd;
+}
+
+static inline double Gamma(double v) {
+  // SimpleGamma must be used when using an intensity_target with butteraugli
+  // to get values above 255.0.
+  // GammaPolynomial is faster but may only be used if the maximum input value
+  // is 255.0.
+  // TODO(lode): allow jxl to specify which to use depending on the intensity
+  //             target.
+  return SimpleGamma(v);
+  // return GammaPolynomial(v);
+}
+
+Image3F OpsinDynamicsImage(const Image3F& rgb) {
+  PROFILER_FUNC;
+  Image3F xyb(rgb.xsize(), rgb.ysize());
+  const double kSigma = 1.2;
+  Image3F blurred(rgb.xsize(), rgb.ysize());
+  Blur(rgb.Plane(0), kSigma, 0.0, const_cast<ImageF*>(&blurred.Plane(0)));
+  Blur(rgb.Plane(1), kSigma, 0.0, const_cast<ImageF*>(&blurred.Plane(1)));
+  Blur(rgb.Plane(2), kSigma, 0.0, const_cast<ImageF*>(&blurred.Plane(2)));
+  for (size_t y = 0; y < rgb.ysize(); ++y) {
+    const float* BUTTERAUGLI_RESTRICT row_r = rgb.ConstPlaneRow(0, y);
+    const float* BUTTERAUGLI_RESTRICT row_g = rgb.ConstPlaneRow(1, y);
+    const float* BUTTERAUGLI_RESTRICT row_b = rgb.ConstPlaneRow(2, y);
+    const float* BUTTERAUGLI_RESTRICT row_blurred_r = blurred.ConstPlaneRow(0, y);
+    const float* BUTTERAUGLI_RESTRICT row_blurred_g = blurred.ConstPlaneRow(1, y);
+    const float* BUTTERAUGLI_RESTRICT row_blurred_b = blurred.ConstPlaneRow(2, y);
+    float* BUTTERAUGLI_RESTRICT row_out_x = xyb.PlaneRow(0, y);
+    float* BUTTERAUGLI_RESTRICT row_out_y = xyb.PlaneRow(1, y);
+    float* BUTTERAUGLI_RESTRICT row_out_b = xyb.PlaneRow(2, y);
+    for (size_t x = 0; x < rgb.xsize(); ++x) {
+      float sensitivity[3];
+      {
+        // Calculate sensitivity based on the smoothed image gamma derivative.
+        float pre_mixed0, pre_mixed1, pre_mixed2;
+        OpsinAbsorbance(row_blurred_r[x], row_blurred_g[x], row_blurred_b[x],
+                        &pre_mixed0, &pre_mixed1, &pre_mixed2);
+        if (pre_mixed0 < 1e-4) {
+          pre_mixed0 = 1e-4;
+        }
+        if (pre_mixed1 < 1e-4) {
+          pre_mixed1 = 1e-4;
+        }
+        if (pre_mixed2 < 1e-4) {
+          pre_mixed2 = 1e-4;
+        }
+        sensitivity[0] = Gamma(pre_mixed0) / pre_mixed0;
+        sensitivity[1] = Gamma(pre_mixed1) / pre_mixed1;
+        sensitivity[2] = Gamma(pre_mixed2) / pre_mixed2;
+        if (sensitivity[0] < 1e-4) {
+          sensitivity[0] = 1e-4;
+        }
+        if (sensitivity[1] < 1e-4) {
+          sensitivity[1] = 1e-4;
+        }
+        if (sensitivity[2] < 1e-4) {
+          sensitivity[2] = 1e-4;
+        }
+      }
+      float cur_mixed0, cur_mixed1, cur_mixed2;
+      OpsinAbsorbance(row_r[x], row_g[x], row_b[x], &cur_mixed0, &cur_mixed1,
+                      &cur_mixed2);
+      cur_mixed0 *= sensitivity[0];
+      cur_mixed1 *= sensitivity[1];
+      cur_mixed2 *= sensitivity[2];
+      RgbToXyb(cur_mixed0, cur_mixed1, cur_mixed2,
+               &row_out_x[x], &row_out_y[x], &row_out_b[x]);
+    }
+  }
+  return xyb;
+}
+
+// Make area around zero less important (remove it).
+static BUTTERAUGLI_INLINE float RemoveRangeAroundZero(float w, float x) {
+  return x > w ? x - w : x < -w ? x + w : 0.0f;
+}
+
+// Make area around zero more important (2x it until the limit).
+static BUTTERAUGLI_INLINE float AmplifyRangeAroundZero(float w, float x) {
+  return x > w ? x + w : x < -w ? x - w : 2.0f * x;
+}
+
+// XybLowFreqToVals converts from low-frequency XYB space to the 'vals' space.
+// Vals space can be converted to L2-norm space (Euclidean and normalized)
+// through visual masking.
+template <class V>
+BUTTERAUGLI_INLINE void XybLowFreqToVals(const V& x, const V& y, const V& b_arg,
+                                         V* BUTTERAUGLI_RESTRICT valx,
+                                         V* BUTTERAUGLI_RESTRICT valy,
+                                         V* BUTTERAUGLI_RESTRICT valb) {
+  static const double xmuli = 16.728334267161084;
+  static const double ymuli = 28.850249498135561;
+  static const double bmuli = 44.392179956162011;
+  static const double y_to_b_muli = -0.43716072660089195;
+  const V xmul(xmuli);
+  const V ymul(ymuli);
+  const V bmul(bmuli);
+  const V y_to_b_mul(y_to_b_muli);
+  const V b = b_arg + y_to_b_mul * y;
+  *valb = b * bmul;
+  *valx = x * xmul;
+  *valy = y * ymul;
+}
+
+static ImageF SuppressInBrightAreas(size_t xsize, size_t ysize, double mul,
+                                    double mul2, double reg, const ImageF& hf,
+                                    const ImageF& brightness) {
+  PROFILER_FUNC;
+  ImageF inew(xsize, ysize);
+  for (size_t y = 0; y < ysize; ++y) {
+    const float* rowhf = hf.Row(y);
+    const float* rowbr = brightness.Row(y);
+    float* rownew = inew.Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      float v = rowhf[x];
+      float scaler = mul * reg / (reg + rowbr[x]);
+      rownew[x] = scaler * v;
+    }
+  }
+  return inew;
+}
+
+static float MaximumClamp(float v, float maxval) {
+  static const double kMul = 0.934914340314;
+  if (v >= maxval) {
+    v -= maxval;
+    v *= kMul;
+    v += maxval;
+  } else if (v < -maxval) {
+    v += maxval;
+    v *= kMul;
+    v -= maxval;
+  }
+  return v;
+}
+
+static ImageF MaximumClamping(size_t xsize, size_t ysize, const ImageF& ix,
+                              double yw) {
+  static const double kMul = 0.70036978414;
+  ImageF inew(xsize, ysize);
+  for (size_t y = 0; y < ysize; ++y) {
+    const float* rowx = ix.Row(y);
+    float* rownew = inew.Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      double v = rowx[x];
+      if (v >= yw) {
+        v -= yw;
+        v *= kMul;
+        v += yw;
+      } else if (v < -yw) {
+        v += yw;
+        v *= kMul;
+        v -= yw;
+      }
+      rownew[x] = v;
+    }
+  }
+  return inew;
+}
+
+static ImageF SuppressXByY(size_t xsize, size_t ysize, const ImageF& ix,
+                           const ImageF& iy, const double yw) {
+  static const double s = 0.941388349694;
+  ImageF inew(xsize, ysize);
+  for (size_t y = 0; y < ysize; ++y) {
+    const float* rowx = ix.Row(y);
+    const float* rowy = iy.Row(y);
+    float* rownew = inew.Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      const double xval = rowx[x];
+      const double yval = rowy[x];
+      const double scaler = s + (yw * (1.0 - s)) / (yw + yval * yval);
+      rownew[x] = scaler * xval;
+    }
+  }
+  return inew;
+}
+
+static void SeparateFrequencies(size_t xsize, size_t ysize,
+                                const Image3F& xyb,
+                                PsychoImage& ps) {
+  PROFILER_FUNC;
+  // Extract lf ...
+  static const double kSigmaLf = 7.15593339443;
+  static const double kSigmaHf = 3.22489901262;
+  static const double kSigmaUhf = 1.56416327805;
+  // Border handling is complicated.
+  static const double border_lf = 0.0;
+  static const double border_mf = 0.0;
+  static const double border_hf = 0.0;
+  ps.mf = Image3F(xsize, ysize);
+  ps.hf[0] = ImageF(xsize, ysize);
+  ps.hf[1] = ImageF(xsize, ysize);
+  ps.lf = Image3F(xyb.xsize(), xyb.ysize());
+  ps.mf = Image3F(xyb.xsize(), xyb.ysize());
+  for (int i = 0; i < 3; ++i) {
+    Blur(xyb.Plane(i), kSigmaLf, border_lf,
+         const_cast<ImageF*>(&ps.lf.Plane(i)));
+
+    // ... and keep everything else in mf.
+    for (size_t y = 0; y < ysize; ++y) {
+      for (size_t x = 0; x < xsize; ++x) {
+        ps.mf.PlaneRow(i, y)[x] =
+            xyb.PlaneRow(i, y)[x] - ps.lf.ConstPlaneRow(i, y)[x];
+      }
+    }
+    if (i == 2) {
+      Blur(ps.mf.Plane(i), kSigmaHf, border_mf,
+           const_cast<ImageF*>(&ps.mf.Plane(i)));
+      break;
+    }
+    // Divide mf into mf and hf.
+    for (size_t y = 0; y < ysize; ++y) {
+      float* BUTTERAUGLI_RESTRICT row_mf = ps.mf.PlaneRow(i, y);
+      float* BUTTERAUGLI_RESTRICT row_hf = ps.hf[i].Row(y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row_hf[x] = row_mf[x];
+      }
+    }
+    Blur(ps.mf.Plane(i), kSigmaHf, border_mf,
+         const_cast<ImageF*>(&ps.mf.Plane(i)));
+    static const double kRemoveMfRange = 0.3;
+    static const double kAddMfRange = 0.1;
+    if (i == 0) {
+      for (size_t y = 0; y < ysize; ++y) {
+        float* BUTTERAUGLI_RESTRICT row_mf = ps.mf.PlaneRow(0, y);
+        float* BUTTERAUGLI_RESTRICT row_hf = ps.hf[0].Row(y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row_hf[x] -= row_mf[x];
+          row_mf[x] = RemoveRangeAroundZero(kRemoveMfRange, row_mf[x]);
+        }
+      }
+    } else {
+      for (size_t y = 0; y < ysize; ++y) {
+        float* BUTTERAUGLI_RESTRICT row_mf = ps.mf.PlaneRow(1, y);
+        float* BUTTERAUGLI_RESTRICT row_hf = ps.hf[1].Row(y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row_hf[x] -= row_mf[x];
+          row_mf[x] = AmplifyRangeAroundZero(kAddMfRange, row_mf[x]);
+        }
+      }
+    }
+  }
+  // Suppress red-green by intensity change in the high freq channels.
+  static const double suppress = 286.09942757;
+  ps.hf[0] = SuppressXByY(xsize, ysize, ps.hf[0], ps.hf[1], suppress);
+
+  ps.uhf[0] = ImageF(xsize, ysize);
+  ps.uhf[1] = ImageF(xsize, ysize);
+  for (int i = 0; i < 2; ++i) {
+    // Divide hf into hf and uhf.
+    for (size_t y = 0; y < ysize; ++y) {
+      float* BUTTERAUGLI_RESTRICT row_uhf = ps.uhf[i].Row(y);
+      float* BUTTERAUGLI_RESTRICT row_hf = ps.hf[i].Row(y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row_uhf[x] = row_hf[x];
+      }
+    }
+    Blur(ps.hf[i], kSigmaUhf, border_hf, &ps.hf[i]);
+    static const double kRemoveHfRange = 0.12;
+    static const double kAddHfRange = 0.03;
+    static const double kRemoveUhfRange = 0.08;
+    static const double kAddUhfRange = 0.02;
+    static const double kMaxclampHf = 78.7416747972;
+    static const double kMaxclampUhf = 4.62878535439;
+    static double kMulYHf = 1.16155986803;
+    static double kMulYUhf = 2.32552960949;
+    if (i == 0) {
+      for (size_t y = 0; y < ysize; ++y) {
+        float* BUTTERAUGLI_RESTRICT row_uhf = ps.uhf[0].Row(y);
+        float* BUTTERAUGLI_RESTRICT row_hf = ps.hf[0].Row(y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row_uhf[x] -= row_hf[x];
+          row_hf[x] = RemoveRangeAroundZero(kRemoveHfRange, row_hf[x]);
+          row_uhf[x] = RemoveRangeAroundZero(kRemoveUhfRange, row_uhf[x]);
+        }
+      }
+    } else {
+      for (size_t y = 0; y < ysize; ++y) {
+        float* BUTTERAUGLI_RESTRICT row_uhf = ps.uhf[1].Row(y);
+        float* BUTTERAUGLI_RESTRICT row_hf = ps.hf[1].Row(y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row_uhf[x] -= row_hf[x];
+          row_hf[x] = MaximumClamp(row_hf[x], kMaxclampHf);
+          row_uhf[x] = MaximumClamp(row_uhf[x], kMaxclampUhf);
+          row_uhf[x] *= kMulYUhf;
+          row_hf[x] *= kMulYHf;
+          row_hf[x] = AmplifyRangeAroundZero(kAddHfRange, row_hf[x]);
+          row_uhf[x] = AmplifyRangeAroundZero(kAddUhfRange, row_uhf[x]);
+        }
+      }
+    }
+  }
+  // Modify range around zero code only concerns the high frequency
+  // planes and only the X and Y channels.
+  // Convert low freq xyb to vals space so that we can do a simple squared sum
+  // diff on the low frequencies later.
+  for (size_t y = 0; y < ysize; ++y) {
+    float* BUTTERAUGLI_RESTRICT row_x = ps.lf.PlaneRow(0, y);
+    float* BUTTERAUGLI_RESTRICT row_y = ps.lf.PlaneRow(1, y);
+    float* BUTTERAUGLI_RESTRICT row_b = ps.lf.PlaneRow(2, y);
+    for (size_t x = 0; x < xsize; ++x) {
+      float valx, valy, valb;
+      XybLowFreqToVals(row_x[x], row_y[x], row_b[x], &valx, &valy, &valb);
+      row_x[x] = valx;
+      row_y[x] = valy;
+      row_b[x] = valb;
+    }
+  }
+}
+
+static void L2Diff(const ImageF& i0, const ImageF& i1, const float w,
+                   Image3F* BUTTERAUGLI_RESTRICT diffmap, size_t c) {
+  if (w == 0) {
+    return;
+  }
+  for (size_t y = 0; y < i0.ysize(); ++y) {
+    const float* BUTTERAUGLI_RESTRICT row0 = i0.ConstRow(y);
+    const float* BUTTERAUGLI_RESTRICT row1 = i1.ConstRow(y);
+    float* BUTTERAUGLI_RESTRICT row_diff = diffmap->PlaneRow(c, y);
+    for (size_t x = 0; x < i0.xsize(); ++x) {
+      const float diff = row0[x] - row1[x];
+      row_diff[x] += w * diff * diff;
+    }
+  }
+}
+
+// i0 is the original image.
+// i1 is the deformed copy.
+static void L2DiffAsymmetric(const ImageF& i0, const ImageF& i1, double w_0gt1,
+                             double w_0lt1,
+                             Image3F* BUTTERAUGLI_RESTRICT diffmap, size_t c) {
+  if (w_0gt1 == 0 && w_0lt1 == 0) {
+    return;
+  }
+  w_0gt1 *= 0.8;
+  w_0lt1 *= 0.8;
+  for (size_t y = 0; y < i0.ysize(); ++y) {
+    const float* BUTTERAUGLI_RESTRICT row0 = i0.Row(y);
+    const float* BUTTERAUGLI_RESTRICT row1 = i1.Row(y);
+    float* BUTTERAUGLI_RESTRICT row_diff = diffmap->PlaneRow(c, y);
+    for (size_t x = 0; x < i0.xsize(); ++x) {
+      // Primary symmetric quadratic objective.
+      double diff = row0[x] - row1[x];
+      row_diff[x] += w_0gt1 * diff * diff;
+
+      // Secondary half-open quadratic objectives.
+      const double fabs0 = std::fabs(row0[x]);
+      const double too_small = 0.4 * fabs0;
+      const double too_big = 1.0 * fabs0;
+
+      if (row0[x] < 0) {
+        if (row1[x] > -too_small) {
+          double v = row1[x] + too_small;
+          row_diff[x] += w_0lt1 * v * v;
+        } else if (row1[x] < -too_big) {
+          double v = -row1[x] - too_big;
+          row_diff[x] += w_0lt1 * v * v;
+        }
+      } else {
+        if (row1[x] < too_small) {
+          double v = too_small - row1[x];
+          row_diff[x] += w_0lt1 * v * v;
+        } else if (row1[x] > too_big) {
+          double v = row1[x] - too_big;
+          row_diff[x] += w_0lt1 * v * v;
+        }
+      }
+    }
+  }
+}
+
+ImageF CalculateDiffmap(const ImageF& diffmap_in) {
+  PROFILER_FUNC;
+  // Take square root.
+  ImageF diffmap(diffmap_in.xsize(), diffmap_in.ysize());
+  static const float kInitialSlope = 100.0f;
+  for (size_t y = 0; y < diffmap.ysize(); ++y) {
+    const float* BUTTERAUGLI_RESTRICT row_in = diffmap_in.Row(y);
+    float* BUTTERAUGLI_RESTRICT row_out = diffmap.Row(y);
+    for (size_t x = 0; x < diffmap.xsize(); ++x) {
+      const float orig_val = row_in[x];
+      // TODO(b/29974893): Until that is fixed do not call sqrt on very small
+      // numbers.
+      row_out[x] = (orig_val < (1.0f / (kInitialSlope * kInitialSlope))
+                        ? kInitialSlope * orig_val
+                        : std::sqrt(orig_val));
+    }
+  }
+  return diffmap;
+}
+
+void MaskPsychoImage(const PsychoImage& pi0, const PsychoImage& pi1,
+                     const size_t xsize, const size_t ysize,
+                     MaskImage* BUTTERAUGLI_RESTRICT mask,
+                     MaskImage* BUTTERAUGLI_RESTRICT mask_dc,
+                     ImageF* BUTTERAUGLI_RESTRICT diff_ac) {
+  Image3F mask_xyb0(xsize, ysize);
+  Image3F mask_xyb1(xsize, ysize);
+  static const double muls[4] = {
+    0.0,
+    0.0632641915861,
+    0.308212951541,
+    1.16513324377,
+  };
+  for (int i = 0; i < 2; ++i) {
+    double a = muls[2 * i];
+    double b = muls[2 * i + 1];
+    for (size_t y = 0; y < ysize; ++y) {
+      const float* BUTTERAUGLI_RESTRICT row_hf0 = pi0.hf[i].Row(y);
+      const float* BUTTERAUGLI_RESTRICT row_hf1 = pi1.hf[i].Row(y);
+      const float* BUTTERAUGLI_RESTRICT row_uhf0 = pi0.uhf[i].Row(y);
+      const float* BUTTERAUGLI_RESTRICT row_uhf1 = pi1.uhf[i].Row(y);
+      float* BUTTERAUGLI_RESTRICT row0 = mask_xyb0.PlaneRow(i, y);
+      float* BUTTERAUGLI_RESTRICT row1 = mask_xyb1.PlaneRow(i, y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row0[x] = a * row_uhf0[x] + b * row_hf0[x];
+        row1[x] = a * row_uhf1[x] + b * row_hf1[x];
+      }
+    }
+  }
+  Mask(mask_xyb0, mask_xyb1, mask, mask_dc, diff_ac);
+}
+
+ButteraugliComparator::ButteraugliComparator(const Image3F& rgb0,
+                                             double hf_asymmetry)
+    : xsize_(rgb0.xsize()),
+      ysize_(rgb0.ysize()),
+      hf_asymmetry_(hf_asymmetry),
+      sub_(nullptr) {
+  if (xsize_ < 8 || ysize_ < 8) {
+    return;
+  }
+  Image3F xyb0 = OpsinDynamicsImage(rgb0);
+  SeparateFrequencies(xsize_, ysize_, xyb0, pi0_);
+
+  // Awful recursive construction of samples of different resolution.
+  // This is an after-thought and possibly somewhat parallel in
+  // functionality with the PsychoImage multi-resolution approach.
+  sub_ = new ButteraugliComparator(SubSample2x(rgb0), hf_asymmetry);
+}
+
+ButteraugliComparator::~ButteraugliComparator() {
+  delete sub_;
+}
+
+
+void ButteraugliComparator::Mask(
+    MaskImage* BUTTERAUGLI_RESTRICT mask,
+    MaskImage* BUTTERAUGLI_RESTRICT mask_dc) const {
+  MaskPsychoImage(pi0_, pi0_, xsize_, ysize_, mask, mask_dc, nullptr);
+}
+
+void ButteraugliComparator::Diffmap(const Image3F& rgb1, ImageF& result) const {
+  PROFILER_FUNC;
+  if (xsize_ < 8 || ysize_ < 8) {
+    ZeroFillImage(&result);
+    return;
+  }
+  DiffmapOpsinDynamicsImage(OpsinDynamicsImage(rgb1), result);
+  if (sub_) {
+    if (sub_->xsize_ < 8 || sub_->ysize_ < 8) {
+      return;
+    }
+    ImageF subresult;
+    sub_->DiffmapOpsinDynamicsImage(
+        OpsinDynamicsImage(SubSample2x(rgb1)), subresult);
+    AddSupersampled2x(subresult, 0.5, result);
+  }
+}
+
+void ButteraugliComparator::DiffmapOpsinDynamicsImage(const Image3F& xyb1,
+                                                      ImageF& result) const {
+  PROFILER_FUNC;
+  if (xsize_ < 8 || ysize_ < 8) {
+    ZeroFillImage(&result);
+    return;
+  }
+  PsychoImage pi1;
+  SeparateFrequencies(xsize_, ysize_, xyb1, pi1);
+  result = ImageF(xsize_, ysize_);
+  DiffmapPsychoImage(pi1, result);
+}
+
+void ButteraugliComparator::DiffmapPsychoImage(const PsychoImage& pi1,
+                                               ImageF& result) const {
+  PROFILER_FUNC;
+  if (xsize_ < 8 || ysize_ < 8) {
+    ZeroFillImage(&result);
+    return;
+  }
+  Image3F block_diff_dc(xsize_, ysize_);
+  ZeroFillImage(&block_diff_dc);
+  Image3F block_diff_ac(xsize_, ysize_);
+  ZeroFillImage(&block_diff_ac);
+  static const double wUhfMalta = 8.3230030810258917;
+  static const double norm1Uhf = 81.97966510743784;
+  MaltaDiffMap(pi0_.uhf[1], pi1.uhf[1], wUhfMalta * hf_asymmetry_,
+               wUhfMalta / hf_asymmetry_, norm1Uhf, &block_diff_ac, 1);
+
+  static const double wUhfMaltaX = 51.738409933971198;
+  static const double norm1UhfX = 19.390881354476463;
+  MaltaDiffMap(pi0_.uhf[0], pi1.uhf[0], wUhfMaltaX * hf_asymmetry_,
+               wUhfMaltaX / hf_asymmetry_, norm1UhfX, &block_diff_ac, 0);
+
+  static const double wHfMalta = 240.0908262374156;
+  static const double norm1Hf = 213.151889155;
+  MaltaDiffMapLF(pi0_.hf[1], pi1.hf[1], wHfMalta * std::sqrt(hf_asymmetry_),
+                 wHfMalta / std::sqrt(hf_asymmetry_), norm1Hf, &block_diff_ac,
+                 1);
+
+  static const double wHfMaltaX = 160.0;
+  static const double norm1HfX = 80.0;
+  MaltaDiffMapLF(pi0_.hf[0], pi1.hf[0], wHfMaltaX * std::sqrt(hf_asymmetry_),
+                 wHfMaltaX / std::sqrt(hf_asymmetry_), norm1HfX, &block_diff_ac,
+                 0);
+
+  static const double wMfMalta = 163.56963612738494;
+  static const double norm1Mf = 0.1635008533899469;
+  MaltaDiffMapLF(pi0_.mf.Plane(1), pi1.mf.Plane(1), wMfMalta, wMfMalta, norm1Mf,
+                 &block_diff_ac, 1);
+
+  static const double wMfMaltaX = 6164.558625327204;
+  static const double norm1MfX = 1002.5;
+  MaltaDiffMapLF(pi0_.mf.Plane(0), pi1.mf.Plane(0), wMfMaltaX, wMfMaltaX,
+                 norm1MfX, &block_diff_ac, 0);
+
+  static const double wmul[9] = {
+    32, 5.0, 0, 32, 5, 237.33703833286302, 0.8170086922843028,
+    1.0323708525451885, 5.5346699491372346,
+  };
+  for (size_t c = 0; c < 3; ++c) {
+    if (c < 2) {  // No blue channel error accumulated at HF.
+      L2DiffAsymmetric(pi0_.hf[c], pi1.hf[c], wmul[c] * hf_asymmetry_,
+                       wmul[c] / hf_asymmetry_, &block_diff_ac, c);
+    }
+    L2Diff(pi0_.mf.Plane(c), pi1.mf.Plane(c), wmul[3 + c], &block_diff_ac, c);
+    L2Diff(pi0_.lf.Plane(c), pi1.lf.Plane(c), wmul[6 + c], &block_diff_dc, c);
+  }
+
+  MaskImage mask_xyb_ac;
+  MaskImage mask_xyb_dc;
+  MaskPsychoImage(pi0_, pi1, xsize_, ysize_, &mask_xyb_ac, &mask_xyb_dc,
+                  const_cast<ImageF*>(&block_diff_ac.Plane(1)));
+
+  result = CalculateDiffmap(
+      CombineChannels(mask_xyb_ac, mask_xyb_dc, block_diff_dc, block_diff_ac));
+}
+
+// Allows PaddedMaltaUnit to call either function via overloading.
+struct MaltaTagLF {};
+struct MaltaTag {};
+
+static float MaltaUnit(MaltaTagLF /*tag*/, const float* BUTTERAUGLI_RESTRICT d,
+                       const int xs) {
+  const int xs3 = 3 * xs;
+  float retval = 0;
+  {
+    // x grows, y constant
+    float sum = d[-4] + d[-2] + d[0] + d[2] + d[4];
+    retval += sum * sum;
+  }
+  {
+    // y grows, x constant
+    float sum = d[-xs3 - xs] + d[-xs - xs] + d[0] + d[xs + xs] + d[xs3 + xs];
+    retval += sum * sum;
+  }
+  {
+    // both grow
+    float sum =
+        d[-xs3 - 3] + d[-xs - xs - 2] + d[0] + d[xs + xs + 2] + d[xs3 + 3];
+    retval += sum * sum;
+  }
+  {
+    // y grows, x shrinks
+    float sum =
+        d[-xs3 + 3] + d[-xs - xs + 2] + d[0] + d[xs + xs - 2] + d[xs3 - 3];
+    retval += sum * sum;
+  }
+  {
+    // y grows -4 to 4, x shrinks 1 -> -1
+    float sum = d[-xs3 - xs + 1] + d[-xs - xs + 1] + d[0] + d[xs + xs - 1] +
+                d[xs3 + xs - 1];
+    retval += sum * sum;
+  }
+  {
+    //  y grows -4 to 4, x grows -1 -> 1
+    float sum = d[-xs3 - xs - 1] + d[-xs - xs - 1] + d[0] + d[xs + xs + 1] +
+                d[xs3 + xs + 1];
+    retval += sum * sum;
+  }
+  {
+    // x grows -4 to 4, y grows -1 to 1
+    float sum = d[-4 - xs] + d[-2 - xs] + d[0] + d[2 + xs] + d[4 + xs];
+    retval += sum * sum;
+  }
+  {
+    // x grows -4 to 4, y shrinks 1 to -1
+    float sum = d[-4 + xs] + d[-2 + xs] + d[0] + d[2 - xs] + d[4 - xs];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1__*______
+       2___*_____
+       3_________
+       4____0____
+       5_________
+       6_____*___
+       7______*__
+       8_________ */
+    float sum =
+        d[-xs3 - 2] + d[-xs - xs - 1] + d[0] + d[xs + xs + 1] + d[xs3 + 2];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1______*__
+       2_____*___
+       3_________
+       4____0____
+       5_________
+       6___*_____
+       7__*______
+       8_________ */
+    float sum =
+        d[-xs3 + 2] + d[-xs - xs + 1] + d[0] + d[xs + xs - 1] + d[xs3 - 2];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1_________
+       2_*_______
+       3__*______
+       4____0____
+       5______*__
+       6_______*_
+       7_________
+       8_________ */
+    float sum =
+        d[-xs - xs - 3] + d[-xs - 2] + d[0] + d[xs + 2] + d[xs + xs + 3];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1_________
+       2_______*_
+       3______*__
+       4____0____
+       5__*______
+       6_*_______
+       7_________
+       8_________ */
+    float sum =
+        d[-xs - xs + 3] + d[-xs + 2] + d[0] + d[xs - 2] + d[xs + xs - 3];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1_________
+       2________*
+       3______*__
+       4____0____
+       5__*______
+       6*________
+       7_________
+       8_________ */
+
+    float sum =
+        d[xs + xs - 4] + d[xs - 2] + d[0] + d[-xs + 2] + d[-xs - xs + 4];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1_________
+       2*________
+       3__*______
+       4____0____
+       5______*__
+       6________*
+       7_________
+       8_________ */
+    float sum =
+        d[-xs - xs - 4] + d[-xs - 2] + d[0] + d[xs + 2] + d[xs + xs + 4];
+    retval += sum * sum;
+  }
+  {
+    /* 0__*______
+       1_________
+       2___*_____
+       3_________
+       4____0____
+       5_________
+       6_____*___
+       7_________
+       8______*__ */
+    float sum = d[-xs3 - xs - 2] + d[-xs - xs - 1] + d[0] + d[xs + xs + 1] +
+                d[xs3 + xs + 2];
+    retval += sum * sum;
+  }
+  {
+    /* 0______*__
+       1_________
+       2_____*___
+       3_________
+       4____0____
+       5_________
+       6___*_____
+       7_________
+       8__*______ */
+    float sum = d[-xs3 - xs + 2] + d[-xs - xs + 1] + d[0] + d[xs + xs - 1] +
+                d[xs3 + xs - 2];
+    retval += sum * sum;
+  }
+  return retval;
+}
+
+static float MaltaUnit(MaltaTag /*tag*/, const float* BUTTERAUGLI_RESTRICT d,
+                       const int xs) {
+  const int xs3 = 3 * xs;
+  float retval = 0;
+  {
+    // x grows, y constant
+    float sum =
+        d[-4] + d[-3] + d[-2] + d[-1] + d[0] + d[1] + d[2] + d[3] + d[4];
+    retval += sum * sum;
+  }
+  {
+    // y grows, x constant
+    float sum = d[-xs3 - xs] + d[-xs3] + d[-xs - xs] + d[-xs] + d[0] + d[xs] +
+                d[xs + xs] + d[xs3] + d[xs3 + xs];
+    retval += sum * sum;
+  }
+  {
+    // both grow
+    float sum = d[-xs3 - 3] + d[-xs - xs - 2] + d[-xs - 1] + d[0] + d[xs + 1] +
+                d[xs + xs + 2] + d[xs3 + 3];
+    retval += sum * sum;
+  }
+  {
+    // y grows, x shrinks
+    float sum = d[-xs3 + 3] + d[-xs - xs + 2] + d[-xs + 1] + d[0] + d[xs - 1] +
+                d[xs + xs - 2] + d[xs3 - 3];
+    retval += sum * sum;
+  }
+  {
+    // y grows -4 to 4, x shrinks 1 -> -1
+    float sum = d[-xs3 - xs + 1] + d[-xs3 + 1] + d[-xs - xs + 1] + d[-xs] +
+                d[0] + d[xs] + d[xs + xs - 1] + d[xs3 - 1] + d[xs3 + xs - 1];
+    retval += sum * sum;
+  }
+  {
+    //  y grows -4 to 4, x grows -1 -> 1
+    float sum = d[-xs3 - xs - 1] + d[-xs3 - 1] + d[-xs - xs - 1] + d[-xs] +
+                d[0] + d[xs] + d[xs + xs + 1] + d[xs3 + 1] + d[xs3 + xs + 1];
+    retval += sum * sum;
+  }
+  {
+    // x grows -4 to 4, y grows -1 to 1
+    float sum = d[-4 - xs] + d[-3 - xs] + d[-2 - xs] + d[-1] + d[0] + d[1] +
+                d[2 + xs] + d[3 + xs] + d[4 + xs];
+    retval += sum * sum;
+  }
+  {
+    // x grows -4 to 4, y shrinks 1 to -1
+    float sum = d[-4 + xs] + d[-3 + xs] + d[-2 + xs] + d[-1] + d[0] + d[1] +
+                d[2 - xs] + d[3 - xs] + d[4 - xs];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1__*______
+       2___*_____
+       3___*_____
+       4____0____
+       5_____*___
+       6_____*___
+       7______*__
+       8_________ */
+    float sum = d[-xs3 - 2] + d[-xs - xs - 1] + d[-xs - 1] + d[0] + d[xs + 1] +
+                d[xs + xs + 1] + d[xs3 + 2];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1______*__
+       2_____*___
+       3_____*___
+       4____0____
+       5___*_____
+       6___*_____
+       7__*______
+       8_________ */
+    float sum = d[-xs3 + 2] + d[-xs - xs + 1] + d[-xs + 1] + d[0] + d[xs - 1] +
+                d[xs + xs - 1] + d[xs3 - 2];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1_________
+       2_*_______
+       3__**_____
+       4____0____
+       5_____**__
+       6_______*_
+       7_________
+       8_________ */
+    float sum = d[-xs - xs - 3] + d[-xs - 2] + d[-xs - 1] + d[0] + d[xs + 1] +
+                d[xs + 2] + d[xs + xs + 3];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1_________
+       2_______*_
+       3_____**__
+       4____0____
+       5__**_____
+       6_*_______
+       7_________
+       8_________ */
+    float sum = d[-xs - xs + 3] + d[-xs + 2] + d[-xs + 1] + d[0] + d[xs - 1] +
+                d[xs - 2] + d[xs + xs - 3];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1_________
+       2_________
+       3______***
+       4___*0*___
+       5***______
+       6_________
+       7_________
+       8_________ */
+
+    float sum = d[xs - 4] + d[xs - 3] + d[xs - 2] + d[-1] +
+                d[0] + d[1] + d[-xs + 2] + d[-xs + 3] + d[-xs + 4];
+    retval += sum * sum;
+  }
+  {
+    /* 0_________
+       1_________
+       2_________
+       3***______
+       4___*0*___
+       5______***
+       6_________
+       7_________
+       8_________ */
+    float sum = d[-xs - 4] + d[-xs - 3] + d[-xs - 2] + d[-1] +
+                d[0] + d[1] + d[xs + 2] + d[xs + 3] + d[xs + 4];
+    retval += sum * sum;
+  }
+  {
+    /* 0___*_____
+       1___*_____
+       2___*_____
+       3____*____
+       4____0____
+       5____*____
+       6_____*___
+       7_____*___
+       8_____*___ */
+    float sum = d[-xs3 - xs - 1] + d[-xs3 - 1] + d[-xs - xs - 1] + d[-xs] +
+                d[0] + d[xs] + d[xs + xs + 1] + d[xs3 + 1] + d[xs3 + xs + 1];
+    retval += sum * sum;
+  }
+  {
+    /* 0_____*___
+       1_____*___
+       2____ *___
+       3____*____
+       4____0____
+       5____*____
+       6___*_____
+       7___*_____
+       8___*_____ */
+    float sum = d[-xs3 - xs + 1] + d[-xs3 + 1] + d[-xs - xs + 1] + d[-xs] +
+                d[0] + d[xs] + d[xs + xs - 1] + d[xs3 - 1] + d[xs3 + xs - 1];
+    retval += sum * sum;
+  }
+  return retval;
+}
+
+// Returns MaltaUnit. "fastMode" avoids bounds-checks when x0 and y0 are known
+// to be far enough from the image borders. "diffs" is a packed image.
+template <bool fastMode, class Tag>
+static BUTTERAUGLI_INLINE float PaddedMaltaUnit(
+    const float* BUTTERAUGLI_RESTRICT diffs, const size_t x0, const size_t y0,
+    const size_t xsize_, const size_t ysize_) {
+  int ix0 = y0 * xsize_ + x0;
+  const float* BUTTERAUGLI_RESTRICT d = &diffs[ix0];
+  if (fastMode ||
+      (x0 >= 4 && y0 >= 4 && x0 < (xsize_ - 4) && y0 < (ysize_ - 4))) {
+    return MaltaUnit(Tag(), d, xsize_);
+  }
+
+  float borderimage[9 * 9];
+  for (int dy = 0; dy < 9; ++dy) {
+    int y = y0 + dy - 4;
+    if (y < 0 || static_cast<size_t>(y) >= ysize_) {
+      for (int dx = 0; dx < 9; ++dx) {
+        borderimage[dy * 9 + dx] = 0.0f;
+      }
+    } else {
+      for (int dx = 0; dx < 9; ++dx) {
+        int x = x0 + dx - 4;
+        if (x < 0 || static_cast<size_t>(x) >= xsize_) {
+          borderimage[dy * 9 + dx] = 0.0f;
+        } else {
+          borderimage[dy * 9 + dx] = diffs[y * xsize_ + x];
+        }
+      }
+    }
+  }
+  return MaltaUnit(Tag(), &borderimage[4 * 9 + 4], 9);
+}
+
+template <class Tag>
+static void MaltaDiffMapImpl(const ImageF& lum0, const ImageF& lum1,
+                             const size_t xsize_, const size_t ysize_,
+                             const double w_0gt1, const double w_0lt1,
+                             const double norm1, const double len,
+                             const double mulli,
+                             Image3F* BUTTERAUGLI_RESTRICT block_diff_ac,
+                             size_t c) {
+  const float kWeight0 = 0.5;
+  const float kWeight1 = 0.33;
+
+  const double w_pre0gt1 = mulli * std::sqrt(kWeight0 * w_0gt1) / (len * 2 + 1);
+  const double w_pre0lt1 = mulli * std::sqrt(kWeight1 * w_0lt1) / (len * 2 + 1);
+  const float norm2_0gt1 = w_pre0gt1 * norm1;
+  const float norm2_0lt1 = w_pre0lt1 * norm1;
+
+  std::vector<float> diffs(ysize_ * xsize_);
+  for (size_t y = 0, ix = 0; y < ysize_; ++y) {
+    const float* BUTTERAUGLI_RESTRICT row0 = lum0.Row(y);
+    const float* BUTTERAUGLI_RESTRICT row1 = lum1.Row(y);
+    for (size_t x = 0; x < xsize_; ++x, ++ix) {
+      const float absval = 0.5f * (std::abs(row0[x]) + std::abs(row1[x]));
+      const float diff = row0[x] - row1[x];
+      const float scaler = norm2_0gt1 / (static_cast<float>(norm1) + absval);
+
+      // Primary symmetric quadratic objective.
+      diffs[ix] = scaler * diff;
+
+      const float scaler2 = norm2_0lt1 / (static_cast<float>(norm1) + absval);
+      const double fabs0 = std::fabs(row0[x]);
+
+      // Secondary half-open quadratic objectives.
+      const double too_small = 0.55 * fabs0;
+      const double too_big = 1.05 * fabs0;
+
+      if (row0[x] < 0) {
+        if (row1[x] > -too_small) {
+          double impact = scaler2 * (row1[x] + too_small);
+          if (diff < 0) {
+            diffs[ix] -= impact;
+          } else {
+            diffs[ix] += impact;
+          }
+        } else if (row1[x] < -too_big) {
+          double impact = scaler2 * (-row1[x] - too_big);
+          if (diff < 0) {
+            diffs[ix] -= impact;
+          } else {
+            diffs[ix] += impact;
+          }
+        }
+      } else {
+        if (row1[x] < too_small) {
+          double impact = scaler2 * (too_small - row1[x]);
+          if (diff < 0) {
+            diffs[ix] -= impact;
+          } else {
+            diffs[ix] += impact;
+          }
+        } else if (row1[x] > too_big) {
+          double impact = scaler2 * (row1[x] - too_big);
+          if (diff < 0) {
+            diffs[ix] -= impact;
+          } else {
+            diffs[ix] += impact;
+          }
+        }
+      }
+    }
+  }
+
+  size_t y0 = 0;
+  // Top
+  for (; y0 < 4; ++y0) {
+    float* BUTTERAUGLI_RESTRICT row_diff = block_diff_ac->PlaneRow(c, y0);
+    for (size_t x0 = 0; x0 < xsize_; ++x0) {
+      row_diff[x0] +=
+          PaddedMaltaUnit<false, Tag>(&diffs[0], x0, y0, xsize_, ysize_);
+    }
+  }
+
+  // Middle
+  for (; y0 < ysize_ - 4; ++y0) {
+    float* BUTTERAUGLI_RESTRICT row_diff = block_diff_ac->PlaneRow(c, y0);
+    size_t x0 = 0;
+    for (; x0 < 4; ++x0) {
+      row_diff[x0] +=
+          PaddedMaltaUnit<false, Tag>(&diffs[0], x0, y0, xsize_, ysize_);
+    }
+    for (; x0 < xsize_ - 4; ++x0) {
+      row_diff[x0] +=
+          PaddedMaltaUnit<true, Tag>(&diffs[0], x0, y0, xsize_, ysize_);
+    }
+
+    for (; x0 < xsize_; ++x0) {
+      row_diff[x0] +=
+          PaddedMaltaUnit<false, Tag>(&diffs[0], x0, y0, xsize_, ysize_);
+    }
+  }
+
+  // Bottom
+  for (; y0 < ysize_; ++y0) {
+    float* BUTTERAUGLI_RESTRICT row_diff = block_diff_ac->PlaneRow(c, y0);
+    for (size_t x0 = 0; x0 < xsize_; ++x0) {
+      row_diff[x0] +=
+          PaddedMaltaUnit<false, Tag>(&diffs[0], x0, y0, xsize_, ysize_);
+    }
+  }
+}
+
+void ButteraugliComparator::MaltaDiffMap(
+    const ImageF& lum0, const ImageF& lum1, const double w_0gt1,
+    const double w_0lt1, const double norm1,
+    Image3F* BUTTERAUGLI_RESTRICT block_diff_ac, size_t c) const {
+  PROFILER_FUNC;
+  const double len = 3.75;
+  static const double mulli = 0.359826387683;
+  MaltaDiffMapImpl<MaltaTag>(lum0, lum1, xsize_, ysize_, w_0gt1, w_0lt1, norm1,
+                             len, mulli, block_diff_ac, c);
+}
+
+void ButteraugliComparator::MaltaDiffMapLF(
+    const ImageF& lum0, const ImageF& lum1, const double w_0gt1,
+    const double w_0lt1, const double norm1,
+    Image3F* BUTTERAUGLI_RESTRICT block_diff_ac, size_t c) const {
+  PROFILER_FUNC;
+  const double len = 3.75;
+  static const double mulli = 0.737143715861;
+  MaltaDiffMapImpl<MaltaTagLF>(lum0, lum1, xsize_, ysize_, w_0gt1, w_0lt1,
+                               norm1, len, mulli, block_diff_ac, c);
+}
+
+ImageF ButteraugliComparator::CombineChannels(
+    const MaskImage& mask_xyb, const MaskImage& mask_xyb_dc,
+    const Image3F& block_diff_dc, const Image3F& block_diff_ac) const {
+  PROFILER_FUNC;
+  ImageF result(xsize_, ysize_);
+  for (size_t y = 0; y < ysize_; ++y) {
+    float* BUTTERAUGLI_RESTRICT row_out = result.Row(y);
+    for (size_t x = 0; x < xsize_; ++x) {
+      float mask[2] = {
+        mask_xyb.mask_x.Row(y)[x],
+        mask_xyb.mask_yb.Row(y)[x],
+      };
+      float dc_mask[2] = {
+        mask_xyb_dc.mask_x.Row(y)[x],
+        mask_xyb_dc.mask_yb.Row(y)[x],
+      };
+      float diff_dc[3];
+      float diff_ac[3];
+      for (int i = 0; i < 3; ++i) {
+        diff_dc[i] = block_diff_dc.PlaneRow(i, y)[x];
+        diff_ac[i] = block_diff_ac.PlaneRow(i, y)[x];
+      }
+      row_out[x] = MaskColor(diff_dc, dc_mask) + MaskColor(diff_ac, mask);
+    }
+  }
+  return result;
+}
+
+double ButteraugliScoreFromDiffmap(const ImageF& diffmap) {
+  PROFILER_FUNC;
+  float retval = 0.0f;
+  for (size_t y = 0; y < diffmap.ysize(); ++y) {
+    const float* BUTTERAUGLI_RESTRICT row = diffmap.ConstRow(y);
+    for (size_t x = 0; x < diffmap.xsize(); ++x) {
+      retval = std::max(retval, row[x]);
+    }
+  }
+  return retval;
+}
+
+// ===== Functions used by Mask only =====
+static std::array<double, 512> MakeMask(double extmul, double extoff,
+                                        double mul, double offset,
+                                        double scaler) {
+  std::array<double, 512> lut;
+  for (size_t i = 0; i < lut.size(); ++i) {
+    const double c = mul / ((0.01 * scaler * i) + offset);
+    lut[i] = kGlobalScale * (1.0 + extmul * (c + extoff));
+    if (lut[i] < 1e-5) {
+      lut[i] = 1e-5;
+    }
+    JXL_DASSERT(lut[i] >= 0.0);
+    lut[i] *= lut[i];
+  }
+  return lut;
+}
+
+double MaskX(double delta) {
+  static const double extmul = 1.9741728897212369;
+  static const double extoff = 2.0124797155052101;
+  static const double offset = 0.26118960206891428;
+  static const double scaler = 386.58249095811573;
+  static const double mul = 4.0096602151572194;
+  static const std::array<double, 512> lut =
+                MakeMask(extmul, extoff, mul, offset, scaler);
+  return InterpolateClampNegative(lut.data(), lut.size(), delta);
+}
+
+double MaskY(double delta) {
+  static const double extmul = 3.2760071347758579;
+  static const double extoff = -3.3377693215128232;
+  static const double offset = 1.9592295279427681;
+  static const double scaler = 1.4422616544932898;
+  static const double mul = 9.6463717073130066;
+  static const std::array<double, 512> lut =
+      MakeMask(extmul, extoff, mul, offset, scaler);
+  return InterpolateClampNegative(lut.data(), lut.size(), delta);
+}
+
+double MaskDcX(double delta) {
+  static const double extmul = 7.9213692914149458;
+  static const double extoff = 2.5312740226978478;
+  static const double offset = 0.1060403567938254;
+  static const double scaler = 43.546844862805052;
+  static const double mul = 0.32110246816137022;
+  static const std::array<double, 512> lut =
+      MakeMask(extmul, extoff, mul, offset, scaler);
+  return InterpolateClampNegative(lut.data(), lut.size(), delta);
+}
+
+double MaskDcY(double delta) {
+  static const double extmul = 0.019875068816088557;
+  static const double extoff = 0.1;
+  static const double offset = 0.1;
+  static const double scaler = 7.0;
+  static const double mul = 9.8744858184185791;
+  static const std::array<double, 512> lut =
+      MakeMask(extmul, extoff, mul, offset, scaler);
+  return InterpolateClampNegative(lut.data(), lut.size(), delta);
+}
+
+ImageF DiffPrecomputeX(const ImageF& xyb0, const ImageF& xyb1,
+                       float mul, float cutoff) {
+  PROFILER_FUNC;
+  const size_t xsize = xyb0.xsize();
+  const size_t ysize = xyb0.ysize();
+  ImageF result(xsize, ysize);
+  size_t x1, y1;
+  size_t x2, y2;
+  for (size_t y = 0; y < ysize; ++y) {
+    if (y + 1 < ysize) {
+      y2 = y + 1;
+    } else if (y > 0) {
+      y2 = y - 1;
+    } else {
+      y2 = y;
+    }
+    if (y == 0 && ysize >= 2) {
+      y1 = y + 1;
+    } else if (y > 0) {
+      y1 = y - 1;
+    } else {
+      y1 = y;
+    }
+    const float* BUTTERAUGLI_RESTRICT row0_in = xyb0.Row(y);
+    const float* BUTTERAUGLI_RESTRICT row1_in = xyb1.Row(y);
+    const float* BUTTERAUGLI_RESTRICT row0_in1 = xyb0.Row(y1);
+    const float* BUTTERAUGLI_RESTRICT row1_in1 = xyb1.Row(y1);
+    const float* BUTTERAUGLI_RESTRICT row0_in2 = xyb0.Row(y2);
+    const float* BUTTERAUGLI_RESTRICT row1_in2 = xyb1.Row(y2);
+    float* BUTTERAUGLI_RESTRICT row_out = result.Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      if (x + 1 < xsize) {
+        x2 = x + 1;
+      } else if (x > 0) {
+        x2 = x - 1;
+      } else {
+        x2 = x;
+      }
+      if (x == 0 && xsize >= 2) {
+        x1 = x + 1;
+      } else if (x > 0) {
+        x1 = x - 1;
+      } else {
+        x1 = x;
+      }
+      double sup0 = (std::fabs(row0_in[x] - row0_in[x2]) +
+                     std::fabs(row0_in[x] - row0_in2[x]) +
+                     std::fabs(row0_in[x] - row0_in[x1]) +
+                     std::fabs(row0_in[x] - row0_in1[x]) +
+                     3 * (std::fabs(row0_in2[x] - row0_in1[x]) +
+                          std::fabs(row0_in[x1] - row0_in[x2])));
+      double sup1 = (std::fabs(row1_in[x] - row1_in[x2]) +
+                     std::fabs(row1_in[x] - row1_in2[x]) +
+                     std::fabs(row1_in[x] - row1_in[x1]) +
+                     std::fabs(row1_in[x] - row1_in1[x]) +
+                     3 * (std::fabs(row1_in2[x] - row1_in1[x]) +
+                          std::fabs(row1_in[x1] - row1_in[x2])));
+
+      row_out[x] = mul * std::min(sup0, sup1);
+      if (row_out[x] >= cutoff) {
+        row_out[x] = cutoff;
+      }
+      {
+        static const double limit = 0.5 * cutoff;
+        if (row_out[x] >= limit) {
+          row_out[x] += limit;
+          row_out[x] *= 0.5;
+        }
+      }
+      {
+        static const double limit = 0.25 * cutoff;
+        if (row_out[x] >= limit) {
+          row_out[x] += limit;
+          row_out[x] *= 0.5;
+        }
+      }
+    }
+  }
+  return result;
+}
+
+// Precalculates masking for y channel, giving masks for
+// both images back so that they can be used for similarity comparisons
+// too.
+void DiffPrecomputeY(const ImageF& xyb0, const ImageF& xyb1,
+                     float mul, float mul2,
+                     ImageF *out0, ImageF *out1) {
+  PROFILER_FUNC;
+  const size_t xsize = xyb0.xsize();
+  const size_t ysize = xyb0.ysize();
+  size_t x1, y1;
+  size_t x2, y2;
+  for (size_t y = 0; y < ysize; ++y) {
+    if (y + 1 < ysize) {
+      y2 = y + 1;
+    } else if (y > 0) {
+      y2 = y - 1;
+    } else {
+      y2 = y;
+    }
+    if (y == 0 && ysize >= 2) {
+      y1 = y + 1;
+    } else if (y > 0) {
+      y1 = y - 1;
+    } else {
+      y1 = y;
+    }
+    const float* BUTTERAUGLI_RESTRICT row0_in = xyb0.Row(y);
+    const float* BUTTERAUGLI_RESTRICT row1_in = xyb1.Row(y);
+    const float* BUTTERAUGLI_RESTRICT row0_in1 = xyb0.Row(y1);
+    const float* BUTTERAUGLI_RESTRICT row1_in1 = xyb1.Row(y1);
+    const float* BUTTERAUGLI_RESTRICT row0_in2 = xyb0.Row(y2);
+    const float* BUTTERAUGLI_RESTRICT row1_in2 = xyb1.Row(y2);
+    float* BUTTERAUGLI_RESTRICT row_out0 = out0->Row(y);
+    float* BUTTERAUGLI_RESTRICT row_out1 = out1->Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      if (x + 1 < xsize) {
+        x2 = x + 1;
+      } else if (x > 0) {
+        x2 = x - 1;
+      } else {
+        x2 = x;
+      }
+      if (x == 0 && xsize >= 2) {
+        x1 = x + 1;
+      } else if (x > 0) {
+        x1 = x - 1;
+      } else {
+        x1 = x;
+      }
+      double sup0 = (std::fabs(row0_in[x] - row0_in[x2]) +
+                     std::fabs(row0_in[x] - row0_in2[x]) +
+                     std::fabs(row0_in[x] - row0_in[x1]) +
+                     std::fabs(row0_in[x] - row0_in1[x]) +
+                     3 * (std::fabs(row0_in2[x] - row0_in1[x]) +
+                          std::fabs(row0_in[x1] - row0_in[x2])));
+      double sup1 = (std::fabs(row1_in[x] - row1_in[x2]) +
+                     std::fabs(row1_in[x] - row1_in2[x]) +
+                     std::fabs(row1_in[x] - row1_in[x1]) +
+                     std::fabs(row1_in[x] - row1_in1[x]) +
+                     3 * (std::fabs(row1_in2[x] - row1_in1[x]) +
+                          std::fabs(row1_in[x1] - row1_in[x2])));
+      // kBias makes log behave more linearly.
+      static const double kBias = 7;
+      row_out0[x] = mul * (log(sup0 * sup0 * mul2 + kBias) - log(kBias));
+      row_out1[x] = mul * (log(sup1 * sup1 * mul2 + kBias) - log(kBias));
+    }
+  }
+}
+
+void Mask(const Image3F& xyb0, const Image3F& xyb1,
+          MaskImage* BUTTERAUGLI_RESTRICT mask,
+          MaskImage* BUTTERAUGLI_RESTRICT mask_dc,
+          ImageF* BUTTERAUGLI_RESTRICT diff_ac) {
+  PROFILER_FUNC;
+  const size_t xsize = xyb0.xsize();
+  const size_t ysize = xyb0.ysize();
+  *mask = MaskImage(xsize, ysize);
+  *mask_dc = MaskImage(xsize, ysize);
+  static const double muls[2] = {
+    0.580411307192999992,
+    0.236069675367,
+  };
+  double normalizer = {
+    1.0 / (muls[0] + muls[1]),
+  };
+  static const double r0 = 1.63479141169;
+  static const double r1 = 5.5;
+  static const double r2 = 8.0;
+  static const double border_ratio = 0;
+
+  {
+    // X component
+    static const double mul = 0.533043878407;
+    static const double cutoff = 0.5;
+    ImageF diff = DiffPrecomputeX(xyb0.Plane(0), xyb1.Plane(0), mul, cutoff);
+    ImageF blurred(diff.xsize(), diff.ysize());
+    Blur(diff, r2, border_ratio, &blurred);
+    for (size_t y = 0; y < ysize; ++y) {
+      for (size_t x = 0; x < xsize; ++x) {
+        mask->mask_x.Row(y)[x] = blurred.Row(y)[x];
+      }
+    }
+  }
+  {
+    // Y component
+    static const double mul = 0.559;
+    static const double mul2 = 1.0;
+    ImageF diff0(xsize, ysize);
+    ImageF diff1(xsize, ysize);
+    ImageF blurred0_a(xsize, ysize);
+    ImageF blurred0_b(xsize, ysize);
+    ImageF blurred1_a(xsize, ysize);
+    ImageF blurred1_b(xsize, ysize);
+    DiffPrecomputeY(xyb0.Plane(1), xyb1.Plane(1), mul, mul2, &diff0, &diff1);
+    Blur(diff0, r0, border_ratio, &blurred0_a);
+    Blur(diff0, r1, border_ratio, &blurred0_b);
+    Blur(diff1, r0, border_ratio, &blurred1_a);
+    Blur(diff1, r1, border_ratio, &blurred1_b);
+    for (size_t y = 0; y < ysize; ++y) {
+      for (size_t x = 0; x < xsize; ++x) {
+        const double val = normalizer * (
+            muls[0] * blurred1_a.Row(y)[x] +
+            muls[1] * blurred1_b.Row(y)[x]);
+        mask->mask_yb.Row(y)[x] = val;
+        if (diff_ac != nullptr) {
+          static const double kMaskToErrorMul = 0.2;
+          double va = blurred0_a.Row(y)[x] - blurred1_a.Row(y)[x];
+          double wa = kMaskToErrorMul * normalizer * muls[0] * va;
+          double vb = blurred0_b.Row(y)[x] - blurred1_b.Row(y)[x];
+          double wb = kMaskToErrorMul * normalizer * muls[1] * vb;
+          diff_ac->Row(y)[x] += wa * wa + wb * wb;
+        }
+      }
+    }
+  }
+  // B component
+  static const double w00 = 564.23707674490015;
+  static const double w11 = 11.480266246995546;
+  static const double p1_to_p0 = 0.022922908053854666;
+
+  for (size_t y = 0; y < ysize; ++y) {
+    for (size_t x = 0; x < xsize; ++x) {
+      const double s0 = mask->mask_x.Row(y)[x];
+      const double s1 = mask->mask_yb.Row(y)[x];
+      const double p1 = w11 * s1;
+      const double p0 = w00 * s0 + p1_to_p0 * p1;
+
+      mask->mask_x.Row(y)[x] = MaskX(p0);
+      mask->mask_yb.Row(y)[x] = MaskY(p1);
+      mask_dc->mask_x.Row(y)[x] = MaskDcX(p0);
+      mask_dc->mask_yb.Row(y)[x] = MaskDcY(p1);
+    }
+  }
+}
+
+bool ButteraugliDiffmap(const Image3F& rgb0, const Image3F& rgb1,
+                        double hf_asymmetry, ImageF& result_image) {
+  PROFILER_FUNC;
+  const size_t xsize = rgb0.xsize();
+  const size_t ysize = rgb0.ysize();
+  if (xsize < 1 || ysize < 1) {
+    return JXL_FAILURE("Zero-sized image");
+  }
+  if (!SameSize(rgb0, rgb1)) {
+    return JXL_FAILURE("Size mismatch");
+  }
+  static const int kMax = 8;
+  if (xsize < kMax || ysize < kMax) {
+    // Butteraugli values for small (where xsize or ysize is smaller
+    // than 8 pixels) images are non-sensical, but most likely it is
+    // less disruptive to try to compute something than just give up.
+    // Temporarily extend the borders of the image to fit 8 x 8 size.
+    size_t xborder = xsize < kMax ? (kMax - xsize) / 2 : 0;
+    size_t yborder = ysize < kMax ? (kMax - ysize) / 2 : 0;
+    size_t xscaled = std::max<size_t>(kMax, xsize);
+    size_t yscaled = std::max<size_t>(kMax, ysize);
+    Image3F scaled0(xscaled, yscaled);
+    Image3F scaled1(xscaled, yscaled);
+    for (int i = 0; i < 3; ++i) {
+      for (size_t y = 0; y < yscaled; ++y) {
+        for (size_t x = 0; x < xscaled; ++x) {
+          size_t x2 =
+              std::min<size_t>(xsize - 1, std::max<size_t>(0, x - xborder));
+          size_t y2 =
+              std::min<size_t>(ysize - 1, std::max<size_t>(0, y - yborder));
+          scaled0.PlaneRow(i, y)[x] = rgb0.PlaneRow(i, y2)[x2];
+          scaled1.PlaneRow(i, y)[x] = rgb1.PlaneRow(i, y2)[x2];
+        }
+      }
+    }
+    ImageF diffmap_scaled;
+    const bool ok =
+        ButteraugliDiffmap(scaled0, scaled1, hf_asymmetry, diffmap_scaled);
+    result_image = ImageF(xsize, ysize);
+    for (size_t y = 0; y < ysize; ++y) {
+      for (size_t x = 0; x < xsize; ++x) {
+        result_image.Row(y)[x] = diffmap_scaled.Row(y + yborder)[x + xborder];
+      }
+    }
+    return ok;
+  }
+  ButteraugliComparator butteraugli(rgb0, hf_asymmetry);
+  butteraugli.Diffmap(rgb1, result_image);
+  return true;
+}
+
+bool ButteraugliInterface(const Image3F& rgb0, const Image3F& rgb1,
+                          float hf_asymmetry, ImageF& diffmap,
+                          double& diffvalue) {
+  if (!ButteraugliDiffmap(rgb0, rgb1, hf_asymmetry, diffmap)) {
+    return false;
+  }
+  diffvalue = ButteraugliScoreFromDiffmap(diffmap);
+  return true;
+}
+
+double ButteraugliFuzzyClass(double score) {
+  static const double fuzzy_width_up = 4.8;
+  static const double fuzzy_width_down = 4.8;
+  static const double m0 = 2.0;
+  static const double scaler = 0.7777;
+  double val;
+  if (score < 1.0) {
+    // val in [scaler .. 2.0]
+    val = m0 / (1.0 + exp((score - 1.0) * fuzzy_width_down));
+    val -= 1.0;  // from [1 .. 2] to [0 .. 1]
+    val *= 2.0 - scaler;  // from [0 .. 1] to [0 .. 2.0 - scaler]
+    val += scaler;  // from [0 .. 2.0 - scaler] to [scaler .. 2.0]
+  } else {
+    // val in [0 .. scaler]
+    val = m0 / (1.0 + exp((score - 1.0) * fuzzy_width_up));
+    val *= scaler;
+  }
+  return val;
+}
+
+// #define PRINT_OUT_NORMALIZATION
+
+double ButteraugliFuzzyInverse(double seek) {
+  double pos = 0;
+  // NOLINTNEXTLINE(clang-analyzer-security.FloatLoopCounter)
+  for (double range = 1.0; range >= 1e-10; range *= 0.5) {
+    double cur = ButteraugliFuzzyClass(pos);
+    if (cur < seek) {
+      pos -= range;
+    } else {
+      pos += range;
+    }
+  }
+#ifdef PRINT_OUT_NORMALIZATION
+  if (seek == 1.0) {
+    fprintf(stderr, "Fuzzy inverse %g\n", pos);
+  }
+#endif
+  return pos;
+}
+
+#ifdef PRINT_OUT_NORMALIZATION
+static double print_out_normalization = ButteraugliFuzzyInverse(1.0);
+#endif
+
+
+
+namespace {
+
+void ScoreToRgb(double score, double good_threshold, double bad_threshold,
+                uint8_t rgb[3]) {
+  double heatmap[12][3] = {
+      {0, 0, 0},
+      {0, 0, 1},
+      {0, 1, 1},
+      {0, 1, 0},  // Good level
+      {1, 1, 0},
+      {1, 0, 0},  // Bad level
+      {1, 0, 1},
+      {0.5, 0.5, 1.0},
+      {1.0, 0.5, 0.5},  // Pastel colors for the very bad quality range.
+      {1.0, 1.0, 0.5},
+      {1, 1, 1},
+      {1, 1, 1},  // Last color repeated to have a solid range of white.
+  };
+  if (score < good_threshold) {
+    score = (score / good_threshold) * 0.3;
+  } else if (score < bad_threshold) {
+    score = 0.3 +
+            (score - good_threshold) / (bad_threshold - good_threshold) * 0.15;
+  } else {
+    score = 0.45 + (score - bad_threshold) / (bad_threshold * 12) * 0.5;
+  }
+  static const int kTableSize = sizeof(heatmap) / sizeof(heatmap[0]);
+  score = std::min<double>(std::max<double>(score * (kTableSize - 1), 0.0),
+                           kTableSize - 2);
+  int ix = static_cast<int>(score);
+  ix = std::min(std::max(0, ix), kTableSize - 2);  // Handle NaN
+  double mix = score - ix;
+  for (int i = 0; i < 3; ++i) {
+    double v = mix * heatmap[ix + 1][i] + (1 - mix) * heatmap[ix][i];
+    rgb[i] = static_cast<uint8_t>(255 * pow(v, 0.5) + 0.5);
+  }
+}
+
+}  // namespace
+
+Image3B CreateHeatMapImage(const ImageF& distmap, double good_threshold,
+                           double bad_threshold) {
+  Image3B heatmap(distmap.xsize(), distmap.ysize());
+  for (size_t y = 0; y < distmap.ysize(); ++y) {
+    const float* BUTTERAUGLI_RESTRICT row_distmap = distmap.ConstRow(y);
+    uint8_t* BUTTERAUGLI_RESTRICT row_h0 = heatmap.PlaneRow(0, y);
+    uint8_t* BUTTERAUGLI_RESTRICT row_h1 = heatmap.PlaneRow(1, y);
+    uint8_t* BUTTERAUGLI_RESTRICT row_h2 = heatmap.PlaneRow(2, y);
+    for (size_t x = 0; x < distmap.xsize(); ++x) {
+      const float d = row_distmap[x];
+      uint8_t rgb[3];
+      ScoreToRgb(d, good_threshold, bad_threshold, rgb);
+      row_h0[x] = rgb[0];
+      row_h1[x] = rgb[1];
+      row_h2[x] = rgb[2];
+    }
+  }
+  return heatmap;
+}
+
+}  // namespace butteraugli
+}  // namespace jxl
diff --git a/jxl/butteraugli/butteraugli.h b/jxl/butteraugli/butteraugli.h
new file mode 100644
index 000000000..78b86173d
--- /dev/null
+++ b/jxl/butteraugli/butteraugli.h
@@ -0,0 +1,330 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// Author: Jyrki Alakuijala (jyrki.alakuijala@gmail.com)
+
+#ifndef JXL_BUTTERAUGLI_BUTTERAUGLI_H_
+#define JXL_BUTTERAUGLI_BUTTERAUGLI_H_
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <cmath>
+#include <memory>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/image.h"
+#include "jxl/image_ops.h"
+
+#define BUTTERAUGLI_ENABLE_CHECKS 0
+#define BUTTERAUGLI_RESTRICT JXL_RESTRICT
+
+// This is the main interface to butteraugli image similarity
+// analysis function.
+
+namespace jxl {
+namespace butteraugli {
+
+// ButteraugliInterface defines the public interface for butteraugli.
+//
+// It calculates the difference between rgb0 and rgb1.
+//
+// rgb0 and rgb1 contain the images. rgb0[c][px] and rgb1[c][px] contains
+// the red image for c == 0, green for c == 1, blue for c == 2. Location index
+// px is calculated as y * xsize + x.
+//
+// Value of pixels of images rgb0 and rgb1 need to be represented as raw
+// intensity. Most image formats store gamma corrected intensity in pixel
+// values. This gamma correction has to be removed, by applying the following
+// function:
+// butteraugli_val = 255.0 * pow(png_val / 255.0, gamma);
+// A typical value of gamma is 2.2. It is usually stored in the image header.
+// Take care not to confuse that value with its inverse. The gamma value should
+// be always greater than one.
+// Butteraugli does not work as intended if the caller does not perform
+// gamma correction.
+//
+// hf_asymmetry is a multiplier for penalizing new HF artifacts more than
+// blurring away features (1.0 -> neutral).
+//
+// diffmap will contain an image of the size xsize * ysize, containing
+// localized differences for values px (indexed with the px the same as rgb0
+// and rgb1). diffvalue will give a global score of similarity.
+//
+// A diffvalue smaller than kButteraugliGood indicates that images can be
+// observed as the same image.
+// diffvalue larger than kButteraugliBad indicates that a difference between
+// the images can be observed.
+// A diffvalue between kButteraugliGood and kButteraugliBad indicates that
+// a subtle difference can be observed between the images.
+//
+// Returns true on success.
+
+bool ButteraugliInterface(const Image3F &rgb0,
+                          const Image3F &rgb1,
+                          float hf_asymmetry,
+                          ImageF &diffmap,
+                          double &diffvalue);
+
+// Converts the butteraugli score into fuzzy class values that are continuous
+// at the class boundary. The class boundary location is based on human
+// raters, but the slope is arbitrary. Particularly, it does not reflect
+// the expectation value of probabilities of the human raters. It is just
+// expected that a smoother class boundary will allow for higher-level
+// optimization algorithms to work faster.
+//
+// Returns 2.0 for a perfect match, and 1.0 for 'ok', 0.0 for bad. Because the
+// scoring is fuzzy, a butteraugli score of 0.96 would return a class of
+// around 1.9.
+double ButteraugliFuzzyClass(double score);
+
+// Input values should be in range 0 (bad) to 2 (good). Use
+// kButteraugliNormalization as normalization.
+double ButteraugliFuzzyInverse(double seek);
+
+// Implementation details, don't use anything below or your code will
+// break in the future.
+
+#ifdef _MSC_VER
+#define BUTTERAUGLI_INLINE __forceinline
+#else
+#define BUTTERAUGLI_INLINE inline
+#endif
+
+#ifdef __clang__
+// Early versions of Clang did not support __builtin_assume_aligned.
+#define BUTTERAUGLI_HAS_ASSUME_ALIGNED __has_builtin(__builtin_assume_aligned)
+#elif defined(__GNUC__)
+#define BUTTERAUGLI_HAS_ASSUME_ALIGNED 1
+#else
+#define BUTTERAUGLI_HAS_ASSUME_ALIGNED 0
+#endif
+
+// Returns a void* pointer which the compiler then assumes is N-byte aligned.
+// Example: float* JXL_RESTRICT aligned = (float*)JXL_ASSUME_ALIGNED(in, 32);
+//
+// The assignment semantics are required by GCC/Clang. ICC provides an in-place
+// __assume_aligned, whereas MSVC's __assume appears unsuitable.
+#if BUTTERAUGLI_HAS_ASSUME_ALIGNED
+#define BUTTERAUGLI_ASSUME_ALIGNED(ptr, align) __builtin_assume_aligned((ptr), (align))
+#else
+#define BUTTERAUGLI_ASSUME_ALIGNED(ptr, align) (ptr)
+#endif  // BUTTERAUGLI_HAS_ASSUME_ALIGNED
+
+struct MaskImage {
+  MaskImage() {}
+  MaskImage(int xs, int ys) :
+      mask_x(xs, ys),
+      mask_yb(xs, ys) {}
+  ImageF mask_x;
+  ImageF mask_yb;
+};
+
+struct PsychoImage {
+  ImageF uhf[2];  // XY
+  ImageF hf[2];   // XY
+  Image3F mf;     // XYB
+  Image3F lf;     // XYB
+};
+
+class ButteraugliComparator {
+ public:
+  ButteraugliComparator(const Image3F &rgb0, double hf_asymmetry);
+  virtual ~ButteraugliComparator();
+
+  // Computes the butteraugli map between the original image given in the
+  // constructor and the distorted image give here.
+  void Diffmap(const Image3F &rgb1, ImageF &result) const;
+
+  // Same as above, but OpsinDynamicsImage() was already applied.
+  void DiffmapOpsinDynamicsImage(const Image3F &xyb1, ImageF &result) const;
+
+  // Same as above, but the frequency decomposition was already applied.
+  void DiffmapPsychoImage(const PsychoImage& ps1, ImageF &result) const;
+
+void Mask(MaskImage *BUTTERAUGLI_RESTRICT mask, MaskImage *BUTTERAUGLI_RESTRICT mask_dc) const;
+
+ private:
+  void MaltaDiffMapLF(const ImageF &y0, const ImageF &y1, double w_0gt1,
+                      double w_0lt1, double normalization,
+                      Image3F *BUTTERAUGLI_RESTRICT block_diff_ac,
+                      size_t c) const;
+
+  void MaltaDiffMap(const ImageF &y0, const ImageF &y1, double w_0gt1,
+                    double w_0lt1, double normalization,
+                    Image3F *BUTTERAUGLI_RESTRICT block_diff_ac,
+                    size_t c) const;
+
+  ImageF CombineChannels(const MaskImage &mask_ac,
+                         const MaskImage &mask_dc,
+                         const Image3F &block_diff_dc,
+                         const Image3F &block_diff_ac) const;
+
+  const size_t xsize_;
+  const size_t ysize_;
+  float hf_asymmetry_;
+  PsychoImage pi0_;
+  ButteraugliComparator *sub_;
+};
+
+bool ButteraugliDiffmap(const Image3F &rgb0, const Image3F &rgb1,
+                        double hf_asymmetry, ImageF &diffmap);
+
+double ButteraugliScoreFromDiffmap(const ImageF& distmap);
+
+// Generate rgb-representation of the distance between two images.
+Image3B CreateHeatMapImage(const ImageF &distmap, double good_threshold,
+                           double bad_threshold);
+
+// Compute values of local frequency and dc masking based on the activity
+// in the two images.
+void Mask(const Image3F &xyb0, const Image3F &xyb1,
+          MaskImage *BUTTERAUGLI_RESTRICT mask,
+          MaskImage *BUTTERAUGLI_RESTRICT mask_dc, ImageF *diff_ac = nullptr);
+
+template <class V>
+BUTTERAUGLI_INLINE void RgbToXyb(const V &r, const V &g, const V &b,
+                                 V *JXL_RESTRICT valx, V *JXL_RESTRICT valy,
+                                 V *JXL_RESTRICT valb) {
+  *valx = r - g;
+  *valy = r + g;
+  *valb = b;
+}
+
+template <class V>
+BUTTERAUGLI_INLINE void OpsinAbsorbance(const V &in0, const V &in1,
+                                        const V &in2, V *JXL_RESTRICT out0,
+                                        V *JXL_RESTRICT out1,
+                                        V *JXL_RESTRICT out2) {
+  // https://en.wikipedia.org/wiki/Photopsin absorbance modeling.
+  static const double mixi0 = 0.29956550340058319;
+  static const double mixi1 = 0.63373087833825936;
+  static const double mixi2 = 0.077705617820981968;
+  static const double mixi3 = 1.7557483643287353;
+  static const double mixi4 = 0.22158691104574774;
+  static const double mixi5 = 0.69391388044116142;
+  static const double mixi6 = 0.0987313588422;
+  static const double mixi7 = 1.7557483643287353;
+  static const double mixi8 = 0.02;
+  static const double mixi9 = 0.02;
+  static const double mixi10 = 0.20480129041026129;
+  static const double mixi11 = 12.226454707163354;
+
+  const V mix0(mixi0);
+  const V mix1(mixi1);
+  const V mix2(mixi2);
+  const V mix3(mixi3);
+  const V mix4(mixi4);
+  const V mix5(mixi5);
+  const V mix6(mixi6);
+  const V mix7(mixi7);
+  const V mix8(mixi8);
+  const V mix9(mixi9);
+  const V mix10(mixi10);
+  const V mix11(mixi11);
+
+  *out0 = mix0 * in0 + mix1 * in1 + mix2 * in2 + mix3;
+  *out1 = mix4 * in0 + mix5 * in1 + mix6 * in2 + mix7;
+  *out2 = mix8 * in0 + mix9 * in1 + mix10 * in2 + mix11;
+}
+
+Image3F OpsinDynamicsImage(const Image3F &rgb);
+
+ImageF Blur(const ImageF& in, float sigma, float border_ratio);
+
+double SimpleGamma(double v);
+
+double GammaMinArg();
+double GammaMaxArg();
+
+// Polynomial evaluation via Clenshaw's scheme (similar to Horner's).
+// Template enables compile-time unrolling of the recursion, but must reside
+// outside of a class due to the specialization.
+template <int INDEX>
+static inline void ClenshawRecursion(const double x, const double *coefficients,
+                                     double *b1, double *b2) {
+  const double x_b1 = x * (*b1);
+  const double t = (x_b1 + x_b1) - (*b2) + coefficients[INDEX];
+  *b2 = *b1;
+  *b1 = t;
+
+  ClenshawRecursion<INDEX - 1>(x, coefficients, b1, b2);
+}
+
+// Base case
+template <>
+inline void ClenshawRecursion<0>(
+    const double x, const double *coefficients, double *b1,
+    double *b2) {  // NOLINT(readability-non-const-parameter)
+  const double x_b1 = x * (*b1);
+  // The final iteration differs - no 2 * x_b1 here.
+  *b1 = x_b1 - (*b2) + coefficients[0];
+}
+
+// Rational polynomial := dividing two polynomial evaluations. These are easier
+// to find than minimax polynomials.
+struct RationalPolynomial {
+  template <int N>
+  static double EvaluatePolynomial(const double x,
+                                   const double (&coefficients)[N]) {
+    double b1 = 0.0;
+    double b2 = 0.0;
+    ClenshawRecursion<N - 1>(x, coefficients, &b1, &b2);
+    return b1;
+  }
+
+  // Evaluates the polynomial at x (in [min_value, max_value]).
+  inline double operator()(const double x) const {
+    // First normalize to [0, 1].
+    const double x01 = (x - min_value) / (max_value - min_value);
+    // And then to [-1, 1] domain of Chebyshev polynomials.
+    const double xc = 2.0 * x01 - 1.0;
+
+    const double yp = EvaluatePolynomial(xc, p);
+    const double yq = EvaluatePolynomial(xc, q);
+    if (yq == 0.0) return 0.0;
+    return static_cast<float>(yp / yq);
+  }
+
+  // Domain of the polynomials; they are undefined elsewhere.
+  double min_value;
+  double max_value;
+
+  // Coefficients of T_n (Chebyshev polynomials of the first kind).
+  // Degree 5/5 is a compromise between accuracy (0.1%) and numerical stability.
+  double p[5 + 1];
+  double q[5 + 1];
+};
+
+static inline double GammaPolynomial(double value) {
+  static const RationalPolynomial r = {
+    0.971783, 590.188894,
+    {
+      98.7821300963361, 164.273222212631, 92.948112871376,
+      33.8165311212688, 6.91626704983562, 0.556380877028234
+    },
+    {
+      1, 1.64339473427892, 0.89392405219969, 0.298947051776379,
+      0.0507146002577288, 0.00226495093949756
+    }};
+  return r(value);
+}
+
+}  // namespace butteraugli
+}  // namespace jxl
+
+#endif  // JXL_BUTTERAUGLI_BUTTERAUGLI_H_
diff --git a/jxl/byte_order_test.cc b/jxl/byte_order_test.cc
new file mode 100644
index 000000000..84a8805af
--- /dev/null
+++ b/jxl/byte_order_test.cc
@@ -0,0 +1,62 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/byte_order.h"
+
+#include "gtest/gtest.h"
+
+namespace jxl {
+namespace {
+
+TEST(ByteOrderTest, TestRoundTripBE16) {
+  const uint32_t in = 0x1234;
+  uint8_t buf[2];
+  StoreBE16(in, buf);
+  EXPECT_EQ(in, LoadBE16(buf));
+  EXPECT_NE(in, LoadLE16(buf));
+}
+
+TEST(ByteOrderTest, TestRoundTripLE16) {
+  const uint32_t in = 0x1234;
+  uint8_t buf[2];
+  StoreLE16(in, buf);
+  EXPECT_EQ(in, LoadLE16(buf));
+  EXPECT_NE(in, LoadBE16(buf));
+}
+
+TEST(ByteOrderTest, TestRoundTripBE32) {
+  const uint32_t in = 0xFEDCBA98u;
+  uint8_t buf[4];
+  StoreBE32(in, buf);
+  EXPECT_EQ(in, LoadBE32(buf));
+  EXPECT_NE(in, LoadLE32(buf));
+}
+
+TEST(ByteOrderTest, TestRoundTripLE32) {
+  const uint32_t in = 0xFEDCBA98u;
+  uint8_t buf[4];
+  StoreLE32(in, buf);
+  EXPECT_EQ(in, LoadLE32(buf));
+  EXPECT_NE(in, LoadBE32(buf));
+}
+
+TEST(ByteOrderTest, TestRoundTripLE64) {
+  const uint64_t in = 0xFEDCBA9876543210ull;
+  uint8_t buf[8];
+  StoreLE64(in, buf);
+  EXPECT_EQ(in, LoadLE64(buf));
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/chroma_from_luma.cc b/jxl/chroma_from_luma.cc
new file mode 100644
index 000000000..04217e2fd
--- /dev/null
+++ b/jxl/chroma_from_luma.cc
@@ -0,0 +1,350 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/chroma_from_luma.h"
+
+#include <stdlib.h>
+
+#include <algorithm>
+#include <array>
+#include <hwy/static_targets.h>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/dct_util.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/image_ops.h"
+#include "jxl/modular/encoding/encoding.h"
+#include "jxl/predictor.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+namespace {
+
+template <int MAIN_CHANNEL, int SIDE_CHANNEL, int SCALE, int OFFSET>
+void FindBestCorrelation(const Image3F& dct, ImageB* JXL_RESTRICT map,
+                         int* JXL_RESTRICT dc, float base,
+                         const DequantMatrices& dequant, ThreadPool* pool) {
+  constexpr float kScale = SCALE;
+  constexpr float kZeroThresh =
+      kScale * kZeroBiasDefault[SIDE_CHANNEL] *
+      0.9999f;  // just epsilon less for better rounding
+  const float* const JXL_RESTRICT qm =
+      dequant.InvMatrix(AcStrategy::Type::DCT, SIDE_CHANNEL);
+  std::vector<int32_t> d_num_zeros_thread;
+  auto process_row_init = [&d_num_zeros_thread](size_t num_threads) {
+    d_num_zeros_thread.resize(256 * num_threads);
+    return true;
+  };
+  auto process_row = [&](int ty, int thread) {
+    uint8_t* JXL_RESTRICT row_out = map->Row(ty);
+    for (size_t tx = 0; tx < map->xsize(); ++tx) {
+      const size_t y0 = ty * kColorTileDimInBlocks;
+      const size_t x0 = tx * kColorTileDimInBlocks * kDCTBlockSize;
+      const size_t y1 =
+          std::min<size_t>(y0 + kColorTileDimInBlocks, dct.ysize());
+      const size_t x1 = std::min<size_t>(
+          x0 + kColorTileDimInBlocks * kDCTBlockSize, dct.xsize());
+      int32_t d_num_zeros[257] = {0};
+      for (size_t y = y0; y < y1; ++y) {
+        const float* JXL_RESTRICT row_m = dct.ConstPlaneRow(MAIN_CHANNEL, y);
+        const float* JXL_RESTRICT row_s = dct.ConstPlaneRow(SIDE_CHANNEL, y);
+        for (size_t x = x0; x < x1; ++x) {
+          const float scaled_m = row_m[x] * qm[x % kDCTBlockSize];
+          const float scaled_s = kScale * row_s[x] * qm[x % kDCTBlockSize] +
+                                 (OFFSET - base * kScale) * scaled_m;
+          // Increment num_zeros[idx] if
+          //   std::abs(scaled_s - (idx - OFFSET) *
+          //   scaled_m) < kZeroThresh
+          if (std::abs(scaled_m) > 1e-8f) {
+            float from;
+            float to;
+            if (scaled_m > 0) {
+              from = (scaled_s - kZeroThresh) / scaled_m;
+              to = (scaled_s + kZeroThresh) / scaled_m;
+            } else {
+              from = (scaled_s + kZeroThresh) / scaled_m;
+              to = (scaled_s - kZeroThresh) / scaled_m;
+            }
+            if (from < 0.0f) {
+              from = 0.0f;
+            }
+            if (to > 255.0f) {
+              to = 255.0f;
+            }
+            // Instead of clamping the both values
+            // we just check that range is sane.
+            if (from <= to) {
+              d_num_zeros_thread[256 * thread +
+                                 static_cast<int>(std::ceil(from))]++;
+              if (to < 255.f)
+                d_num_zeros_thread[256 * thread +
+                                   static_cast<int>(std::floor(to + 1))]--;
+              if (x % kDCTBlockSize == 0) continue;
+              d_num_zeros[static_cast<int>(std::ceil(from))]++;
+              d_num_zeros[static_cast<int>(std::floor(to + 1))]--;
+            }
+          }
+        }
+      }
+      int best = 0;
+      int32_t best_sum = 0;
+      FindIndexOfSumMaximum(d_num_zeros, 256, &best, &best_sum);
+      row_out[tx] = best;
+    }
+  };
+
+  RunOnPool(pool, 0, map->ysize(), process_row_init, process_row,
+            "FindCorrelation");
+
+  size_t num_threads = d_num_zeros_thread.size() / 256;
+  int32_t d_num_zeros_global[256] = {0};
+  for (size_t t = 0; t < num_threads; t++) {
+    for (size_t i = 0; i < 256; i++) {
+      d_num_zeros_global[i] += d_num_zeros_thread[t * 256 + i];
+    }
+  }
+  int global_best = 0;
+  int32_t global_sum = 0;
+  FindIndexOfSumMaximum(d_num_zeros_global, 256, &global_best, &global_sum);
+  *dc = global_best;
+}
+
+}  // namespace
+
+ColorCorrelationMap::ColorCorrelationMap(size_t xsize, size_t ysize, bool XYB)
+    : ytox_map(DivCeil(xsize, kColorTileDim), DivCeil(ysize, kColorTileDim)),
+      ytob_map(DivCeil(xsize, kColorTileDim), DivCeil(ysize, kColorTileDim)) {
+  FillImage(kColorOffset, &ytox_map);
+  FillImage(kColorOffset, &ytob_map);
+  if (!XYB) {
+    base_correlation_b_ = 0;
+  }
+  RecomputeDCFactors();
+}
+
+void FindBestColorCorrelationMap(const Image3F& opsin,
+                                 const DequantMatrices& dequant,
+                                 ThreadPool* pool, ColorCorrelationMap* cmap) {
+  PROFILER_ZONE("enc YTo* correlation");
+
+  const size_t xsize_blocks = opsin.xsize() / kBlockDim;
+  const size_t ysize_blocks = opsin.ysize() / kBlockDim;
+  Image3F dct(xsize_blocks * kDCTBlockSize, ysize_blocks);
+  TransposedScaledDCT(opsin, &dct);
+
+  int32_t ytob_dc = kColorOffset;
+  int32_t ytox_dc = kColorOffset;
+
+  FindBestCorrelation</* from Y */ 1, /* to B */ 2, kDefaultColorFactor,
+                      kColorOffset>(dct, &cmap->ytob_map, &ytob_dc,
+                                    cmap->YtoBRatio(kColorOffset), dequant,
+                                    pool);
+  FindBestCorrelation</* from Y */ 1, /* to X */ 0, kDefaultColorFactor,
+                      kColorOffset>(dct, &cmap->ytox_map, &ytox_dc,
+                                    cmap->YtoXRatio(kColorOffset), dequant,
+                                    pool);
+  cmap->SetYToBDC(ytob_dc);
+  cmap->SetYToXDC(ytox_dc);
+}
+
+class ColorCorrelationMapCoder {
+ public:
+  // TODO(veluca): change predictors?
+  template <typename T>
+  using Predictor = predictor::ComputeResiduals<
+      T, predictor::PackSignedRange<0, 255>,
+      predictor::Predictors2<predictor::YPredictor, predictor::YPredictor>>;
+
+  enum {
+    kNumResidualContexts = 8,
+    kContextsPerChannel = kNumResidualContexts + 4,
+    kNumContexts = 2 * kContextsPerChannel,
+  };
+
+  static int Context(size_t c, size_t correct, size_t badness) {
+    if (correct == 0) {
+      JXL_ASSERT(badness != 0);
+      badness = (badness + 1) >> 1;
+      size_t badness_offset =
+          std::min<size_t>(badness, kNumResidualContexts) - 1;
+      return kContextsPerChannel * c + badness_offset;
+    }
+    return kContextsPerChannel * c + kNumResidualContexts +
+           CeilLog2Nonzero(9 - correct);
+  }
+  static_assert(kNumContexts == kCmapContexts,
+                "Invalid number of cmap contexts");
+
+  struct Decoder : public Predictor<Decoder> {
+    BitReader* JXL_RESTRICT br;
+    ANSSymbolReader* decoder;
+    const std::vector<uint8_t>* context_map;
+    uint8_t* JXL_RESTRICT rows[2];
+    size_t stride;
+    size_t base_context;
+    AuxOut* JXL_RESTRICT aux_out;
+
+    Decoder(BitReader* JXL_RESTRICT br, ANSSymbolReader* decoder,
+            const std::vector<uint8_t>* context_map,
+            uint8_t* JXL_RESTRICT rows[2], size_t stride, size_t base_context,
+            AuxOut* JXL_RESTRICT aux_out)
+        : br(br),
+          decoder(decoder),
+          context_map(context_map),
+          rows{rows[0], rows[1]},
+          stride(stride),
+          base_context(base_context),
+          aux_out(aux_out) {}
+
+    void Decode(size_t xsize, size_t ysize) {
+      Predictor<Decoder>::Run(xsize, ysize, aux_out);
+    }
+
+    HWY_ATTR JXL_INLINE void Prediction(
+        size_t x, size_t y, const int32_t* JXL_RESTRICT predictions,
+        const uint32_t* JXL_RESTRICT num_correct,
+        const uint32_t* JXL_RESTRICT min_error, int32_t* JXL_RESTRICT decoded) {
+      for (size_t c = 0; c < 2; c++) {
+        size_t ctx = base_context + Context(c, num_correct[c], min_error[c]);
+
+        uint32_t residual = ReadHybridUint(ctx, br, decoder, *context_map);
+
+        if (x == 0 && y == 0) {
+          decoded[c] = kColorOffset + UnpackSigned(residual);
+        } else {
+          decoded[c] = predictions[c] + UnpackSigned(residual);
+        }
+
+        rows[c][y * stride + x] = decoded[c];
+      }
+    }
+  };
+
+  struct Encoder : public Predictor<Encoder> {
+    std::vector<Token>* JXL_RESTRICT tokens;
+    const uint8_t* JXL_RESTRICT rows[2];
+    size_t stride;
+    size_t base_context;
+    AuxOut* JXL_RESTRICT aux_out;
+
+    Encoder(std::vector<Token>* JXL_RESTRICT tokens,
+            const uint8_t* JXL_RESTRICT rows[2], size_t stride,
+            size_t base_context, AuxOut* JXL_RESTRICT aux_out)
+        : tokens(tokens),
+          rows{rows[0], rows[1]},
+          stride(stride),
+          base_context(base_context),
+          aux_out(aux_out) {}
+
+    void Encode(size_t xsize, size_t ysize) {
+      Predictor<Encoder>::Run(xsize, ysize, aux_out);
+    }
+
+    HWY_ATTR JXL_INLINE void Prediction(
+        size_t x, size_t y, const int32_t* JXL_RESTRICT predictions,
+        const uint32_t* JXL_RESTRICT num_correct,
+        const uint32_t* JXL_RESTRICT min_error, int32_t* JXL_RESTRICT decoded) {
+      for (size_t c = 0; c < 2; c++) {
+        size_t ctx = base_context + Context(c, num_correct[c], min_error[c]);
+
+        decoded[c] = rows[c][y * stride + x];
+
+        uint32_t residual;
+        if (x == 0 && y == 0) {
+          residual = PackSigned(decoded[c] - kColorOffset);
+        } else {
+          residual = PackSigned(decoded[c] - predictions[c]);
+        }
+
+        TokenizeHybridUint(ctx, residual, tokens);
+      }
+    }
+  };
+};
+
+Status DecodeColorMap(BitReader* JXL_RESTRICT br, ANSSymbolReader* decoder,
+                      const std::vector<uint8_t>& context_map,
+                      ColorCorrelationMap* cmap, const Rect& rect,
+                      size_t base_context, AuxOut* JXL_RESTRICT aux_out) {
+  uint8_t* JXL_RESTRICT rows[2] = {
+      rect.Row(&cmap->ytox_map, 0),
+      rect.Row(&cmap->ytob_map, 0),
+  };
+  const size_t stride = cmap->ytob_map.PixelsPerRow();
+
+  ColorCorrelationMapCoder::Decoder(br, decoder, &context_map, rows, stride,
+                                    base_context, aux_out)
+      .Decode(rect.xsize(), rect.ysize());
+  return true;
+}
+
+bool DecodeFullColorMap(BitReader* JXL_RESTRICT br, ColorCorrelationMap* cmap,
+                        bool use_new_cmap) {
+  if (use_new_cmap) {
+    if (!br->JumpToByteBoundary()) return false;
+    const Span<const uint8_t> span = br->GetSpan();
+    Rect rect0(cmap->ytob_map);
+    size_t pos = 0;
+    if (!modular_rect_decompress_2(span, &pos, &cmap->ytox_map, &cmap->ytob_map,
+                                   rect0, kColorOffset, kColorOffset))
+      return JXL_FAILURE("Failed to decode color map");
+    br->SkipBits(8 * pos);
+  } else {
+    // no longer exists
+  }
+  return true;
+}
+
+void EncodeColorMap(const ColorCorrelationMap& cmap, const Rect& rect,
+                    std::vector<Token>* tokens, size_t base_context,
+                    AuxOut* JXL_RESTRICT aux_out) {
+  const uint8_t* JXL_RESTRICT rows[2] = {
+      rect.ConstRow(cmap.ytox_map, 0),
+      rect.ConstRow(cmap.ytob_map, 0),
+  };
+  const size_t stride = cmap.ytob_map.PixelsPerRow();
+  ColorCorrelationMapCoder::Encoder(tokens, rows, stride, base_context, aux_out)
+      .Encode(rect.xsize(), rect.ysize());
+}
+
+void EncodeFullColorMap(const ColorCorrelationMap& cmap, const Rect& rect,
+                        BitWriter* writer, size_t layer, AuxOut* aux_out,
+                        bool use_new_cmap) {
+  if (use_new_cmap) {
+    PaddedBytes enc_co;
+    modular_options cfopts;
+    set_default_modular_options(cfopts);
+    cfopts.nb_repeats = 1;
+    modular_rect_compress_2(cmap.ytox_map, cmap.ytob_map, rect, &enc_co,
+                            &cfopts, kColorOffset, kColorOffset);
+
+    if (aux_out != nullptr)
+      aux_out->layers[layer].total_bits += enc_co.size() * 8;
+
+    writer->ZeroPadToByte();
+    *writer += enc_co;
+
+  } else {
+    // no longer exists
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/chroma_from_luma.h b/jxl/chroma_from_luma.h
new file mode 100644
index 000000000..50f4f8b0c
--- /dev/null
+++ b/jxl/chroma_from_luma.h
@@ -0,0 +1,189 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_CHROMA_FROM_LUMA_H_
+#define JXL_CHROMA_FROM_LUMA_H_
+
+// Chroma-from-luma, computed using heuristics to determine the best linear
+// model for the X and B channels from the Y channel.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/static_targets.h>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/field_encodings.h"
+#include "jxl/fields.h"
+#include "jxl/image.h"
+#include "jxl/opsin_params.h"
+#include "jxl/quant_weights.h"
+
+namespace jxl {
+
+// Tile is the rectangular grid of blocks that share color correlation
+// parameters ("factor_x/b" such that residual_b = blue - Y * factor_b).
+static constexpr size_t kColorTileDim = 64;
+
+static_assert(kColorTileDim % kBlockDim == 0,
+              "Color tile dim should be divisible by block dim");
+static constexpr size_t kColorTileDimInBlocks = kColorTileDim / kBlockDim;
+
+static_assert(kGroupDimInBlocks % kColorTileDimInBlocks == 0,
+              "Group dim should be divisible by color tile dim");
+
+static constexpr const uint8_t kColorOffset = 127;
+static constexpr const uint8_t kDefaultColorFactor = 84;
+
+static constexpr U32Enc kColorFactorDist(Val(kDefaultColorFactor), Val(256),
+                                         BitsOffset(2, 8), BitsOffset(258, 12));
+
+struct ColorCorrelationMap {
+  ColorCorrelationMap() = default;
+  // xsize/ysize are in pixels
+  // set XYB=false to do something close to no-op cmap (needed for now since
+  // cmap is mandatory)
+  ColorCorrelationMap(size_t xsize, size_t ysize, bool XYB = true);
+
+  float YtoXRatio(int32_t x_factor) const {
+    return base_correlation_x_ + (x_factor - kColorOffset) * color_scale_;
+  }
+
+  float YtoBRatio(int32_t b_factor) const {
+    return base_correlation_b_ + (b_factor - kColorOffset) * color_scale_;
+  }
+
+  void EncodeDC(BitWriter* writer, size_t layer, AuxOut* aux_out) const {
+    BitWriter::Allotment allotment(writer, 1 + 2 * kBitsPerByte + 12 + 32);
+    if (ytox_dc_ == kColorOffset && ytob_dc_ == kColorOffset &&
+        color_factor_ == kDefaultColorFactor && base_correlation_x_ == 0.0f &&
+        base_correlation_b_ == kYToBRatio) {
+      writer->Write(1, 1);
+      ReclaimAndCharge(writer, &allotment, layer, aux_out);
+      return;
+    }
+    writer->Write(1, 0);
+    JXL_CHECK(U32Coder::Write(kColorFactorDist, color_factor_, writer));
+    JXL_CHECK(F16Coder::Write(base_correlation_x_, writer));
+    JXL_CHECK(F16Coder::Write(base_correlation_b_, writer));
+    writer->Write(kBitsPerByte, ytox_dc_);
+    writer->Write(kBitsPerByte, ytob_dc_);
+    ReclaimAndCharge(writer, &allotment, layer, aux_out);
+  }
+
+  HWY_ATTR Status DecodeDC(BitReader* br) {
+    if (br->ReadFixedBits<1>() == 1) {
+      // All default.
+      return true;
+    }
+    SetColorFactor(U32Coder::Read(kColorFactorDist, br));
+    JXL_RETURN_IF_ERROR(F16Coder::Read(br, &base_correlation_x_));
+    JXL_RETURN_IF_ERROR(F16Coder::Read(br, &base_correlation_b_));
+    ytox_dc_ = br->ReadFixedBits<kBitsPerByte>();
+    ytob_dc_ = br->ReadFixedBits<kBitsPerByte>();
+    RecomputeDCFactors();
+    return true;
+  }
+
+  void SetColorFactor(uint32_t factor) {
+    color_factor_ = factor;
+    color_scale_ = 1.0f / color_factor_;
+    RecomputeDCFactors();
+  }
+
+  void SetYToBDC(int32_t ytob_dc) {
+    ytob_dc_ = ytob_dc;
+    RecomputeDCFactors();
+  }
+  void SetYToXDC(int32_t ytox_dc) {
+    ytox_dc_ = ytox_dc;
+    RecomputeDCFactors();
+  }
+
+  const float* DCFactors() const { return dc_factors_; }
+
+  void RecomputeDCFactors() {
+    dc_factors_[0] = YtoXRatio(ytox_dc_);
+    dc_factors_[2] = YtoBRatio(ytob_dc_);
+  }
+
+  ImageB ytox_map;
+  ImageB ytob_map;
+
+ private:
+  // range of factor: -1.51 to +1.52
+  uint32_t color_factor_ = kDefaultColorFactor;
+  float color_scale_ = 1.0f / color_factor_;
+  float base_correlation_x_ = 0.0f;
+  float base_correlation_b_ = kYToBRatio;
+  int32_t ytox_dc_ = kColorOffset;
+  int32_t ytob_dc_ = kColorOffset;
+  HWY_ALIGN float dc_factors_[4] = {};
+};
+
+void FindBestColorCorrelationMap(const Image3F& opsin,
+                                 const DequantMatrices& dequant,
+                                 ThreadPool* pool, ColorCorrelationMap* cmap);
+
+void EncodeColorMap(const ColorCorrelationMap& cmap, const Rect& rect,
+                    std::vector<Token>* tokens, size_t base_context,
+                    AuxOut* JXL_RESTRICT aux_out);
+
+Status DecodeColorMap(BitReader* JXL_RESTRICT br, ANSSymbolReader* decoder,
+                      const std::vector<uint8_t>& context_map,
+                      ColorCorrelationMap* cmap, const Rect& rect,
+                      size_t base_context, AuxOut* JXL_RESTRICT aux_out);
+
+// Declared here to avoid including predictor.h.
+static constexpr size_t kCmapContexts = 24;
+
+void EncodeFullColorMap(const ColorCorrelationMap& cmap, const Rect& rect,
+                        BitWriter* writer, size_t layer, AuxOut* aux_out,
+                        bool use_new_cmap);
+
+bool DecodeFullColorMap(BitReader* JXL_RESTRICT br, ColorCorrelationMap* cmap,
+                        bool use_new_cmap);
+
+template <typename V, typename R>
+inline void FindIndexOfSumMaximum(const V* array, const size_t len, R* idx,
+                                  V* sum) {
+  JXL_ASSERT(len > 0);
+  V maxval = 0;
+  V val = 0;
+  R maxidx = 0;
+  for (size_t i = 0; i < len; ++i) {
+    val += array[i];
+    if (val > maxval) {
+      maxval = val;
+      maxidx = i;
+    }
+  }
+  *idx = maxidx;
+  *sum = maxval;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_CHROMA_FROM_LUMA_H_
diff --git a/jxl/codec_in_out.h b/jxl/codec_in_out.h
new file mode 100644
index 000000000..87433f16f
--- /dev/null
+++ b/jxl/codec_in_out.h
@@ -0,0 +1,212 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_CODEC_IN_OUT_H_
+#define JXL_CODEC_IN_OUT_H_
+
+// Holds inputs/outputs for decoding/encoding images.
+
+#include <stddef.h>
+
+#include <utility>
+#include <vector>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/common.h"
+#include "jxl/frame_header.h"
+#include "jxl/headers.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+
+// Per-channel interval, used to convert between (full-range) external and
+// (bounded or unbounded) temp values. See external_image.cc for the definitions
+// of temp/external.
+struct CodecInterval {
+  CodecInterval() = default;
+  constexpr CodecInterval(float min, float max) : min(min), width(max - min) {}
+  // Defaults for temp.
+  float min = 0.0f;
+  float width = 1.0f;
+};
+
+using CodecIntervals = std::array<CodecInterval, 4>;  // RGB[A] or Y[A]
+
+// Allows passing arbitrary metadata to decoders (required for PNM).
+class DecoderHints {
+ public:
+  // key=color_space, value=Description(c/pp): specify the ColorEncoding of
+  //   the pixels for decoding. Otherwise, if the codec did not obtain an ICC
+  //   profile from the image, assume sRGB.
+  //
+  // Strings are taken from the command line, so avoid spaces for convenience.
+  void Add(const std::string& key, const std::string& value) {
+    kv_.emplace_back(key, value);
+  }
+
+  // Calls func(key, value) in order of Add.
+  template <class Func>
+  void Foreach(const Func& func) const {
+    for (const KeyValue& kv : kv_) {
+      func(kv.key, kv.value);
+    }
+  }
+
+ private:
+  // Splitting into key/value avoids parsing in each codec.
+  struct KeyValue {
+    KeyValue(std::string key, std::string value)
+        : key(std::move(key)), value(std::move(value)) {}
+
+    std::string key;
+    std::string value;
+  };
+
+  std::vector<KeyValue> kv_;
+};
+
+// Optional text/EXIF metadata.
+struct Blobs {
+  PaddedBytes exif;
+  PaddedBytes iptc;
+  PaddedBytes jumbf;
+  PaddedBytes xmp;
+};
+
+// Holds a preview, a main image or one or more frames, plus the inputs/outputs
+// to/from decoding/encoding.
+class CodecInOut {
+ public:
+  CodecInOut() : preview_frame(&metadata) {
+    frames.reserve(1);
+    frames.emplace_back(&metadata);
+  }
+
+  // Move-only.
+  CodecInOut(CodecInOut&&) = default;
+  CodecInOut& operator=(CodecInOut&&) = default;
+
+  ImageBundle& Main() {
+    JXL_DASSERT(frames.size() == 1);
+    return frames[0];
+  }
+  const ImageBundle& Main() const {
+    JXL_DASSERT(frames.size() == 1);
+    return frames[0];
+  }
+
+  // If c_current.IsGray(), all planes must be identical.
+  void SetFromImage(Image3F&& color, const ColorEncoding& c_current) {
+    Main().SetFromImage(std::move(color), c_current);
+  }
+
+  void CheckMetadata() const {
+    JXL_CHECK(metadata.bits_per_sample != 0);
+    JXL_CHECK(!metadata.color_encoding.icc.empty());
+
+    if (preview_frame.xsize() != 0) preview_frame.VerifyMetadata();
+    JXL_CHECK(preview_frame.metadata() == &metadata);
+
+    for (const ImageBundle& ib : frames) {
+      ib.VerifyMetadata();
+      JXL_CHECK(ib.metadata() == &metadata);
+    }
+  }
+
+  size_t xsize() const { return Main().xsize(); }
+  size_t ysize() const { return Main().ysize(); }
+  void ShrinkTo(size_t xsize, size_t ysize) {
+    // preview is unaffected.
+    for (ImageBundle& ib : frames) {
+      ib.ShrinkTo(xsize, ysize);
+    }
+  }
+
+  template <typename T>
+  Status VerifyDimensions(T xs, T ys) const {
+    if (xs == 0 || ys == 0) return JXL_FAILURE("Empty image.");
+    if (xs > dec_max_xsize) return JXL_FAILURE("Image too wide.");
+    if (ys > dec_max_ysize) return JXL_FAILURE("Image too tall.");
+
+    const uint64_t num_pixels = uint64_t(xs) * ys;
+    if (num_pixels > dec_max_pixels) return JXL_FAILURE("Image too big.");
+
+    return true;
+  }
+
+  // Calls TransformTo for each ImageBundle (preview/frames).
+  Status TransformTo(const ColorEncoding& c_desired,
+                     ThreadPool* pool = nullptr) {
+    if (metadata.m2.have_preview) {
+      JXL_RETURN_IF_ERROR(preview_frame.TransformTo(c_desired, pool));
+    }
+    for (ImageBundle& ib : frames) {
+      JXL_RETURN_IF_ERROR(ib.TransformTo(c_desired, pool));
+    }
+    return true;
+  }
+
+  // -- ENCODER OUTPUT:
+
+  // Size [bytes] of encoded bitstream after encoding / before decoding.
+  mutable size_t enc_size;
+
+  // Encoder-specific function of its bits_per_sample argument. Used to compute
+  // error tolerance in round trips.
+  mutable size_t enc_bits_per_sample;
+
+  // -- DECODER INPUT:
+
+  // Upper limit on pixel dimensions/area, enforced by DimensionsWithinLimits
+  // (called from decoders). Fuzzers set smaller values to limit memory use.
+  uint32_t dec_max_xsize = 0xFFFFFFFFu;
+  uint32_t dec_max_ysize = 0xFFFFFFFFu;
+  uint64_t dec_max_pixels = ~0ull;
+
+  // Used to set c_current for codecs that lack color space metadata.
+  DecoderHints dec_hints;
+
+  // -- DECODER OUTPUT
+
+  // Total number of pixels decoded (may differ from #frames * xsize * ysize
+  // if frames are cropped)
+  uint64_t dec_pixels = 0;
+
+  // -- DECODER OUTPUT, ENCODER INPUT:
+
+  // Metadata stored into / retrieved from bitstreams.
+
+  Blobs blobs;
+  ImageMetadata metadata;  // applies to preview and all frames
+
+  // If metadata.have_preview:
+  PreviewHeader preview;
+  ImageBundle preview_frame;
+
+  // If metadata.have_animation:
+  AnimationHeader animation;
+  std::vector<AnimationFrame> animation_frames;
+
+  std::vector<ImageBundle> frames;  // size=1 if !metadata.have_animation
+
+  bool use_sjpeg = false;
+  // If the image should be written to a JPEG, use this quality for encoding.
+  // If the image was read from a JPEG, this represents the
+  size_t jpeg_quality;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_CODEC_IN_OUT_H_
diff --git a/jxl/coeff_order.cc b/jxl/coeff_order.cc
new file mode 100644
index 000000000..4ee86359d
--- /dev/null
+++ b/jxl/coeff_order.cc
@@ -0,0 +1,347 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/coeff_order.h"
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <vector>
+
+#include "jxl/ans_params.h"
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/span.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/lehmer_code.h"
+#include "jxl/modular/encoding/encoding.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+namespace {
+void SetDefaultOrder(AcStrategy acs, coeff_order_t* JXL_RESTRICT order) {
+  PROFILER_FUNC;
+  const size_t size =
+      kDCTBlockSize * acs.covered_blocks_x() * acs.covered_blocks_y();
+  const coeff_order_t* natural_coeff_order = acs.NaturalCoeffOrder();
+  for (size_t k = 0; k < size; ++k) {
+    order[k] = natural_coeff_order[k];
+  }
+}
+}  // namespace
+
+uint32_t ComputeUsedOrders(const SpeedTier speed,
+                           const AcStrategyImage& ac_strategy,
+                           const Rect& rect) {
+  // Use default orders for small images.
+  if (ac_strategy.xsize() < 5 && ac_strategy.ysize() < 5) return 0;
+
+  // Only uses DCT8 = 0, so bitfield = 1.
+  if (speed == SpeedTier::kFalcon) return 1;
+
+  uint32_t ret = 0;
+  size_t xsize_blocks = rect.xsize();
+  size_t ysize_blocks = rect.ysize();
+  // TODO(janwas): parallelize
+  for (size_t by = 0; by < ysize_blocks; ++by) {
+    AcStrategyRow acs_row = ac_strategy.ConstRow(rect, by);
+    for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+      int ord = kStrategyOrder[acs_row[bx].RawStrategy()];
+      ret |= 1u << ord;
+    }
+  }
+  return ret;
+}
+
+void ComputeCoeffOrder(const ACImage3& acs, const AcStrategyImage& ac_strategy,
+                       const FrameDimensions& frame_dim, uint32_t used_orders,
+                       coeff_order_t* JXL_RESTRICT order) {
+  int32_t num_zeros[3 * AcStrategy::kNumValidStrategies *
+                    AcStrategy::kMaxCoeffArea] = {};
+  // No need to compute number of zero coefficients if all orders are the
+  // default.
+  if (used_orders != 0) {
+    // Count number of zero coefficients, separately for each DCT band.
+    for (size_t group_index = 0; group_index < frame_dim.num_groups;
+         group_index++) {
+      const size_t gx = group_index % frame_dim.xsize_groups;
+      const size_t gy = group_index / frame_dim.xsize_groups;
+      const Rect rect(gx * kGroupDimInBlocks, gy * kGroupDimInBlocks,
+                      kGroupDimInBlocks, kGroupDimInBlocks,
+                      frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+      const ac_qcoeff_t* JXL_RESTRICT rows[3] = {
+          acs.ConstPlaneRow(0, group_index),
+          acs.ConstPlaneRow(1, group_index),
+          acs.ConstPlaneRow(2, group_index),
+      };
+      size_t ac_offset = 0;
+
+      for (size_t by = 0; by < rect.ysize(); ++by) {
+        AcStrategyRow acs_row = ac_strategy.ConstRow(rect, by);
+        for (size_t bx = 0; bx < rect.xsize(); ++bx) {
+          AcStrategy acs = acs_row[bx];
+          if (!acs.IsFirstBlock()) continue;
+          size_t size = kDCTBlockSize << acs.log2_covered_blocks();
+          for (size_t c = 0; c < 3; ++c) {
+            const size_t order_offset =
+                (kStrategyOrder[acs.RawStrategy()] * 3 + c) *
+                AcStrategy::kMaxCoeffArea;
+            for (size_t k = 0; k < size; k++) {
+              if (rows[c][ac_offset + k] == 0) {
+                num_zeros[order_offset + k]++;
+              }
+            }
+            // Ensure LLFs are first in the order.
+            size_t cx = acs.covered_blocks_x();
+            size_t cy = acs.covered_blocks_y();
+            CoefficientLayout(&cy, &cx);
+            for (size_t iy = 0; iy < cy; iy++) {
+              for (size_t ix = 0; ix < cx; ix++) {
+                num_zeros[order_offset + iy * kBlockDim * cx + ix] = -1;
+              }
+            }
+          }
+          ac_offset += size;
+        }
+      }
+    }
+  }
+
+  uint16_t computed = 0;
+  for (uint8_t o = 0; o < AcStrategy::kNumValidStrategies; ++o) {
+    uint8_t ord = kStrategyOrder[o];
+    if (computed & (1 << ord)) continue;
+    computed |= 1 << ord;
+    // Ensure natural coefficient order is not permuted if the order is
+    // not transmitted.
+    if ((1 << ord) & ~used_orders) {
+      for (size_t c = 0; c < 3; c++) {
+        SetDefaultOrder(AcStrategy::FromRawStrategy(o),
+                        &order[(3 * ord + c) * AcStrategy::kMaxCoeffArea]);
+      }
+      continue;
+    }
+    AcStrategy acs = AcStrategy::FromRawStrategy(o);
+    size_t sz = kDCTBlockSize * acs.covered_blocks_x() * acs.covered_blocks_y();
+    const coeff_order_t* natural_coeff_order = acs.NaturalCoeffOrder();
+
+    for (uint8_t c = 0; c < 3; c++) {
+      struct PosAndCount {
+        uint32_t pos;
+        uint32_t count;
+      };
+
+      // Apply zig-zag order.
+      PosAndCount pos_and_val[AcStrategy::kMaxCoeffArea];
+      size_t offset = (ord * 3 + c) * AcStrategy::kMaxCoeffArea;
+      for (size_t i = 0; i < sz; ++i) {
+        size_t pos = natural_coeff_order[i];
+        pos_and_val[i].pos = pos;
+        // We don't care for the exact number -> quantize number of zeros,
+        // to get less permuted order.
+        pos_and_val[i].count = num_zeros[offset + pos] / std::sqrt(sz) + 0.1;
+      }
+
+      // Stable-sort -> elements with same number of zeros will preserve their
+      // order.
+      auto comparator = [](const PosAndCount& a, const PosAndCount& b) -> bool {
+        return a.count < b.count;
+      };
+      std::stable_sort(pos_and_val, pos_and_val + sz, comparator);
+
+      // Grab indices.
+      for (size_t i = 0; i < sz; ++i) {
+        order[(ord * 3 + c) * AcStrategy::kMaxCoeffArea + i] =
+            pos_and_val[i].pos;
+      }
+    }
+  }
+}
+namespace {
+constexpr uint32_t kPermutationContexts = 8;
+uint32_t Context(uint32_t val) {
+  uint32_t nbits, bits;
+  EncodeVarLenUint(val, &nbits, &bits);
+  return std::min(nbits, kPermutationContexts - 1);
+}
+
+void TokenizePermutation(const coeff_order_t* JXL_RESTRICT order, size_t skip,
+                         size_t size, std::vector<Token>* tokens) {
+  std::vector<LehmerT> lehmer(size);
+  std::vector<uint32_t> temp(size + 1);
+  ComputeLehmerCode(order, temp.data(), size, lehmer.data());
+  size_t end = size;
+  while (end > skip && lehmer[end - 1] == 0) {
+    --end;
+  }
+  uint32_t nbits, bits;
+  EncodeVarLenUint(end - skip, &nbits, &bits);
+  tokens->emplace_back(Context(size), nbits, nbits, bits);
+  uint32_t last = 0;
+  for (size_t i = skip; i < end; ++i) {
+    EncodeVarLenUint(lehmer[i], &nbits, &bits);
+    tokens->emplace_back(Context(last), nbits, nbits, bits);
+    last = lehmer[i];
+  }
+}
+
+HWY_ATTR void ReadPermutation(size_t skip, size_t size, coeff_order_t* order,
+                              BitReader* br, ANSSymbolReader* reader,
+                              const std::vector<uint8_t>& context_map) {
+  std::vector<LehmerT> lehmer(size);
+  std::vector<uint32_t> temp(size);
+  uint32_t nbits = reader->ReadSymbol(context_map[Context(size)], br);
+  uint32_t bits = br->ReadBits(nbits);
+  uint32_t end = DecodeVarLenUint(nbits, bits) + skip;
+  uint32_t last = 0;
+  for (size_t i = skip; i < end; ++i) {
+    br->Refill();  // covers ReadSymbolWithoutRefill + ReadBits
+    uint32_t nbits =
+        reader->ReadSymbolWithoutRefill(context_map[Context(last)], br);
+    uint32_t bits = br->ReadBits(nbits);
+    lehmer[i] = DecodeVarLenUint(nbits, bits);
+    last = lehmer[i];
+  }
+  DecodeLehmerCode(lehmer.data(), temp.data(), size, order);
+}
+
+}  // namespace
+
+Status DecodePermutation(size_t skip, size_t size, coeff_order_t* order,
+                         BitReader* br) {
+  std::vector<uint8_t> context_map;
+  ANSCode code;
+  JXL_RETURN_IF_ERROR(DecodeHistograms(
+      br, kPermutationContexts, ANS_MAX_ALPHA_SIZE, &code, &context_map));
+  ANSSymbolReader reader(&code, br);
+  ReadPermutation(skip, size, order, br, &reader, context_map);
+  if (!reader.CheckANSFinalState()) {
+    return JXL_FAILURE("Invalid ANS stream");
+  }
+  return true;
+}
+
+void EncodePermutation(const coeff_order_t* JXL_RESTRICT order, size_t skip,
+                       size_t size, BitWriter* writer, int layer,
+                       AuxOut* aux_out) {
+  std::vector<Token> tokens;
+  TokenizePermutation(order, skip, size, &tokens);
+  std::vector<uint8_t> context_map;
+  EntropyEncodingData codes;
+  BuildAndEncodeHistograms(HistogramParams(), kPermutationContexts, {tokens},
+                           &codes, &context_map, writer, layer, aux_out);
+  WriteTokens(tokens, codes, context_map, writer, layer, aux_out);
+}
+
+namespace {
+void EncodeCoeffOrder(const coeff_order_t* JXL_RESTRICT order, AcStrategy acs,
+                      std::vector<Token>* tokens) {
+  const size_t llf = acs.covered_blocks_x() * acs.covered_blocks_y();
+  const size_t size = kDCTBlockSize * llf;
+  coeff_order_t order_zigzag[AcStrategy::kMaxCoeffArea];
+  const coeff_order_t* natural_coeff_order_lut = acs.NaturalCoeffOrderLut();
+  for (size_t i = 0; i < size; ++i) {
+    order_zigzag[i] = natural_coeff_order_lut[order[i]];
+  }
+  TokenizePermutation(order_zigzag, llf, size, tokens);
+}
+}  // namespace
+
+void EncodeCoeffOrders(uint16_t used_orders, const coeff_order_t* order,
+                       BitWriter* writer, size_t layer, AuxOut* aux_out) {
+  uint16_t computed = 0;
+  std::vector<Token> tokens;
+  for (uint8_t o = 0; o < AcStrategy::kNumValidStrategies; ++o) {
+    uint8_t ord = kStrategyOrder[o];
+    if (computed & (1 << ord)) continue;
+    computed |= 1 << ord;
+    if ((used_orders & (1 << ord)) == 0) continue;
+    AcStrategy acs = AcStrategy::FromRawStrategy(o);
+    for (size_t c = 0; c < 3; c++) {
+      EncodeCoeffOrder(&order[(3 * ord + c) * AcStrategy::kMaxCoeffArea], acs,
+                       &tokens);
+    }
+  }
+  // Do not write anything if no order is used.
+  if (used_orders != 0) {
+    std::vector<uint8_t> context_map;
+    EntropyEncodingData codes;
+    BuildAndEncodeHistograms(HistogramParams(), kPermutationContexts, {tokens},
+                             &codes, &context_map, writer, layer, aux_out);
+    WriteTokens(tokens, codes, context_map, writer, layer, aux_out);
+  }
+}
+
+namespace {
+
+Status DecodeCoeffOrder(AcStrategy acs, coeff_order_t* order, BitReader* br,
+                        ANSSymbolReader* reader,
+                        const std::vector<uint8_t>& context_map) {
+  PROFILER_FUNC;
+  const size_t llf = acs.covered_blocks_x() * acs.covered_blocks_y();
+  const size_t size = kDCTBlockSize * llf;
+
+  ReadPermutation(llf, size, order, br, reader, context_map);
+  const coeff_order_t* natural_coeff_order = acs.NaturalCoeffOrder();
+  for (size_t k = 0; k < size; ++k) {
+    order[k] = natural_coeff_order[order[k]];
+  }
+  return true;
+}
+
+}  // namespace
+
+Status DecodeCoeffOrders(uint16_t used_orders, coeff_order_t* order,
+                         BitReader* br) {
+  uint16_t computed = 0;
+  std::vector<uint8_t> context_map;
+  ANSCode code;
+  std::unique_ptr<ANSSymbolReader> reader;
+  // Bitstream does not have histograms if no coefficient order is used.
+  if (used_orders != 0) {
+    JXL_RETURN_IF_ERROR(DecodeHistograms(
+        br, kPermutationContexts, ANS_MAX_ALPHA_SIZE, &code, &context_map));
+    reader = make_unique<ANSSymbolReader>(&code, br);
+  }
+  for (uint8_t o = 0; o < AcStrategy::kNumValidStrategies; ++o) {
+    uint8_t ord = kStrategyOrder[o];
+    if (computed & (1 << ord)) continue;
+    computed |= 1 << ord;
+    AcStrategy acs = AcStrategy::FromRawStrategy(o);
+    if ((used_orders & (1 << ord)) == 0) {
+      for (size_t c = 0; c < 3; c++) {
+        SetDefaultOrder(acs, &order[(3 * ord + c) * AcStrategy::kMaxCoeffArea]);
+      }
+    } else {
+      for (size_t c = 0; c < 3; c++) {
+        JXL_RETURN_IF_ERROR(DecodeCoeffOrder(
+            acs, &order[(3 * ord + c) * AcStrategy::kMaxCoeffArea], br,
+            reader.get(), context_map));
+      }
+    }
+  }
+  if (used_orders && !reader->CheckANSFinalState()) {
+    return JXL_FAILURE("Invalid ANS stream");
+  }
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/coeff_order.h b/jxl/coeff_order.h
new file mode 100644
index 000000000..5fa16d872
--- /dev/null
+++ b/jxl/coeff_order.h
@@ -0,0 +1,76 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_COEFF_ORDER_H_
+#define JXL_COEFF_ORDER_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/common.h"
+#include "jxl/dct_util.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_params.h"
+
+namespace jxl {
+
+static constexpr size_t kCoeffOrderSize =
+    3 * kNumOrders * AcStrategy::kMaxCoeffArea;
+
+// Mapping from AC strategy to order bucket. Strategies with different natural
+// orders must have different buckets.
+constexpr uint8_t kStrategyOrder[AcStrategy::kNumValidStrategies] = {
+    0, 1, 1, 1, 2, 3, 4, 4, 5, 5, 6, 6, 1, 1, 1, 1, 1, 1};
+
+static_assert(AcStrategy::kNumValidStrategies == 18,
+              "Update this array when adding or removing AC strategies.");
+
+// Orders that are actually used in part of image. `rect` is in block units.
+uint32_t ComputeUsedOrders(SpeedTier speed, const AcStrategyImage& ac_strategy,
+                           const Rect& rect);
+
+// Modify zig-zag order, so that DCT bands with more zeros go later.
+// Order of DCT bands with same number of zeros is untouched, so
+// permutation will be cheaper to encode.
+void ComputeCoeffOrder(const ACImage3& acs, const AcStrategyImage& ac_strategy,
+                       const FrameDimensions& frame_dim, uint32_t used_orders,
+                       coeff_order_t* JXL_RESTRICT order);
+
+void EncodeCoeffOrders(uint16_t used_orders,
+                       const coeff_order_t* JXL_RESTRICT order,
+                       BitWriter* writer, size_t layer,
+                       AuxOut* JXL_RESTRICT aux_out);
+
+Status DecodeCoeffOrders(uint16_t used_orders, coeff_order_t* order,
+                         BitReader* br);
+
+// Encoding/decoding of a single permutation. `size`: number of elements in the
+// permutation. `skip`: number of elements to skip from the *beginning* of the
+// permutation.
+void EncodePermutation(const coeff_order_t* JXL_RESTRICT order, size_t skip,
+                       size_t size, BitWriter* writer, int layer,
+                       AuxOut* aux_out);
+
+Status DecodePermutation(size_t skip, size_t size, coeff_order_t* order,
+                         BitReader* br);
+
+}  // namespace jxl
+
+#endif  // JXL_COEFF_ORDER_H_
diff --git a/jxl/coeff_order_fwd.h b/jxl/coeff_order_fwd.h
new file mode 100644
index 000000000..176bcad27
--- /dev/null
+++ b/jxl/coeff_order_fwd.h
@@ -0,0 +1,56 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_COEFF_ORDER_FWD_H_
+#define JXL_COEFF_ORDER_FWD_H_
+
+// Breaks circular dependency between ac_strategy and coeff_order.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "base/compiler_specific.h"
+
+namespace jxl {
+
+// Needs at least 10 bits. A 32-bit type speeds up DecodeAC by 2% at the cost of
+// more memory.
+using coeff_order_t = uint32_t;
+
+// Maximum number of orders to be used. Note that this needs to be multiplied by
+// the number of channels. One per "size class" (plus one extra for DCT8),
+// shared between transforms of size XxY and of size YxX.
+constexpr uint8_t kNumOrders = 7;
+
+// DCT coefficients are laid out in such a way that the number of rows of
+// coefficients is always the smaller coordinate.
+JXL_INLINE constexpr size_t CoefficientRows(size_t rows, size_t columns) {
+  return rows < columns ? rows : columns;
+}
+
+JXL_INLINE constexpr size_t CoefficientColumns(size_t rows, size_t columns) {
+  return rows < columns ? columns : rows;
+}
+
+JXL_INLINE void CoefficientLayout(size_t* JXL_RESTRICT rows,
+                                  size_t* JXL_RESTRICT columns) {
+  size_t r = *rows;
+  size_t c = *columns;
+  *rows = CoefficientRows(r, c);
+  *columns = CoefficientColumns(r, c);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_COEFF_ORDER_FWD_H_
diff --git a/jxl/coeff_order_test.cc b/jxl/coeff_order_test.cc
new file mode 100644
index 000000000..8eff10d03
--- /dev/null
+++ b/jxl/coeff_order_test.cc
@@ -0,0 +1,110 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/coeff_order.h"
+
+#include <stdio.h>
+
+#include <algorithm>
+#include <numeric>  // iota
+#include <random>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/base/span.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/dec_bit_reader.h"
+
+namespace jxl {
+namespace {
+
+size_t RoundtripPermutation(coeff_order_t* perm, coeff_order_t* out,
+                            size_t len) {
+  BitWriter writer;
+  EncodePermutation(perm, 0, len, &writer, 0, nullptr);
+  writer.ZeroPadToByte();
+  BitReader reader(writer.GetSpan());
+  EXPECT_TRUE(DecodePermutation(0, len, out, &reader));
+  EXPECT_TRUE(reader.Close());
+  return writer.GetSpan().size();
+}
+
+enum Permutation { kIdentity, kFewSwaps, kFewSlides, kRandom };
+
+#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \
+    defined(THREAD_SANITIZER)
+constexpr size_t kNumReps = 128;
+#else
+constexpr size_t kNumReps = 1024;
+#endif
+constexpr size_t kSwaps = 32;
+
+void TestPermutation(Permutation kind, size_t len) {
+  std::vector<coeff_order_t> perm(len);
+  std::iota(perm.begin(), perm.end(), 0);
+  std::mt19937 rng;
+  if (kind == kFewSwaps) {
+    std::uniform_int_distribution<size_t> dist(0, len - 1);
+    for (size_t i = 0; i < kSwaps; i++) {
+      size_t a = dist(rng);
+      size_t b = dist(rng);
+      std::swap(perm[a], perm[b]);
+    }
+  }
+  if (kind == kFewSlides) {
+    std::uniform_int_distribution<size_t> dist(0, len - 1);
+    for (size_t i = 0; i < kSwaps; i++) {
+      size_t a = dist(rng);
+      size_t b = dist(rng);
+      size_t from = std::min(a, b);
+      size_t to = std::max(a, b);
+      size_t start = perm[from];
+      for (size_t j = from; j < to; j++) {
+        perm[j] = perm[j + 1];
+      }
+      perm[to] = start;
+    }
+  }
+  if (kind == kRandom) {
+    std::shuffle(perm.begin(), perm.end(), rng);
+  }
+  std::vector<coeff_order_t> out(len);
+  size_t size = 0;
+  for (size_t i = 0; i < kNumReps; i++) {
+    size = RoundtripPermutation(perm.data(), out.data(), len);
+    for (size_t idx = 0; idx < len; idx++) {
+      EXPECT_EQ(perm[idx], out[idx]);
+    }
+  }
+  printf("Encoded size: %zu\n", size);
+}
+
+TEST(CoeffOrderTest, IdentitySmall) { TestPermutation(kIdentity, 256); }
+TEST(CoeffOrderTest, kFewSlidesSmall) { TestPermutation(kFewSlides, 256); }
+TEST(CoeffOrderTest, FewSwapsSmall) { TestPermutation(kFewSwaps, 256); }
+TEST(CoeffOrderTest, RandomSmall) { TestPermutation(kRandom, 256); }
+
+TEST(CoeffOrderTest, IdentityMedium) { TestPermutation(kIdentity, 1 << 12); }
+TEST(CoeffOrderTest, FewSlidesMedium) { TestPermutation(kFewSlides, 1 << 12); }
+TEST(CoeffOrderTest, FewSwapsMedium) { TestPermutation(kFewSwaps, 1 << 12); }
+TEST(CoeffOrderTest, RandomMedium) { TestPermutation(kRandom, 1 << 12); }
+
+TEST(CoeffOrderTest, IdentityBig) { TestPermutation(kIdentity, 1 << 16); }
+TEST(CoeffOrderTest, FewSlidesBig) { TestPermutation(kFewSlides, 1 << 16); }
+TEST(CoeffOrderTest, FewSwapsBig) { TestPermutation(kFewSwaps, 1 << 16); }
+TEST(CoeffOrderTest, RandomBig) { TestPermutation(kRandom, 1 << 16); }
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/color_encoding.cc b/jxl/color_encoding.cc
new file mode 100644
index 000000000..5dfa59452
--- /dev/null
+++ b/jxl/color_encoding.cc
@@ -0,0 +1,508 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/color_encoding.h"
+
+#include <cmath>
+
+#include "jxl/fields.h"
+
+namespace jxl {
+namespace {
+
+// These strings are baked into Description - do not change.
+
+std::string ToString(ColorSpace color_space) {
+  switch (color_space) {
+    case ColorSpace::kRGB:
+      return "RGB";
+    case ColorSpace::kGray:
+      return "Gra";
+    case ColorSpace::kXYB:
+      return "XYB";
+    case ColorSpace::kUnknown:
+      return "CS?";
+  }
+  // Should not happen - visitor fails if enum is invalid.
+  JXL_ABORT("Invalid ColorSpace %u", static_cast<uint32_t>(color_space));
+}
+
+std::string ToString(WhitePoint white_point) {
+  switch (white_point) {
+    case WhitePoint::kD65:
+      return "D65";
+    case WhitePoint::kCustom:
+      return "Cst";
+    case WhitePoint::kE:
+      return "EER";
+    case WhitePoint::kDCI:
+      return "DCI";
+  }
+  // Should not happen - visitor fails if enum is invalid.
+  JXL_ABORT("Invalid WhitePoint %u", static_cast<uint32_t>(white_point));
+}
+
+std::string ToString(Primaries primaries) {
+  switch (primaries) {
+    case Primaries::kSRGB:
+      return "SRG";
+    case Primaries::k2100:
+      return "202";
+    case Primaries::kP3:
+      return "DCI";
+    case Primaries::kCustom:
+      return "Cst";
+  }
+  // Should not happen - visitor fails if enum is invalid.
+  JXL_ABORT("Invalid Primaries %u", static_cast<uint32_t>(primaries));
+}
+
+std::string ToString(TransferFunction transfer_function) {
+  switch (transfer_function) {
+    case TransferFunction::kSRGB:
+      return "SRG";
+    case TransferFunction::kLinear:
+      return "Lin";
+    case TransferFunction::k709:
+      return "709";
+    case TransferFunction::kPQ:
+      return "PeQ";
+    case TransferFunction::kHLG:
+      return "HLG";
+    case TransferFunction::kDCI:
+      return "DCI";
+    case TransferFunction::kUnknown:
+      return "TF?";
+  }
+  // Should not happen - visitor fails if enum is invalid.
+  JXL_ABORT("Invalid TransferFunction %u",
+            static_cast<uint32_t>(transfer_function));
+}
+
+std::string ToString(RenderingIntent rendering_intent) {
+  switch (rendering_intent) {
+    case RenderingIntent::kPerceptual:
+      return "Per";
+    case RenderingIntent::kRelative:
+      return "Rel";
+    case RenderingIntent::kSaturation:
+      return "Sat";
+    case RenderingIntent::kAbsolute:
+      return "Abs";
+  }
+  // Should not happen - visitor fails if enum is invalid.
+  JXL_ABORT("Invalid RenderingIntent %u",
+            static_cast<uint32_t>(rendering_intent));
+}
+
+template <typename Enum>
+Status ParseEnum(const std::string& token, Enum* value) {
+  std::string str;
+  for (Enum e : Values<Enum>()) {
+    if (ToString(e) == token) {
+      *value = e;
+      return true;
+    }
+  }
+  return false;
+}
+
+class Tokenizer {
+ public:
+  Tokenizer(const std::string* input, char separator)
+      : input_(input), separator_(separator) {}
+
+  Status Next(std::string* JXL_RESTRICT next) {
+    const size_t end = input_->find(separator_, start_);
+    if (end == std::string::npos) {
+      *next = input_->substr(start_);  // rest of string
+    } else {
+      *next = input_->substr(start_, end - start_);
+    }
+    if (next->empty()) return JXL_FAILURE("Missing token");
+    start_ = end + 1;
+    return true;
+  }
+
+ private:
+  const std::string* const input_;  // not owned
+  const char separator_;
+  size_t start_ = 0;  // of next token
+};
+
+Status ParseDouble(Tokenizer* tokenizer, double* JXL_RESTRICT d) {
+  std::string num;
+  JXL_RETURN_IF_ERROR(tokenizer->Next(&num));
+  *d = std::stod(num);
+  return true;
+}
+
+Status ParseColorSpace(Tokenizer* JXL_RESTRICT tokenizer,
+                       ColorEncoding* JXL_RESTRICT c) {
+  std::string str;
+  JXL_RETURN_IF_ERROR(tokenizer->Next(&str));
+  ColorSpace cs;
+  if (ParseEnum(str, &cs)) {
+    c->SetColorSpace(cs);
+    return true;
+  }
+
+  return JXL_FAILURE("Unknown ColorSpace %s", str.c_str());
+}
+
+Status ParseWhitePoint(Tokenizer* JXL_RESTRICT tokenizer,
+                       ColorEncoding* JXL_RESTRICT c) {
+  if (c->ImplicitWhitePoint()) return true;
+
+  std::string str;
+  JXL_RETURN_IF_ERROR(tokenizer->Next(&str));
+  if (ParseEnum(str, &c->white_point)) return true;
+
+  CIExy xy;
+  Tokenizer xy_tokenizer(&str, ';');
+  JXL_RETURN_IF_ERROR(ParseDouble(&xy_tokenizer, &xy.x));
+  JXL_RETURN_IF_ERROR(ParseDouble(&xy_tokenizer, &xy.y));
+  if (c->SetWhitePoint(xy)) return true;
+
+  return JXL_FAILURE("Invalid white point %s", str.c_str());
+}
+
+Status ParsePrimaries(Tokenizer* JXL_RESTRICT tokenizer,
+                      ColorEncoding* JXL_RESTRICT c) {
+  if (!c->HasPrimaries()) return true;
+
+  std::string str;
+  JXL_RETURN_IF_ERROR(tokenizer->Next(&str));
+  if (ParseEnum(str, &c->primaries)) return true;
+
+  PrimariesCIExy xy;
+  Tokenizer xy_tokenizer(&str, ';');
+  JXL_RETURN_IF_ERROR(ParseDouble(&xy_tokenizer, &xy.r.x));
+  JXL_RETURN_IF_ERROR(ParseDouble(&xy_tokenizer, &xy.r.y));
+  JXL_RETURN_IF_ERROR(ParseDouble(&xy_tokenizer, &xy.g.x));
+  JXL_RETURN_IF_ERROR(ParseDouble(&xy_tokenizer, &xy.g.y));
+  JXL_RETURN_IF_ERROR(ParseDouble(&xy_tokenizer, &xy.b.x));
+  JXL_RETURN_IF_ERROR(ParseDouble(&xy_tokenizer, &xy.b.y));
+  if (c->SetPrimaries(xy)) return true;
+
+  return JXL_FAILURE("Invalid primaries %s", str.c_str());
+}
+
+Status ParseRenderingIntent(Tokenizer* JXL_RESTRICT tokenizer,
+                            ColorEncoding* JXL_RESTRICT c) {
+  std::string str;
+  JXL_RETURN_IF_ERROR(tokenizer->Next(&str));
+  if (ParseEnum(str, &c->rendering_intent)) return true;
+
+  return JXL_FAILURE("Invalid RenderingIntent %s\n", str.c_str());
+}
+
+Status ParseTransferFunction(Tokenizer* JXL_RESTRICT tokenizer,
+                             ColorEncoding* JXL_RESTRICT c) {
+  if (c->tf.SetImplicit()) return true;
+
+  std::string str;
+  JXL_RETURN_IF_ERROR(tokenizer->Next(&str));
+  TransferFunction transfer_function;
+  if (ParseEnum(str, &transfer_function)) {
+    c->tf.SetTransferFunction(transfer_function);
+    return true;
+  }
+
+  if (str[0] == 'g') {
+    const double gamma = std::stod(str.substr(1));
+    if (c->tf.SetGamma(gamma)) return true;
+  }
+
+  return JXL_FAILURE("Invalid gamma %s", str.c_str());
+}
+
+static double F64FromI32(const int32_t i) { return i * 1E-6; }
+static int32_t I32FromF64(const double f) {
+  return static_cast<int32_t>(std::round(f * 1E6));
+}
+
+}  // namespace
+
+CIExy Customxy::Get() const {
+  CIExy xy;
+  xy.x = F64FromI32(x);
+  xy.y = F64FromI32(y);
+  return xy;
+}
+
+Status Customxy::Set(const CIExy& xy) {
+  x = I32FromF64(xy.x);
+  y = I32FromF64(xy.y);
+  size_t extension_bits, total_bits;
+  if (!Bundle::CanEncode(*this, &extension_bits, &total_bits)) {
+    return JXL_FAILURE("Unable to encode XY %f %f", xy.x, xy.y);
+  }
+  return true;
+}
+
+bool CustomTransferFunction::SetImplicit() {
+  if (nonserialized_color_space == ColorSpace::kXYB) {
+    if (!SetGamma(1.0 / 3)) JXL_ASSERT(false);
+    return true;
+  }
+  return false;
+}
+
+Status CustomTransferFunction::SetGamma(double gamma) {
+  if (gamma <= 0.0 || gamma > 1.0) {
+    return JXL_FAILURE("Invalid gamma %f", gamma);
+  }
+
+  have_gamma_ = false;
+  if (ApproxEq(gamma, 1.0)) {
+    transfer_function_ = TransferFunction::kLinear;
+    return true;
+  }
+  if (ApproxEq(gamma, 1.0 / 2.6)) {
+    transfer_function_ = TransferFunction::kDCI;
+    return true;
+  }
+  // Don't translate 0.45.. to kSRGB nor k709 - that might change pixel
+  // values because those curves also have a linear part.
+
+  have_gamma_ = true;
+  gamma_ = std::round(gamma * kGammaMul);
+  transfer_function_ = TransferFunction::kUnknown;
+  return true;
+}
+
+CIExy ColorEncoding::GetWhitePoint() const {
+  CIExy xy;
+  switch (white_point) {
+    case WhitePoint::kCustom:
+      return white_.Get();
+
+    case WhitePoint::kD65:
+      xy.x = 0.3127;
+      xy.y = 0.3290;
+      return xy;
+
+    case WhitePoint::kDCI:
+      // From https://ieeexplore.ieee.org/document/7290729 C.2 page 11
+      xy.x = 0.314;
+      xy.y = 0.351;
+      return xy;
+
+    case WhitePoint::kE:
+      xy.x = xy.y = 1.0 / 3;
+      return xy;
+  }
+  JXL_ABORT("Invalid WhitePoint %u", static_cast<uint32_t>(white_point));
+}
+
+Status ColorEncoding::SetWhitePoint(const CIExy& xy) {
+  if (xy.x == 0.0 || xy.y == 0.0) {
+    return JXL_FAILURE("Invalid white point %f %f", xy.x, xy.y);
+  }
+  if (ApproxEq(xy.x, 0.3127) && ApproxEq(xy.y, 0.3290)) {
+    white_point = WhitePoint::kD65;
+    return true;
+  }
+  if (ApproxEq(xy.x, 1.0 / 3) && ApproxEq(xy.y, 1.0 / 3)) {
+    white_point = WhitePoint::kE;
+    return true;
+  }
+  if (ApproxEq(xy.x, 0.314) && ApproxEq(xy.y, 0.351)) {
+    white_point = WhitePoint::kDCI;
+    return true;
+  }
+  white_point = WhitePoint::kCustom;
+  return white_.Set(xy);
+}
+
+PrimariesCIExy ColorEncoding::GetPrimaries() const {
+  JXL_ASSERT(HasPrimaries());
+  PrimariesCIExy xy;
+  switch (primaries) {
+    case Primaries::kCustom:
+      xy.r = red_.Get();
+      xy.g = green_.Get();
+      xy.b = blue_.Get();
+      return xy;
+
+    case Primaries::kSRGB:
+      xy.r.x = 0.639998686;
+      xy.r.y = 0.330010138;
+      xy.g.x = 0.300003784;
+      xy.g.y = 0.600003357;
+      xy.b.x = 0.150002046;
+      xy.b.y = 0.059997204;
+      return xy;
+
+    case Primaries::k2100:
+      xy.r.x = 0.708;
+      xy.r.y = 0.292;
+      xy.g.x = 0.170;
+      xy.g.y = 0.797;
+      xy.b.x = 0.131;
+      xy.b.y = 0.046;
+      return xy;
+
+    case Primaries::kP3:
+      xy.r.x = 0.680;
+      xy.r.y = 0.320;
+      xy.g.x = 0.265;
+      xy.g.y = 0.690;
+      xy.b.x = 0.150;
+      xy.b.y = 0.060;
+      return xy;
+  }
+  JXL_ABORT("Invalid Primaries %u", static_cast<uint32_t>(primaries));
+}
+
+Status ColorEncoding::SetPrimaries(const PrimariesCIExy& xy) {
+  JXL_ASSERT(HasPrimaries());
+  if (xy.r.x == 0.0 || xy.r.y == 0.0 || xy.g.x == 0.0 || xy.g.y == 0.0 ||
+      xy.b.x == 0.0 || xy.b.y == 0.0) {
+    return JXL_FAILURE("Invalid primaries %f %f %f %f %f %f", xy.r.x, xy.r.y,
+                       xy.g.x, xy.g.y, xy.b.x, xy.b.y);
+  }
+
+  if (ApproxEq(xy.r.x, 0.64) && ApproxEq(xy.r.y, 0.33) &&
+      ApproxEq(xy.g.x, 0.30) && ApproxEq(xy.g.y, 0.60) &&
+      ApproxEq(xy.b.x, 0.15) && ApproxEq(xy.b.y, 0.06)) {
+    primaries = Primaries::kSRGB;
+    return true;
+  }
+
+  if (ApproxEq(xy.r.x, 0.708) && ApproxEq(xy.r.y, 0.292) &&
+      ApproxEq(xy.g.x, 0.170) && ApproxEq(xy.g.y, 0.797) &&
+      ApproxEq(xy.b.x, 0.131) && ApproxEq(xy.b.y, 0.046)) {
+    primaries = Primaries::k2100;
+    return true;
+  }
+  if (ApproxEq(xy.r.x, 0.680) && ApproxEq(xy.r.y, 0.320) &&
+      ApproxEq(xy.g.x, 0.265) && ApproxEq(xy.g.y, 0.690) &&
+      ApproxEq(xy.b.x, 0.150) && ApproxEq(xy.b.y, 0.060)) {
+    primaries = Primaries::kP3;
+    return true;
+  }
+
+  primaries = Primaries::kCustom;
+  JXL_RETURN_IF_ERROR(red_.Set(xy.r));
+  JXL_RETURN_IF_ERROR(green_.Set(xy.g));
+  JXL_RETURN_IF_ERROR(blue_.Set(xy.b));
+  return true;
+}
+
+std::string Description(const ColorEncoding& c_in) {
+  // Copy required for Implicit*
+  ColorEncoding c = c_in;
+
+  std::string d = ToString(c.GetColorSpace());
+
+  if (!c.ImplicitWhitePoint()) {
+    d += '_';
+    if (c.white_point == WhitePoint::kCustom) {
+      const CIExy wp = c.GetWhitePoint();
+      d += std::to_string(wp.x) + ';';
+      d += std::to_string(wp.y);
+    } else {
+      d += ToString(c.white_point);
+    }
+  }
+
+  if (c.HasPrimaries()) {
+    d += '_';
+    if (c.primaries == Primaries::kCustom) {
+      const PrimariesCIExy pr = c.GetPrimaries();
+      d += std::to_string(pr.r.x) + ';';
+      d += std::to_string(pr.r.y) + ';';
+      d += std::to_string(pr.g.x) + ';';
+      d += std::to_string(pr.g.y) + ';';
+      d += std::to_string(pr.b.x) + ';';
+      d += std::to_string(pr.b.y);
+    } else {
+      d += ToString(c.primaries);
+    }
+  }
+
+  d += '_';
+  d += ToString(c.rendering_intent);
+
+  if (!c.tf.SetImplicit()) {
+    d += '_';
+    if (c.tf.IsGamma()) {
+      d += 'g';
+      d += std::to_string(c.tf.GetGamma());
+    } else {
+      d += ToString(c.tf.GetTransferFunction());
+    }
+  }
+
+  return d;
+}
+
+Status ParseDescription(const std::string& description,
+                        ColorEncoding* JXL_RESTRICT c) {
+  Tokenizer tokenizer(&description, '_');
+  JXL_RETURN_IF_ERROR(ParseColorSpace(&tokenizer, c));
+  JXL_RETURN_IF_ERROR(ParseWhitePoint(&tokenizer, c));
+  JXL_RETURN_IF_ERROR(ParsePrimaries(&tokenizer, c));
+  JXL_RETURN_IF_ERROR(ParseRenderingIntent(&tokenizer, c));
+  JXL_RETURN_IF_ERROR(ParseTransferFunction(&tokenizer, c));
+  return true;
+}
+
+std::vector<ColorEncoding> AllEncodings() {
+  std::vector<ColorEncoding> all_encodings;
+  all_encodings.reserve(300);
+  ColorEncoding c;
+
+  for (ColorSpace cs : Values<ColorSpace>()) {
+    if (cs == ColorSpace::kUnknown || cs == ColorSpace::kXYB) continue;
+    c.SetColorSpace(cs);
+
+    for (WhitePoint wp : Values<WhitePoint>()) {
+      if (wp == WhitePoint::kCustom) continue;
+      if (c.ImplicitWhitePoint() && c.white_point != wp) continue;
+      c.white_point = wp;
+
+      for (Primaries primaries : Values<Primaries>()) {
+        if (primaries == Primaries::kCustom) continue;
+        if (!c.HasPrimaries()) continue;
+        c.primaries = primaries;
+
+        for (TransferFunction tf : Values<TransferFunction>()) {
+          if (tf == TransferFunction::kUnknown) continue;
+          if (c.tf.SetImplicit() &&
+              (c.tf.IsGamma() || c.tf.GetTransferFunction() != tf)) {
+            continue;
+          }
+          c.tf.SetTransferFunction(tf);
+
+          for (RenderingIntent ri : Values<RenderingIntent>()) {
+            c.rendering_intent = ri;
+
+            all_encodings.push_back(c);
+          }
+        }
+      }
+    }
+  }
+
+  return all_encodings;
+}
+
+Customxy::Customxy() { Bundle::Init(this); }
+CustomTransferFunction::CustomTransferFunction() { Bundle::Init(this); }
+ColorEncoding::ColorEncoding() { Bundle::Init(this); }
+
+}  // namespace jxl
diff --git a/jxl/color_encoding.h b/jxl/color_encoding.h
new file mode 100644
index 000000000..23d42a208
--- /dev/null
+++ b/jxl/color_encoding.h
@@ -0,0 +1,445 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_COLOR_ENCODING_H_
+#define JXL_COLOR_ENCODING_H_
+
+// Metadata for color space conversions.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <cmath>  // std::abs
+#include <ostream>
+#include <string>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/field_encodings.h"
+
+namespace jxl {
+
+// (All CIE units are for the standard 1931 2 degree observer)
+
+enum class ColorSpace : uint32_t { kRGB, kGray, kXYB, kUnknown };
+
+static inline const char* EnumName(ColorSpace /*unused*/) {
+  return "ColorSpace";
+}
+static inline constexpr uint64_t EnumBits(ColorSpace /*unused*/) {
+  using CS = ColorSpace;
+  return MakeBit(CS::kRGB) | MakeBit(CS::kGray) | MakeBit(CS::kXYB) |
+         MakeBit(CS::kUnknown);
+}
+
+// Values from CICP ColourPrimaries.
+enum class WhitePoint : uint32_t {
+  kD65 = 1,     // sRGB/BT.709/Display P3/BT.2020
+  kCustom = 2,  // Actual values encoded in separate fields
+  kE = 10,      // XYZ
+  kDCI = 11,    // DCI-P3
+};
+
+static inline const char* EnumName(WhitePoint /*unused*/) {
+  return "WhitePoint";
+}
+static inline constexpr uint64_t EnumBits(WhitePoint /*unused*/) {
+  return MakeBit(WhitePoint::kD65) | MakeBit(WhitePoint::kCustom) |
+         MakeBit(WhitePoint::kE) | MakeBit(WhitePoint::kDCI);
+}
+
+// Values from CICP ColourPrimaries
+enum class Primaries : uint32_t {
+  kSRGB = 1,    // Same as BT.709
+  kCustom = 2,  // Actual values encoded in separate fields
+  k2100 = 9,    // Same as BT.2020
+  kP3 = 11,
+};
+
+static inline const char* EnumName(Primaries /*unused*/) { return "Primaries"; }
+static inline constexpr uint64_t EnumBits(Primaries /*unused*/) {
+  using Pr = Primaries;
+  return MakeBit(Pr::kSRGB) | MakeBit(Pr::kCustom) | MakeBit(Pr::k2100) |
+         MakeBit(Pr::kP3);
+}
+
+// Values from CICP TransferCharacteristics
+enum TransferFunction : uint32_t {
+  k709 = 1,
+  kUnknown = 2,
+  kLinear = 8,
+  kSRGB = 13,
+  kPQ = 16,   // from BT.2100
+  kDCI = 17,  // from SMPTE RP 431-2 reference projector
+  kHLG = 18,  // from BT.2100
+};
+
+static inline const char* EnumName(TransferFunction /*unused*/) {
+  return "TransferFunction";
+}
+static inline constexpr uint64_t EnumBits(TransferFunction /*unused*/) {
+  using TF = TransferFunction;
+  return MakeBit(TF::k709) | MakeBit(TF::kLinear) | MakeBit(TF::kSRGB) |
+         MakeBit(TF::kPQ) | MakeBit(TF::kDCI) | MakeBit(TF::kHLG) |
+         MakeBit(TF::kUnknown);
+}
+
+enum class RenderingIntent : uint32_t {
+  // Values match ICC sRGB encodings.
+  kPerceptual = 0,  // good for photos, requires a profile with LUT.
+  kRelative,        // good for logos.
+  kSaturation,      // perhaps useful for CG with fully saturated colors.
+  kAbsolute,        // leaves white point unchanged; good for proofing.
+};
+
+static inline const char* EnumName(RenderingIntent /*unused*/) {
+  return "RenderingIntent";
+}
+static inline constexpr uint64_t EnumBits(RenderingIntent /*unused*/) {
+  using RI = RenderingIntent;
+  return MakeBit(RI::kPerceptual) | MakeBit(RI::kRelative) |
+         MakeBit(RI::kSaturation) | MakeBit(RI::kAbsolute);
+}
+
+// Chromaticity (Y is omitted because it is 1 for primaries/white points)
+struct CIExy {
+  double x = 0.0;
+  double y = 0.0;
+};
+
+struct PrimariesCIExy {
+  CIExy r;
+  CIExy g;
+  CIExy b;
+};
+
+// Serializable form of CIExy.
+struct Customxy {
+  Customxy();
+  static const char* Name() { return "Customxy"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->S32(Bits(19), BitsOffset(19, 524288), BitsOffset(20, 1048576),
+                 BitsOffset(21, 2097152), 0, &x);
+    visitor->S32(Bits(19), BitsOffset(19, 524288), BitsOffset(20, 1048576),
+                 BitsOffset(21, 2097152), 0, &y);
+    return true;
+  }
+
+  CIExy Get() const;
+  // Returns false if x or y do not fit in the encoding.
+  Status Set(const CIExy& xy);
+
+  int32_t x;
+  int32_t y;
+};
+
+struct CustomTransferFunction {
+  CustomTransferFunction();
+  static const char* Name() { return "CustomTransferFunction"; }
+
+  // Sets fields and returns true if nonserialized_color_space has an implicit
+  // transfer function, otherwise leaves fields unchanged and returns false.
+  bool SetImplicit();
+
+  // Gamma: only used for PNG inputs
+  bool IsGamma() const { return have_gamma_; }
+  double GetGamma() const {
+    JXL_ASSERT(IsGamma());
+    return gamma_ * 1E-7;  // (0, 1)
+  }
+  Status SetGamma(double gamma);
+
+  TransferFunction GetTransferFunction() const {
+    JXL_ASSERT(!IsGamma());
+    return transfer_function_;
+  }
+  void SetTransferFunction(const TransferFunction tf) {
+    have_gamma_ = false;
+    transfer_function_ = tf;
+  }
+
+  bool IsUnknown() const {
+    return !have_gamma_ && (transfer_function_ == TransferFunction::kUnknown);
+  }
+  bool IsSRGB() const {
+    return !have_gamma_ && (transfer_function_ == TransferFunction::kSRGB);
+  }
+  bool IsLinear() const {
+    return !have_gamma_ && (transfer_function_ == TransferFunction::kLinear);
+  }
+  bool IsPQ() const {
+    return !have_gamma_ && (transfer_function_ == TransferFunction::kPQ);
+  }
+  bool IsHLG() const {
+    return !have_gamma_ && (transfer_function_ == TransferFunction::kHLG);
+  }
+  bool IsSame(const CustomTransferFunction& other) const {
+    if (have_gamma_ != other.have_gamma_) return false;
+    if (have_gamma_) {
+      if (gamma_ != other.gamma_) return false;
+    } else {
+      if (transfer_function_ != other.transfer_function_) return false;
+    }
+    return true;
+  }
+
+  template <class Visitor>
+  bool VisitFields(Visitor* JXL_RESTRICT visitor) {
+    if (visitor->Conditional(!SetImplicit())) {
+      visitor->Bool(false, &have_gamma_);
+
+      if (visitor->Conditional(have_gamma_)) {
+        visitor->Bits(24, kGammaMul, &gamma_);
+        if (gamma_ > kGammaMul) {
+          return JXL_FAILURE("Invalid gamma %u", gamma_);
+        }
+      }
+
+      if (visitor->Conditional(!have_gamma_)) {
+        JXL_RETURN_IF_ERROR(
+            visitor->Enum(TransferFunction::kSRGB, &transfer_function_));
+      }
+    }
+
+    return true;
+  }
+
+  // Must be set before calling VisitFields!
+  ColorSpace nonserialized_color_space = ColorSpace::kRGB;
+
+ private:
+  static constexpr uint32_t kGammaMul = 10000000;
+
+  bool have_gamma_;
+
+  // OETF exponent to go from linear to gamma-compressed.
+  uint32_t gamma_;  // Only used if have_gamma_.
+
+  // Can be kUnknown.
+  TransferFunction transfer_function_;  // Only used if !have_gamma_.
+};
+
+// Compact encoding of data required to interpret and translate pixels to a
+// known color space. Stored in Metadata.
+struct ColorEncoding {
+  ColorEncoding();
+  static const char* Name() { return "ColorEncoding"; }
+
+  bool IsGray() const { return color_space_ == ColorSpace::kGray; }
+  size_t Channels() const { return IsGray() ? 1 : 3; }
+
+  // Returns false if the field is invalid and unusable.
+  bool HasPrimaries() const {
+    return !IsGray() && color_space_ != ColorSpace::kXYB;
+  }
+
+  // Returns true after setting the field to a value defined by color_space,
+  // otherwise false and leaves the field unchanged.
+  bool ImplicitWhitePoint() {
+    if (color_space_ == ColorSpace::kXYB) {
+      white_point = WhitePoint::kD65;
+      return true;
+    }
+    return false;
+  }
+
+  bool IsSRGB() const {
+    if (!IsGray() && color_space_ != ColorSpace::kRGB) return false;
+    if (white_point != WhitePoint::kD65) return false;
+    if (primaries != Primaries::kSRGB) return false;
+    if (!tf.IsSRGB()) return false;
+    return true;
+  }
+
+  void SetSRGB(const ColorSpace cs) {
+    icc.clear();
+    received_icc = opaque_icc = false;
+    JXL_ASSERT(cs == ColorSpace::kGray || cs == ColorSpace::kRGB);
+    color_space_ = cs;
+    white_point = WhitePoint::kD65;
+    primaries = Primaries::kSRGB;
+    tf.SetTransferFunction(TransferFunction::kSRGB);
+    rendering_intent = RenderingIntent::kRelative;
+  }
+
+  template <class Visitor>
+  bool VisitFields(Visitor* JXL_RESTRICT visitor) {
+    // NOTE: `icc` is not serialized here to avoid dependency on icc_codec.h.
+
+    if (visitor->AllDefault(*this, &all_default)) return true;
+
+    visitor->Bool(false, &received_icc);
+
+    if (visitor->Conditional(received_icc)) {
+      visitor->Bool(false, &opaque_icc);
+    }
+
+    // NOTE: use most likely values as default so ImageMetadata.all_default is
+    // true. We set ColorSpace/TransferFunction to kUnknown below if opaque_icc.
+
+    if (visitor->Conditional(!opaque_icc)) {
+      JXL_RETURN_IF_ERROR(visitor->Enum(ColorSpace::kRGB, &color_space_));
+
+      if (visitor->Conditional(!ImplicitWhitePoint())) {
+        JXL_RETURN_IF_ERROR(visitor->Enum(WhitePoint::kD65, &white_point));
+        if (visitor->Conditional(white_point == WhitePoint::kCustom)) {
+          JXL_RETURN_IF_ERROR(visitor->VisitNested(&white_));
+        }
+      }
+
+      if (visitor->Conditional(HasPrimaries())) {
+        JXL_RETURN_IF_ERROR(visitor->Enum(Primaries::kSRGB, &primaries));
+        if (visitor->Conditional(primaries == Primaries::kCustom)) {
+          JXL_RETURN_IF_ERROR(visitor->VisitNested(&red_));
+          JXL_RETURN_IF_ERROR(visitor->VisitNested(&green_));
+          JXL_RETURN_IF_ERROR(visitor->VisitNested(&blue_));
+        }
+      }
+
+      JXL_RETURN_IF_ERROR(visitor->VisitNested(&tf));
+
+      JXL_RETURN_IF_ERROR(
+          visitor->Enum(RenderingIntent::kRelative, &rendering_intent));
+    }
+
+    if (opaque_icc) {
+      // Set correct 'defaults' (see above) but only when reading.
+      if (visitor->IsReading()) {
+        color_space_ = ColorSpace::kUnknown;
+        tf.SetTransferFunction(TransferFunction::kUnknown);
+      }
+    } else {
+      // Not opaque implies all fields should be known.
+      if (color_space_ == ColorSpace::kUnknown || tf.IsUnknown()) {
+        return JXL_FAILURE("Opaque %d but cs %u and tf %u%s", opaque_icc,
+                           color_space_,
+                           tf.IsGamma() ? 0 : tf.GetTransferFunction(),
+                           tf.IsGamma() ? "(gamma)" : "");
+      }
+    }
+
+    return true;
+  }
+
+  // Accessors ensure tf.nonserialized_color_space is updated at the same time.
+  ColorSpace GetColorSpace() const { return color_space_; }
+  void SetColorSpace(const ColorSpace cs) {
+    color_space_ = cs;
+    tf.nonserialized_color_space = cs;
+  }
+
+  CIExy GetWhitePoint() const;
+  Status SetWhitePoint(const CIExy& xy);
+
+  PrimariesCIExy GetPrimaries() const;
+  Status SetPrimaries(const PrimariesCIExy& xy);
+
+  // Checks if the color spaces (including white point / primaries) are the
+  // same, but ignores the transfer function, rendering intent and ICC bytes.
+  bool SameColorSpace(const ColorEncoding& other) const {
+    if (color_space_ != other.color_space_) return false;
+
+    if (white_point != other.white_point) return false;
+    if (white_point == WhitePoint::kCustom) {
+      if (white_.x != other.white_.x || white_.y != other.white_.y)
+        return false;
+    }
+
+    if (HasPrimaries() != other.HasPrimaries()) return false;
+    if (HasPrimaries()) {
+      if (primaries != other.primaries) return false;
+      if (primaries == Primaries::kCustom) {
+        if (red_.x != other.red_.x || red_.y != other.red_.y) return false;
+        if (green_.x != other.green_.x || green_.y != other.green_.y)
+          return false;
+        if (blue_.x != other.blue_.x || blue_.y != other.blue_.y) return false;
+      }
+    }
+    return true;
+  }
+
+  // Checks if the color space and transfer function are the same, ignoring
+  // rendering intent and ICC bytes
+  bool SameColorEncoding(const ColorEncoding& other) const {
+    return SameColorSpace(other) && tf.IsSame(other.tf);
+  }
+
+  mutable bool all_default;
+
+  // Either empty, or a valid ICC profile.
+  PaddedBytes icc;
+
+  // icc is non-empty AND it comes from an external source.
+  bool received_icc;
+
+  // received_icc AND the ICC cannot be described by ColorSpace and/or
+  // TransferFunction. If true, subsequent fields are default-initialized!
+  bool opaque_icc;
+
+  // Not serialized if opaque_icc. Fully describes the ICC if !opaque_icc.
+  // SetProfile will either fail or initialize all fields.
+
+  WhitePoint white_point;
+  Primaries primaries;  // Only valid if HasPrimaries()
+  CustomTransferFunction tf;
+  RenderingIntent rendering_intent;
+
+ private:
+  ColorSpace color_space_;  // Can be kUnknown
+
+  // Only used if white_point == kCustom.
+  Customxy white_;
+
+  // Only used if primaries == kCustom.
+  Customxy red_;
+  Customxy green_;
+  Customxy blue_;
+};
+
+// Returns whether the two inputs are approximately equal.
+static inline bool ApproxEq(const double a, const double b,
+#if JPEGXL_ENABLE_SKCMS
+                            double max_l1 = 1E-3) {
+#else
+                            double max_l1 = 8E-5) {
+#endif
+  // Threshold should be sufficient for ICC's 15-bit fixed-point numbers.
+  // We have seen differences of 7.1E-5 with lcms2 and 1E-3 with skcms.
+  return std::abs(a - b) <= max_l1;
+}
+
+// Returns a representation of the ColorEncoding fields (not icc).
+// Example description: "RGB_D65_SRG_Rel_Lin"
+std::string Description(const ColorEncoding& c);
+Status ParseDescription(const std::string& description,
+                        ColorEncoding* JXL_RESTRICT c);
+
+// Returns ColorEncoding with empty ICC profile. Caller must use
+// ColorEncoding::CreateProfile() to generate a profile.
+std::vector<ColorEncoding> AllEncodings();
+
+// Define the operator<< for tests.
+static inline ::std::ostream& operator<<(::std::ostream& os,
+                                         const ColorEncoding& c) {
+  return os << "ColorEncoding/" << Description(c);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_COLOR_ENCODING_H_
diff --git a/jxl/color_encoding_test.cc b/jxl/color_encoding_test.cc
new file mode 100644
index 000000000..f2670c13b
--- /dev/null
+++ b/jxl/color_encoding_test.cc
@@ -0,0 +1,110 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/color_encoding.h"
+
+#include <stdio.h>
+
+#include "gtest/gtest.h"
+
+namespace jxl {
+namespace {
+
+TEST(ColorEncodingTest, RoundTripAll) {
+  for (const ColorEncoding& c_original : AllEncodings()) {
+    // Verify Set(Get) yields the same white point/primaries/gamma.
+    {
+      ColorEncoding c;
+      EXPECT_TRUE(c.SetWhitePoint(c_original.GetWhitePoint()));
+      EXPECT_EQ(c_original.white_point, c.white_point);
+    }
+    {
+      ColorEncoding c;
+      EXPECT_TRUE(c.SetPrimaries(c_original.GetPrimaries()));
+      EXPECT_EQ(c_original.primaries, c.primaries);
+    }
+    if (c_original.tf.IsGamma()) {
+      ColorEncoding c;
+      EXPECT_TRUE(c.tf.SetGamma(c_original.tf.GetGamma()));
+      EXPECT_TRUE(c_original.tf.IsSame(c.tf));
+    }
+
+    // Verify ParseDescription(Description) yields the same ColorEncoding
+    {
+      const std::string description = Description(c_original);
+      printf("%s\n", description.c_str());
+      ColorEncoding c;
+      EXPECT_TRUE(ParseDescription(description, &c));
+      EXPECT_TRUE(c_original.SameColorEncoding(c));
+    }
+  }
+}
+
+// Verify Set(Get) for specific custom values
+
+TEST(ColorEncodingTest, CustomWhitePoint) {
+  ColorEncoding c;
+  // Nonsensical values
+  CIExy xy_in;
+  xy_in.x = 0.8;
+  xy_in.y = 0.01;
+  EXPECT_TRUE(c.SetWhitePoint(xy_in));
+  const CIExy xy = c.GetWhitePoint();
+
+  ColorEncoding c2;
+  EXPECT_TRUE(c2.SetWhitePoint(xy));
+  EXPECT_TRUE(c.SameColorSpace(c2));
+}
+
+TEST(ColorEncodingTest, CustomPrimaries) {
+  ColorEncoding c;
+  PrimariesCIExy xy_in;
+  // Nonsensical values
+  xy_in.r.x = -0.01;
+  xy_in.r.y = 0.2;
+  xy_in.g.x = 0.4;
+  xy_in.g.y = 0.401;
+  xy_in.b.x = 1.1;
+  xy_in.b.y = -1.2;
+  EXPECT_TRUE(c.SetPrimaries(xy_in));
+  const PrimariesCIExy xy = c.GetPrimaries();
+
+  ColorEncoding c2;
+  EXPECT_TRUE(c2.SetPrimaries(xy));
+  EXPECT_TRUE(c.SameColorSpace(c2));
+}
+
+TEST(ColorEncodingTest, CustomGamma) {
+  ColorEncoding c;
+#ifndef JXL_CRASH_ON_ERROR
+  EXPECT_FALSE(c.tf.SetGamma(0.0));
+  EXPECT_FALSE(c.tf.SetGamma(-1E-6));
+  EXPECT_FALSE(c.tf.SetGamma(1.001));
+#endif
+  EXPECT_TRUE(c.tf.SetGamma(1.0));
+  EXPECT_FALSE(c.tf.IsGamma());
+  EXPECT_TRUE(c.tf.IsLinear());
+
+  EXPECT_TRUE(c.tf.SetGamma(0.123));
+  EXPECT_TRUE(c.tf.IsGamma());
+  const double gamma = c.tf.GetGamma();
+
+  ColorEncoding c2;
+  EXPECT_TRUE(c2.tf.SetGamma(gamma));
+  EXPECT_TRUE(c.SameColorEncoding(c2));
+  EXPECT_TRUE(c2.tf.IsGamma());
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/color_management.cc b/jxl/color_management.cc
new file mode 100644
index 000000000..47a2980b7
--- /dev/null
+++ b/jxl/color_management.cc
@@ -0,0 +1,1684 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/color_management.h"
+
+#include <math.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <array>
+#include <atomic>
+#include <hwy/static_targets.h>
+#include <memory>
+#include <mutex>
+#include <string>
+#include <utility>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/field_encodings.h"
+#include "jxl/linalg.h"
+#include "jxl/rational_polynomial.h"
+#if JPEGXL_ENABLE_SKCMS
+#include "skcms.h"
+#else  // JPEGXL_ENABLE_SKCMS
+#include "lcms2.h"
+#include "lcms2_plugin.h"
+#endif  // JPEGXL_ENABLE_SKCMS
+
+namespace jxl {
+namespace {
+
+#define JXL_CMS_VERBOSE 0
+
+// Define to 1 on OS X as a workaround for older LCMS lacking MD5.
+#define JXL_CMS_OLD_VERSION 0
+
+// cms functions (even *THR) are not thread-safe, except cmsDoTransform.
+// To ensure all functions are covered without frequent lock-taking nor risk of
+// recursive lock, we lock in the top-level APIs.
+std::mutex lcms_mutex;
+
+#if JPEGXL_ENABLE_SKCMS
+JXL_MUST_USE_RESULT CIExy CIExyFromXYZ(const float XYZ[3]) {
+  const float factor = 1.f / (XYZ[0] + XYZ[1] + XYZ[2]);
+  CIExy xy;
+  xy.x = XYZ[0] * factor;
+  xy.y = XYZ[1] * factor;
+  return xy;
+}
+
+void CIEXYZFromWhiteCIExy(const CIExy& xy, float XYZ[3]) {
+  // Target Y = 1.
+  const float factor = 1 / xy.y;
+  XYZ[0] = xy.x * factor;
+  XYZ[1] = 1;
+  XYZ[2] = (1 - xy.x - xy.y) * factor;
+}
+#else  // JPEGXL_ENABLE_SKCMS
+// (LCMS interface requires xyY but we omit the Y for white points/primaries.)
+
+JXL_MUST_USE_RESULT CIExy CIExyFromxyY(const cmsCIExyY& xyY) {
+  CIExy xy;
+  xy.x = xyY.x;
+  xy.y = xyY.y;
+  return xy;
+}
+
+JXL_MUST_USE_RESULT CIExy CIExyFromXYZ(const cmsCIEXYZ& XYZ) {
+  cmsCIExyY xyY;
+  cmsXYZ2xyY(/*Dest=*/&xyY, /*Source=*/&XYZ);
+  return CIExyFromxyY(xyY);
+}
+
+JXL_MUST_USE_RESULT cmsCIEXYZ D50_XYZ() {
+  // Quantized D50 as stored in ICC profiles.
+  return {0.96420288, 1.0, 0.82490540};
+}
+
+JXL_MUST_USE_RESULT cmsCIExyY xyYFromCIExy(const CIExy& xy) {
+  const cmsCIExyY xyY = {xy.x, xy.y, 1.0};
+  return xyY;
+}
+
+// RAII
+
+struct ProfileDeleter {
+  void operator()(void* p) { cmsCloseProfile(p); }
+};
+using Profile = std::unique_ptr<void, ProfileDeleter>;
+
+struct TransformDeleter {
+  void operator()(void* p) { cmsDeleteTransform(p); }
+};
+using Transform = std::unique_ptr<void, TransformDeleter>;
+
+struct CurveDeleter {
+  void operator()(cmsToneCurve* p) { cmsFreeToneCurve(p); }
+};
+using Curve = std::unique_ptr<cmsToneCurve, CurveDeleter>;
+
+Status CreateProfileXYZ(const cmsContext context,
+                        Profile* JXL_RESTRICT profile) {
+  profile->reset(cmsCreateXYZProfileTHR(context));
+  if (profile->get() == nullptr) return JXL_FAILURE("Failed to create XYZ");
+  return true;
+}
+
+// Multi-Localized Unicode string
+class MLU {
+ public:
+  MLU(const cmsContext context, const char* ascii)
+      : mlu_(cmsMLUalloc(context, 0)) {
+    if (!cmsMLUsetASCII(mlu_, "en", "US", ascii)) {
+      JXL_NOTIFY_ERROR("Failed to set ASCII");
+    }
+  }
+  ~MLU() { cmsMLUfree(mlu_); }
+
+  MLU(const MLU&) = delete;
+  MLU& operator=(const MLU&) = delete;
+  MLU(MLU&&) = delete;
+  MLU& operator=(MLU&&) = delete;
+
+  cmsMLU* get() const { return mlu_; }
+
+ private:
+  cmsMLU* mlu_;
+};
+
+// Sets header and required tags; called by EncodeProfile.
+Status SetTags(const cmsContext context, const Profile& profile,
+               const std::string& profile_description) {
+  cmsHPROFILE p = profile.get();
+
+  // Header
+  cmsSetHeaderFlags(p, 1);  // embedded
+
+  const MLU copyright(
+      context,
+      "Copyright 2018 Google LLC, CC-BY-SA 3.0 Unported license"
+      "(https://creativecommons.org/licenses/by-sa/3.0/legalcode)");
+  const MLU manufacturer(context, "Google");
+  const MLU model(context, "Image codec");
+  const MLU description(context, profile_description.c_str());
+
+  // Required tags
+  bool all_ok = true;
+  all_ok &= cmsWriteTag(p, cmsSigCopyrightTag, copyright.get());
+  all_ok &= cmsWriteTag(p, cmsSigDeviceMfgDescTag, manufacturer.get());
+  all_ok &= cmsWriteTag(p, cmsSigDeviceModelDescTag, model.get());
+  all_ok &= cmsWriteTag(p, cmsSigProfileDescriptionTag, description.get());
+
+  if (!all_ok) return JXL_FAILURE("Failed to write header/tags");
+  return true;
+}
+
+// Replacement for cmsMD5computeID, which re-encodes the profile and in so
+// doing changes the creator, resulting in different checksums than here.
+// See https://github.com/mm2/Little-CMS/issues/181. This implementation uses
+// newer LCMS features but the checksum matches displaycal's expectation.
+void ComputeProfileID(const cmsContext context, PaddedBytes icc,
+                      uint8_t id[16]) {
+#if !JXL_CMS_OLD_VERSION
+  cmsProfileID profile_id;
+  // Per http://www.color.org/specification/ICC1v43_2010-12.pdf section 7.2.18
+  JXL_ASSERT(icc.size() > 99);
+  memset(icc.data() + 44, 0, 4);   // Profile flags.
+  memset(icc.data() + 64, 0, 4);   // Rendering intent.
+  memset(icc.data() + 84, 0, 16);  // Profile ID.
+  cmsHANDLE md5_handle = cmsMD5alloc(context);
+  cmsMD5add(md5_handle, icc.data(), icc.size());
+  cmsMD5finish(&profile_id, md5_handle);
+  memcpy(id, profile_id.ID8, 16);
+#endif  // !JXL_CMS_OLD_VERSION
+}
+
+Status EncodeProfile(const cmsContext context, const Profile& profile,
+                     const std::string& description, PaddedBytes* icc) {
+  JXL_RETURN_IF_ERROR(SetTags(context, profile, description));
+
+  cmsUInt32Number size = 0;
+  if (!cmsSaveProfileToMem(profile.get(), nullptr, &size)) {
+    return JXL_FAILURE("Failed to get profile size");
+  }
+  JXL_ASSERT(size != 0);
+
+  icc->resize(size);
+  if (!cmsSaveProfileToMem(profile.get(), icc->data(), &size)) {
+    return JXL_FAILURE("Failed to encode profile");
+  }
+  JXL_ASSERT(size == icc->size());
+  ComputeProfileID(context, *icc, icc->data() + 84);
+  return true;
+}
+#endif  // !JPEGXL_ENABLE_SKCMS
+
+#if JPEGXL_ENABLE_SKCMS
+// IMPORTANT: icc must outlive profile.
+Status DecodeProfile(const PaddedBytes& icc, skcms_ICCProfile* const profile) {
+  if (!skcms_Parse(icc.data(), icc.size(), profile)) {
+    return JXL_FAILURE("Failed to parse ICC profile with %zu bytes",
+                       icc.size());
+  }
+  return true;
+}
+#else   // JPEGXL_ENABLE_SKCMS
+Status DecodeProfile(const cmsContext context, const PaddedBytes& icc,
+                     Profile* profile) {
+  profile->reset(cmsOpenProfileFromMemTHR(context, icc.data(), icc.size()));
+  if (profile->get() == nullptr) {
+    return JXL_FAILURE("Failed to decode profile");
+  }
+
+  // WARNING: due to the LCMS MD5 issue mentioned above, many existing
+  // profiles have incorrect MD5, so do not even bother checking them nor
+  // generating warning clutter.
+
+  return true;
+}
+#endif  // JPEGXL_ENABLE_SKCMS
+
+// Definitions for BT.2100-2 transfer functions:
+// "display" is linear light (nits) normalized to [0, 1].
+// "encoded" is a nonlinear encoding (e.g. PQ) in [0, 1].
+// "scene" is a linear function of photon counts, normalized to [0, 1].
+
+// Despite the stated ranges, we need unbounded transfer functions: see
+// http://www.littlecms.com/CIC18_UnboundedCMM.pdf. Inputs can be negative or
+// above 1 due to chromatic adaptation. To avoid severe round-trip errors caused
+// by clamping, we mirror negative inputs via copysign (f(-x) = -f(x), see
+// https://developer.apple.com/documentation/coregraphics/cgcolorspace/1644735-extendedsrgb)
+// and extend the function domains above 1.
+
+// Hybrid Log-Gamma.
+class TF_HLG {
+ public:
+  // EOTF. e = encoded.
+  JXL_INLINE double DisplayFromEncoded(const double e) const {
+    const double lifted = e * (1.0 - kBeta) + kBeta;
+    return OOTF(InvOETF(lifted));
+  }
+
+  // Inverse EOTF. d = display.
+  JXL_INLINE double EncodedFromDisplay(const double d) const {
+    const double lifted = OETF(InvOOTF(d));
+    const double e = (lifted - kBeta) * (1.0 / (1.0 - kBeta));
+    return e;
+  }
+
+ private:
+  // OETF (defines the HLG approach). s = scene, returns encoded.
+  JXL_INLINE double OETF(double s) const {
+    if (s == 0.0) return 0.0;
+    const double original_sign = s;
+    s = std::abs(s);
+
+    if (s <= kDiv12) return std::copysign(std::sqrt(3.0 * s), original_sign);
+
+    const double e = kA * std::log(12 * s - kB) + kC;
+    JXL_ASSERT(e > 0.0);
+    return std::copysign(e, original_sign);
+  }
+
+  // e = encoded, returns scene.
+  JXL_INLINE double InvOETF(double e) const {
+    if (e == 0.0) return 0.0;
+    const double original_sign = e;
+    e = std::abs(e);
+
+    if (e <= 0.5) return std::copysign(e * e * (1.0 / 3), original_sign);
+
+    const double s = (std::exp((e - kC) * kRA) + kB) * kDiv12;
+    JXL_ASSERT(s >= 0);
+    return std::copysign(s, original_sign);
+  }
+
+  // s = scene, returns display.
+  JXL_INLINE double OOTF(const double s) const {
+    // The actual (red channel) OOTF is RD = alpha * YS^(gamma-1) * RS, where
+    // YS = 0.2627 * RS + 0.6780 * GS + 0.0593 * BS. Let alpha = 1 so we return
+    // "display" (normalized [0, 1]) instead of nits. Our transfer function
+    // interface does not allow a dependency on YS. Fortunately, the system
+    // gamma at 334 nits is 1.0, so this reduces to RD = RS.
+    return s;
+  }
+
+  // d = display, returns scene.
+  JXL_INLINE double InvOOTF(const double d) const {
+    return d;  // see OOTF().
+  }
+
+  // Assume 1000:1 contrast @ 200 nits => gamma 0.9
+  static constexpr double kBeta = 0.04;  // = sqrt(3 * contrast^(1/gamma))
+
+  static constexpr double kA = 0.17883277;
+  static constexpr double kRA = 1.0 / kA;
+  static constexpr double kB = 1 - 4 * kA;
+  static constexpr double kC = 0.5599107295;
+  static constexpr double kDiv12 = 1.0 / 12;
+};
+
+// Perceptual Quantization
+class TF_PQ {
+ public:
+  // EOTF (defines the PQ approach). e = encoded.
+  JXL_INLINE double DisplayFromEncoded(double e) const {
+    if (e == 0.0) return 0.0;
+    const double original_sign = e;
+    e = std::abs(e);
+
+    const double xp = std::pow(e, 1.0 / kM2);
+    const double num = std::max(xp - kC1, 0.0);
+    const double den = kC2 - kC3 * xp;
+    JXL_ASSERT(den != 0.0);
+    const double d = std::pow(num / den, 1.0 / kM1);
+    JXL_ASSERT(d >= 0.0);  // Equal for e ~= 1E-9
+    return std::copysign(d, original_sign);
+  }
+
+  // Inverse EOTF. d = display.
+  JXL_INLINE double EncodedFromDisplay(double d) const {
+    if (d == 0.0) return 0.0;
+    const double original_sign = d;
+    d = std::abs(d);
+
+    const double xp = std::pow(d, kM1);
+    const double num = kC1 + xp * kC2;
+    const double den = 1.0 + xp * kC3;
+    const double e = std::pow(num / den, kM2);
+    JXL_ASSERT(e > 0.0);
+    return std::copysign(e, original_sign);
+  }
+
+ private:
+  static constexpr double kM1 = 2610.0 / 16384;
+  static constexpr double kM2 = (2523.0 / 4096) * 128;
+  static constexpr double kC1 = 3424.0 / 4096;
+  static constexpr double kC2 = (2413.0 / 4096) * 32;
+  static constexpr double kC3 = (2392.0 / 4096) * 32;
+};
+
+// sRGB
+class TF_SRGB {
+ public:
+  template <typename V>
+  HWY_ATTR JXL_INLINE V DisplayFromEncoded(V x) const {
+    const HWY_FULL(float) d;
+    const HWY_FULL(uint32_t) du;
+    const V kSign = BitCast(d, Set(du, 0x80000000u));
+    const V original_sign = x & kSign;
+    x = AndNot(kSign, x);  // abs
+
+    // Computed via af_cheb_rational (k=100); replicated 4x.
+    HWY_ALIGN constexpr float p[(4 + 1) * 4] = {
+        2.200248328e-04, 2.200248328e-04, 2.200248328e-04, 2.200248328e-04,
+        1.043637593e-02, 1.043637593e-02, 1.043637593e-02, 1.043637593e-02,
+        1.624820318e-01, 1.624820318e-01, 1.624820318e-01, 1.624820318e-01,
+        7.961564959e-01, 7.961564959e-01, 7.961564959e-01, 7.961564959e-01,
+        8.210152774e-01, 8.210152774e-01, 8.210152774e-01, 8.210152774e-01,
+    };
+    HWY_ALIGN constexpr float q[(4 + 1) * 4] = {
+        2.631846970e-01,  2.631846970e-01,  2.631846970e-01,  2.631846970e-01,
+        1.076976492e+00,  1.076976492e+00,  1.076976492e+00,  1.076976492e+00,
+        4.987528350e-01,  4.987528350e-01,  4.987528350e-01,  4.987528350e-01,
+        -5.512498495e-02, -5.512498495e-02, -5.512498495e-02, -5.512498495e-02,
+        6.521209011e-03,  6.521209011e-03,  6.521209011e-03,  6.521209011e-03,
+    };
+    const V linear = x * Set(d, kLowDivInv);
+    const V poly = EvalRationalPolynomial(x, p, q);
+    const V magnitude =
+        IfThenElse(x > Set(d, kThreshSRGBToLinear), poly, linear);
+    return AndNot(kSign, magnitude) | original_sign;
+  }
+
+  template <class V>
+  HWY_ATTR JXL_INLINE V EncodedFromDisplay(V x) const {
+    const HWY_FULL(float) d;
+    const HWY_FULL(uint32_t) du;
+    const V kSign = BitCast(d, Set(du, 0x80000000u));
+    const V original_sign = x & kSign;
+    x = AndNot(kSign, x);  // abs
+
+    // Computed via af_cheb_rational (k=100); replicated 4x.
+    HWY_ALIGN constexpr float p[(4 + 1) * 4] = {
+        -5.135152395e-04, -5.135152395e-04, -5.135152395e-04, -5.135152395e-04,
+        5.287254571e-03,  5.287254571e-03,  5.287254571e-03,  5.287254571e-03,
+        3.903842876e-01,  3.903842876e-01,  3.903842876e-01,  3.903842876e-01,
+        1.474205315e+00,  1.474205315e+00,  1.474205315e+00,  1.474205315e+00,
+        7.352629620e-01,  7.352629620e-01,  7.352629620e-01,  7.352629620e-01,
+    };
+    HWY_ALIGN constexpr float q[(4 + 1) * 4] = {
+        1.004519624e-02, 1.004519624e-02, 1.004519624e-02, 1.004519624e-02,
+        3.036675394e-01, 3.036675394e-01, 3.036675394e-01, 3.036675394e-01,
+        1.340816930e+00, 1.340816930e+00, 1.340816930e+00, 1.340816930e+00,
+        9.258482155e-01, 9.258482155e-01, 9.258482155e-01, 9.258482155e-01,
+        2.424867759e-02, 2.424867759e-02, 2.424867759e-02, 2.424867759e-02,
+    };
+    const V linear = x * Set(d, kLowDiv);
+    const V poly = EvalRationalPolynomial(Sqrt(x), p, q);
+    const V magnitude =
+        IfThenElse(x > Set(d, kThreshLinearToSRGB), poly, linear);
+    return AndNot(kSign, magnitude) | original_sign;
+  }
+
+ private:
+  static constexpr float kThreshSRGBToLinear = 0.04045f;
+  static constexpr float kThreshLinearToSRGB = 0.0031308f;
+  static constexpr float kLowDiv = 12.92f;
+  static constexpr float kLowDivInv = 1.0f / kLowDiv;
+};
+
+#if JPEGXL_ENABLE_SKCMS
+
+// NOTE: this is only used to provide a reasonable ICC profile that other
+// software can read. Our own transforms use ExtraTF instead because that is
+// more precise and supports unbounded mode.
+template <class Func>
+std::vector<uint16_t> CreateTableCurve(uint32_t N, const Func& func) {
+  JXL_ASSERT(N <= 4096);  // ICC MFT2 only allows 4K entries
+  // No point using float - LCMS converts to 16-bit for A2B/MFT.
+  std::vector<uint16_t> table(N);
+  for (uint32_t i = 0; i < N; ++i) {
+    const float x = static_cast<float>(i) / (N - 1);  // 1.0 at index N - 1.
+    // LCMS requires EOTF (e.g. 2.4 exponent).
+    double y = func.DisplayFromEncoded(static_cast<double>(x));
+    JXL_ASSERT(y >= 0.0);
+    // Clamp to table range - necessary for HLG.
+    if (y > 1.0) y = 1.0;
+    // 1.0 corresponds to table value 0xFFFF.
+    table[i] = static_cast<uint16_t>(std::round(y * 65535.0));
+  }
+  return table;
+}
+
+void ICCComputeMD5(const PaddedBytes& data, uint8_t sum[16]) {
+  PaddedBytes data64 = data;
+  data64.push_back(128);
+  // Add bytes such that ((size + 8) & 63) == 0.
+  size_t extra = ((64 - ((data64.size() + 8) & 63)) & 63);
+  data64.resize(data64.size() + extra, 0);
+  for (uint64_t i = 0; i < 64; i += 8) {
+    data64.push_back((data.size() << 3u) >> i);
+  }
+
+  static const uint32_t sineparts[64] = {
+      0xd76aa478, 0xe8c7b756, 0x242070db, 0xc1bdceee, 0xf57c0faf, 0x4787c62a,
+      0xa8304613, 0xfd469501, 0x698098d8, 0x8b44f7af, 0xffff5bb1, 0x895cd7be,
+      0x6b901122, 0xfd987193, 0xa679438e, 0x49b40821, 0xf61e2562, 0xc040b340,
+      0x265e5a51, 0xe9b6c7aa, 0xd62f105d, 0x02441453, 0xd8a1e681, 0xe7d3fbc8,
+      0x21e1cde6, 0xc33707d6, 0xf4d50d87, 0x455a14ed, 0xa9e3e905, 0xfcefa3f8,
+      0x676f02d9, 0x8d2a4c8a, 0xfffa3942, 0x8771f681, 0x6d9d6122, 0xfde5380c,
+      0xa4beea44, 0x4bdecfa9, 0xf6bb4b60, 0xbebfbc70, 0x289b7ec6, 0xeaa127fa,
+      0xd4ef3085, 0x04881d05, 0xd9d4d039, 0xe6db99e5, 0x1fa27cf8, 0xc4ac5665,
+      0xf4292244, 0x432aff97, 0xab9423a7, 0xfc93a039, 0x655b59c3, 0x8f0ccc92,
+      0xffeff47d, 0x85845dd1, 0x6fa87e4f, 0xfe2ce6e0, 0xa3014314, 0x4e0811a1,
+      0xf7537e82, 0xbd3af235, 0x2ad7d2bb, 0xeb86d391,
+  };
+  static const uint32_t shift[64] = {
+      7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22, 7, 12, 17, 22,
+      5, 9,  14, 20, 5, 9,  14, 20, 5, 9,  14, 20, 5, 9,  14, 20,
+      4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23, 4, 11, 16, 23,
+      6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21, 6, 10, 15, 21,
+  };
+
+  uint32_t a0 = 0x67452301, b0 = 0xefcdab89, c0 = 0x98badcfe, d0 = 0x10325476;
+
+  for (size_t i = 0; i < data64.size(); i += 64) {
+    uint32_t a = a0, b = b0, c = c0, d = d0, f, g;
+    for (size_t j = 0; j < 64; j++) {
+      if (j < 16) {
+        f = (b & c) | ((~b) & d);
+        g = j;
+      } else if (j < 32) {
+        f = (d & b) | ((~d) & c);
+        g = (5 * j + 1) & 0xf;
+      } else if (j < 48) {
+        f = b ^ c ^ d;
+        g = (3 * j + 5) & 0xf;
+      } else {
+        f = c ^ (b | (~d));
+        g = (7 * j) & 0xf;
+      }
+      uint32_t dg0 = data64[i + g * 4 + 0], dg1 = data64[i + g * 4 + 1],
+               dg2 = data64[i + g * 4 + 2], dg3 = data64[i + g * 4 + 3];
+      uint32_t u = dg0 | (dg1 << 8u) | (dg2 << 16u) | (dg3 << 24u);
+      f += a + sineparts[j] + u;
+      a = d;
+      d = c;
+      c = b;
+      b += (f << shift[j]) | (f >> (32u - shift[j]));
+    }
+    a0 += a;
+    b0 += b;
+    c0 += c;
+    d0 += d;
+  }
+  sum[0] = a0;
+  sum[1] = a0 >> 8u;
+  sum[2] = a0 >> 16u;
+  sum[3] = a0 >> 24u;
+  sum[4] = b0;
+  sum[5] = b0 >> 8u;
+  sum[6] = b0 >> 16u;
+  sum[7] = b0 >> 24u;
+  sum[8] = c0;
+  sum[9] = c0 >> 8u;
+  sum[10] = c0 >> 16u;
+  sum[11] = c0 >> 24u;
+  sum[12] = d0;
+  sum[13] = d0 >> 8u;
+  sum[14] = d0 >> 16u;
+  sum[15] = d0 >> 24u;
+}
+
+Status CreateICCChadMatrix(CIExy w, float result[9]) {
+  skcms_Matrix3x3 m;
+  JXL_RETURN_IF_ERROR(skcms_AdaptToXYZD50(w.x, w.y, &m));
+  memcpy(result, m.vals, sizeof(float) * 9);
+  return true;
+}
+
+// Creates RGB to XYZ matrix given RGB primaries and whitepoint in xy.
+Status CreateICCRGBMatrix(CIExy r, CIExy g, CIExy b, CIExy w, float result[9]) {
+  skcms_Matrix3x3 m;
+  JXL_RETURN_IF_ERROR(
+      skcms_PrimariesToXYZD50(r.x, r.y, g.x, g.y, b.x, b.y, w.x, w.y, &m));
+  memcpy(result, m.vals, sizeof(float) * 9);
+  return true;
+}
+
+void WriteICCUint32(uint32_t value, size_t pos, PaddedBytes* JXL_RESTRICT icc) {
+  if (icc->size() < pos + 4) icc->resize(pos + 4);
+  (*icc)[pos + 0] = (value >> 24u) & 255;
+  (*icc)[pos + 1] = (value >> 16u) & 255;
+  (*icc)[pos + 2] = (value >> 8u) & 255;
+  (*icc)[pos + 3] = value & 255;
+}
+
+void WriteICCUint16(uint16_t value, size_t pos, PaddedBytes* JXL_RESTRICT icc) {
+  if (icc->size() < pos + 2) icc->resize(pos + 2);
+  (*icc)[pos + 0] = (value >> 8u) & 255;
+  (*icc)[pos + 1] = value & 255;
+}
+
+// Writes a 4-character tag
+void WriteICCTag(const char* value, size_t pos, PaddedBytes* JXL_RESTRICT icc) {
+  if (icc->size() < pos + 4) icc->resize(pos + 4);
+  memcpy(icc->data() + pos, value, 4);
+}
+
+void WriteICCS15Fixed16(float value, size_t pos,
+                        PaddedBytes* JXL_RESTRICT icc) {
+  int32_t i = value * 65536.0f + 0.5f;
+  // Use two's complement
+  uint32_t u = static_cast<uint32_t>(i);
+  WriteICCUint32(u, pos, icc);
+}
+
+Status CreateICCHeader(const ColorEncoding& c,
+                       PaddedBytes* JXL_RESTRICT header) {
+  // TODO(lode): choose color management engine name, e.g. "skia" if
+  // integrated in skia.
+  static const char* kCmm = "jxl ";
+
+  header->resize(128, 0);
+
+  WriteICCUint32(0, 0, header);  // size, correct value filled in at end
+  WriteICCTag(kCmm, 4, header);
+  WriteICCUint32(0x04300000u, 8, header);
+  WriteICCTag("mntr", 12, header);
+  WriteICCTag(c.IsGray() ? "GRAY" : "RGB ", 16, header);
+  WriteICCTag("XYZ ", 20, header);
+
+  // Three uint32_t's date/time encoding.
+  // TODO(lode): encode actual date and time, this is a placeholder
+  uint32_t year = 2019, month = 12, day = 1;
+  uint32_t hour = 0, minute = 0, second = 0;
+  WriteICCUint16(year, 24, header);
+  WriteICCUint16(month, 26, header);
+  WriteICCUint16(day, 28, header);
+  WriteICCUint16(hour, 30, header);
+  WriteICCUint16(minute, 32, header);
+  WriteICCUint16(second, 34, header);
+
+  WriteICCTag("acsp", 36, header);
+  WriteICCTag("APPL", 40, header);
+  WriteICCUint32(0, 44, header);  // flags
+  WriteICCUint32(0, 48, header);  // device manufacturer
+  WriteICCUint32(0, 52, header);  // device model
+  WriteICCUint32(0, 56, header);  // device attributes
+  WriteICCUint32(0, 60, header);  // device attributes
+  WriteICCUint32(static_cast<uint32_t>(c.rendering_intent), 64, header);
+
+  // Mandatory D50 white point of profile connection space
+  WriteICCUint32(0x0000f6d6, 68, header);
+  WriteICCUint32(0x00010000, 72, header);
+  WriteICCUint32(0x0000d32d, 76, header);
+
+  WriteICCTag(kCmm, 80, header);
+
+  return true;
+}
+
+void AddToICCTagTable(const char* tag, size_t offset, size_t size,
+                      PaddedBytes* JXL_RESTRICT tagtable,
+                      std::vector<size_t>* offsets) {
+  WriteICCTag(tag, tagtable->size(), tagtable);
+  // writing true offset deferred to later
+  WriteICCUint32(0, tagtable->size(), tagtable);
+  offsets->push_back(offset);
+  WriteICCUint32(size, tagtable->size(), tagtable);
+}
+
+void FinalizeICCTag(PaddedBytes* JXL_RESTRICT tags, size_t* offset,
+                    size_t* size) {
+  while ((tags->size() & 3) != 0) {
+    tags->push_back(0);
+  }
+  *offset += *size;
+  *size = tags->size() - *offset;
+}
+
+// The input text must be ASCII, writing other characters to UTF-16 is not
+// implemented.
+void CreateICCMlucTag(const std::string& text, PaddedBytes* JXL_RESTRICT tags) {
+  WriteICCTag("mluc", tags->size(), tags);
+  WriteICCUint32(0, tags->size(), tags);
+  WriteICCUint32(1, tags->size(), tags);
+  WriteICCUint32(12, tags->size(), tags);
+  WriteICCTag("enUS", tags->size(), tags);
+  WriteICCUint32(text.size() * 2, tags->size(), tags);
+  WriteICCUint32(28, tags->size(), tags);
+  for (size_t i = 0; i < text.size(); i++) {
+    tags->push_back(0);  // prepend 0 for UTF-16
+    tags->push_back(text[i]);
+  }
+}
+
+void CreateICCXYZTag(float xyz[3], PaddedBytes* JXL_RESTRICT tags) {
+  WriteICCTag("XYZ ", tags->size(), tags);
+  WriteICCUint32(0, tags->size(), tags);
+  WriteICCS15Fixed16(xyz[0], tags->size(), tags);
+  WriteICCS15Fixed16(xyz[1], tags->size(), tags);
+  WriteICCS15Fixed16(xyz[2], tags->size(), tags);
+}
+
+void CreateICCChadTag(float chad[9], PaddedBytes* JXL_RESTRICT tags) {
+  WriteICCTag("sf32", tags->size(), tags);
+  WriteICCUint32(0, tags->size(), tags);
+  for (size_t i = 0; i < 9; i++) {
+    WriteICCS15Fixed16(chad[i], tags->size(), tags);
+  }
+}
+
+void CreateICCCurvCurvTag(const std::vector<uint16_t>& curve,
+                          PaddedBytes* JXL_RESTRICT tags) {
+  size_t pos = tags->size();
+  tags->resize(tags->size() + 12 + curve.size() * 2, 0);
+  WriteICCTag("curv", pos, tags);
+  WriteICCUint32(0, pos + 4, tags);
+  WriteICCUint32(curve.size(), pos + 8, tags);
+  for (size_t i = 0; i < curve.size(); i++) {
+    WriteICCUint16(curve[i], pos + 12 + i * 2, tags);
+  }
+}
+
+void CreateICCCurvParaTag(std::vector<float> params, size_t curve_type,
+                          PaddedBytes* JXL_RESTRICT tags) {
+  WriteICCTag("para", tags->size(), tags);
+  WriteICCUint32(0, tags->size(), tags);
+  WriteICCUint16(curve_type, tags->size(), tags);
+  WriteICCUint16(0, tags->size(), tags);
+  for (size_t i = 0; i < params.size(); i++) {
+    WriteICCS15Fixed16(params[i], tags->size(), tags);
+  }
+}
+
+Status MaybeCreateProfile(const ColorEncoding& c,
+                          PaddedBytes* JXL_RESTRICT icc) {
+  PaddedBytes header, tagtable, tags;
+
+  // Unknown color_space/transfer_function.
+  if (c.opaque_icc) return false;  // Not an error
+
+  switch (c.GetColorSpace()) {
+    case ColorSpace::kRGB:
+    case ColorSpace::kGray:
+      break;  // OK
+    case ColorSpace::kXYB:
+      return JXL_FAILURE("XYB ICC not yet implemented");
+    case ColorSpace::kUnknown:
+    default:
+      return JXL_FAILURE("Unknown CS despite !opaque_icc");
+  }
+
+  JXL_RETURN_IF_ERROR(CreateICCHeader(c, &header));
+
+  std::vector<size_t> offsets;
+  // tag count, deferred to later
+  WriteICCUint32(0, tagtable.size(), &tagtable);
+
+  size_t tag_offset = 0, tag_size = 0;
+
+  CreateICCMlucTag(Description(c), &tags);
+  FinalizeICCTag(&tags, &tag_offset, &tag_size);
+  AddToICCTagTable("desc", tag_offset, tag_size, &tagtable, &offsets);
+
+  const std::string copyright =
+      "Copyright 2019 Google LLC, CC-BY-SA 3.0 Unported "
+      "license(https://creativecommons.org/licenses/by-sa/3.0/legalcode)";
+  CreateICCMlucTag(copyright, &tags);
+  FinalizeICCTag(&tags, &tag_offset, &tag_size);
+  AddToICCTagTable("cprt", tag_offset, tag_size, &tagtable, &offsets);
+
+  float d50[3] = {0.964203, 1.0, 0.824905};
+  float wtpt[3];
+  CIEXYZFromWhiteCIExy(c.GetWhitePoint(), wtpt);
+  CreateICCXYZTag(c.IsGray() ? wtpt : d50, &tags);
+  FinalizeICCTag(&tags, &tag_offset, &tag_size);
+  AddToICCTagTable("wtpt", tag_offset, tag_size, &tagtable, &offsets);
+
+  if (!c.IsGray()) {
+    // Chromatic adaptation matrix
+    float chad[9];
+    JXL_RETURN_IF_ERROR(CreateICCChadMatrix(c.GetWhitePoint(), chad));
+
+    const PrimariesCIExy primaries = c.GetPrimaries();
+    float m[9];
+    JXL_RETURN_IF_ERROR(CreateICCRGBMatrix(primaries.r, primaries.g,
+                                           primaries.b, c.GetWhitePoint(), m));
+    float r[3] = {m[0], m[3], m[6]};
+    float g[3] = {m[1], m[4], m[7]};
+    float b[3] = {m[2], m[5], m[8]};
+
+    CreateICCChadTag(chad, &tags);
+    FinalizeICCTag(&tags, &tag_offset, &tag_size);
+    AddToICCTagTable("chad", tag_offset, tag_size, &tagtable, &offsets);
+
+    CreateICCXYZTag(r, &tags);
+    FinalizeICCTag(&tags, &tag_offset, &tag_size);
+    AddToICCTagTable("rXYZ", tag_offset, tag_size, &tagtable, &offsets);
+
+    CreateICCXYZTag(g, &tags);
+    FinalizeICCTag(&tags, &tag_offset, &tag_size);
+    AddToICCTagTable("gXYZ", tag_offset, tag_size, &tagtable, &offsets);
+
+    CreateICCXYZTag(b, &tags);
+    FinalizeICCTag(&tags, &tag_offset, &tag_size);
+    AddToICCTagTable("bXYZ", tag_offset, tag_size, &tagtable, &offsets);
+  }
+
+  if (c.tf.IsGamma()) {
+    float gamma = 1.0 / c.tf.GetGamma();
+    CreateICCCurvParaTag({gamma, 1.0, 0.0, 1.0, 0.0}, 3, &tags);
+  } else {
+    switch (c.tf.GetTransferFunction()) {
+      case TransferFunction::kHLG:
+        CreateICCCurvCurvTag(CreateTableCurve(4096, TF_HLG()), &tags);
+        break;
+      case TransferFunction::kPQ:
+        CreateICCCurvCurvTag(CreateTableCurve(4096, TF_PQ()), &tags);
+        break;
+      case TransferFunction::kSRGB:
+        CreateICCCurvParaTag(
+            {2.4, 1.0 / 1.055, 0.055 / 1.055, 1.0 / 12.92, 0.04045}, 3, &tags);
+        break;
+      case TransferFunction::k709:
+        CreateICCCurvParaTag(
+            {1.0 / 0.45, 1.0 / 1.099, 0.099 / 1.099, 1.0 / 4.5, 0.081}, 3,
+            &tags);
+        break;
+      case TransferFunction::kLinear:
+        CreateICCCurvParaTag({1.0, 1.0, 0.0, 1.0, 0.0}, 3, &tags);
+        break;
+      case TransferFunction::kDCI:
+        CreateICCCurvParaTag({2.6, 1.0, 0.0, 1.0, 0.0}, 3, &tags);
+        break;
+      case TransferFunction::kUnknown:
+      default:
+        JXL_ABORT("Unknown TF despite !opaque_icc");
+    }
+  }
+  FinalizeICCTag(&tags, &tag_offset, &tag_size);
+  if (c.IsGray()) {
+    AddToICCTagTable("kTRC", tag_offset, tag_size, &tagtable, &offsets);
+  } else {
+    AddToICCTagTable("rTRC", tag_offset, tag_size, &tagtable, &offsets);
+    AddToICCTagTable("gTRC", tag_offset, tag_size, &tagtable, &offsets);
+    AddToICCTagTable("bTRC", tag_offset, tag_size, &tagtable, &offsets);
+  }
+
+  // Tag count
+  WriteICCUint32(offsets.size(), 0, &tagtable);
+  for (size_t i = 0; i < offsets.size(); i++) {
+    WriteICCUint32(offsets[i] + header.size() + tagtable.size(), 4 + 12 * i + 4,
+                   &tagtable);
+  }
+
+  // ICC profile size
+  WriteICCUint32(header.size() + tagtable.size() + tags.size(), 0, &header);
+
+  *icc = header;
+  icc->append(tagtable);
+  icc->append(tags);
+
+  // The MD5 checksum must be computed on the profile with profile flags,
+  // rendering intent, and region of the checksum itself, set to 0.
+  // TODO(lode): manually verify with a reliable tool that this creates correct
+  // signature (profile id) for ICC profiles.
+  PaddedBytes icc_sum = *icc;
+  memset(icc_sum.data() + 44, 0, 4);
+  memset(icc_sum.data() + 64, 0, 4);
+  uint8_t checksum[16];
+  ICCComputeMD5(icc_sum, checksum);
+
+  memcpy(icc->data() + 84, checksum, sizeof(checksum));
+
+  return true;
+}
+
+Status MaybeCreateProfile(const ColorEncoding& c, skcms_ICCProfile* profile,
+                          PaddedBytes* bytes) {
+  // Unknown color_space/transfer_function.
+  if (c.opaque_icc) return false;  // Not an error
+
+  JXL_RETURN_IF_ERROR(MaybeCreateProfile(c, bytes));
+  JXL_RETURN_IF_ERROR(skcms_Parse(bytes->data(), bytes->size(), profile));
+
+  return true;
+}
+
+#else  // JPEGXL_ENABLE_SKCMS
+
+// NOTE: this is only used to provide a reasonable ICC profile that other
+// software can read. Our own transforms use ExtraTF instead because that is
+// more precise and supports unbounded mode.
+template <class Func>
+cmsToneCurve* CreateTableCurve(const cmsContext context, uint32_t N,
+                               const Func& func) {
+  JXL_ASSERT(N <= 4096);  // ICC MFT2 only allows 4K entries
+  // No point using float - LCMS converts to 16-bit for A2B/MFT.
+  std::vector<uint16_t> table;
+  table.reserve(N);
+  for (uint32_t i = 0; i < N; ++i) {
+    const float x = static_cast<float>(i) / (N - 1);  // 1.0 at index N - 1.
+    // LCMS requires EOTF (e.g. 2.4 exponent).
+    double y = func.DisplayFromEncoded(static_cast<double>(x));
+    JXL_ASSERT(y >= 0.0);
+    // Clamp to table range - necessary for HLG.
+    if (y > 1.0) y = 1.0;
+    // 1.0 corresponds to table value 0xFFFF.
+    table.push_back(static_cast<uint16_t>(std::round(y * 65535.0)));
+  }
+  return cmsBuildTabulatedToneCurve16(context, N, table.data());
+}
+
+Curve CreateCurve(const cmsContext context, const ColorEncoding& c) {
+  // Exponential with linear part. Note that the LittleCMS API reference and
+  // tutorial disagree on the type number.
+  const cmsUInt32Number type = 4;
+  // First is EOTF exponent as expected by LCMS (c.GetGamma is the OETF, so take
+  // the reciprocal). Then a,b,c,d: (a*x + b)^gamma, or c*x if x < d.
+  std::array<cmsFloat64Number, 5> params;
+
+  if (c.tf.IsGamma()) {
+    params = {1.0 / c.tf.GetGamma(), 1.0, 0.0, 1.0, 0.0};
+    return Curve(cmsBuildParametricToneCurve(context, type, params.data()));
+  }
+  switch (c.tf.GetTransferFunction()) {
+    case TransferFunction::kHLG:
+      return Curve(CreateTableCurve(context, 4096, TF_HLG()));
+    case TransferFunction::kPQ:
+      return Curve(CreateTableCurve(context, 4096, TF_PQ()));
+
+    case TransferFunction::kSRGB:
+      params = {2.4, 1.0 / 1.055, 0.055 / 1.055, 1.0 / 12.92, 0.04045};
+      break;
+    case TransferFunction::k709:
+      params = {1.0 / 0.45, 1.0 / 1.099, 0.099 / 1.099, 1.0 / 4.5, 0.081};
+      break;
+    case TransferFunction::kLinear:
+      params = {1.0, 1.0, 0.0, 1.0, 0.0};
+      break;
+    case TransferFunction::kDCI:
+      params = {2.6, 1.0, 0.0, 1.0, 0.0};
+      break;
+    case TransferFunction::kUnknown:
+      JXL_ABORT("Unknown TF despite !opaque_icc");
+  }
+  // WARNING: using cmsBuildGamma results in a bounded curve - LittleCMS
+  // clamps negative outputs to zero. To retain unbounded mode, we use the
+  // same parametric curve type as sRGB.
+  return Curve(cmsBuildParametricToneCurve(context, type, params.data()));
+}
+
+Profile MaybeCreateProfileRGB(const cmsContext context, const ColorEncoding& c,
+                              const cmsCIExyY& wp_xyY, const Curve& curve) {
+  const PrimariesCIExy primaries = c.GetPrimaries();
+  const cmsCIExyYTRIPLE primaries_xyY = {xyYFromCIExy(primaries.r),
+                                         xyYFromCIExy(primaries.g),
+                                         xyYFromCIExy(primaries.b)};
+  cmsToneCurve* curves[3] = {curve.get(), curve.get(), curve.get()};
+  return Profile(
+      cmsCreateRGBProfileTHR(context, &wp_xyY, &primaries_xyY, curves));
+}
+
+// Serializes the profile before use to ensure all values are quantized.
+Status MaybeCreateProfile(const cmsContext context, const ColorEncoding& c,
+                          PaddedBytes* JXL_RESTRICT icc) {
+  // Unknown color_space/transfer_function.
+  if (c.opaque_icc) return false;  // Not an error
+
+  // (If color_space == kRGB, we'll use this curve for all channels.)
+  const Curve curve = CreateCurve(context, c);
+  if (curve == nullptr) return JXL_FAILURE("Failed to create curve");
+
+  const cmsCIExyY wp_xyY = xyYFromCIExy(c.GetWhitePoint());
+
+  Profile profile;
+  switch (c.GetColorSpace()) {
+    case ColorSpace::kRGB:
+      profile = MaybeCreateProfileRGB(context, c, wp_xyY, curve);
+      break;
+    case ColorSpace::kGray:
+      profile.reset(cmsCreateGrayProfileTHR(context, &wp_xyY, curve.get()));
+      break;
+    case ColorSpace::kXYB:
+      JXL_FAILURE("XYB ICC not yet implemented");
+      break;
+    case ColorSpace::kUnknown:
+      return JXL_FAILURE("Unknown CS despite !opaque_icc");
+  }
+  if (profile.get() == nullptr) {
+    return JXL_FAILURE("Failed to create profile (cs = %u)", c.GetColorSpace());
+  }
+
+  // ICC uses the same values.
+  cmsSetHeaderRenderingIntent(profile.get(),
+                              static_cast<cmsUInt32Number>(c.rendering_intent));
+
+  return EncodeProfile(context, profile, Description(c), icc);
+}
+
+#endif  // JPEGXL_ENABLE_SKCMS
+
+#if JPEGXL_ENABLE_SKCMS
+
+// Sets *opaque_icc = true if color space is unknown.
+ColorSpace ColorSpaceFromProfile(const skcms_ICCProfile& profile,
+                                 bool* opaque_icc) {
+  switch (profile.data_color_space) {
+    case skcms_Signature_RGB:
+      return ColorSpace::kRGB;
+    case skcms_Signature_Gray:
+      return ColorSpace::kGray;
+    default:
+      *opaque_icc = true;
+      return ColorSpace::kUnknown;
+  }
+}
+
+// "profile1" is pre-decoded to save time in DetectTransferFunction.
+Status ProfileEquivalentToICC(const skcms_ICCProfile& profile1,
+                              const PaddedBytes& icc) {
+  skcms_ICCProfile profile2;
+  JXL_RETURN_IF_ERROR(skcms_Parse(icc.data(), icc.size(), &profile2));
+  return skcms_ApproximatelyEqualProfiles(&profile1, &profile2);
+}
+
+// vector_out := matmul(matrix, vector_in)
+void MatrixProduct(const skcms_Matrix3x3& matrix, const float vector_in[3],
+                   float vector_out[3]) {
+  for (int i = 0; i < 3; ++i) {
+    vector_out[i] = 0;
+    for (int j = 0; j < 3; ++j) {
+      vector_out[i] += matrix.vals[i][j] * vector_in[j];
+    }
+  }
+}
+
+// Returns white point that was specified when creating the profile.
+JXL_MUST_USE_RESULT CIExy UnadaptedWhitePoint(const skcms_ICCProfile& profile) {
+  float media_white_point_XYZ[3];
+  JXL_CHECK(skcms_GetWTPT(&profile, media_white_point_XYZ));
+  skcms_Matrix3x3 CHAD;
+  if (!skcms_GetCHAD(&profile, &CHAD)) {
+    // If there is no chromatic adaptation matrix, it means that the white point
+    // is already unadapted.
+    return CIExyFromXYZ(media_white_point_XYZ);
+  }
+  // Otherwise, it has been adapted to the PCS white point using said matrix,
+  // and the adaptation needs to be undone.
+  skcms_Matrix3x3 inverse_CHAD;
+  JXL_CHECK(skcms_Matrix3x3_invert(&CHAD, &inverse_CHAD));
+  float unadapted_white_point_XYZ[3];
+  MatrixProduct(inverse_CHAD, media_white_point_XYZ, unadapted_white_point_XYZ);
+  return CIExyFromXYZ(unadapted_white_point_XYZ);
+}
+
+Status IdentifyPrimaries(const skcms_ICCProfile& profile,
+                         const CIExy& wp_unadapted, ColorEncoding* c) {
+  if (!c->HasPrimaries()) return true;
+
+  skcms_Matrix3x3 CHAD, inverse_CHAD;
+  if (skcms_GetCHAD(&profile, &CHAD)) {
+    JXL_RETURN_IF_ERROR(skcms_Matrix3x3_invert(&CHAD, &inverse_CHAD));
+  } else {
+    static constexpr skcms_Matrix3x3 kLMSFromXYZ = {
+        {{0.8951, 0.2664, -0.1614},
+         {-0.7502, 1.7135, 0.0367},
+         {0.0389, -0.0685, 1.0296}}};
+    static constexpr skcms_Matrix3x3 kXYZFromLMS = {
+        {{0.9869929, -0.1470543, 0.1599627},
+         {0.4323053, 0.5183603, 0.0492912},
+         {-0.0085287, 0.0400428, 0.9684867}}};
+    static constexpr float kWpD50XYZ[3] = {0.96420288, 1.0, 0.82490540};
+    float wp_unadapted_XYZ[3];
+    CIEXYZFromWhiteCIExy(wp_unadapted, wp_unadapted_XYZ);
+    float wp_D50_LMS[3], wp_unadapted_LMS[3];
+    MatrixProduct(kLMSFromXYZ, kWpD50XYZ, wp_D50_LMS);
+    MatrixProduct(kLMSFromXYZ, wp_unadapted_XYZ, wp_unadapted_LMS);
+    inverse_CHAD = {{{wp_unadapted_LMS[0] / wp_D50_LMS[0], 0, 0},
+                     {0, wp_unadapted_LMS[1] / wp_D50_LMS[1], 0},
+                     {0, 0, wp_unadapted_LMS[2] / wp_D50_LMS[2]}}};
+    inverse_CHAD = skcms_Matrix3x3_concat(&kXYZFromLMS, &inverse_CHAD);
+    inverse_CHAD = skcms_Matrix3x3_concat(&inverse_CHAD, &kLMSFromXYZ);
+  }
+
+  float XYZ[3];
+  PrimariesCIExy primaries;
+  CIExy* const chromaticities[] = {&primaries.r, &primaries.g, &primaries.b};
+  for (int i = 0; i < 3; ++i) {
+    float RGB[3] = {};
+    RGB[i] = 1;
+    skcms_Transform(RGB, skcms_PixelFormat_RGB_fff, skcms_AlphaFormat_Opaque,
+                    &profile, XYZ, skcms_PixelFormat_RGB_fff,
+                    skcms_AlphaFormat_Opaque, skcms_XYZD50_profile(), 1);
+    float unadapted_XYZ[3];
+    MatrixProduct(inverse_CHAD, XYZ, unadapted_XYZ);
+    *chromaticities[i] = CIExyFromXYZ(unadapted_XYZ);
+  }
+  return c->SetPrimaries(primaries);
+}
+
+void DetectTransferFunction(const skcms_ICCProfile& profile,
+                            ColorEncoding* JXL_RESTRICT c) {
+  if (c->tf.SetImplicit()) return;
+
+  for (TransferFunction tf : Values<TransferFunction>()) {
+    // Can only create profile from known transfer function.
+    if (tf == TransferFunction::kUnknown) continue;
+
+    c->tf.SetTransferFunction(tf);
+
+    skcms_ICCProfile profile_test;
+    PaddedBytes bytes;
+    if (MaybeCreateProfile(*c, &profile_test, &bytes) &&
+        skcms_ApproximatelyEqualProfiles(&profile, &profile_test)) {
+      return;
+    }
+  }
+
+  c->tf.SetTransferFunction(TransferFunction::kUnknown);
+  c->opaque_icc = true;
+}
+
+#else  // JPEGXL_ENABLE_SKCMS
+
+uint32_t Type32(const ColorEncoding& c) {
+  if (c.IsGray()) return TYPE_GRAY_FLT;
+  return TYPE_RGB_FLT;
+}
+
+uint32_t Type64(const ColorEncoding& c) {
+  if (c.IsGray()) return TYPE_GRAY_DBL;
+  return TYPE_RGB_DBL;
+}
+
+// Sets *opaque_icc = true if color space is unknown.
+ColorSpace ColorSpaceFromProfile(const Profile& profile, bool* opaque_icc) {
+  switch (cmsGetColorSpace(profile.get())) {
+    case cmsSigRgbData:
+      return ColorSpace::kRGB;
+    case cmsSigGrayData:
+      return ColorSpace::kGray;
+    default:
+      *opaque_icc = true;
+      return ColorSpace::kUnknown;
+  }
+}
+
+// "profile1" is pre-decoded to save time in DetectTransferFunction.
+Status ProfileEquivalentToICC(const cmsContext context, const Profile& profile1,
+                              const PaddedBytes& icc, const ColorEncoding& c) {
+  const uint32_t type_src = Type64(c);
+
+  Profile profile2;
+  JXL_RETURN_IF_ERROR(DecodeProfile(context, icc, &profile2));
+
+  Profile profile_xyz;
+  JXL_RETURN_IF_ERROR(CreateProfileXYZ(context, &profile_xyz));
+
+  const uint32_t intent = INTENT_RELATIVE_COLORIMETRIC;
+  const uint32_t flags = cmsFLAGS_NOOPTIMIZE | cmsFLAGS_BLACKPOINTCOMPENSATION |
+                         cmsFLAGS_HIGHRESPRECALC;
+  Transform xform1(cmsCreateTransformTHR(context, profile1.get(), type_src,
+                                         profile_xyz.get(), TYPE_XYZ_DBL,
+                                         intent, flags));
+  Transform xform2(cmsCreateTransformTHR(context, profile2.get(), type_src,
+                                         profile_xyz.get(), TYPE_XYZ_DBL,
+                                         intent, flags));
+  if (xform1 == nullptr || xform2 == nullptr) {
+    return JXL_FAILURE("Failed to create transform");
+  }
+
+  double in[3];
+  double out1[3];
+  double out2[3];
+
+  // Uniformly spaced samples from very dark to almost fully bright.
+  const double init = 1E-3;
+  const double step = 0.2;
+
+  if (c.IsGray()) {
+    // Finer sampling and replicate each component.
+    for (in[0] = init; in[0] < 1.0; in[0] += step / 8) {
+      cmsDoTransform(xform1.get(), in, out1, 1);
+      cmsDoTransform(xform2.get(), in, out2, 1);
+      if (!ApproxEq(out1[0], out2[0], 2E-4)) {
+        return false;
+      }
+    }
+  } else {
+    for (in[0] = init; in[0] < 1.0; in[0] += step) {
+      for (in[1] = init; in[1] < 1.0; in[1] += step) {
+        for (in[2] = init; in[2] < 1.0; in[2] += step) {
+          cmsDoTransform(xform1.get(), in, out1, 1);
+          cmsDoTransform(xform2.get(), in, out2, 1);
+          for (size_t i = 0; i < 3; ++i) {
+            if (!ApproxEq(out1[i], out2[i], 2E-4)) {
+              return false;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  return true;
+}
+
+// Returns white point that was specified when creating the profile.
+// NOTE: we can't just use cmsSigMediaWhitePointTag because its interpretation
+// differs between ICC versions.
+JXL_MUST_USE_RESULT cmsCIEXYZ UnadaptedWhitePoint(const cmsContext context,
+                                                  const Profile& profile,
+                                                  const ColorEncoding& c) {
+  cmsCIEXYZ XYZ = {1.0, 1.0, 1.0};
+
+  Profile profile_xyz;
+  if (!CreateProfileXYZ(context, &profile_xyz)) return XYZ;
+  // Array arguments are one per profile.
+  cmsHPROFILE profiles[2] = {profile.get(), profile_xyz.get()};
+  // Leave white point unchanged - that is what we're trying to extract.
+  cmsUInt32Number intents[2] = {INTENT_ABSOLUTE_COLORIMETRIC,
+                                INTENT_ABSOLUTE_COLORIMETRIC};
+  cmsBool black_compensation[2] = {0, 0};
+  cmsFloat64Number adaption[2] = {0.0, 0.0};
+  // Only transforming a single pixel, so skip expensive optimizations.
+  cmsUInt32Number flags = cmsFLAGS_NOOPTIMIZE | cmsFLAGS_HIGHRESPRECALC;
+  Transform xform(cmsCreateExtendedTransform(
+      context, 2, profiles, black_compensation, intents, adaption, nullptr, 0,
+      Type64(c), TYPE_XYZ_DBL, flags));
+  if (!xform) return XYZ;  // TODO(lode): return error
+
+  // xy are relative, so magnitude does not matter if we ignore output Y.
+  const cmsFloat64Number in[3] = {1.0, 1.0, 1.0};
+  cmsDoTransform(xform.get(), in, &XYZ.X, 1);
+  return XYZ;
+}
+
+Status IdentifyPrimaries(const Profile& profile, const cmsCIEXYZ& wp_unadapted,
+                         ColorEncoding* c) {
+  if (!c->HasPrimaries()) return true;
+
+  // These were adapted to the profile illuminant before storing in the profile.
+  const cmsCIEXYZ* adapted_r = static_cast<const cmsCIEXYZ*>(
+      cmsReadTag(profile.get(), cmsSigRedColorantTag));
+  const cmsCIEXYZ* adapted_g = static_cast<const cmsCIEXYZ*>(
+      cmsReadTag(profile.get(), cmsSigGreenColorantTag));
+  const cmsCIEXYZ* adapted_b = static_cast<const cmsCIEXYZ*>(
+      cmsReadTag(profile.get(), cmsSigBlueColorantTag));
+  if (adapted_r == nullptr || adapted_g == nullptr || adapted_b == nullptr) {
+    return JXL_FAILURE("Failed to retrieve colorants");
+  }
+
+  // TODO(janwas): no longer assume Bradford and D50.
+  // Undo the chromatic adaptation.
+  const cmsCIEXYZ d50 = D50_XYZ();
+
+  cmsCIEXYZ r, g, b;
+  cmsAdaptToIlluminant(&r, &d50, &wp_unadapted, adapted_r);
+  cmsAdaptToIlluminant(&g, &d50, &wp_unadapted, adapted_g);
+  cmsAdaptToIlluminant(&b, &d50, &wp_unadapted, adapted_b);
+
+  const PrimariesCIExy rgb = {CIExyFromXYZ(r), CIExyFromXYZ(g),
+                              CIExyFromXYZ(b)};
+  return c->SetPrimaries(rgb);
+}
+
+void DetectTransferFunction(const cmsContext context, const Profile& profile,
+                            ColorEncoding* JXL_RESTRICT c) {
+  if (c->tf.SetImplicit()) return;
+
+  for (TransferFunction tf : Values<TransferFunction>()) {
+    // Can only create profile from known transfer function.
+    if (tf == TransferFunction::kUnknown) continue;
+
+    c->tf.SetTransferFunction(tf);
+
+    PaddedBytes icc_test;
+    if (MaybeCreateProfile(context, *c, &icc_test) &&
+        ProfileEquivalentToICC(context, profile, icc_test, *c)) {
+      return;
+    }
+  }
+
+  c->tf.SetTransferFunction(TransferFunction::kUnknown);
+  c->opaque_icc = true;
+}
+
+void ErrorHandler(cmsContext context, cmsUInt32Number code, const char* text) {
+  JXL_WARNING("LCMS error %u: %s", code, text);
+}
+
+// Returns a context for the current thread, creating it if necessary.
+cmsContext GetContext() {
+  static thread_local void* context_;
+  if (context_ == nullptr) {
+    context_ = cmsCreateContext(nullptr, nullptr);
+    JXL_ASSERT(context_ != nullptr);
+
+    cmsSetLogErrorHandlerTHR(static_cast<cmsContext>(context_), &ErrorHandler);
+  }
+  return static_cast<cmsContext>(context_);
+}
+
+#endif  // JPEGXL_ENABLE_SKCMS
+
+}  // namespace
+
+// All functions that call lcms directly (except ColorSpaceTransform::Run) must
+// lock lcms_mutex.
+
+Status ColorManagement::SetProfile(PaddedBytes&& icc,
+                                   ColorEncoding* JXL_RESTRICT c) {
+  c->icc.clear();
+  if (icc.empty()) return false;
+
+#if JPEGXL_ENABLE_SKCMS
+  skcms_ICCProfile profile;
+  JXL_RETURN_IF_ERROR(skcms_Parse(icc.data(), icc.size(), &profile));
+
+  // skcms does not return the rendering intent, so get it from the file. It
+  // is encoded as big-endian 32-bit integer in bytes 60..63.
+  if (icc.size() < 128) {
+    return JXL_FAILURE("ICC file too small");
+  }
+  uint32_t rendering_intent = icc[67];
+  if (rendering_intent > 3 || icc[64] != 0 || icc[65] != 0 || icc[66] != 0) {
+    return JXL_FAILURE("Invalid rendering intent %u\n", rendering_intent);
+  }
+
+  c->icc = std::move(icc);
+  c->received_icc = true;
+  c->opaque_icc = false;  // May be set to true below.
+
+  c->SetColorSpace(ColorSpaceFromProfile(profile, &c->opaque_icc));
+
+  const CIExy wp_unadapted = UnadaptedWhitePoint(profile);
+  JXL_RETURN_IF_ERROR(c->SetWhitePoint(wp_unadapted));
+
+  // Relies on color_space.
+  JXL_RETURN_IF_ERROR(IdentifyPrimaries(profile, wp_unadapted, c));
+
+  // Relies on color_space/white point/primaries being set already.
+  DetectTransferFunction(profile, c);
+  // ICC and RenderingIntent have the same values (0..3).
+  c->rendering_intent = static_cast<RenderingIntent>(rendering_intent);
+#else   // JPEGXL_ENABLE_SKCMS
+
+  std::lock_guard<std::mutex> guard(lcms_mutex);
+  const cmsContext context = GetContext();
+
+  Profile profile;
+  JXL_RETURN_IF_ERROR(DecodeProfile(context, icc, &profile));
+
+  const cmsUInt32Number rendering_intent =
+      cmsGetHeaderRenderingIntent(profile.get());
+  if (rendering_intent > 3) {
+    return JXL_FAILURE("Invalid rendering intent %u\n", rendering_intent);
+  }
+
+  c->icc = std::move(icc);
+  c->received_icc = true;
+  c->opaque_icc = false;  // May be set to true below.
+
+  c->SetColorSpace(ColorSpaceFromProfile(profile, &c->opaque_icc));
+
+  const cmsCIEXYZ wp_unadapted = UnadaptedWhitePoint(context, profile, *c);
+  JXL_RETURN_IF_ERROR(c->SetWhitePoint(CIExyFromXYZ(wp_unadapted)));
+
+  // Relies on color_space.
+  JXL_RETURN_IF_ERROR(IdentifyPrimaries(profile, wp_unadapted, c));
+
+  // Relies on color_space/white point/primaries being set already.
+  DetectTransferFunction(context, profile, c);
+
+  // ICC and RenderingIntent have the same values (0..3).
+  c->rendering_intent = static_cast<RenderingIntent>(rendering_intent);
+#endif  // JPEGXL_ENABLE_SKCMS
+
+  if (!c->opaque_icc) {
+    // Ensure that it is safe to skip the profile, i.e. the reconstructed
+    // profile will be equivalent:
+    PaddedBytes icc_new;
+#if JPEGXL_ENABLE_SKCMS
+    JXL_RETURN_IF_ERROR(MaybeCreateProfile(*c, &icc_new));
+    if (!ProfileEquivalentToICC(profile, icc_new)) {
+#else   // JPEGXL_ENABLE_SKCMS
+    JXL_RETURN_IF_ERROR(MaybeCreateProfile(context, *c, &icc_new));
+    if (!ProfileEquivalentToICC(context, profile, icc_new, *c)) {
+#endif  // JPEGXL_ENABLE_SKCMS
+      return JXL_FAILURE("Generated profile does not match");
+    }
+  }
+
+  return true;
+}
+
+Status ColorManagement::CreateProfile(ColorEncoding* JXL_RESTRICT c) {
+  std::lock_guard<std::mutex> guard(lcms_mutex);
+  c->icc.clear();
+  c->received_icc = c->opaque_icc = false;
+
+#if JPEGXL_ENABLE_SKCMS
+  if (!MaybeCreateProfile(*c, &c->icc)) {
+#else   // JPEGXL_ENABLE_SKCMS
+  const cmsContext context = GetContext();
+  if (!MaybeCreateProfile(context, *c, &c->icc)) {
+#endif  // JPEGXL_ENABLE_SKCMS
+    return JXL_FAILURE("Failed to create profile from fields");
+  }
+  return true;
+}
+
+namespace {
+
+void InitC2Once(const Primaries pr, const TransferFunction tf,
+                std::atomic<int>* initialized,
+                std::array<ColorEncoding, 2>* c2) {
+  // Thread-safe double-checked locking.
+  int done = initialized->load(std::memory_order_acquire);
+  if (done == 1) return;
+  static std::mutex mutex;
+  std::lock_guard<std::mutex> lock(mutex);
+  done = initialized->load(std::memory_order_relaxed);
+  if (done == 1) return;
+
+  {
+    ColorEncoding* c_rgb = c2->data() + 0;
+    c_rgb->SetColorSpace(ColorSpace::kRGB);
+    c_rgb->white_point = WhitePoint::kD65;
+    c_rgb->primaries = pr;
+    c_rgb->tf.SetTransferFunction(tf);
+    JXL_CHECK(ColorManagement::CreateProfile(c_rgb));
+  }
+
+  {
+    ColorEncoding* c_gray = c2->data() + 1;
+    c_gray->SetColorSpace(ColorSpace::kGray);
+    c_gray->white_point = WhitePoint::kD65;
+    c_gray->primaries = pr;
+    c_gray->tf.SetTransferFunction(tf);
+    JXL_CHECK(ColorManagement::CreateProfile(c_gray));
+  }
+
+  initialized->store(1, std::memory_order_release);
+}
+
+}  // namespace
+
+const ColorEncoding& ColorManagement::SRGB(bool is_gray) {
+  static std::atomic<int> initialized{0};
+  static std::array<ColorEncoding, 2> c2;
+  InitC2Once(Primaries::kSRGB, TransferFunction::kSRGB, &initialized, &c2);
+  return c2[is_gray];
+}
+const ColorEncoding& ColorManagement::LinearSRGB(bool is_gray) {
+  static std::atomic<int> initialized{0};
+  static std::array<ColorEncoding, 2> c2;
+  InitC2Once(Primaries::kSRGB, TransferFunction::kLinear, &initialized, &c2);
+  return c2[is_gray];
+}
+
+ColorSpaceTransform::~ColorSpaceTransform() {
+#if !JPEGXL_ENABLE_SKCMS
+  std::lock_guard<std::mutex> guard(lcms_mutex);
+  for (void* p : transforms_) {
+    TransformDeleter()(p);
+  }
+#endif
+}
+
+Status ColorSpaceTransform::Init(const ColorEncoding& c_src,
+                                 const ColorEncoding& c_dst, size_t xsize,
+                                 const size_t num_threads) {
+  std::lock_guard<std::mutex> guard(lcms_mutex);
+#if JXL_CMS_VERBOSE
+  printf("%s -> %s\n", Description(c_src).c_str(), Description(c_dst).c_str());
+#endif
+
+#if JPEGXL_ENABLE_SKCMS
+  icc_src_ = c_src.icc;
+  icc_dst_ = c_dst.icc;
+  JXL_RETURN_IF_ERROR(DecodeProfile(icc_src_, &profile_src_));
+  JXL_RETURN_IF_ERROR(DecodeProfile(icc_dst_, &profile_dst_));
+#else   // JPEGXL_ENABLE_SKCMS
+  const cmsContext context = GetContext();
+  Profile profile_src, profile_dst;
+  JXL_RETURN_IF_ERROR(DecodeProfile(context, c_src.icc, &profile_src));
+  JXL_RETURN_IF_ERROR(DecodeProfile(context, c_dst.icc, &profile_dst));
+#endif  // JPEGXL_ENABLE_SKCMS
+
+  skip_lcms_ = false;
+  if (c_src.SameColorEncoding(c_dst)) {
+    skip_lcms_ = true;
+#if JXL_CMS_VERBOSE
+    printf("Skip CMS\n");
+#endif
+  }
+
+  // Special-case for BT.2100 HLG/PQ and SRGB <=> linear:
+  const bool src_linear = c_src.tf.IsLinear();
+  const bool dst_linear = c_dst.tf.IsLinear();
+  if (((c_src.tf.IsPQ() || c_src.tf.IsHLG()) && dst_linear) ||
+      ((c_dst.tf.IsPQ() || c_dst.tf.IsHLG()) && src_linear) ||
+      (c_src.tf.IsSRGB() && dst_linear) || (c_dst.tf.IsSRGB() && src_linear)) {
+    // Construct new profiles as if the data were already/still linear.
+    ColorEncoding c_linear_src = c_src;
+    ColorEncoding c_linear_dst = c_dst;
+    c_linear_src.tf.SetTransferFunction(TransferFunction::kLinear);
+    c_linear_dst.tf.SetTransferFunction(TransferFunction::kLinear);
+    PaddedBytes icc_src, icc_dst;
+#if JPEGXL_ENABLE_SKCMS
+    skcms_ICCProfile new_src, new_dst;
+#else   // JPEGXL_ENABLE_SKCMS
+    Profile new_src, new_dst;
+#endif  // JPEGXL_ENABLE_SKCMS
+        // Only enable ExtraTF if profile creation succeeded.
+#if JPEGXL_ENABLE_SKCMS
+    if (MaybeCreateProfile(c_linear_src, &new_src, &icc_src) &&
+        MaybeCreateProfile(c_linear_dst, &new_dst, &icc_dst)) {
+#else   // JPEGXL_ENABLE_SKCMS
+    if (MaybeCreateProfile(context, c_linear_src, &icc_src) &&
+        MaybeCreateProfile(context, c_linear_dst, &icc_dst) &&
+        DecodeProfile(context, icc_src, &new_src) &&
+        DecodeProfile(context, icc_dst, &new_dst)) {
+#endif  // JPEGXL_ENABLE_SKCMS
+      if (c_src.SameColorSpace(c_dst)) {
+        skip_lcms_ = true;
+      }
+#if JXL_CMS_VERBOSE
+      printf("Special linear <-> HLG/PQ/sRGB; skip=%d\n", skip_lcms_);
+#endif
+#if JPEGXL_ENABLE_SKCMS
+      icc_src_ = PaddedBytes();
+      profile_src_ = new_src;
+      icc_dst_ = PaddedBytes();
+      profile_dst_ = new_dst;
+#else   // JPEGXL_ENABLE_SKCMS
+      profile_src.swap(new_src);
+      profile_dst.swap(new_dst);
+#endif  // JPEGXL_ENABLE_SKCMS
+      if (c_dst.tf.IsLinear()) {
+        preprocess_ = c_src.tf.IsSRGB()
+                          ? ExtraTF::kSRGB
+                          : (c_src.tf.IsPQ() ? ExtraTF::kPQ : ExtraTF::kHLG);
+      } else {
+        JXL_ASSERT(c_src.tf.IsLinear());
+        postprocess_ = c_dst.tf.IsSRGB()
+                           ? ExtraTF::kSRGB
+                           : (c_dst.tf.IsPQ() ? ExtraTF::kPQ : ExtraTF::kHLG);
+      }
+    } else {
+      JXL_WARNING("Failed to create extra linear profiles");
+    }
+  }
+
+#if JPEGXL_ENABLE_SKCMS
+  if (!skcms_MakeUsableAsDestination(&profile_dst_)) {
+    return JXL_FAILURE(
+        "Failed to make %s usable as a color transform destination",
+        Description(c_dst).c_str());
+  }
+#endif  // JPEGXL_ENABLE_SKCMS
+
+  // Not including alpha channel (copied separately).
+  const size_t channels_src = c_src.Channels();
+  const size_t channels_dst = c_dst.Channels();
+  JXL_CHECK(channels_src == channels_dst);
+#if JXL_CMS_VERBOSE
+  printf("Channels: %zu; Threads: %zu\n", channels_src, num_threads);
+#endif
+
+#if !JPEGXL_ENABLE_SKCMS
+  // Type includes color space (XYZ vs RGB), so can be different.
+  const uint32_t type_src = Type32(c_src);
+  const uint32_t type_dst = Type32(c_dst);
+  transforms_.clear();
+  for (size_t i = 0; i < num_threads; ++i) {
+    const uint32_t intent = static_cast<uint32_t>(c_dst.rendering_intent);
+    const uint32_t flags =
+        cmsFLAGS_BLACKPOINTCOMPENSATION | cmsFLAGS_HIGHRESPRECALC;
+    // NOTE: we're using the current thread's context and assuming all state
+    // modified by cmsDoTransform resides in the transform, not the context.
+    transforms_.emplace_back(cmsCreateTransformTHR(context, profile_src.get(),
+                                                   type_src, profile_dst.get(),
+                                                   type_dst, intent, flags));
+    if (transforms_.back() == nullptr) {
+      return JXL_FAILURE("Failed to create transform");
+    }
+  }
+#endif  // !JPEGXL_ENABLE_SKCMS
+
+  // Ideally LCMS would convert directly from External to Image3. However,
+  // cmsDoTransformLineStride only accepts 32-bit BytesPerPlaneIn, whereas our
+  // planes can be more than 4 GiB apart. Hence, transform inputs/outputs must
+  // be interleaved. Calling cmsDoTransform for each pixel is expensive
+  // (indirect call). We therefore transform rows, which requires per-thread
+  // buffers. To avoid separate allocations, we use the rows of an image.
+  // Because LCMS apparently also cannot handle <= 16 bit inputs and 32-bit
+  // outputs (or vice versa), we use floating point input/output.
+  buf_src_ = ImageF(xsize * channels_src, num_threads);
+  buf_dst_ = ImageF(xsize * channels_dst, num_threads);
+  xsize_ = xsize;
+  return true;
+}
+
+HWY_ATTR void ColorSpaceTransform::Run(const size_t thread,
+                                       const float* buf_src, float* buf_dst) {
+  // No lock needed.
+
+  // If ExtraTF, we need a writable buffer; otherwise, only READ from buf_src.
+  float* const xform_src = (preprocess_ == ExtraTF::kNone)
+                               ? const_cast<float*>(buf_src)
+                               : buf_src_.Row(thread);
+
+#if JXL_CMS_VERBOSE >= 2
+  const size_t kX = 0;  // pixel index, multiplied by 3 for RGB
+#endif
+
+  switch (preprocess_) {
+    case ExtraTF::kNone:
+      break;
+    case ExtraTF::kPQ:
+      for (size_t i = 0; i < buf_src_.xsize(); ++i) {
+        xform_src[i] = static_cast<float>(
+            TF_PQ().DisplayFromEncoded(static_cast<double>(buf_src[i])));
+      }
+#if JXL_CMS_VERBOSE >= 2
+      printf("pre in %.4f %.4f %.4f undoPQ %.4f %.4f %.4f\n", buf_src[3 * kX],
+             buf_src[3 * kX + 1], buf_src[3 * kX + 2], xform_src[3 * kX],
+             xform_src[3 * kX + 1], xform_src[3 * kX + 2]);
+#endif
+      break;
+    case ExtraTF::kHLG:
+      for (size_t i = 0; i < buf_src_.xsize(); ++i) {
+        xform_src[i] = static_cast<float>(
+            TF_HLG().DisplayFromEncoded(static_cast<double>(buf_src[i])));
+      }
+#if JXL_CMS_VERBOSE >= 2
+      printf("pre in %.4f %.4f %.4f undoHLG %.4f %.4f %.4f\n", buf_src[3 * kX],
+             buf_src[3 * kX + 1], buf_src[3 * kX + 2], xform_src[3 * kX],
+             xform_src[3 * kX + 1], xform_src[3 * kX + 2]);
+#endif
+      break;
+    case ExtraTF::kSRGB:
+      HWY_FULL(float) df;
+      for (size_t i = 0; i < buf_src_.xsize(); i += df.N) {
+        const auto val = Load(df, buf_src + i);
+        const auto result = TF_SRGB().DisplayFromEncoded(val);
+        Store(result, df, xform_src + i);
+      }
+#if JXL_CMS_VERBOSE >= 2
+      printf("pre in %.4f %.4f %.4f undoSRGB %.4f %.4f %.4f\n", buf_src[3 * kX],
+             buf_src[3 * kX + 1], buf_src[3 * kX + 2], xform_src[3 * kX],
+             xform_src[3 * kX + 1], xform_src[3 * kX + 2]);
+#endif
+      break;
+  }
+
+#if JXL_CMS_VERBOSE >= 2
+  // Save inputs for printing before in-place transforms overwrite them.
+  const float in0 = xform_src[3 * kX + 0];
+  const float in1 = xform_src[3 * kX + 1];
+  const float in2 = xform_src[3 * kX + 2];
+#endif
+
+  if (skip_lcms_) {
+    if (buf_dst != xform_src) {
+      memcpy(buf_dst, xform_src, buf_dst_.xsize() * sizeof(*buf_dst));
+    }  // else: in-place, no need to copy
+  } else {
+#if JPEGXL_ENABLE_SKCMS
+    JXL_CHECK(skcms_Transform(xform_src, skcms_PixelFormat_RGB_fff,
+                              skcms_AlphaFormat_Opaque, &profile_src_, buf_dst,
+                              skcms_PixelFormat_RGB_fff,
+                              skcms_AlphaFormat_Opaque, &profile_dst_, xsize_));
+#else   // JPEGXL_ENABLE_SKCMS
+    JXL_DASSERT(thread < transforms_.size());
+    cmsHTRANSFORM xform = transforms_[thread];
+    cmsDoTransform(xform, xform_src, buf_dst,
+                   static_cast<cmsUInt32Number>(xsize_));
+#endif  // JPEGXL_ENABLE_SKCMS
+  }
+#if JXL_CMS_VERBOSE >= 2
+  printf("xform skip%d: %.4f %.4f %.4f (%p) -> (%p) %.4f %.4f %.4f\n",
+         skip_lcms_, in0, in1, in2, xform_src, buf_dst, buf_dst[3 * kX],
+         buf_dst[3 * kX + 1], buf_dst[3 * kX + 2]);
+#endif
+
+  switch (postprocess_) {
+    case ExtraTF::kNone:
+      break;
+    case ExtraTF::kPQ:
+      for (size_t i = 0; i < buf_dst_.xsize(); ++i) {
+        buf_dst[i] = static_cast<float>(
+            TF_PQ().EncodedFromDisplay(static_cast<double>(buf_dst[i])));
+      }
+#if JXL_CMS_VERBOSE >= 2
+      printf("after PQ enc %.4f %.4f %.4f\n", buf_dst[3 * kX],
+             buf_dst[3 * kX + 1], buf_dst[3 * kX + 2]);
+#endif
+      break;
+    case ExtraTF::kHLG:
+      for (size_t i = 0; i < buf_dst_.xsize(); ++i) {
+        buf_dst[i] = static_cast<float>(
+            TF_HLG().EncodedFromDisplay(static_cast<double>(buf_dst[i])));
+      }
+#if JXL_CMS_VERBOSE >= 2
+      printf("after HLG enc %.4f %.4f %.4f\n", buf_dst[3 * kX],
+             buf_dst[3 * kX + 1], buf_dst[3 * kX + 2]);
+#endif
+      break;
+    case ExtraTF::kSRGB:
+      HWY_FULL(float) df;
+      for (size_t i = 0; i < buf_dst_.xsize(); i += df.N) {
+        const auto val = Load(df, buf_dst + i);
+        const auto result = TF_SRGB().EncodedFromDisplay(val);
+        Store(result, df, buf_dst + i);
+      }
+#if JXL_CMS_VERBOSE >= 2
+      printf("after SRGB enc %.4f %.4f %.4f\n", buf_dst[3 * kX],
+             buf_dst[3 * kX + 1], buf_dst[3 * kX + 2]);
+#endif
+      break;
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/color_management.h b/jxl/color_management.h
new file mode 100644
index 000000000..9718ff131
--- /dev/null
+++ b/jxl/color_management.h
@@ -0,0 +1,104 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_COLOR_MANAGEMENT_H_
+#define JXL_COLOR_MANAGEMENT_H_
+
+// ICC profiles and color space conversions.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <memory>
+#include <vector>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/color_encoding.h"
+#include "jxl/common.h"
+#include "jxl/image.h"
+#if JPEGXL_ENABLE_SKCMS
+#include "skcms.h"
+#endif
+
+namespace jxl {
+
+// Thread-safe monostate.
+struct ColorManagement {
+  // Returns false and clears c->icc if `icc` is empty or decoding it fails.
+  // Otherwise, moves `icc` into c->icc and sets other fields based on its
+  // contents. c->opaque_icc is true if any ICC field cannot be represented.
+  static Status SetProfile(PaddedBytes&& icc, ColorEncoding* c);
+
+  // Returns true if c->icc was successfully created based on the other fields.
+  // Called by codecs that provide their own non-ICC metadata. Returning false
+  // indicates the profile is lost/empty and ColorSpaceTransform will fail.
+  static Status CreateProfile(ColorEncoding* c);
+
+  // Returns ready-to-use color encodings:
+  static const ColorEncoding& SRGB(bool is_gray = false);
+  static const ColorEncoding& LinearSRGB(bool is_gray = false);
+};
+
+// Run is thread-safe.
+class ColorSpaceTransform {
+ public:
+  ColorSpaceTransform() = default;
+  ~ColorSpaceTransform();
+
+  // Cannot copy (transforms_ holds pointers).
+  ColorSpaceTransform(const ColorSpaceTransform&) = delete;
+  ColorSpaceTransform& operator=(const ColorSpaceTransform&) = delete;
+
+  // "Constructor"; allocates for up to `num_threads`, or returns false.
+  Status Init(const ColorEncoding& c_src, const ColorEncoding& c_dst,
+              size_t xsize, size_t num_threads);
+
+  float* BufSrc(const size_t thread) { return buf_src_.Row(thread); }
+
+  float* BufDst(const size_t thread) { return buf_dst_.Row(thread); }
+
+  // buf_X can either be from BufX() or caller-allocated, interleaved storage.
+  // `thread` must be less than the `num_threads` passed to Init.
+  void Run(size_t thread, const float* buf_src, float* buf_dst);
+
+ private:
+  enum class ExtraTF {
+    kNone,
+    kPQ,
+    kHLG,
+    kSRGB,
+  };
+
+#if JPEGXL_ENABLE_SKCMS
+  // Parsed skcms_ICCProfiles retain pointers to the original data.
+  PaddedBytes icc_src_, icc_dst_;
+  skcms_ICCProfile profile_src_, profile_dst_;
+#else
+  // One per thread - cannot share because of caching.
+  std::vector<void*> transforms_;
+#endif
+
+  ImageF buf_src_;
+  ImageF buf_dst_;
+  size_t xsize_;
+  bool skip_lcms_ = false;
+  ExtraTF preprocess_ = ExtraTF::kNone;
+  ExtraTF postprocess_ = ExtraTF::kNone;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_COLOR_MANAGEMENT_H_
diff --git a/jxl/color_management_test.cc b/jxl/color_management_test.cc
new file mode 100644
index 000000000..448fa46e5
--- /dev/null
+++ b/jxl/color_management_test.cc
@@ -0,0 +1,240 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/color_management.h"
+
+#include <stdint.h>
+#include <stdio.h>
+
+#include <algorithm>
+#include <new>
+#include <string>
+#include <utility>
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/file_io.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/image_test_utils.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+
+std::ostream& operator<<(std::ostream& os, const CIExy& xy) {
+  return os << "{x=" << xy.x << ", y=" << xy.y << "}";
+}
+
+std::ostream& operator<<(std::ostream& os, const PrimariesCIExy& primaries) {
+  return os << "{r=" << primaries.r << ", g=" << primaries.g
+            << ", b=" << primaries.b << "}";
+}
+
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::FloatNear;
+
+// Small enough to be fast. If changed, must update Generate*.
+static constexpr size_t kWidth = 16;
+
+struct Globals {
+  static constexpr size_t kNumThreads = 0;  // only have a single row.
+
+  Globals() : pool(kNumThreads) {
+    in_gray = GenerateGray();
+    in_color = GenerateColor();
+    out_gray = ImageF(kWidth, 1);
+    out_color = ImageF(kWidth * 3, 1);
+
+    c_native = ColorManagement::LinearSRGB(/*is_gray=*/false);
+    c_gray = ColorManagement::LinearSRGB(/*is_gray=*/true);
+  }
+
+ private:
+  static ImageF GenerateGray() {
+    ImageF gray(kWidth, 1);
+    float* JXL_RESTRICT row = gray.Row(0);
+    // Increasing left to right
+    for (int32_t x = 0; x < kWidth; ++x) {
+      row[x] = x * 1.0f / (kWidth - 1);  // [0, 1]
+    }
+    return gray;
+  }
+
+  static ImageF GenerateColor() {
+    ImageF image(kWidth * 3, 1);
+    float* JXL_RESTRICT interleaved = image.Row(0);
+    std::fill(interleaved, interleaved + kWidth * 3, 0.0f);
+
+    // [0, 4): neutral
+    for (int32_t x = 0; x < 4; ++x) {
+      interleaved[3 * x + 0] = x * 1.0f / 3;  // [0, 1]
+      interleaved[3 * x + 2] = interleaved[3 * x + 1] = interleaved[3 * x + 0];
+    }
+
+    // [4, 13): pure RGB with low/medium/high saturation
+    for (int32_t c = 0; c < 3; ++c) {
+      interleaved[3 * (4 + c) + c] = 0.08f + c * 0.01f;
+      interleaved[3 * (7 + c) + c] = 0.75f + c * 0.01f;
+      interleaved[3 * (10 + c) + c] = 1.0f;
+    }
+
+    // [13, 16): impure, not quite saturated RGB
+    interleaved[3 * 13 + 0] = 0.86f;
+    interleaved[3 * 13 + 2] = interleaved[3 * 13 + 1] = 0.16f;
+    interleaved[3 * 14 + 1] = 0.87f;
+    interleaved[3 * 14 + 2] = interleaved[3 * 14 + 0] = 0.16f;
+    interleaved[3 * 15 + 2] = 0.88f;
+    interleaved[3 * 15 + 1] = interleaved[3 * 15 + 0] = 0.16f;
+
+    return image;
+  }
+
+ public:
+  ThreadPoolInternal pool;
+
+  // ImageF so we can use VerifyRelativeError; all are interleaved RGB.
+  ImageF in_gray;
+  ImageF in_color;
+  ImageF out_gray;
+  ImageF out_color;
+  ColorEncoding c_native;
+  ColorEncoding c_gray;
+};
+static Globals* g;
+
+class ColorManagementTest : public ::testing::TestWithParam<ColorEncoding> {
+ public:
+  static void SetUpTestSuite() { g = new Globals; }
+  static void TearDownTestSuite() { delete g; }
+
+  static void VerifySameFields(const ColorEncoding& c,
+                               const ColorEncoding& c2) {
+    ASSERT_EQ(c.rendering_intent, c2.rendering_intent);
+    ASSERT_EQ(c.GetColorSpace(), c2.GetColorSpace());
+    ASSERT_EQ(c.white_point, c2.white_point);
+    if (c.HasPrimaries()) {
+      ASSERT_EQ(c.primaries, c2.primaries);
+    }
+    ASSERT_TRUE(c.tf.IsSame(c2.tf));
+  }
+
+  // "Same" pixels after converting g->c_native -> c -> g->c_native.
+  static void VerifyPixelRoundTrip(const ColorEncoding& c) {
+    const ColorEncoding& c_native = c.IsGray() ? g->c_gray : g->c_native;
+    ColorSpaceTransform xform_fwd;
+    ColorSpaceTransform xform_rev;
+    ASSERT_TRUE(xform_fwd.Init(c_native, c, kWidth, g->pool.NumThreads()));
+    ASSERT_TRUE(xform_rev.Init(c, c_native, kWidth, g->pool.NumThreads()));
+
+    const size_t thread = 0;
+    const ImageF& in = c.IsGray() ? g->in_gray : g->in_color;
+    ImageF* JXL_RESTRICT out = c.IsGray() ? &g->out_gray : &g->out_color;
+    xform_fwd.Run(thread, in.Row(0), xform_fwd.BufDst(thread));
+    xform_rev.Run(thread, xform_fwd.BufDst(thread), out->Row(0));
+
+#if JPEGXL_ENABLE_SKCMS
+    double max_l1 = 5E-3;
+    double max_rel = 6E-3;
+#else
+    double max_l1 = 5E-5;
+    // Most are lower; reached 3E-7 with D60 AP0.
+    double max_rel = 4E-7;
+#endif
+    if (c.IsGray()) max_rel = 2E-5;
+    VerifyRelativeError(in, *out, max_l1, max_rel);
+  }
+};
+INSTANTIATE_TEST_CASE_P(ColorManagementTestInstantiation, ColorManagementTest,
+                        ::testing::ValuesIn(AllEncodings()));
+
+// Exercises the ColorManagement interface for ALL ColorEncoding synthesizable
+// via enums.
+TEST_P(ColorManagementTest, VerifyAllProfiles) {
+  ColorEncoding c = GetParam();
+  printf("%s\n", Description(c).c_str());
+
+  // Can create profile.
+  ASSERT_TRUE(ColorManagement::CreateProfile(&c));
+
+  // Can set an equivalent ColorEncoding from the generated ICC profile.
+  ColorEncoding c3;
+  ASSERT_TRUE(ColorManagement::SetProfile(std::move(c.icc), &c3));
+  VerifySameFields(c, c3);
+  // (need a profile for VerifyPixelRoundTrip.)
+  c.icc = std::move(c3.icc);
+
+  VerifyPixelRoundTrip(c);
+}
+
+testing::Matcher<CIExy> CIExyIs(const double x, const double y) {
+  static constexpr double kMaxError = 1e-4;
+  return testing::AllOf(
+      testing::Field("x", &CIExy::x, testing::DoubleNear(x, kMaxError)),
+      testing::Field("y", &CIExy::y, testing::DoubleNear(y, kMaxError)));
+}
+
+testing::Matcher<PrimariesCIExy> PrimariesAre(
+    const testing::Matcher<CIExy>& r, const testing::Matcher<CIExy>& g,
+    const testing::Matcher<CIExy>& b) {
+  return testing::AllOf(testing::Field("r", &PrimariesCIExy::r, r),
+                        testing::Field("g", &PrimariesCIExy::g, g),
+                        testing::Field("b", &PrimariesCIExy::b, b));
+}
+
+TEST_F(ColorManagementTest, sRGBChromaticity) {
+  const ColorEncoding sRGB = ColorManagement::SRGB();
+  EXPECT_THAT(sRGB.GetWhitePoint(), CIExyIs(0.3127, 0.3290));
+  EXPECT_THAT(sRGB.GetPrimaries(),
+              PrimariesAre(CIExyIs(0.64, 0.33), CIExyIs(0.30, 0.60),
+                           CIExyIs(0.15, 0.06)));
+}
+
+TEST_F(ColorManagementTest, D2700Chromaticity) {
+  PaddedBytes icc;
+  ASSERT_TRUE(jxl::ReadFile(
+      GetTestDataPath("jxl/color_management/sRGB-D2700.icc"), &icc));
+  ColorEncoding sRGB_D2700;
+  ASSERT_TRUE(jxl::ColorManagement::SetProfile(std::move(icc), &sRGB_D2700));
+
+  EXPECT_THAT(sRGB_D2700.GetWhitePoint(), CIExyIs(0.45986, 0.41060));
+  // The illuminant-relative chromaticities of this profile's primaries are the
+  // same as for sRGB. It is the PCS-relative chromaticities that would be
+  // different.
+  EXPECT_THAT(sRGB_D2700.GetPrimaries(),
+              PrimariesAre(CIExyIs(0.64, 0.33), CIExyIs(0.30, 0.60),
+                           CIExyIs(0.15, 0.06)));
+}
+
+TEST_F(ColorManagementTest, D2700ToSRGB) {
+  PaddedBytes icc;
+  ASSERT_TRUE(jxl::ReadFile(
+      GetTestDataPath("jxl/color_management/sRGB-D2700.icc"), &icc));
+  ColorEncoding sRGB_D2700;
+  ASSERT_TRUE(jxl::ColorManagement::SetProfile(std::move(icc), &sRGB_D2700));
+
+  ColorSpaceTransform transform;
+  ASSERT_TRUE(transform.Init(sRGB_D2700, ColorManagement::SRGB(), 1, 1));
+  const float sRGB_D2700_values[3] = {0.863, 0.737, 0.490};
+  float sRGB_values[3];
+  transform.Run(0, sRGB_D2700_values, sRGB_values);
+  EXPECT_THAT(sRGB_values,
+              ElementsAre(FloatNear(0.914, 1e-3), FloatNear(0.745, 1e-3),
+                          FloatNear(0.601, 1e-3)));
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/common.h b/jxl/common.h
new file mode 100644
index 000000000..ef1ff0ddb
--- /dev/null
+++ b/jxl/common.h
@@ -0,0 +1,137 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_COMMON_H_
+#define JXL_COMMON_H_
+
+// Shared constants and helper functions.
+
+#include <stddef.h>
+
+#include <memory>  // unique_ptr
+
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+// Some enums and typedefs used by more than one header file.
+enum class BlockType : uint8_t {
+  kAdd = 0,
+  kTowardsBlack = 1,
+  kTowardsWhite = 2,
+  kNumBlockTypes,
+};
+
+constexpr size_t kBitsPerByte = 8;  // more clear than CHAR_BIT
+
+constexpr inline size_t RoundUpBitsToByteMultiple(size_t bits) {
+  return (bits + 7) & ~size_t(7);
+}
+
+constexpr inline size_t RoundUpToBlockDim(size_t dim) {
+  return (dim + 7) & ~size_t(7);
+}
+
+template <typename T1, typename T2>
+constexpr inline T1 DivCeil(T1 a, T2 b) {
+  return (a + b - 1) / b;
+}
+
+constexpr double kPi = 3.14159265358979323846264338327950288;
+
+// Reasonable default for sRGB, matches common monitors. Butteraugli was tuned
+// for this, we scale darker/brighter inputs accordingly.
+static constexpr int kDefaultIntensityTarget = 250;
+static constexpr float kIntensityMultiplier = 1.0f / kDefaultIntensityTarget;
+
+template <typename T>
+constexpr T Pi(T multiplier) {
+  return static_cast<T>(multiplier * kPi);
+}
+
+// Block is the square grid of pixels to which an "energy compaction"
+// transformation (e.g. DCT) is applied. Each block has its own AC quantizer.
+constexpr size_t kBlockDim = 8;
+
+constexpr size_t kDCTBlockSize = kBlockDim * kBlockDim;
+
+// Group is the rectangular grid of blocks that can be decoded in parallel. This
+// is different for DC.
+// TODO(jon) : signal kDcGroupDimInBlocks and kGroupDim (and make them
+// variables),
+//             allowing powers of two between (say) 64 and 1024
+constexpr size_t kDcGroupDimInBlocks = 256;
+constexpr size_t kDcGroupDim = kDcGroupDimInBlocks * kBlockDim;
+// 512x512 DC = 4096x4096, enough for a 4K frame (3840x2160)
+// (setting it to 256 results in four DC groups of size 256x256, 224x256,
+// 256x14, 224x14)
+constexpr size_t kGroupDim = 256;
+static_assert(kGroupDim % kBlockDim == 0,
+              "Group dim should be divisible by block dim");
+constexpr size_t kGroupDimInBlocks = kGroupDim / kBlockDim;
+
+// Maximum number of passes in an image.
+constexpr size_t kMaxNumPasses = 11;
+
+// Maximum number of reference frames.
+constexpr size_t kMaxNumReferenceFrames = 3;
+
+// Dimensions of a frame, in pixels, and other derived dimensions.
+// Initially set from Preview/SizeHeader, may be overridden by AnimationFrame.
+struct FrameDimensions {
+  void Set(size_t xsize, size_t ysize) {
+    this->xsize = xsize;
+    this->ysize = ysize;
+    xsize_blocks = DivCeil(xsize, kBlockDim);
+    ysize_blocks = DivCeil(ysize, kBlockDim);
+    xsize_padded = xsize_blocks * kBlockDim;
+    ysize_padded = ysize_blocks * kBlockDim;
+    xsize_groups = DivCeil(xsize, kGroupDim);
+    ysize_groups = DivCeil(ysize, kGroupDim);
+    xsize_dc_groups = DivCeil(xsize, kDcGroupDim);
+    ysize_dc_groups = DivCeil(ysize, kDcGroupDim);
+    num_groups = xsize_groups * ysize_groups;
+    num_dc_groups = xsize_dc_groups * ysize_dc_groups;
+  }
+
+  size_t xsize;
+  size_t ysize;
+  size_t xsize_padded;
+  size_t ysize_padded;
+  size_t xsize_blocks;
+  size_t ysize_blocks;
+  size_t xsize_groups;
+  size_t ysize_groups;
+  size_t xsize_dc_groups;
+  size_t ysize_dc_groups;
+  size_t num_groups;
+  size_t num_dc_groups;
+};
+
+// Can't rely on C++14 yet.
+template <typename T, typename... Args>
+std::unique_ptr<T> make_unique(Args&&... args) {
+  return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
+}
+
+// This leads to somewhat better code than pointer arithmetic.
+template <typename T>
+JXL_INLINE T* JXL_RESTRICT ByteOffset(T* JXL_RESTRICT base,
+                                      const intptr_t byte_offset) {
+  const uintptr_t base_addr = reinterpret_cast<uintptr_t>(base);
+  return reinterpret_cast<T*>(base_addr + byte_offset);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_COMMON_H_
diff --git a/jxl/compressed_dc.cc b/jxl/compressed_dc.cc
new file mode 100644
index 000000000..b6767b782
--- /dev/null
+++ b/jxl/compressed_dc.cc
@@ -0,0 +1,809 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/compressed_dc.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <array>
+#include <hwy/scalar.h>
+#include <hwy/static_targets.h>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/ans_params.h"
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/common.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_cache.h"
+#include "jxl/enc_cache.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/image.h"
+#include "jxl/predictor.h"
+
+namespace jxl {
+namespace {
+
+// Approximate mapping of actual residual to encoded residual, used for
+// computing context.
+
+constexpr size_t kResidualLut[4][8] = {
+    {0, 1, 2, 3, 4, 5, 6, 7},
+    {0, 1, 2, 3, 3, 4, 4, 5},
+    {0, 1, 2, 3, 3, 3, 4, 4},
+    {0, 1, 2, 3, 3, 3, 4, 4},
+};
+
+// NOLINTNEXTLINE(clang-analyzer-optin.performance.Padding)
+class DcCoderBase {
+ public:
+  enum {
+    kNumResidualContexts = 8,
+    kContextsPerChannel = kNumResidualContexts + 4,
+    kNumContexts = 3 * kContextsPerChannel,
+  };
+
+  uint32_t JXL_INLINE ResidualToEncoded(size_t c, size_t residual) {
+    if (residual > 7) {
+      return ((residual + (1 << extra_levels_[c]) / 2) >> extra_levels_[c]) +
+             extra_levels_[c];
+    }
+    return kResidualLut[extra_levels_[c]][residual];
+  }
+
+  size_t Context(size_t c, size_t correct, size_t badness) {
+    if (correct == 0) {
+      JXL_ASSERT(badness != 0);
+      // Discard error sign. Assumes that badness comes from PackSigned.
+      badness = (badness + 1) >> 1;
+      badness = ResidualToEncoded(c, badness);
+      size_t badness_offset =
+          std::min<uint32_t>(badness, kNumResidualContexts) - 1;
+      return kContextsPerChannel * c + badness_offset;
+    }
+    return kContextsPerChannel * c + kNumResidualContexts +
+           CeilLog2Nonzero(9 - correct);
+  }
+
+  // Stores the temporary buffer row in the final image.
+  void FlushRow(size_t y) {
+    for (size_t c = 0; c < 3; c++) {
+      float* JXL_RESTRICT dc_dec_row =
+          dc_dec_rows_base_[c] + y * dc_dec_stride_;
+      float* JXL_RESTRICT dc_quant_field_row =
+          dc_quant_field_rows_base_[c] + y * dc_quant_field_stride_;
+      for (size_t x = 0; x < xsize_; x++) {
+        dc_quant_field_row[x] = dc_quant_field_[4 * x + c];
+        dc_dec_row[x] = dc_dec_[4 * x + c];
+      }
+    }
+  }
+
+  void StartRow(size_t y) {
+    if (y != 0) FlushRow(y - 1);
+  }
+
+  float GetDcValue(size_t c, size_t y, size_t x) {
+    dc_dec_[4 * x + c] = 0;
+    return dc_rows_[c][y * dc_stride_ + x] -
+           dc_dec_[4 * x + 1] * cmap_factor_[c];
+  }
+
+  DcCoderBase(size_t xsize, const uint32_t* JXL_RESTRICT extra_levels,
+              const float* JXL_RESTRICT mul_dc,
+              const float* JXL_RESTRICT inv_mul_dc,
+              const float* JXL_RESTRICT cmap_factor,
+              const float* JXL_RESTRICT* JXL_RESTRICT dc_rows, size_t dc_stride,
+              float** JXL_RESTRICT dc_dec_rows, size_t dc_dec_stride,
+              float** JXL_RESTRICT dc_quant_field_rows,
+              size_t dc_quant_field_stride)
+      : xsize_(xsize),
+        mul_dc_{mul_dc[0] / (1 << extra_levels[0]),
+                mul_dc[1] / (1 << extra_levels[1]),
+                mul_dc[2] / (1 << extra_levels[2]), 0},
+        inv_mul_dc_(inv_mul_dc),
+        cmap_factor_(cmap_factor),
+        dc_rows_(dc_rows),
+        dc_stride_(dc_stride),
+        dc_dec_rows_base_(dc_dec_rows),
+        dc_dec_stride_(dc_dec_stride),
+        dc_quant_field_rows_base_(dc_quant_field_rows),
+        dc_quant_field_stride_(dc_quant_field_stride),
+        extra_levels_(extra_levels) {}
+
+  float JXL_INLINE Dequantize(size_t c, int val) { return val * mul_dc_[c]; }
+
+  int Quantize(size_t c, float val) {
+    JXL_DASSERT(val >= 0);
+    val *= inv_mul_dc_[c];
+    if (extra_levels_[c] == 0) {
+      return static_cast<int>(val + 0.5f);
+    }
+    if (extra_levels_[c] == 1) {
+      if (val <= 0.25f) {
+        return 0;
+      }
+      if (val < 0.75f) {
+        return 1;
+      }
+      return static_cast<int>(val + 0.5f) + 1;
+    }
+    if (extra_levels_[c] == 2) {
+      if (val <= 0.125f) {
+        return 0;
+      }
+      if (val < 0.375f) {
+        return 1;
+      }
+      if (val < 0.75f) {
+        return 2;
+      }
+      return static_cast<int>(val + 0.5f) + 2;
+    }
+    JXL_DASSERT(extra_levels_[c] == 3);
+    if (val <= 0.0625f) {
+      return 0;
+    }
+    if (val < 0.1875f) {
+      return 1;
+    }
+    if (val < 0.375f) {
+      return 2;
+    }
+    if (val < 0.75f) {
+      return 3;
+    }
+    return static_cast<int>(val + 0.5f) + 3;
+  }
+
+  uint32_t ComputeResidual(size_t c, int32_t predicted, float actual) {
+    float fpred = Dequantize(c, predicted);
+    float fdelta = actual - fpred;
+    bool negative = fdelta < 0;
+    fdelta = std::abs(fdelta);
+
+    int32_t delta = Quantize(c, fdelta);
+
+    delta = negative ? -delta : delta;
+    return PackSigned(delta);
+  }
+
+  int32_t JXL_INLINE ApplyResidual(size_t c, int32_t predicted,
+                                   uint32_t residual,
+                                   float* JXL_RESTRICT interval) {
+    int32_t delta = UnpackSigned(residual);
+    bool negative = delta < 0;
+    delta = std::abs(delta);
+
+    if (static_cast<uint32_t>(delta) > extra_levels_[c]) {
+      delta = (delta - extra_levels_[c]) << extra_levels_[c];
+      *interval *= 1 << extra_levels_[c];
+    } else if (delta != 0) {
+      delta = 1 << (delta - 1);
+      *interval *= delta;
+    }
+
+    delta = negative ? -delta : delta;
+    return predicted + delta;
+  }
+
+  // Encoder only
+  void ComputeDecoded(size_t c, size_t x, size_t y,
+                      const int32_t* JXL_RESTRICT predictions,
+                      const uint32_t* JXL_RESTRICT residuals,
+                      int32_t* JXL_RESTRICT decoded) {
+    int32_t prediction = predictions[c];
+
+    dc_quant_field_[4 * x + c] = mul_dc_[c];
+    dc_dec_[4 * x + c] = 0;
+
+    int quant =
+        ApplyResidual(c, prediction, residuals[c], &dc_quant_field_[4 * x + c]);
+
+    decoded[c] = quant;
+
+    float dcd = Dequantize(c, quant);
+    dc_dec_[4 * x + c] = dcd + dc_dec_[4 * x + 1] * cmap_factor_[c];
+  }
+
+  HWY_ATTR void ComputeDecoded3(size_t x, size_t y,
+                                const int32_t* JXL_RESTRICT predictions,
+                                const uint32_t* JXL_RESTRICT residuals,
+                                int32_t* JXL_RESTRICT decoded) {
+#if HWY_HAS_VARIABLE_SHIFT && HWY_BITS >= 128
+    using DU = HWY_CAPPED(uint32_t, 4);
+    using DI = HWY_CAPPED(int32_t, 4);
+    using DF = HWY_CAPPED(float, 4);
+    const DI di;
+    const DU du;
+    const auto residual = Load(du, residuals);
+
+    // UnpackSigned
+    const auto one = Set(du, 1);
+    const auto is_negative = TestBit(residual, one);  // sign bit = LSB
+    // Workaround for >> rounding to neg infinity instead of zero:
+    // If positive, LSB is 0 so the add disappears after the shift.
+    // If negative, LSB is 1 so the add carries and we have incremented the
+    // shifted result by one.
+    const auto delta = hwy::ShiftRight<1>(residual + one);
+
+    // Expand extra levels
+    const auto extra_levels = Load(du, extra_levels_);
+    // Build mask of "delta > extra_levels"
+    const auto is_hi = MaskFromVec(
+        BitCast(du, hwy::ShiftRight<31>(BitCast(di, extra_levels - delta))));
+    const auto delta_hi = (delta - extra_levels) << extra_levels;
+    const auto is_zero = delta == Zero(du);
+    const auto deltam1 = delta - one;
+    const auto delta_lo = one << deltam1;
+    const auto final_delta =
+        IfThenElse(is_hi, delta_hi, IfThenZeroElse(is_zero, delta_lo));
+
+    // Apply prediction
+    const auto prediction = BitCast(du, Load(di, predictions));
+    const auto quantized =
+        BitCast(di, IfThenElse(is_negative, prediction - final_delta,
+                               prediction + final_delta));
+    // Not aligned in predictor.h.
+    StoreU(quantized, di, decoded);
+
+    const auto mul = Load(DF(), mul_dc_);
+    // Compute adjusted quant interval
+    const auto mul_factor_hi = one << extra_levels;
+    const auto mul_factor = BitCast(
+        di,
+        IfThenElse(is_hi, mul_factor_hi, IfThenElse(is_zero, one, delta_lo)));
+    const auto adj_mul = mul * ConvertTo(DF(), mul_factor);
+    Store(adj_mul, DF(), dc_quant_field_ + 4 * x);
+
+    const auto dequant = ConvertTo(DF(), quantized) * mul;
+    const auto y_dequant = hwy::Broadcast<1>(dequant);
+    const auto correlated =
+        MulAdd(Load(DF(), cmap_factor_), y_dequant, dequant);
+    Store(correlated, DF(), dc_dec_ + 4 * x);
+#else
+    for (size_t c : {1, 0, 2}) {
+      ComputeDecoded(c, x, y, predictions, residuals, decoded);
+    }
+#endif
+  }
+
+ private:
+  const size_t xsize_;
+  HWY_ALIGN float const mul_dc_[4];
+  const float* JXL_RESTRICT inv_mul_dc_;
+  const float* JXL_RESTRICT cmap_factor_;
+
+  // Input (encoder-side) values.
+  const float* JXL_RESTRICT* JXL_RESTRICT dc_rows_;
+  const size_t dc_stride_;
+
+  // Output (decoder-side) values.
+  float** JXL_RESTRICT dc_dec_rows_base_;
+  const size_t dc_dec_stride_;
+
+  float** JXL_RESTRICT dc_quant_field_rows_base_;
+  const size_t dc_quant_field_stride_;
+
+  // One padding value every 3 to have aligned stores.
+  HWY_ALIGN float dc_dec_[4 * kDcGroupDimInBlocks];
+  HWY_ALIGN float dc_quant_field_[4 * kDcGroupDimInBlocks];
+
+  const uint32_t* JXL_RESTRICT const extra_levels_;
+};
+
+class DCEncoder : public DcPredictor<DCEncoder> {
+ public:
+  DCEncoder(size_t xsize, const uint32_t* JXL_RESTRICT extra_levels,
+            const float* JXL_RESTRICT mul_dc,
+            const float* JXL_RESTRICT inv_mul_dc,
+            const float* JXL_RESTRICT cmap_factor,
+            const float* JXL_RESTRICT* JXL_RESTRICT dc_rows, size_t dc_stride,
+            float** JXL_RESTRICT dc_dec_rows, size_t dc_dec_stride,
+            float** JXL_RESTRICT dc_quant_field_rows,
+            size_t dc_quant_field_stride)
+      : predictor_(xsize, extra_levels, mul_dc, inv_mul_dc, cmap_factor,
+                   dc_rows, dc_stride, dc_dec_rows, dc_dec_stride,
+                   dc_quant_field_rows, dc_quant_field_stride) {}
+
+  HWY_ATTR void Run(size_t xsize, size_t ysize,
+                    std::vector<Token>* JXL_RESTRICT tokens,
+                    AuxOut* JXL_RESTRICT aux_out) {
+    tokens_ = tokens;
+    DcPredictor<DCEncoder>::Run(xsize, ysize, aux_out);
+    predictor_.FlushRow(ysize - 1);
+  }
+
+  HWY_ATTR JXL_INLINE void Prediction(size_t x, size_t y,
+                                      const int32_t* JXL_RESTRICT predictions,
+                                      const uint32_t* JXL_RESTRICT num_correct,
+                                      const uint32_t* JXL_RESTRICT min_error,
+                                      int32_t* JXL_RESTRICT decoded) {
+    uint32_t residuals[3];
+    // TODO(veluca): investigate possible SIMD-fication of this code.
+
+    int32_t prediction = predictions[1];
+    float dcv = predictor_.GetDcValue(1, y, x);
+    residuals[1] = predictor_.ComputeResidual(1, prediction, dcv);
+    predictor_.ComputeDecoded(1, x, y, predictions, residuals, decoded);
+
+    prediction = predictions[0];
+    dcv = predictor_.GetDcValue(0, y, x);
+    residuals[0] = predictor_.ComputeResidual(0, prediction, dcv);
+    predictor_.ComputeDecoded(0, x, y, predictions, residuals, decoded);
+
+    prediction = predictions[2];
+    dcv = predictor_.GetDcValue(2, y, x);
+    residuals[2] = predictor_.ComputeResidual(2, prediction, dcv);
+    predictor_.ComputeDecoded(2, x, y, predictions, residuals, decoded);
+
+    int ctx = predictor_.Context(1, num_correct[1], min_error[1]);
+    TokenizeHybridUint(ctx, residuals[1], tokens_);
+    ctx = predictor_.Context(0, num_correct[0], min_error[0]);
+    TokenizeHybridUint(ctx, residuals[0], tokens_);
+    ctx = predictor_.Context(2, num_correct[2], min_error[2]);
+    TokenizeHybridUint(ctx, residuals[2], tokens_);
+  }
+
+  void StartRow(size_t y) { predictor_.StartRow(y); }
+
+ private:
+  std::vector<Token>* JXL_RESTRICT tokens_;
+  DcCoderBase predictor_;
+};
+
+class DCDecoder : public DcPredictor<DCDecoder> {
+ public:
+  DCDecoder(size_t xsize, const uint32_t* JXL_RESTRICT extra_levels,
+            const float* JXL_RESTRICT mul_dc,
+            const float* JXL_RESTRICT cmap_factor,
+            float** JXL_RESTRICT dc_dec_rows, size_t dc_dec_stride,
+            float** JXL_RESTRICT dc_quant_field_rows,
+            size_t dc_quant_field_stride)
+      : predictor_(xsize, extra_levels, mul_dc, nullptr, cmap_factor, nullptr,
+                   0, dc_dec_rows, dc_dec_stride, dc_quant_field_rows,
+                   dc_quant_field_stride) {}
+
+  HWY_ATTR void Run(size_t xsize, size_t ysize, BitReader* JXL_RESTRICT br,
+                    ANSSymbolReader* JXL_RESTRICT decoder,
+                    const std::vector<uint8_t>& JXL_RESTRICT context_map,
+                    AuxOut* JXL_RESTRICT aux_out) {
+    br_ = br;
+    decoder_ = decoder;
+    context_map_ = &context_map;
+    DcPredictor<DCDecoder>::Run(xsize, ysize, aux_out);
+    predictor_.FlushRow(ysize - 1);
+  }
+
+  HWY_ATTR JXL_INLINE void Prediction(size_t x, size_t y,
+                                      const int32_t* JXL_RESTRICT predictions,
+                                      const uint32_t* JXL_RESTRICT num_correct,
+                                      const uint32_t* JXL_RESTRICT min_error,
+                                      int32_t* JXL_RESTRICT decoded) {
+    HWY_ALIGN uint32_t residuals[4];
+    residuals[3] = 0;
+
+    for (size_t c : {1, 0, 2}) {
+      int ctx = predictor_.Context(c, num_correct[c], min_error[c]);
+      residuals[c] = ReadHybridUint(ctx, br_, decoder_, *context_map_);
+    }
+    predictor_.ComputeDecoded3(x, y, predictions, residuals, decoded);
+  }
+  void StartRow(size_t y) { predictor_.StartRow(y); }
+
+ private:
+  BitReader* JXL_RESTRICT br_;
+  ANSSymbolReader* JXL_RESTRICT decoder_;
+  const std::vector<uint8_t>* JXL_RESTRICT context_map_;
+  DcCoderBase predictor_;
+};
+
+void RunDCEncoder(bool progressive16, const uint32_t* JXL_RESTRICT extra_levels,
+                  const Rect& rect, const float* JXL_RESTRICT mul_dc,
+                  const float* JXL_RESTRICT inv_mul_dc,
+                  const float* JXL_RESTRICT cmap_factor,
+                  const float* JXL_RESTRICT* JXL_RESTRICT downsampled_dc_rows,
+                  size_t downsampled_dc_stride,
+                  const float* JXL_RESTRICT* JXL_RESTRICT dc_rows,
+                  size_t dc_stride, float** JXL_RESTRICT dc_dec_rows,
+                  size_t dc_dec_stride,
+                  float** JXL_RESTRICT dc_quant_field_rows,
+                  size_t dc_quant_field_stride, std::vector<Token>* tokens,
+                  AuxOut* JXL_RESTRICT aux_out) {}
+
+void RunDCDecoder(bool progressive16, const uint32_t* JXL_RESTRICT extra_levels,
+                  const Rect& rect, const float* JXL_RESTRICT mul_dc,
+                  const float* JXL_RESTRICT cmap_factor,
+                  const float* JXL_RESTRICT* JXL_RESTRICT downsampled_dc_rows,
+                  size_t downsampled_dc_stride,
+                  float** JXL_RESTRICT dc_dec_rows, size_t dc_dec_stride,
+                  float** JXL_RESTRICT dc_quant_field_rows,
+                  size_t dc_quant_field_stride, BitReader* JXL_RESTRICT br,
+                  ANSSymbolReader* JXL_RESTRICT decoder,
+                  const std::vector<uint8_t>& context_map,
+                  AuxOut* JXL_RESTRICT aux_out) {}
+
+namespace {
+HWY_ALIGN const uint32_t kExtraLevels[4][4] = {
+    {0, 0, 0, /*unused*/ 0},
+    {1, 1, 1, /*unused*/ 0},
+    {1, 2, 1, /*unused*/ 0},
+    {2, 3, 2, /*unused*/ 0},
+};
+}
+
+}  // namespace
+
+void TokenizeDC(size_t group_index, const Image3F& dc,
+                PassesEncoderState* JXL_RESTRICT enc_state, AuxOut* aux_out) {
+  const Rect rect = enc_state->shared.DCGroupRect(group_index);
+  const Quantizer& quantizer = enc_state->shared.quantizer;
+  const ColorCorrelationMap& cmap = enc_state->shared.cmap;
+  Image3F* dc_dec = &enc_state->shared.dc_storage;
+  Image3F* dc_quant_field = &enc_state->shared.dc_quant_field;
+  const float* JXL_RESTRICT dc_rows[3] = {rect.ConstPlaneRow(dc, 0, 0),
+                                          rect.ConstPlaneRow(dc, 1, 0),
+                                          rect.ConstPlaneRow(dc, 2, 0)};
+  const size_t dc_stride = dc.PixelsPerRow();
+  // Not restrict: the predictor will create aliasing pointers in these arrays,
+  // and they are not used directly anyway.
+  float* dc_dec_rows[3] = {rect.PlaneRow(dc_dec, 0, 0),
+                           rect.PlaneRow(dc_dec, 1, 0),
+                           rect.PlaneRow(dc_dec, 2, 0)};
+  const size_t dc_dec_stride = dc_dec->PixelsPerRow();
+
+  float* dc_quant_field_rows[3] = {rect.PlaneRow(dc_quant_field, 0, 0),
+                                   rect.PlaneRow(dc_quant_field, 1, 0),
+                                   rect.PlaneRow(dc_quant_field, 2, 0)};
+  const size_t dc_quant_field_stride = dc_quant_field->PixelsPerRow();
+
+  DCEncoder coder(rect.xsize(),
+                  kExtraLevels[enc_state->extra_dc_levels[group_index]],
+                  quantizer.MulDC(), quantizer.InvMulDC(), cmap.DCFactors(),
+                  dc_rows, dc_stride, dc_dec_rows, dc_dec_stride,
+                  dc_quant_field_rows, dc_quant_field_stride);
+  coder.Run(rect.xsize(), rect.ysize(), &enc_state->dc_tokens[group_index],
+            aux_out);
+}
+
+void DecodeDC(BitReader* br, ANSSymbolReader* decoder,
+              const std::vector<uint8_t>& context_map, const Rect& rect,
+              const float* mul_dc, const float* cmap_factor, int extra_levels,
+              Image3F* JXL_RESTRICT dc, Image3F* JXL_RESTRICT dc_quant_field,
+              AuxOut* aux_out) {
+  PROFILER_FUNC;
+  // Not restrict: the predictor will create aliasing pointers in these arrays,
+  // and they are not used directly anyway.
+  float* dc_rows[3] = {rect.PlaneRow(dc, 0, 0), rect.PlaneRow(dc, 1, 0),
+                       rect.PlaneRow(dc, 2, 0)};
+  const size_t dc_stride = dc->PixelsPerRow();
+  float* dc_quant_field_rows[3] = {rect.PlaneRow(dc_quant_field, 0, 0),
+                                   rect.PlaneRow(dc_quant_field, 1, 0),
+                                   rect.PlaneRow(dc_quant_field, 2, 0)};
+  const size_t dc_quant_field_stride = dc_quant_field->PixelsPerRow();
+
+  DCDecoder coder(rect.xsize(), kExtraLevels[extra_levels], mul_dc, cmap_factor,
+                  dc_rows, dc_stride, dc_quant_field_rows,
+                  dc_quant_field_stride);
+  coder.Run(rect.xsize(), rect.ysize(), br, decoder, context_map, aux_out);
+}
+
+constexpr size_t kCmapBaseContext = DcCoderBase::kNumContexts;
+constexpr size_t kControlFieldBaseContext = kCmapBaseContext + kCmapContexts;
+constexpr size_t kNumDCContexts =
+    kControlFieldBaseContext + kNumControlFieldContexts;
+
+Status EncodeDCGroup(const PassesEncoderState& enc_state, size_t group_idx,
+                     BitWriter* writer, AuxOut* aux_out) {
+  const Rect rect = enc_state.shared.DCGroupRect(group_idx);
+  // Single set of tokens to avoid overhead of multiple ANS streams.
+  std::vector<std::vector<Token>> tokens(1);
+
+  if (!(enc_state.shared.frame_header.flags & FrameHeader::kUseDcFrame)) {
+    JXL_ASSERT(enc_state.extra_dc_levels[group_idx] < 4);
+    // Copy DC tokens from enc cache.
+    tokens[0] = enc_state.dc_tokens[group_idx];
+
+    BitWriter::Allotment allotment(writer, 1 + 2 * kBitsPerByte);
+    // Write number of extra DC levels.
+    writer->Write(2, enc_state.extra_dc_levels[group_idx]);
+    ReclaimAndCharge(writer, &allotment, kLayerDC, aux_out);
+  }
+
+  if (!enc_state.shared.experiments.use_new_cmap) {
+    JXL_ASSERT(rect.x0() % kColorTileDimInBlocks == 0);
+    JXL_ASSERT(rect.y0() % kColorTileDimInBlocks == 0);
+    Rect cmap_rect(rect.x0() / kColorTileDimInBlocks,
+                   rect.y0() / kColorTileDimInBlocks,
+                   DivCeil(rect.xsize(), kColorTileDimInBlocks),
+                   DivCeil(rect.ysize(), kColorTileDimInBlocks));
+
+    EncodeColorMap(enc_state.shared.cmap, cmap_rect, &tokens[0],
+                   kCmapBaseContext, aux_out);
+  }
+
+  TokenizeAcStrategy(rect, enc_state.shared.ac_strategy, &tokens[0],
+                     kControlFieldBaseContext);
+
+  TokenizeQuantField(rect, enc_state.shared.raw_quant_field,
+                     enc_state.shared.ac_strategy, &tokens[0],
+                     kControlFieldBaseContext + kAcStrategyContexts);
+
+  if (enc_state.shared.image_features.loop_filter.epf) {
+    TokenizeARParameters(
+        rect, enc_state.shared.epf_sharpness, enc_state.shared.ac_strategy,
+        &tokens[0],
+        kControlFieldBaseContext + kAcStrategyContexts + kQuantFieldContexts);
+  }
+
+  std::vector<uint8_t> context_map;
+  EntropyEncodingData codes;
+  HistogramParams hist_params(enc_state.cparams.speed_tier, kNumDCContexts);
+  BuildAndEncodeHistograms(hist_params, kNumDCContexts, tokens, &codes,
+                           &context_map, writer, kLayerDC, aux_out);
+  WriteTokens(tokens[0], codes, context_map, writer, kLayerDC, aux_out);
+
+  return true;
+}
+
+// `rect`: block units.
+HWY_ATTR Status DecodeDCGroup(BitReader* reader, size_t group_idx,
+                              PassesDecoderState* dec_state, AuxOut* aux_out) {
+  PROFILER_FUNC;
+  int extra_dc_levels = 0;
+
+  if (!(dec_state->shared->frame_header.flags & FrameHeader::kUseDcFrame)) {
+    extra_dc_levels = reader->ReadFixedBits<2>();
+  }
+
+  std::vector<uint8_t> context_map;
+  ANSCode code;
+  JXL_RETURN_IF_ERROR(DecodeHistograms(
+      reader, kNumDCContexts, ANS_MAX_ALPHA_SIZE, &code, &context_map));
+  ANSSymbolReader decoder(&code, reader);
+
+  const Rect rect = dec_state->shared->DCGroupRect(group_idx);
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+  Rect rect0(0, 0, xsize, ysize);
+  if (!(dec_state->shared->frame_header.flags & FrameHeader::kUseDcFrame)) {
+    DecodeDC(reader, &decoder, context_map, rect,
+             dec_state->shared->quantizer.MulDC(),
+             dec_state->shared->cmap.DCFactors(), extra_dc_levels,
+             &dec_state->shared_storage.dc_storage,
+             &dec_state->shared_storage.dc_quant_field, aux_out);
+  }
+
+  if (!dec_state->shared_storage.experiments.use_new_cmap) {
+    JXL_ASSERT(rect.x0() % kColorTileDimInBlocks == 0);
+    JXL_ASSERT(rect.y0() % kColorTileDimInBlocks == 0);
+    Rect cmap_rect(rect.x0() / kColorTileDimInBlocks,
+                   rect.y0() / kColorTileDimInBlocks,
+                   DivCeil(rect.xsize(), kColorTileDimInBlocks),
+                   DivCeil(rect.ysize(), kColorTileDimInBlocks));
+
+    // Not all default
+    JXL_RETURN_IF_ERROR(DecodeColorMap(reader, &decoder, context_map,
+                                       &dec_state->shared_storage.cmap,
+                                       cmap_rect, kCmapBaseContext, aux_out));
+  }
+
+  if (!DecodeAcStrategy(reader, &decoder, context_map, rect,
+                        &dec_state->shared_storage.ac_strategy,
+                        kControlFieldBaseContext)) {
+    return JXL_FAILURE("Failed to decode AcStrategy.");
+  }
+
+  if (!DecodeQuantField(reader, &decoder, context_map, rect,
+                        dec_state->shared_storage.ac_strategy,
+                        &dec_state->shared_storage.raw_quant_field,
+                        kControlFieldBaseContext + kAcStrategyContexts)) {
+    return JXL_FAILURE("Failed to decode QuantField.");
+  }
+
+  if (dec_state->shared->image_features.loop_filter.epf &&
+      !DecodeARParameters(reader, &decoder, context_map, rect,
+                          dec_state->shared_storage.ac_strategy,
+                          &dec_state->shared_storage.epf_sharpness,
+                          kControlFieldBaseContext + kAcStrategyContexts +
+                              kQuantFieldContexts)) {
+    return JXL_FAILURE("Failed to decode ARParameters.");
+  }
+
+  if (!decoder.CheckANSFinalState()) {
+    return JXL_FAILURE("DC group: ANS checksum failure.");
+  }
+
+  return true;
+}
+
+namespace {
+using D = HWY_FULL(float);
+using DScalar = hwy::Scalar<float>;
+using V = hwy::VT<D>;
+
+// TODO(veluca): optimize constants.
+const float w1 = 0.20345139757231578f;
+const float w2 = 0.0334829185968739f;
+const float w0 = 1.0f - 4.0f * (w1 + w2);
+
+template <class V>
+HWY_ATTR V MaxWorkaround(V a, V b) {
+#if (HWY_ALL_TARGETS & HWY_AVX512) && HWY_COMPILER_CLANG <= 800
+  // Prevents "Do not know how to split the result of this operator" error
+  return IfThenElse(a > b, a, b);
+#else
+  return Max(a, b);
+#endif
+}
+
+template <typename D>
+HWY_ATTR JXL_INLINE void ComputePixelChannel(
+    const D d, const float* JXL_RESTRICT dc_quant_row,
+    const float* JXL_RESTRICT row_top, const float* JXL_RESTRICT row,
+    const float* JXL_RESTRICT row_bottom, hwy::VT<D>* JXL_RESTRICT mc,
+    hwy::VT<D>* JXL_RESTRICT sm, hwy::VT<D>* JXL_RESTRICT gap, size_t x) {
+  const auto tl = LoadU(d, row_top + x - 1);
+  const auto tc = Load(d, row_top + x);
+  const auto tr = LoadU(d, row_top + x + 1);
+
+  const auto ml = LoadU(d, row + x - 1);
+  *mc = Load(d, row + x);
+  const auto mr = LoadU(d, row + x + 1);
+
+  const auto bl = LoadU(d, row_bottom + x - 1);
+  const auto bc = Load(d, row_bottom + x);
+  const auto br = LoadU(d, row_bottom + x + 1);
+
+  const auto w_center = Set(d, w0);
+  const auto w_side = Set(d, w1);
+  const auto w_corner = Set(d, w2);
+
+  const auto corner = tl + tr + bl + br;
+  const auto side = ml + mr + tc + bc;
+  *sm = corner * w_corner + side * w_side + *mc * w_center;
+
+  constexpr uint32_t kMask = 0x7FFFFFFF;
+  float mask;
+  memcpy(&mask, &kMask, sizeof(float));
+  const auto sign_mask = Set(d, mask);
+
+  const auto dc_quant = Load(d, dc_quant_row + x);
+  *gap = MaxWorkaround(*gap, sign_mask & ((*mc - *sm) / dc_quant));
+}
+
+template <typename D>
+HWY_ATTR JXL_INLINE void ComputePixel(
+    const float* JXL_RESTRICT* JXL_RESTRICT dc_quant_rows,
+    const float* JXL_RESTRICT* JXL_RESTRICT rows_top,
+    const float* JXL_RESTRICT* JXL_RESTRICT rows,
+    const float* JXL_RESTRICT* JXL_RESTRICT rows_bottom,
+    float* JXL_RESTRICT* JXL_RESTRICT out_rows, size_t x) {
+  const D d;
+  auto mc_x = Undefined(d);
+  auto mc_y = Undefined(d);
+  auto mc_b = Undefined(d);
+  auto sm_x = Undefined(d);
+  auto sm_y = Undefined(d);
+  auto sm_b = Undefined(d);
+  auto gap = Set(d, 0.5f);
+  ComputePixelChannel(d, dc_quant_rows[0], rows_top[0], rows[0], rows_bottom[0],
+                      &mc_x, &sm_x, &gap, x);
+  ComputePixelChannel(d, dc_quant_rows[1], rows_top[1], rows[1], rows_bottom[1],
+                      &mc_y, &sm_y, &gap, x);
+  ComputePixelChannel(d, dc_quant_rows[2], rows_top[2], rows[2], rows_bottom[2],
+                      &mc_b, &sm_b, &gap, x);
+  auto factor = MulAdd(Set(d, -4.0f), gap, Set(d, 3.0f));
+  factor = ZeroIfNegative(factor);
+
+  auto out = MulAdd(sm_x - mc_x, factor, mc_x);
+  Store(out, d, out_rows[0] + x);
+  out = MulAdd(sm_y - mc_y, factor, mc_y);
+  Store(out, d, out_rows[1] + x);
+  out = MulAdd(sm_b - mc_b, factor, mc_b);
+  Store(out, d, out_rows[2] + x);
+}
+
+}  // namespace
+
+HWY_ATTR void AdaptiveDCSmoothing(const Image3F& dc_quant_field, Image3F* dc,
+                                  ThreadPool* pool) {
+  const size_t xsize = dc->xsize();
+  const size_t ysize = dc->ysize();
+  if (ysize <= 2 || xsize <= 2) return;
+
+  // TODO(veluca): use tile-based processing?
+  // TODO(veluca): decide if changes to the y channel should be propagated to
+  // the x and b channels through color correlation.
+  JXL_ASSERT(w1 + w2 < 0.25f);
+
+  PROFILER_FUNC;
+
+  Image3F smoothed(xsize, ysize);
+  // Fill in borders that the loop below will not. First and last are unused.
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t y : {size_t(0), ysize - 1}) {
+      memcpy(smoothed.PlaneRow(c, y), dc->PlaneRow(c, y),
+             xsize * sizeof(float));
+    }
+  }
+  auto process_row = [&](int y, int /*thread*/) HWY_ATTR {
+    const float* JXL_RESTRICT rows_top[3]{
+        dc->ConstPlaneRow(0, y - 1),
+        dc->ConstPlaneRow(1, y - 1),
+        dc->ConstPlaneRow(2, y - 1),
+    };
+    const float* JXL_RESTRICT rows[3] = {
+        dc->ConstPlaneRow(0, y),
+        dc->ConstPlaneRow(1, y),
+        dc->ConstPlaneRow(2, y),
+    };
+    const float* JXL_RESTRICT rows_bottom[3] = {
+        dc->ConstPlaneRow(0, y + 1),
+        dc->ConstPlaneRow(1, y + 1),
+        dc->ConstPlaneRow(2, y + 1),
+    };
+    const float* JXL_RESTRICT rows_dc_quant[3] = {
+        dc_quant_field.ConstPlaneRow(0, y),
+        dc_quant_field.ConstPlaneRow(1, y),
+        dc_quant_field.ConstPlaneRow(2, y),
+    };
+    float* JXL_RESTRICT rows_out[3] = {
+        smoothed.PlaneRow(0, y),
+        smoothed.PlaneRow(1, y),
+        smoothed.PlaneRow(2, y),
+    };
+    for (size_t x : {size_t(0), xsize - 1}) {
+      for (size_t c = 0; c < 3; c++) {
+        rows_out[c][x] = rows[c][x];
+      }
+    }
+
+    size_t x = 1;
+    // First pixels
+    constexpr size_t kVectorSize = D::N;
+    for (; x < std::min(kVectorSize, xsize - 1); x++) {
+      ComputePixel<DScalar>(rows_dc_quant, rows_top, rows, rows_bottom,
+                            rows_out, x);
+    }
+    // Full vectors.
+    for (; x + D::N <= xsize - 1; x += D::N) {
+      ComputePixel<D>(rows_dc_quant, rows_top, rows, rows_bottom, rows_out, x);
+    }
+    // Last pixels.
+    for (; x < xsize - 1; x++) {
+      ComputePixel<DScalar>(rows_dc_quant, rows_top, rows, rows_bottom,
+                            rows_out, x);
+    }
+  };
+  RunOnPool(pool, 1, ysize - 1, ThreadPool::SkipInit(), process_row,
+            "DCSmoothingRow");
+  dc->Swap(smoothed);
+}
+
+}  // namespace jxl
diff --git a/jxl/compressed_dc.h b/jxl/compressed_dc.h
new file mode 100644
index 000000000..1b10b94d4
--- /dev/null
+++ b/jxl/compressed_dc.h
@@ -0,0 +1,62 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_COMPRESSED_DC_H_
+#define JXL_COMPRESSED_DC_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/static_targets.h>
+#include <vector>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_cache.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_cache.h"
+#include "jxl/image.h"
+
+// DC handling functions: encoding and decoding of DC to and from bitstream, and
+// related function to initialize the per-group decoder cache.
+
+namespace jxl {
+
+void TokenizeDC(size_t group_index, const Image3F& dc,
+                PassesEncoderState* JXL_RESTRICT enc_state, AuxOut* aux_out);
+
+// Encodes the DC-related information from enc_state: quantized dc itself
+// and gradient map.
+Status EncodeDCGroup(const PassesEncoderState& enc_state, size_t group_idx,
+                     BitWriter* writer, AuxOut* aux_out);
+
+// Decodes and dequantizes DC.
+Status DecodeDCGroup(BitReader* reader, size_t group_idx,
+                     PassesDecoderState* dec_state, AuxOut* aux_out);
+
+// Fill border of DC image.
+void InitializeDCBorder(Image3F* JXL_RESTRICT dc);
+
+// Smooth DC in already-smooth areas, to counteract banding.
+HWY_ATTR void AdaptiveDCSmoothing(const Image3F& dc_quant_field, Image3F* dc,
+                                  ThreadPool* pool);
+
+}  // namespace jxl
+
+#endif  // JXL_COMPRESSED_DC_H_
diff --git a/jxl/compressed_dc_test.cc b/jxl/compressed_dc_test.cc
new file mode 100644
index 000000000..816ace617
--- /dev/null
+++ b/jxl/compressed_dc_test.cc
@@ -0,0 +1,135 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_file.h"
+#include "jxl/enc_params.h"
+#include "jxl/extras/codec.h"
+#include "jxl/image.h"
+#include "jxl/image_test_utils.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+
+struct CompressedDCTestParams {
+  explicit CompressedDCTestParams(const double butteraugli_distance,
+                                  const bool fast_mode = false,
+                                  const bool shrink8 = false)
+      : butteraugli_distance(butteraugli_distance),
+        fast_mode(fast_mode),
+        shrink8(shrink8) {}
+  double butteraugli_distance;
+  bool fast_mode;
+  bool shrink8;
+};
+
+std::ostream& operator<<(std::ostream& os, CompressedDCTestParams params) {
+  auto previous_flags = os.flags();
+  os << std::boolalpha;
+  os << "CompressedDCTestParams{/*butteraugli_distance=*/"
+     << params.butteraugli_distance << ", /*fast_mode=*/" << params.fast_mode
+     << ", /*shrink8=*/" << params.shrink8 << "}";
+  os.flags(previous_flags);
+  return os;
+}
+
+class CompressedDCTest : public testing::TestWithParam<CompressedDCTestParams> {
+ protected:
+  // Returns compressed size [bytes].
+  static void Roundtrip(CodecInOut* io, const CompressParams& cparams,
+                        const DecompressParams& dparams, ThreadPool* pool) {
+    PaddedBytes compressed;
+
+    Image3F encoding_dc;
+    Image3F encoding_dec;
+    AuxOut encoding_info;
+    encoding_info.testing_aux.dc = &encoding_dc;
+    encoding_info.testing_aux.decoded = &encoding_dec;
+
+    PassesEncoderState enc_state;
+    EXPECT_TRUE(
+        EncodeFile(cparams, io, &enc_state, &compressed, &encoding_info, pool));
+
+    Image3F decoding_dc;
+    AuxOut decoding_info;
+    decoding_info.testing_aux.dc = &decoding_dc;
+
+    EXPECT_TRUE(DecodeFile(dparams, compressed, io, &decoding_info, pool));
+
+#ifdef __aarch64__
+    // Roundtrip precision is a bit worse in arm64.
+    // TODO(veluca): Investigate why this is the case.
+    const float kErrorThreshold = 6e-6;
+#else
+    const float kErrorThreshold = 1e-6;
+#endif
+
+    VerifyRelativeError(decoding_dc, encoding_dc, kErrorThreshold,
+                        kErrorThreshold);
+    VerifyRelativeError(io->Main().color(), encoding_dec,
+                                  kErrorThreshold, kErrorThreshold);
+  }
+};
+
+INSTANTIATE_TEST_SUITE_P(
+    CompressedDCTestInstantiation, CompressedDCTest,
+    testing::Values(CompressedDCTestParams{/*butteraugli_distance=*/1.0,
+                                           /*fast_mode=*/false,
+                                           /*shrink8=*/true},
+                    CompressedDCTestParams{/*butteraugli_distance=*/1.0},
+                    CompressedDCTestParams{/*butteraugli_distance=*/1.5},
+                    CompressedDCTestParams{/*butteraugli_distance=*/2.0},
+                    CompressedDCTestParams{/*butteraugli_distance=*/1.0,
+                                           /*fast_mode=*/true},
+                    CompressedDCTestParams{/*butteraugli_distance=*/1.5,
+                                           /*fast_mode=*/true},
+                    CompressedDCTestParams{/*butteraugli_distance=*/2.0,
+                                           /*fast_mode=*/true}));
+
+TEST_P(CompressedDCTest, Roundtrip) {
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ThreadPoolInternal pool(8);
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  const CompressedDCTestParams& params = GetParam();
+
+  if (params.shrink8) {
+    io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+  }
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = params.butteraugli_distance;
+  if (params.fast_mode) {
+    cparams.speed_tier = SpeedTier::kSquirrel;
+  }
+  DecompressParams dparams;
+
+  Roundtrip(&io, cparams, dparams, &pool);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/compressed_image_test.cc b/jxl/compressed_image_test.cc
new file mode 100644
index 000000000..68899f93f
--- /dev/null
+++ b/jxl/compressed_image_test.cc
@@ -0,0 +1,120 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <algorithm>
+#include <string>
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "jxl/ac_strategy.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/enc_adaptive_quantization.h"
+#include "jxl/enc_butteraugli_comparator.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_params.h"
+#include "jxl/enc_xyb.h"
+#include "jxl/extras/codec.h"
+#include "jxl/frame_header.h"
+#include "jxl/gaborish.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/loop_filter.h"
+#include "jxl/multiframe.h"
+#include "jxl/passes_state.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+
+// Verifies ReconOpsinImage reconstructs with low butteraugli distance.
+void RunRGBRoundTrip(float distance, bool fast) {
+  ThreadPoolInternal pool(4);
+
+  const std::string& pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  JXL_CHECK(SetFromFile(pathname, &io, &pool));
+  // This test can only handle a single group.
+  io.ShrinkTo(std::min(io.xsize(), kGroupDim), std::min(io.ysize(), kGroupDim));
+
+  Image3F opsin(io.xsize(), io.ysize());
+  ToXYB(io.Main(), 1.0f, &pool, &opsin);
+  opsin = PadImageToMultiple(opsin, kBlockDim);
+  opsin = GaborishInverse(opsin, 1.0, &pool);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = distance;
+  if (fast) {
+    cparams.speed_tier = SpeedTier::kSquirrel;
+  }
+
+  FrameHeader frame_header;
+  frame_header.animation_frame.nonserialized_have_timecode = false;
+  LoopFilter loop_filter;
+  loop_filter.gab = true;
+  loop_filter.gab_custom = true;
+  loop_filter.gab_x_weight1 = 0.11501538179658321;
+  loop_filter.gab_x_weight2 = 0.089979079587015454;
+  loop_filter.gab_y_weight1 = 0.11501538179658321;
+  loop_filter.gab_y_weight2 = 0.089979079587015454;
+  loop_filter.gab_b_weight1 = 0.11501538179658321;
+  loop_filter.gab_b_weight2 = 0.089979079587015454;
+  loop_filter.epf = false;
+
+  FrameDimensions frame_dim;
+  frame_dim.Set(opsin.xsize(), opsin.ysize());
+  Multiframe multiframe;
+  PassesEncoderState enc_state;
+  JXL_CHECK(InitializePassesSharedState(frame_header, loop_filter, io.metadata,
+                                        frame_dim, &multiframe,
+                                        &enc_state.shared));
+
+  enc_state.x_qm_multiplier = 1.0f;
+  enc_state.shared.quantizer.SetQuant(4.0f, 4.0f,
+                                      &enc_state.shared.raw_quant_field);
+  enc_state.shared.ac_strategy.FillDCT8();
+  enc_state.cparams = cparams;
+  ZeroFillImage(&enc_state.shared.epf_sharpness);
+  Image3F recon = RoundtripImage(opsin, &enc_state, &pool);
+
+  CodecInOut io1;
+  io1.metadata.bits_per_sample = io.metadata.bits_per_sample;
+  io1.metadata.color_encoding = ColorManagement::LinearSRGB();
+  io1.SetFromImage(std::move(recon), io1.metadata.color_encoding);
+
+  EXPECT_LE(ButteraugliDistance(io, io1, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, &pool),
+            1.2);
+}
+
+TEST(CompressedImageTest, RGBRoundTrip_1) { RunRGBRoundTrip(1.0, false); }
+
+TEST(CompressedImageTest, RGBRoundTrip_1_fast) { RunRGBRoundTrip(1.0, true); }
+
+TEST(CompressedImageTest, RGBRoundTrip_2) { RunRGBRoundTrip(2.0, false); }
+
+TEST(CompressedImageTest, RGBRoundTrip_2_fast) { RunRGBRoundTrip(2.0, true); }
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/convolve.h b/jxl/convolve.h
new file mode 100644
index 000000000..4aaa16618
--- /dev/null
+++ b/jxl/convolve.h
@@ -0,0 +1,1265 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_CONVOLVE_H_
+#define JXL_CONVOLVE_H_
+
+// Fast SIMD 2D convolution.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <hwy/arch.h>
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/image.h"
+#include "jxl/image_ops.h"
+
+namespace jxl {
+
+// Usable by any 3x3 kernel; applied as-is without flipping.
+struct Weights3x3 {
+  // top/middle/bottom left/center/right, replicated 4x via HWY_REP4.
+  float tl[4];
+  float tc[4];
+  float tr[4];
+  float ml[4];
+  float mc[4];
+  float mr[4];
+  float bl[4];
+  float bc[4];
+  float br[4];
+};
+
+struct WeightsSeparable5 {
+  // Horizontal 1D, distances 0..2, each replicated 4x.
+  float horz[3 * 4];
+  float vert[3 * 4];
+};
+
+struct WeightsSeparable7 {
+  // Horizontal 1D, distances 0..3, each replicated 4x.
+  float horz[4 * 4];
+  float vert[4 * 4];
+};
+
+// For code-folding.
+namespace kernel {
+
+// Holds weights computed at runtime (e.g. inverse of another kernel).
+class Variable3 {
+ public:
+  explicit Variable3(const float tl, const float tc, const float tr,
+                     const float ml, const float mc, const float mr,
+                     const float bl, const float bc, const float br) {
+    for (size_t i = 0; i < 4; ++i) {
+      weights_.tl[i] = tl;
+      weights_.tc[i] = tc;
+      weights_.tr[i] = tr;
+      weights_.ml[i] = ml;
+      weights_.mc[i] = mc;
+      weights_.mr[i] = mr;
+      weights_.bl[i] = bl;
+      weights_.bc[i] = bc;
+      weights_.br[i] = br;
+    }
+  }
+
+  JXL_INLINE const Weights3x3& Weights() const { return weights_; }
+
+ private:
+  Weights3x3 weights_;
+};
+
+// Concentrates energy in low-frequency components (e.g. for antialiasing).
+struct Lowpass3 {
+  JXL_INLINE const Weights3x3& Weights() const {
+    // Computed by research/convolve_weights.py's cubic spline approximations of
+    // prolate spheroidal wave functions.
+    constexpr float w0 = 0.36208932f;
+    constexpr float w1 = 0.12820096f;
+    constexpr float w2 = 0.03127668f;
+    static constexpr Weights3x3 weights = {
+        {HWY_REP4(w2)}, {HWY_REP4(w1)}, {HWY_REP4(w2)},
+        {HWY_REP4(w1)}, {HWY_REP4(w0)}, {HWY_REP4(w1)},
+        {HWY_REP4(w2)}, {HWY_REP4(w1)}, {HWY_REP4(w2)}};
+    return weights;
+  }
+};
+
+struct Lowpass5 {
+  JXL_INLINE const WeightsSeparable5& Weights() const {
+    constexpr float w0 = 0.41714928f;
+    constexpr float w1 = 0.25539268f;
+    constexpr float w2 = 0.03603267f;
+    static constexpr WeightsSeparable5 weights = {
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)},
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}};
+    return weights;
+  }
+};
+
+struct Gaussian5Sigma1 {
+  JXL_INLINE const WeightsSeparable5& Weights() const {
+    constexpr float w0 = 0.38774f;
+    constexpr float w1 = 0.24477f;
+    constexpr float w2 = 0.06136f;
+    static constexpr WeightsSeparable5 weights = {
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)},
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}};
+    return weights;
+  }
+};
+
+struct Gaussian5Sigma2 {
+  JXL_INLINE const WeightsSeparable5& Weights() const {
+    constexpr float w0 = 0.250301f;
+    constexpr float w1 = 0.221461f;
+    constexpr float w2 = 0.153388f;
+    static constexpr WeightsSeparable5 weights = {
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)},
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}};
+    return weights;
+  }
+};
+
+struct Gaussian7Sigma8 {
+  JXL_INLINE const WeightsSeparable7& Weights() const {
+    constexpr float w0 = 0.147332f;
+    constexpr float w1 = 0.146187f;
+    constexpr float w2 = 0.142805f;
+    constexpr float w3 = 0.137341f;
+    static constexpr WeightsSeparable7 weights = {
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2), HWY_REP4(w3)},
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2), HWY_REP4(w3)}};
+    return weights;
+  }
+};
+
+}  // namespace kernel
+
+// Non-vectorized implementations for validation.
+namespace slow {
+
+// Separable kernels, any radius.
+template <int64_t kRadius, class Wrap>
+class SeparableConvolution {
+ public:
+  template <class Kernel>
+  static void Run(const ImageF& in, const Rect& rect, const Kernel& kernel,
+                  ImageF* out) {
+    PROFILER_ZONE("slow::Separable::Run");
+    const float* horz_weights = &kernel.Weights().horz[0];
+    const float* vert_weights = &kernel.Weights().vert[0];
+    for (size_t y = 0; y < rect.ysize(); ++y) {
+      float* const JXL_RESTRICT row_out = out->Row(y);
+      for (size_t x = 0; x < rect.xsize(); ++x) {
+        row_out[x] = ConvolvePixel(in, rect, x, y, horz_weights, vert_weights);
+      }
+    }
+  }
+
+  template <class Kernel>
+  static void Run(const Image3F& in, const Rect& rect, const Kernel& kernel,
+                  Image3F* out) {
+    for (size_t c = 0; c < 3; ++c) {
+      Run(in.Plane(c), rect, kernel, const_cast<ImageF*>(&out->Plane(c)));
+    }
+  }
+
+ private:
+  static float ConvolvePixel(const ImageF& in, const Rect& rect,
+                             const int64_t x, const int64_t y,
+                             const float* JXL_RESTRICT horz_weights,
+                             const float* JXL_RESTRICT vert_weights) {
+    const size_t xsize = rect.xsize();
+    const size_t ysize = rect.ysize();
+
+    float mul = 0.0f;
+    for (int dy = -kRadius; dy <= kRadius; ++dy) {
+      const float wy = vert_weights[std::abs(dy) * 4];
+      const size_t sy = Wrap()(y + dy, ysize);
+      JXL_CHECK(sy < ysize);
+      const float* const JXL_RESTRICT row = rect.ConstRow(in, sy);
+      for (int dx = -kRadius; dx <= kRadius; ++dx) {
+        const float wx = horz_weights[std::abs(dx) * 4];
+        const size_t sx = Wrap()(x + dx, xsize);
+        JXL_CHECK(sx < xsize);
+        mul += row[sx] * wx * wy;
+      }
+    }
+    return mul;
+  }
+};
+
+// Hardcoded 5x5 "Laplacian".
+template <int64_t kRadius, class Wrap>
+struct Laplacian5 {
+  static_assert(kRadius == 2, "Wrong kRadius");
+
+  template <class Kernel>
+  static void Run(const ImageF& in, const Rect& rect, const Kernel& kernel,
+                  ImageF* out) {
+    PROFILER_ZONE("slow::Laplacian5::Run");
+    JXL_CHECK(SameSize(rect, *out));
+
+    const size_t xsize = rect.xsize();
+    const size_t ysize = rect.ysize();
+    for (int64_t y = 0; y < ysize; ++y) {
+      const float* const JXL_RESTRICT row_t =
+          rect.ConstRow(in, Wrap()(y - 2, ysize));
+      const float* const JXL_RESTRICT row_m = rect.ConstRow(in, y);
+      const float* const JXL_RESTRICT row_b =
+          rect.ConstRow(in, Wrap()(y + 2, ysize));
+      float* const JXL_RESTRICT row_out = out->Row(y);
+
+      for (int64_t x = 0; x < xsize; ++x) {
+        const int64_t xm2 = Wrap()(x - 2, xsize);
+        const int64_t xp2 = Wrap()(x + 2, xsize);
+        float r = 0.0f;
+        r += /*               */ 1.0f * row_t[x];
+        r += 1.0f * row_m[xm2] - 4.0f * row_m[x] + 1.0f * row_m[xp2];
+        r += /*               */ 1.0f * row_b[x];
+        row_out[x] = r;
+      }
+    }
+  }
+
+  template <class Kernel>
+  static void Run(const Image3F& in, const Rect& rect, const Kernel& kernel,
+                  Image3F* out) {
+    for (size_t c = 0; c < 3; ++c) {
+      Run(in.Plane(c), rect, kernel, const_cast<ImageF*>(&out->Plane(c)));
+    }
+  }
+};
+
+template <int64_t kRadius, class Wrap>
+struct General3x3Convolution {
+  static_assert(kRadius == 1, "Wrong kRadius");
+
+  template <class Kernel>
+  static void Run(const ImageF& in, const Rect& rect, const Kernel& kernel,
+                  ImageF* out) {
+    PROFILER_ZONE("slow::General3::Run");
+    JXL_CHECK(SameSize(rect, *out));
+    const Weights3x3& weights = kernel.Weights();
+
+    const size_t xsize = rect.xsize();
+    const size_t ysize = rect.ysize();
+    for (int64_t y = 0; y < ysize; ++y) {
+      const float* const JXL_RESTRICT row_t =
+          rect.ConstRow(in, Wrap()(y - 1, ysize));
+      const float* const JXL_RESTRICT row_m = rect.ConstRow(in, y);
+      const float* const JXL_RESTRICT row_b =
+          rect.ConstRow(in, Wrap()(y + 1, ysize));
+      float* const JXL_RESTRICT row_out = out->Row(y);
+
+      for (int64_t x = 0; x < xsize; ++x) {
+        const int64_t xm1 = Wrap()(x - 1, xsize);
+        const int64_t xp1 = Wrap()(x + 1, xsize);
+        const float tl = row_t[xm1];
+        const float ml = row_m[xm1];
+        const float bl = row_b[xm1];
+        const float tr = row_t[xp1];
+        const float mr = row_m[xp1];
+        const float br = row_b[xp1];
+        float r = 0.0f;
+        r += tl * weights.tl[0] + row_t[x] * weights.tc[0] + tr * weights.tr[0];
+        r += ml * weights.ml[0] + row_m[x] * weights.mc[0] + mr * weights.mr[0];
+        r += bl * weights.bl[0] + row_b[x] * weights.bc[0] + br * weights.br[0];
+        row_out[x] = r;
+      }
+    }
+  }
+
+  template <class Kernel>
+  static void Run(const Image3F& in, const Rect& rect, const Kernel& kernel,
+                  Image3F* out) {
+    for (size_t c = 0; c < 3; ++c) {
+      Run(in.Plane(c), rect, kernel, const_cast<ImageF*>(&out->Plane(c)));
+    }
+  }
+};
+
+// Slow N*R^2 algorithm in case weights are not separable, but avoids
+// bounds-checking overhead for interior pixels. Weights are the lower-right
+// quadrant of the kernel and need not be pre-normalized.
+template <int64_t kRadius, class Wrap>
+class SymmetricConvolution {
+ public:
+  static void Run(const ImageF& in, const Rect& rect,
+                  const float (&weights)[(kRadius + 1) * (kRadius + 1)],
+                  ThreadPool* pool, ImageF* JXL_RESTRICT out) {
+    PROFILER_ZONE("slow::Symmetric::Run");
+
+    float normalized[(kRadius + 1) * (kRadius + 1)];
+    NormalizeWeights(weights, &normalized);
+    const size_t ysize = rect.ysize();
+    RunOnPool(
+        pool, 0, static_cast<int>(ysize), ThreadPool::SkipInit(),
+        [&](const int task, int /*thread*/) {
+          const int64_t iy = task;
+
+          if (iy < kRadius || iy >= ysize - kRadius) {
+            ConvolveRow<Wrap>(in, rect, iy, normalized, out->Row(iy));
+          } else {
+            ConvolveRow<WrapUnchanged>(in, rect, iy, normalized, out->Row(iy));
+          }
+        },
+        "Symmetric conv");
+  }
+
+  static void Run(const Image3F& in, const Rect& rect,
+                  const float (&weights)[(kRadius + 1) * (kRadius + 1)],
+                  ThreadPool* pool, Image3F* JXL_RESTRICT out) {
+    PROFILER_ZONE("slow::Symmetric::Run3");
+
+    float normalized[(kRadius + 1) * (kRadius + 1)];
+    NormalizeWeights(weights, &normalized);
+    const size_t ysize = rect.ysize();
+    RunOnPool(
+        pool, 0, static_cast<int>(ysize), ThreadPool::SkipInit(),
+        [&](const int task, int /*thread*/) {
+          const size_t iy = task;
+
+          if (iy < kRadius || iy >= ysize - kRadius) {
+            for (size_t c = 0; c < 3; ++c) {
+              ConvolveRow<Wrap>(in.Plane(c), rect, iy, normalized,
+                                out->PlaneRow(c, iy));
+            }
+          } else {
+            for (size_t c = 0; c < 3; ++c) {
+              ConvolveRow<WrapUnchanged>(in.Plane(c), rect, iy, normalized,
+                                         out->PlaneRow(c, iy));
+            }
+          }
+        },
+        "Symmetric conv3");
+  }
+
+ private:
+  // Expands quadrant into entire kernel and normalizes.
+  static void NormalizeWeights(
+      const float (&weights)[(kRadius + 1) * (kRadius + 1)],
+      float (*normalized)[(kRadius + 1) * (kRadius + 1)]) {
+    double sum = 0.0;
+    for (int64_t ky = -kRadius; ky <= kRadius; ky++) {
+      const int64_t wy = std::abs(ky);
+      for (int64_t kx = -kRadius; kx <= kRadius; kx++) {
+        const int64_t wx = std::abs(kx);
+        sum += weights[wy * (kRadius + 1) + wx];
+      }
+    }
+    const float mul = sum == 0.0f ? 1.0f : 1.0 / sum;
+    for (size_t i = 0; i < (kRadius + 1) * (kRadius + 1); ++i) {
+      (*normalized)[i] = weights[i] * mul;
+    }
+  }
+
+  template <class WrapX, class WrapY>
+  static float ConvolvePixel(
+      const ImageF& in, const Rect& rect, const int64_t ix, const int64_t iy,
+      const float (&weights)[(kRadius + 1) * (kRadius + 1)]) {
+    float sum = 0.0;
+
+    // ix: image; kx: kernel; wx: weight
+    for (int64_t ky = -kRadius; ky <= kRadius; ky++) {
+      const int64_t wy = std::abs(ky);
+      const int64_t y = WrapY()(iy + ky, rect.ysize());
+      const float* JXL_RESTRICT row_in = in.ConstRow(y);
+
+      for (int64_t kx = -kRadius; kx <= kRadius; kx++) {
+        const int64_t wx = std::abs(kx);
+        const int64_t x = WrapX()(ix + kx, rect.xsize());
+
+        sum += row_in[x] * weights[wy * (kRadius + 1) + wx];
+      }
+    }
+    return sum;
+  }
+
+  template <class WrapY>
+  static inline void ConvolveRow(
+      const ImageF& in, const Rect& rect, const int64_t iy,
+      const float (&weights)[(kRadius + 1) * (kRadius + 1)],
+      float* JXL_RESTRICT row_out) {
+    const size_t xsize = rect.xsize();
+    size_t ix = 0;
+    for (; ix < kRadius; ix++) {
+      row_out[ix] = ConvolvePixel<WrapMirror, WrapY>(in, rect, ix, iy, weights);
+    }
+    for (; ix < xsize - kRadius; ix++) {
+      row_out[ix] =
+          ConvolvePixel<WrapUnchanged, WrapY>(in, rect, ix, iy, weights);
+    }
+    for (; ix < xsize; ix++) {
+      row_out[ix] = ConvolvePixel<WrapMirror, WrapY>(in, rect, ix, iy, weights);
+    }
+  }
+};
+
+}  // namespace slow
+
+// Synthesizes left/right neighbors from a vector of center pixels.
+class Neighbors {
+ public:
+  // TODO(janwas): AVX512
+#if HWY_BITS >= 256
+  using D = HWY_CAPPED(float, 8);
+#else
+  using D = HWY_CAPPED(float, 4);
+#endif
+  using V = hwy::VT<D>;
+
+  // Returns l[i] == c[Mirror(i - 1)].
+  static HWY_ATTR JXL_INLINE V FirstL1(const V c) {
+#if HWY_BITS >= 256
+    const D d;
+    HWY_ALIGN constexpr int lanes[8] = {0, 0, 1, 2, 3, 4, 5, 6};
+    const auto indices = SetTableIndices(d, lanes);
+    // c = PONM'LKJI
+    return TableLookupLanes(c, indices);  // ONML'KJII
+#elif HWY_BITS == 128
+    // c = LKJI
+#if HWY_ARCH == HWY_ARCH_X86
+    return V{_mm_shuffle_ps(c.raw, c.raw, _MM_SHUFFLE(2, 1, 0, 0))};  // KJII
+#else
+    const D d;
+    // TODO(deymo): Figure out if this can be optimized using a single vsri
+    // instruction to convert LKJI to KJII.
+    HWY_ALIGN constexpr int lanes[4] = {0, 0, 1, 2};  // KJII
+    const auto indices = SetTableIndices(d, lanes);
+    return TableLookupLanes(c, indices);
+#endif
+#else
+    return c;  // Same (the first mirrored value is the last valid one)
+#endif
+  }
+
+  // Returns l[i] == c[Mirror(i - 2)].
+  static HWY_ATTR JXL_INLINE V FirstL2(const V c) {
+#if HWY_BITS >= 256
+    const D d;
+    HWY_ALIGN constexpr int lanes[8] = {1, 0, 0, 1, 2, 3, 4, 5};
+    const auto indices = SetTableIndices(d, lanes);
+    // c = PONM'LKJI
+    return TableLookupLanes(c, indices);  // NMLK'JIIJ
+#elif HWY_BITS == 128
+    // c = LKJI
+#if HWY_ARCH == HWY_ARCH_X86
+    return V{_mm_shuffle_ps(c.raw, c.raw, _MM_SHUFFLE(1, 0, 0, 1))};  // JIIJ
+#else
+    const D d;
+    HWY_ALIGN constexpr int lanes[4] = {1, 0, 0, 1};  // JIIJ
+    const auto indices = SetTableIndices(d, lanes);
+    return TableLookupLanes(c, indices);
+#endif
+#else
+    const D d;
+    JXL_ASSERT(false);  // unsupported, avoid calling this.
+    return Zero(d);
+#endif
+  }
+
+  // Returns l[i] == c[Mirror(i - 3)].
+  static HWY_ATTR JXL_INLINE V FirstL3(const V c) {
+#if HWY_BITS >= 256
+    const D d;
+    HWY_ALIGN constexpr int lanes[8] = {2, 1, 0, 0, 1, 2, 3, 4};
+    const auto indices = SetTableIndices(d, lanes);
+    // c = PONM'LKJI
+    return TableLookupLanes(c, indices);  // MLKJ'IIJK
+#elif HWY_BITS == 128
+    // c = LKJI
+#if HWY_ARCH == HWY_ARCH_X86
+    return V{_mm_shuffle_ps(c.raw, c.raw, _MM_SHUFFLE(0, 0, 1, 2))};  // IIJK
+#else
+    const D d;
+    HWY_ALIGN constexpr int lanes[4] = {2, 1, 0, 0};  // IIJK
+    const auto indices = SetTableIndices(d, lanes);
+    return TableLookupLanes(c, indices);
+#endif
+#else
+    const D d;
+    JXL_ASSERT(false);  // unsupported, avoid calling this.
+    return Zero(d);
+#endif
+  }
+
+  // Returns r[i] == c[i + 1].
+  static HWY_ATTR JXL_INLINE V R1(const V c, const V n) {
+#if HWY_BITS >= 256
+    // c = PONM'LKJI, n = xxxx'xxxQ
+    const V Q_M = ConcatLoHi(n, c);  // Right-aligned (lower lane)
+    return hwy::CombineShiftRightBytes<4>(Q_M, c);  // QPON'MLKJ
+#elif HWY_BITS == 128
+    // c = LKJI, n = xxxM
+    return hwy::CombineShiftRightBytes<4>(n, c);  // MLKJ
+#else
+    return n;
+#endif
+  }
+
+  // Returns r[i] == c[i + 1].
+  static HWY_ATTR JXL_INLINE V LastR1(const V c) {
+#if HWY_BITS >= 256
+    HWY_ALIGN constexpr uint32_t lanes[8] = {1, 2, 3, 4, 5, 6, 7, 7};
+    const auto indices = Load(HWY_CAPPED(uint32_t, 8)(), lanes);
+    // c = PONM'LKJI
+    return V{_mm256_permutevar8x32_ps(c.raw, indices.raw)};  // PPON'MLKJ
+#elif HWY_BITS == 128
+    // c = LKJI
+    const auto L = hwy::Broadcast<3>(c);
+    return hwy::CombineShiftRightBytes<4>(L, c);  // LLKJ
+#else
+    return c;
+#endif
+  }
+};
+
+// No valid values outside [0, xsize), but the strategy may still safely load
+// the preceding vector, and/or round xsize up to the vector lane count. This
+// avoids needing PadImage.
+// Requires xsize >= kConvolveLanes + kConvolveMaxRadius.
+static constexpr size_t kConvolveMaxRadius = 3;
+
+// For use by SetTableIndices.
+static inline const int32_t* MirrorLanes(const size_t mod) {
+#if HWY_BITS >= 256
+  HWY_CAPPED(float, 8) d;
+  // last  part  mirrored
+  // 01234567| 76543210   loadedReg 76543210 mirroredReg 01234567
+  // 01234567|8 8765432   loadedReg 87654321 mirroredReg 23456788
+  // 01234567|89 987654   loadedReg 98765432 mirroredReg 45678998
+  // 01234567|89A A9876   loadedReg A9876543 mirroredReg 6789AA98
+  // 01234567|89AB BA98
+  // 01234567|89ABC CBA
+  // 01234567|89ABCD DC
+  // 01234567|89ABCDE E   loadedReg EDCBA987 mirroredReg EEDCBA98
+  HWY_ALIGN static constexpr int32_t idx_lanes[d.N * d.N] = {
+      7, 6, 5, 4, 3, 2, 1, 0,  // 0
+      7, 7, 6, 5, 4, 3, 2, 1,  // 1
+      6, 7, 7, 6, 5, 4, 3, 2,  // 2
+      5, 6, 7, 7, 6, 5, 4, 3,  // 3
+      4, 5, 6, 7, 7, 6, 5, 4,  // 4
+      3, 4, 5, 6, 7, 7, 6, 5,  // 5
+      2, 3, 4, 5, 6, 7, 7, 6,  // 6
+      1, 2, 3, 4, 5, 6, 7, 7,  // 7
+  };
+  return idx_lanes + mod * d.N;
+#elif HWY_BITS == 128
+  HWY_CAPPED(float, 4) d;
+  // 0123| 3210   loadedReg 3210 mirroredReg 0123
+  // 0123|4 432   loadedReg 4321 mirroredReg 2344
+  // 0123|45 54   loadedReg 5432 mirroredReg 4554
+  // 0123|456 6   loadedReg 6543 mirroredReg 6654
+  HWY_ALIGN static constexpr int32_t idx_lanes[d.N * d.N] = {
+      3, 2, 1, 0,  // 0
+      3, 3, 2, 1,  // 1
+      2, 3, 3, 2,  // 2
+      1, 2, 3, 3,  // 3
+  };
+  return idx_lanes + mod * d.N;
+#else
+  return nullptr;  // do not call
+#endif
+}
+
+namespace strategy {
+
+struct StrategyBase {
+#if HWY_BITS >= 256
+  using D = HWY_CAPPED(float, 8);
+#else
+  using D = HWY_CAPPED(float, 4);
+#endif
+  using V = hwy::VT<D>;
+};
+
+// 3x3 convolution by symmetric kernel with a single scan through the input.
+class Symmetric3 : public StrategyBase {
+ public:
+  static constexpr int64_t kRadius = 1;
+
+  // Only accesses pixels in [0, xsize).
+  template <size_t kSizeModN, class WrapRow>
+  static HWY_ATTR JXL_INLINE void ConvolveRow(
+      const float* const JXL_RESTRICT row_m, const size_t xsize,
+      const int64_t stride, const WrapRow& wrap_row, const Weights3x3& weights,
+      float* const JXL_RESTRICT row_out) {
+    const D d;
+    // t, m, b = top, middle, bottom row;
+    const float* const JXL_RESTRICT row_t = wrap_row(row_m - stride, stride);
+    const float* const JXL_RESTRICT row_b = wrap_row(row_m + stride, stride);
+
+    // Must load in advance - compiler doesn't understand LoadDup128 and
+    // schedules them too late.
+    const V w0 = LoadDup128(d, weights.mc);
+    const V w1 = LoadDup128(d, weights.tc);
+    const V w2 = LoadDup128(d, weights.tl);
+
+    // l, c, r = left, center, right. Leftmost vector: need FirstL1.
+    {
+      const V tc = LoadU(d, row_t + 0);
+      const V mc = LoadU(d, row_m + 0);
+      const V bc = LoadU(d, row_b + 0);
+      const V tl = Neighbors::FirstL1(tc);
+      const V tr = LoadU(d, row_t + 0 + 1);
+      const V ml = Neighbors::FirstL1(mc);
+      const V mr = LoadU(d, row_m + 0 + 1);
+      const V bl = Neighbors::FirstL1(bc);
+      const V br = LoadU(d, row_b + 0 + 1);
+      const V conv =
+          WeightedSum(tl, tc, tr, ml, mc, mr, bl, bc, br, w0, w1, w2);
+      Store(conv, d, row_out + 0);
+    }
+
+    // Loop as long as we can load enough new values:
+    size_t x = d.N;
+    for (; x + d.N + kRadius <= xsize; x += d.N) {
+      const auto conv = ConvolveValid(row_t, row_m, row_b, x, w0, w1, w2);
+      Store(conv, d, row_out + x);
+    }
+
+    // For final (partial) vector:
+    const V tc = LoadU(d, row_t + x);
+    const V mc = LoadU(d, row_m + x);
+    const V bc = LoadU(d, row_b + x);
+
+    V tr, mr, br;
+#if HWY_BITS == 0
+    tr = tc;  // Single-lane => mirrored right neighbor = center value.
+    mr = mc;
+    br = bc;
+#else
+    if (kSizeModN == 0) {
+      // The above loop didn't handle the last vector because it needs an
+      // additional right neighbor (generated via mirroring).
+      auto mirror = SetTableIndices(d, MirrorLanes(d.N - 1));
+      tr = TableLookupLanes(tc, mirror);
+      mr = TableLookupLanes(mc, mirror);
+      br = TableLookupLanes(bc, mirror);
+    } else {
+      auto mirror = SetTableIndices(d, MirrorLanes((xsize % d.N) - 1));
+      // Loads last valid value into uppermost lane and mirrors.
+      tr = TableLookupLanes(LoadU(d, row_t + xsize - d.N), mirror);
+      mr = TableLookupLanes(LoadU(d, row_m + xsize - d.N), mirror);
+      br = TableLookupLanes(LoadU(d, row_b + xsize - d.N), mirror);
+    }
+#endif
+
+    const V tl = LoadU(d, row_t + x - 1);
+    const V ml = LoadU(d, row_m + x - 1);
+    const V bl = LoadU(d, row_b + x - 1);
+    const V conv = WeightedSum(tl, tc, tr, ml, mc, mr, bl, bc, br, w0, w1, w2);
+    Store(conv, d, row_out + x);
+  }
+
+ private:
+  // Returns sum{x_i * w_i}.
+  template <class V>
+  static HWY_ATTR JXL_INLINE V WeightedSum(const V tl, const V tc, const V tr,
+                                           const V ml, const V mc, const V mr,
+                                           const V bl, const V bc, const V br,
+                                           const V w0, const V w1, const V w2) {
+    const V sum_tb = tc + bc;
+
+    // Faster than 5 mul + 4 FMA.
+    const V mul0 = mc * w0;
+    const V sum_lr = ml + mr;
+
+    const V x1 = sum_tb + sum_lr;
+    const V mul1 = MulAdd(x1, w1, mul0);
+
+    const V sum_t2 = tl + tr;
+    const V sum_b2 = bl + br;
+    const V x2 = sum_t2 + sum_b2;
+    const V mul2 = MulAdd(x2, w2, mul1);
+    return mul2;
+  }
+
+  static HWY_ATTR JXL_INLINE V ConvolveValid(const float* JXL_RESTRICT row_t,
+                                             const float* JXL_RESTRICT row_m,
+                                             const float* JXL_RESTRICT row_b,
+                                             const int64_t x, const V w0,
+                                             const V w1, const V w2) {
+    const D d;
+    const V tc = LoadU(d, row_t + x);
+    const V mc = LoadU(d, row_m + x);
+    const V bc = LoadU(d, row_b + x);
+    const V tl = LoadU(d, row_t + x - 1);
+    const V tr = LoadU(d, row_t + x + 1);
+    const V ml = LoadU(d, row_m + x - 1);
+    const V mr = LoadU(d, row_m + x + 1);
+    const V bl = LoadU(d, row_b + x - 1);
+    const V br = LoadU(d, row_b + x + 1);
+    return WeightedSum(tl, tc, tr, ml, mc, mr, bl, bc, br, w0, w1, w2);
+  }
+};
+
+// 5x5 convolution by separable kernel with a single scan through the input.
+class Separable5 : public StrategyBase {
+ public:
+  static constexpr int64_t kRadius = 2;
+
+  template <size_t kSizeModN, class WrapRow>
+  static HWY_ATTR JXL_INLINE void ConvolveRow(
+      const float* const JXL_RESTRICT row_m, const size_t xsize,
+      const int64_t stride, const WrapRow& wrap_row,
+      const WeightsSeparable5& weights, float* const JXL_RESTRICT row_out) {
+    const D d;
+    const int64_t neg_stride = -stride;  // allows LEA addressing.
+    const float* const JXL_RESTRICT row_t2 =
+        wrap_row(row_m + 2 * neg_stride, stride);
+    const float* const JXL_RESTRICT row_t1 =
+        wrap_row(row_m + 1 * neg_stride, stride);
+    const float* const JXL_RESTRICT row_b1 =
+        wrap_row(row_m + 1 * stride, stride);
+    const float* const JXL_RESTRICT row_b2 =
+        wrap_row(row_m + 2 * stride, stride);
+
+    const V wh0 = LoadDup128(d, weights.horz + 0 * 4);
+    const V wh1 = LoadDup128(d, weights.horz + 1 * 4);
+    const V wh2 = LoadDup128(d, weights.horz + 2 * 4);
+    const V wv0 = LoadDup128(d, weights.vert + 0 * 4);
+    const V wv1 = LoadDup128(d, weights.vert + 1 * 4);
+    const V wv2 = LoadDup128(d, weights.vert + 2 * 4);
+
+    size_t x = 0;
+
+    // Need to loop more than once for scalars (d.N == 1).
+    for (; x < kRadius; x += d.N) {
+      const V conv0 = HorzConvolveFirst(row_m, x, xsize, wh0, wh1, wh2) * wv0;
+
+      const V conv1t = HorzConvolveFirst(row_t1, x, xsize, wh0, wh1, wh2);
+      const V conv1b = HorzConvolveFirst(row_b1, x, xsize, wh0, wh1, wh2);
+      const V conv1 = MulAdd(conv1t + conv1b, wv1, conv0);
+
+      const V conv2t = HorzConvolveFirst(row_t2, x, xsize, wh0, wh1, wh2);
+      const V conv2b = HorzConvolveFirst(row_b2, x, xsize, wh0, wh1, wh2);
+      const V conv2 = MulAdd(conv2t + conv2b, wv2, conv1);
+      Store(conv2, d, row_out + x);
+    }
+
+    // Main loop: load inputs without padding
+    for (; x + d.N + kRadius <= xsize; x += d.N) {
+      const V conv0 = HorzConvolve(row_m + x, wh0, wh1, wh2) * wv0;
+
+      const V conv1t = HorzConvolve(row_t1 + x, wh0, wh1, wh2);
+      const V conv1b = HorzConvolve(row_b1 + x, wh0, wh1, wh2);
+      const V conv1 = MulAdd(conv1t + conv1b, wv1, conv0);
+
+      const V conv2t = HorzConvolve(row_t2 + x, wh0, wh1, wh2);
+      const V conv2b = HorzConvolve(row_b2 + x, wh0, wh1, wh2);
+      const V conv2 = MulAdd(conv2t + conv2b, wv2, conv1);
+      Store(conv2, d, row_out + x);
+    }
+
+    // Last full vector to write (the above loop handled mod >= kRadius)
+#if HWY_BITS == 0
+    while (x < xsize) {
+#else
+    if (kSizeModN < kRadius) {
+#endif
+      const V conv0 =
+          HorzConvolveLast<kSizeModN>(row_m, x, xsize, wh0, wh1, wh2) * wv0;
+
+      const V conv1t =
+          HorzConvolveLast<kSizeModN>(row_t1, x, xsize, wh0, wh1, wh2);
+      const V conv1b =
+          HorzConvolveLast<kSizeModN>(row_b1, x, xsize, wh0, wh1, wh2);
+      const V conv1 = MulAdd(conv1t + conv1b, wv1, conv0);
+
+      const V conv2t =
+          HorzConvolveLast<kSizeModN>(row_t2, x, xsize, wh0, wh1, wh2);
+      const V conv2b =
+          HorzConvolveLast<kSizeModN>(row_b2, x, xsize, wh0, wh1, wh2);
+      const V conv2 = MulAdd(conv2t + conv2b, wv2, conv1);
+      Store(conv2, d, row_out + x);
+      x += d.N;
+    }
+
+    // If mod = 0, the above vector was the last.
+    if (kSizeModN != 0) {
+      for (; x < xsize; ++x) {
+        float mul = 0.0f;
+        for (int64_t dy = -kRadius; dy <= kRadius; ++dy) {
+          const float wy = weights.vert[std::abs(dy) * 4];
+          const float* clamped_row = wrap_row(row_m + dy * stride, stride);
+          for (int64_t dx = -kRadius; dx <= kRadius; ++dx) {
+            const float wx = weights.horz[std::abs(dx) * 4];
+            const int64_t clamped_x = Mirror(x + dx, xsize);
+            mul += clamped_row[clamped_x] * wx * wy;
+          }
+        }
+        row_out[x] = mul;
+      }
+    }
+  }
+
+ private:
+  // Same as HorzConvolve for the first/last vector in a row.
+  static HWY_ATTR JXL_INLINE V HorzConvolveFirst(
+      const float* const JXL_RESTRICT row, const int64_t x, const int64_t xsize,
+      const V wh0, const V wh1, const V wh2) {
+    const D d;
+    const V c = LoadU(d, row + x);
+    const V mul0 = c * wh0;
+
+#if HWY_BITS == 0
+    const V l1 = LoadU(d, row + Mirror(x - 1, xsize));
+    const V l2 = LoadU(d, row + Mirror(x - 2, xsize));
+#else
+    (void)xsize;
+    const V l1 = Neighbors::FirstL1(c);
+    const V l2 = Neighbors::FirstL2(c);
+#endif
+
+    const V r1 = LoadU(d, row + x + 1);
+    const V r2 = LoadU(d, row + x + 2);
+
+    const V mul1 = MulAdd(l1 + r1, wh1, mul0);
+    const V mul2 = MulAdd(l2 + r2, wh2, mul1);
+    return mul2;
+  }
+
+  template <size_t kSizeModN>
+  static HWY_ATTR JXL_INLINE V
+  HorzConvolveLast(const float* const JXL_RESTRICT row, const int64_t x,
+                   const int64_t xsize, const V wh0, const V wh1, const V wh2) {
+    const D d;
+    const V c = LoadU(d, row + x);
+    const V mul0 = c * wh0;
+
+    const V l1 = LoadU(d, row + x - 1);
+    const V l2 = LoadU(d, row + x - 2);
+
+    V r1, r2;
+#if HWY_BITS == 0
+    r1 = LoadU(d, row + Mirror(x + 1, xsize));
+    r2 = LoadU(d, row + Mirror(x + 2, xsize));
+#else
+    if (kSizeModN == 0) {
+      r2 = TableLookupLanes(c, SetTableIndices(d, MirrorLanes(d.N - 2)));
+      r1 = TableLookupLanes(c, SetTableIndices(d, MirrorLanes(d.N - 1)));
+    } else {  // == 1
+      const auto last = LoadU(d, row + xsize - d.N);
+      r2 = TableLookupLanes(last, SetTableIndices(d, MirrorLanes(d.N - 1)));
+      r1 = last;
+    }
+#endif
+
+    // Sum of pixels with Manhattan distance i, multiplied by weights[i].
+    const V sum1 = l1 + r1;
+    const V mul1 = MulAdd(sum1, wh1, mul0);
+    const V sum2 = l2 + r2;
+    const V mul2 = MulAdd(sum2, wh2, mul1);
+    return mul2;
+  }
+
+  // Requires kRadius valid pixels before/after pos.
+  static HWY_ATTR JXL_INLINE V HorzConvolve(const float* const JXL_RESTRICT pos,
+                                            const V wh0, const V wh1,
+                                            const V wh2) {
+    const D d;
+    const V c = LoadU(d, pos);
+    const V mul0 = c * wh0;
+
+    // Loading anew is faster than combining vectors.
+    const V l1 = LoadU(d, pos - 1);
+    const V r1 = LoadU(d, pos + 1);
+    const V l2 = LoadU(d, pos - 2);
+    const V r2 = LoadU(d, pos + 2);
+    // Sum of pixels with Manhattan distance i, multiplied by weights[i].
+    const V sum1 = l1 + r1;
+    const V mul1 = MulAdd(sum1, wh1, mul0);
+    const V sum2 = l2 + r2;
+    const V mul2 = MulAdd(sum2, wh2, mul1);
+    return mul2;
+  }
+};  // namespace strategy
+
+// 7x7 convolution by separable kernel with a single scan through the input.
+class Separable7 : public StrategyBase {
+ public:
+  static constexpr int64_t kRadius = 3;
+
+  template <size_t kSizeModN, class WrapRow>
+  static HWY_ATTR JXL_INLINE void ConvolveRow(
+      const float* const JXL_RESTRICT row_m, const size_t xsize,
+      const int64_t stride, const WrapRow& wrap_row,
+      const WeightsSeparable7& weights, float* const JXL_RESTRICT row_out) {
+    const D d;
+    const int64_t neg_stride = -stride;  // allows LEA addressing.
+    const float* const JXL_RESTRICT row_t3 =
+        wrap_row(row_m + 3 * neg_stride, stride);
+    const float* const JXL_RESTRICT row_t2 =
+        wrap_row(row_m + 2 * neg_stride, stride);
+    const float* const JXL_RESTRICT row_t1 =
+        wrap_row(row_m + 1 * neg_stride, stride);
+    const float* const JXL_RESTRICT row_b1 =
+        wrap_row(row_m + 1 * stride, stride);
+    const float* const JXL_RESTRICT row_b2 =
+        wrap_row(row_m + 2 * stride, stride);
+    const float* const JXL_RESTRICT row_b3 =
+        wrap_row(row_m + 3 * stride, stride);
+
+    const V wh0 = LoadDup128(d, weights.horz + 0 * 4);
+    const V wh1 = LoadDup128(d, weights.horz + 1 * 4);
+    const V wh2 = LoadDup128(d, weights.horz + 2 * 4);
+    const V wh3 = LoadDup128(d, weights.horz + 3 * 4);
+    const V wv0 = LoadDup128(d, weights.vert + 0 * 4);
+    const V wv1 = LoadDup128(d, weights.vert + 1 * 4);
+    const V wv2 = LoadDup128(d, weights.vert + 2 * 4);
+    const V wv3 = LoadDup128(d, weights.vert + 3 * 4);
+
+    size_t x = 0;
+
+    // Need to loop more than once for scalars (d.N == 1).
+    for (; x < kRadius; x += d.N) {
+      const V conv0 =
+          HorzConvolveFirst(row_m, x, xsize, wh0, wh1, wh2, wh3) * wv0;
+
+      const V conv1t = HorzConvolveFirst(row_t1, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv1b = HorzConvolveFirst(row_b1, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv1 = MulAdd(conv1t + conv1b, wv1, conv0);
+
+      const V conv2t = HorzConvolveFirst(row_t2, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv2b = HorzConvolveFirst(row_b2, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv2 = MulAdd(conv2t + conv2b, wv2, conv1);
+
+      const V conv3t = HorzConvolveFirst(row_t3, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv3b = HorzConvolveFirst(row_b3, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv3 = MulAdd(conv3t + conv3b, wv3, conv2);
+
+      Store(conv3, d, row_out + x);
+    }
+
+    // Main loop: load inputs without padding
+    for (; x + d.N + kRadius <= xsize; x += d.N) {
+      const V conv0 = HorzConvolve(row_m + x, wh0, wh1, wh2, wh3) * wv0;
+
+      const V conv1t = HorzConvolve(row_t1 + x, wh0, wh1, wh2, wh3);
+      const V conv1b = HorzConvolve(row_b1 + x, wh0, wh1, wh2, wh3);
+      const V conv1 = MulAdd(conv1t + conv1b, wv1, conv0);
+
+      const V conv2t = HorzConvolve(row_t2 + x, wh0, wh1, wh2, wh3);
+      const V conv2b = HorzConvolve(row_b2 + x, wh0, wh1, wh2, wh3);
+      const V conv2 = MulAdd(conv2t + conv2b, wv2, conv1);
+
+      const V conv3t = HorzConvolve(row_t3 + x, wh0, wh1, wh2, wh3);
+      const V conv3b = HorzConvolve(row_b3 + x, wh0, wh1, wh2, wh3);
+      const V conv3 = MulAdd(conv3t + conv3b, wv3, conv2);
+
+      Store(conv3, d, row_out + x);
+    }
+
+    // Last full vector to write (the above loop handled mod >= kRadius)
+#if HWY_BITS == 0
+    while (x < xsize) {
+#else
+    if (kSizeModN < kRadius) {
+#endif
+      const V conv0 =
+          HorzConvolveLast<kSizeModN>(row_m, x, xsize, wh0, wh1, wh2, wh3) *
+          wv0;
+
+      const V conv1t =
+          HorzConvolveLast<kSizeModN>(row_t1, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv1b =
+          HorzConvolveLast<kSizeModN>(row_b1, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv1 = MulAdd(conv1t + conv1b, wv1, conv0);
+
+      const V conv2t =
+          HorzConvolveLast<kSizeModN>(row_t2, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv2b =
+          HorzConvolveLast<kSizeModN>(row_b2, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv2 = MulAdd(conv2t + conv2b, wv2, conv1);
+
+      const V conv3t =
+          HorzConvolveLast<kSizeModN>(row_t3, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv3b =
+          HorzConvolveLast<kSizeModN>(row_b3, x, xsize, wh0, wh1, wh2, wh3);
+      const V conv3 = MulAdd(conv3t + conv3b, wv3, conv2);
+
+      Store(conv3, d, row_out + x);
+      x += d.N;
+    }
+
+    // If mod = 0, the above vector was the last.
+    if (kSizeModN != 0) {
+      for (; x < xsize; ++x) {
+        float mul = 0.0f;
+        for (int64_t dy = -kRadius; dy <= kRadius; ++dy) {
+          const float wy = weights.vert[std::abs(dy) * 4];
+          const float* clamped_row = wrap_row(row_m + dy * stride, stride);
+          for (int64_t dx = -kRadius; dx <= kRadius; ++dx) {
+            const float wx = weights.horz[std::abs(dx) * 4];
+            const int64_t clamped_x = Mirror(x + dx, xsize);
+            mul += clamped_row[clamped_x] * wx * wy;
+          }
+        }
+        row_out[x] = mul;
+      }
+    }
+  }
+
+ private:
+  // Same as HorzConvolve for the first/last vector in a row.
+  static HWY_ATTR JXL_INLINE V HorzConvolveFirst(
+      const float* const JXL_RESTRICT row, const int64_t x, const int64_t xsize,
+      const V wh0, const V wh1, const V wh2, const V wh3) {
+    const D d;
+    const V c = LoadU(d, row + x);
+    const V mul0 = c * wh0;
+
+#if HWY_BITS == 0
+    const V l1 = LoadU(d, row + Mirror(x - 1, xsize));
+    const V l2 = LoadU(d, row + Mirror(x - 2, xsize));
+    const V l3 = LoadU(d, row + Mirror(x - 3, xsize));
+#else
+    const V l1 = Neighbors::FirstL1(c);
+    const V l2 = Neighbors::FirstL2(c);
+    const V l3 = Neighbors::FirstL3(c);
+#endif
+
+    const V r1 = LoadU(d, row + Mirror(x + 1, xsize));
+    const V r2 = LoadU(d, row + Mirror(x + 2, xsize));
+    const V r3 = LoadU(d, row + Mirror(x + 3, xsize));
+
+    const V mul1 = MulAdd(l1 + r1, wh1, mul0);
+    const V mul2 = MulAdd(l2 + r2, wh2, mul1);
+    const V mul3 = MulAdd(l3 + r3, wh3, mul2);
+    return mul3;
+  }
+
+  template <size_t kSizeModN>
+  static HWY_ATTR JXL_INLINE V HorzConvolveLast(
+      const float* const JXL_RESTRICT row, const int64_t x, const int64_t xsize,
+      const V wh0, const V wh1, const V wh2, const V wh3) {
+    const D d;
+    const V c = LoadU(d, row + x);
+    const V mul0 = c * wh0;
+
+    JXL_DASSERT(x >= kRadius);
+    const V l1 = LoadU(d, row + x - 1);
+    const V l2 = LoadU(d, row + x - 2);
+    const V l3 = LoadU(d, row + x - 3);
+
+    V r1, r2, r3;
+#if HWY_BITS == 0
+    r1 = LoadU(d, row + Mirror(x + 1, xsize));
+    r2 = LoadU(d, row + Mirror(x + 2, xsize));
+    r3 = LoadU(d, row + Mirror(x + 3, xsize));
+#else
+    switch (kSizeModN) {
+      case 0:
+        r3 = TableLookupLanes(c, SetTableIndices(d, MirrorLanes(d.N - 3)));
+        r2 = TableLookupLanes(c, SetTableIndices(d, MirrorLanes(d.N - 2)));
+        r1 = TableLookupLanes(c, SetTableIndices(d, MirrorLanes(d.N - 1)));
+        break;
+      case 1: {
+        const auto last = LoadU(d, row + xsize - d.N);
+        r3 = TableLookupLanes(last, SetTableIndices(d, MirrorLanes(d.N - 2)));
+        r2 = TableLookupLanes(last, SetTableIndices(d, MirrorLanes(d.N - 1)));
+        r1 = last;
+        break;
+      }
+      default:
+        JXL_DASSERT(kSizeModN == 2);
+        {
+          const auto last = LoadU(d, row + xsize - d.N);
+          r3 = TableLookupLanes(last, SetTableIndices(d, MirrorLanes(d.N - 1)));
+          r2 = last;
+          r1 = LoadU(d, row + x + 1);
+          break;
+        }
+    }
+#endif
+
+    // Sum of pixels with Manhattan distance i, multiplied by weights[i].
+    const V sum1 = l1 + r1;
+    const V mul1 = MulAdd(sum1, wh1, mul0);
+    const V sum2 = l2 + r2;
+    const V mul2 = MulAdd(sum2, wh2, mul1);
+    const V sum3 = l3 + r3;
+    const V mul3 = MulAdd(sum3, wh3, mul2);
+    return mul3;
+  }
+
+  // Requires kRadius valid pixels before/after pos.
+  static HWY_ATTR JXL_INLINE V HorzConvolve(const float* const JXL_RESTRICT pos,
+                                            const V wh0, const V wh1,
+                                            const V wh2, const V wh3) {
+    const D d;
+    const V c = LoadU(d, pos);
+    const V mul0 = c * wh0;
+
+    // Loading anew is faster than combining vectors.
+    const V l1 = LoadU(d, pos - 1);
+    const V r1 = LoadU(d, pos + 1);
+    const V l2 = LoadU(d, pos - 2);
+    const V r2 = LoadU(d, pos + 2);
+    const V l3 = LoadU(d, pos - 3);
+    const V r3 = LoadU(d, pos + 3);
+    // Sum of pixels with Manhattan distance i, multiplied by weights[i].
+    const V sum1 = l1 + r1;
+    const V mul1 = MulAdd(sum1, wh1, mul0);
+    const V sum2 = l2 + r2;
+    const V mul2 = MulAdd(sum2, wh2, mul1);
+    const V sum3 = l3 + r3;
+    const V mul3 = MulAdd(sum3, wh3, mul2);
+    return mul3;
+  }
+};  // namespace jxl
+
+}  // namespace strategy
+
+// TODO(janwas): AVX-512
+static constexpr size_t kConvolveLanes = HWY_CAPPED(float, 8)::N;
+// 3x3 kernels require inputs at least this wide - for the first vector, they
+// load right neighbors (N lanes starting from x + 1).
+static constexpr size_t kConvolveMinWidth = kConvolveLanes + 1;
+
+// Single entry point for convolution.
+// "Strategy" (Direct*/Separable*) decides kernel size and how to evaluate it.
+template <class Strategy>
+class ConvolveT {
+  static constexpr int64_t kRadius = Strategy::kRadius;
+
+ public:
+  // "Image" is ImageF or Image3F.
+  template <class Image, class Kernel>
+  static HWY_ATTR void Run(const Image& in, const Rect& rect,
+                           const Kernel& kernel, ThreadPool* pool,
+                           const Image* out) {
+    PROFILER_ZONE("ConvolveT::Run");
+    JXL_CHECK(SameSize(rect, *out));
+    JXL_CHECK(rect.xsize() >= kConvolveMinWidth);
+
+    static_assert(int64_t(kRadius) <= 3,
+                  "Must handle [0, kRadius) and >= kRadius");
+    switch (rect.xsize() % kConvolveLanes) {
+      case 0:
+        return RunRows<0>(in, rect, kernel, pool, out);
+      case 1:
+        return RunRows<1>(in, rect, kernel, pool, out);
+      case 2:
+        return RunRows<2>(in, rect, kernel, pool, out);
+      default:
+        return RunRows<3>(in, rect, kernel, pool, out);
+    }
+  }
+
+ private:
+  template <size_t kSizeModN, class WrapRow, class Kernel>
+  static HWY_ATTR JXL_INLINE void RunRow(const float* JXL_RESTRICT in,
+                                         const size_t xsize,
+                                         const int64_t stride,
+                                         const WrapRow& wrap_row,
+                                         const Kernel& kernel,
+                                         const float* JXL_RESTRICT out) {
+    Strategy::template ConvolveRow<kSizeModN>(
+        in, xsize, stride, wrap_row, kernel.Weights(), const_cast<float*>(out));
+  }
+
+  template <size_t kSizeModN, class Kernel>
+  static HWY_ATTR JXL_INLINE void RunBorderRows(
+      const ImageF& in, const Rect& rect, const int64_t ybegin,
+      const int64_t yend, const Kernel& kernel, const ImageF* out) {
+    const int64_t stride = in.PixelsPerRow();
+    const WrapRowMirror wrap_row(in, rect.ysize());
+    for (int64_t y = ybegin; y < yend; ++y) {
+      RunRow<kSizeModN>(rect.ConstRow(in, y), rect.xsize(), stride, wrap_row,
+                        kernel, out->Row(y));
+    }
+  }
+
+  // Image3F.
+  template <size_t kSizeModN, class Kernel>
+  static HWY_ATTR JXL_INLINE void RunBorderRows(
+      const Image3F& in, const Rect& rect, const int64_t ybegin,
+      const int64_t yend, const Kernel& kernel, const Image3F* out) {
+    const int64_t stride = in.PixelsPerRow();
+    for (int64_t y = ybegin; y < yend; ++y) {
+      for (size_t c = 0; c < 3; ++c) {
+        const WrapRowMirror wrap_row(in.Plane(c), rect.ysize());
+        RunRow<kSizeModN>(rect.ConstPlaneRow(in, c, y), rect.xsize(), stride,
+                          wrap_row, kernel, out->PlaneRow(c, y));
+      }
+    }
+  }
+
+  template <size_t kSizeModN, class Kernel>
+  static HWY_ATTR JXL_INLINE void RunInteriorRows(
+      const ImageF& in, const Rect& rect, const int64_t ybegin,
+      const int64_t yend, const Kernel& kernel, ThreadPool* pool,
+      const ImageF* out) {
+    const int64_t stride = in.PixelsPerRow();
+    RunOnPool(
+        pool, ybegin, yend, ThreadPool::SkipInit(),
+        [&](const int y, int /*thread*/) HWY_ATTR {
+          RunRow<kSizeModN>(rect.ConstRow(in, y), rect.xsize(), stride,
+                            WrapRowUnchanged(), kernel, out->Row(y));
+        },
+        "Convolve");
+  }
+
+  // Image3F.
+  template <size_t kSizeModN, class Kernel>
+  static HWY_ATTR JXL_INLINE void RunInteriorRows(
+      const Image3F& in, const Rect& rect, const int64_t ybegin,
+      const int64_t yend, const Kernel& kernel, ThreadPool* pool,
+      const Image3F* out) {
+    const int64_t stride = in.PixelsPerRow();
+    RunOnPool(
+        pool, ybegin, yend, ThreadPool::SkipInit(),
+        [&](const int y, int /*thread*/) HWY_ATTR {
+          for (size_t c = 0; c < 3; ++c) {
+            RunRow<kSizeModN>(rect.ConstPlaneRow(in, c, y), rect.xsize(),
+                              stride, WrapRowUnchanged(), kernel,
+                              out->PlaneRow(c, y));
+          }
+        },
+        "Convolve3");
+  }
+
+  template <size_t kSizeModN, class Image, class Kernel>
+  static HWY_ATTR JXL_INLINE void RunRows(const Image& in, const Rect& rect,
+                                          const Kernel& kernel,
+                                          ThreadPool* pool, const Image* out) {
+    const int64_t ysize = rect.ysize();
+    RunBorderRows<kSizeModN>(in, rect, 0, std::min(int64_t(kRadius), ysize),
+                             kernel, out);
+    if (ysize > 2 * int64_t(kRadius)) {
+      RunInteriorRows<kSizeModN>(in, rect, int64_t(kRadius),
+                                 ysize - int64_t(kRadius), kernel, pool, out);
+    }
+    if (ysize > int64_t(kRadius)) {
+      RunBorderRows<kSizeModN>(in, rect, ysize - int64_t(kRadius), ysize,
+                               kernel, out);
+    }
+  }
+};
+
+}  // namespace jxl
+
+#endif  // JXL_CONVOLVE_H_
diff --git a/jxl/convolve_test.cc b/jxl/convolve_test.cc
new file mode 100644
index 000000000..a93fbfeb4
--- /dev/null
+++ b/jxl/convolve_test.cc
@@ -0,0 +1,236 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/convolve.h"
+
+#include <random>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/thread_pool_internal.h"
+#define HWY_USE_GTEST  // prevent redefining TEST
+#include <hwy/tests/test_util.h>
+
+#include "jxl/image_ops.h"
+#include "jxl/image_test_utils.h"
+
+#ifndef JXL_DEBUG_CONVOLVE
+#define JXL_DEBUG_CONVOLVE 0
+#endif
+
+namespace jxl {
+namespace {
+
+HWY_ATTR void TestNeighborsImpl() {
+  const Neighbors::D d;
+  const Neighbors::V v = Iota(d, 0);
+  HWY_ALIGN float actual[hwy::kTestMaxVectorSize / sizeof(float)] = {0};
+
+  HWY_ALIGN float first_l1[hwy::kTestMaxVectorSize / sizeof(float)] = {
+      0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14};
+  Store(Neighbors::FirstL1(v), d, actual);
+  EXPECT_EQ(std::vector<float>(first_l1, first_l1 + d.N),
+            std::vector<float>(actual, actual + d.N));
+
+#if HWY_BITS != 0
+  HWY_ALIGN float first_l2[hwy::kTestMaxVectorSize / sizeof(float)] = {
+      1, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13};
+  Store(Neighbors::FirstL2(v), d, actual);
+  EXPECT_EQ(std::vector<float>(first_l2, first_l2 + d.N),
+            std::vector<float>(actual, actual + d.N));
+
+  HWY_ALIGN float first_l3[hwy::kTestMaxVectorSize / sizeof(float)] = {
+      2, 1, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12};
+  Store(Neighbors::FirstL3(v), d, actual);
+  EXPECT_EQ(std::vector<float>(first_l3, first_l3 + d.N),
+            std::vector<float>(actual, actual + d.N));
+#endif
+}
+
+TEST(ConvolveTest, TestNeighbors) { TestNeighborsImpl(); }
+
+// Weights i=0..2 are for Manhattan distance i from center.
+template <int64_t kRadius, class Wrap>
+struct Symmetric3x3Convolution {
+  static_assert(kRadius == 1, "Wrong kRadius");
+
+  template <class Kernel>
+  static void Run(const ImageF& in, const Rect& rect, const Kernel& kernel,
+                  ImageF* out) {
+    PROFILER_ZONE("slow::Symmetric3::Run");
+    JXL_CHECK(SameSize(rect, *out));
+    const Weights3x3& weights = kernel.Weights();
+
+    const size_t xsize = rect.xsize();
+    const size_t ysize = rect.ysize();
+    for (int64_t y = 0; y < ysize; ++y) {
+      const float* const JXL_RESTRICT row_t = in.ConstRow(Wrap()(y - 1, ysize));
+      const float* const JXL_RESTRICT row_m = in.ConstRow(y);
+      const float* const JXL_RESTRICT row_b = in.ConstRow(Wrap()(y + 1, ysize));
+      float* const JXL_RESTRICT row_out = out->Row(y);
+
+      for (int64_t x = 0; x < xsize; ++x) {
+        float mul = row_m[x] * weights.mc[0];
+        const int64_t xm1 = Wrap()(x - 1, xsize);
+        const int64_t xp1 = Wrap()(x + 1, xsize);
+        const float tl = row_t[xm1];
+        const float ml = row_m[xm1];
+        const float bl = row_b[xm1];
+        const float tr = row_t[xp1];
+        const float mr = row_m[xp1];
+        const float br = row_b[xp1];
+        mul += (row_t[x] + row_b[x] + ml + mr) * weights.tc[0];
+        mul += (tl + tr + bl + br) * weights.tl[0];
+        row_out[x] = mul;
+      }
+    }
+  }
+};
+
+template <class Slow, class ImageOrView, class Kernel>
+ImageF SlowConvolve(const ImageOrView& in, const Rect& rect,
+                    const Kernel& kernel) {
+  ImageF out(rect.xsize(), rect.ysize());
+  Slow::Run(in, rect, kernel, &out);
+  return out;
+}
+
+// Compares ConvolveT<> against SlowConvolve.
+template <template <int64_t, class> class SlowT, class Strategy, class Kernel,
+          class Random>
+HWY_ATTR void Verify(const size_t xsize, const size_t ysize,
+                     const Kernel& kernel, ThreadPool* pool, Random* rng) {
+  constexpr size_t kRadius = Strategy::kRadius;
+  static_assert(kRadius <= kConvolveMaxRadius, "Update max radius");
+
+  ImageF in(xsize, ysize);
+  GenerateImage(GeneratorRandom<float, Random>(rng, 1.0f), &in);
+
+  using Slow = SlowT<kRadius, WrapMirror>;
+  const Rect rect(0, 0, xsize, ysize);
+  const ImageF out_expected = SlowConvolve<Slow>(in, rect, kernel);
+
+  ImageF out_actual(xsize, ysize);
+  ConvolveT<Strategy>::Run(in, rect, kernel, pool, &out_actual);
+  VerifyRelativeError(out_expected, out_actual, 1E-5f, 1E-5f);
+}
+
+// Call Verify for null and non-null pool.
+template <template <int64_t, class> class SlowT, class Strategy, class Kernel,
+          class Random>
+HWY_ATTR void VerifyAll(const size_t xsize, const size_t ysize,
+                        const Kernel& kernel, Random* rng, ThreadPool* pool) {
+  JXL_DEBUG(JXL_DEBUG_CONVOLVE, "pool0");
+  Verify<SlowT, Strategy>(xsize, ysize, kernel, /*pool=*/nullptr, rng);
+
+  JXL_DEBUG(JXL_DEBUG_CONVOLVE, "Used pool");
+  Verify<SlowT, Strategy>(xsize, ysize, kernel, pool, rng);
+}
+
+// Compares slow::Symmetric3x3Convolution against slow::SymmetricConvolution.
+template <class Random>
+HWY_ATTR void VerifyConvolveSymmetric3x3(const size_t xsize, const size_t ysize,
+                                         Random* rng) {
+  const size_t kRadius = 1;
+  JXL_CHECK(xsize > kRadius);
+  JXL_CHECK(ysize > kRadius);
+  const Rect rect(0, 0, xsize, ysize);
+
+  ImageF in(xsize, ysize);
+  GenerateImage(GeneratorRandom<float, Random>(rng, 1.0f), &in);
+
+  ImageF out_expected(xsize, ysize);
+  ImageF out_actual(xsize, ysize);
+
+  Symmetric3x3Convolution<1, WrapMirror>::Run(in, rect, kernel::Lowpass3(),
+                                              &out_expected);
+
+  // Expanded form of kernel::Lowpass3: lower-right quadrant.
+  const float weights_symmetric[4] = {0.36208932f, 0.12820096f,  //
+                                      0.12820096f, 0.03127668f};
+  ThreadPool* null_pool = nullptr;
+  slow::SymmetricConvolution<kRadius, WrapClamp>::Run(
+      in, rect, weights_symmetric, null_pool, &out_actual);
+
+  VerifyRelativeError(out_expected, out_actual, 1E-5f, 1E-5f);
+}
+
+// Compares ConvolveT<> against slow::ConvolveSymmetric.
+template <class Random>
+HWY_ATTR void VerifyConvolveSymmetric5x5(const size_t xsize, const size_t ysize,
+                                         Random* rng) {
+  const size_t kRadius = 2;
+  JXL_CHECK(xsize > kRadius);
+  JXL_CHECK(ysize > kRadius);
+  const Rect rect(0, 0, xsize, ysize);
+
+  ImageF in(xsize, ysize);
+  GenerateImage(GeneratorRandom<float, Random>(rng, 1.0f), &in);
+
+  ImageF out_expected(xsize, ysize);
+  ImageF out_actual(xsize, ysize);
+
+  ConvolveT<strategy::Separable5>::Run(in, Rect(in), kernel::Lowpass5(),
+                                       /*pool=*/nullptr, &out_expected);
+
+  // Expanded form of kernel::Lowpass5: lower-right quadrant.
+  const float weights_symmetric[9] = {0.1740135f, 0.1065369f, 0.0150310f,  //
+                                      0.1065369f, 0.0652254f, 0.0092025f,  //
+                                      0.0150310f, 0.0092025f, 0.0012984f};
+  ThreadPool* null_pool = nullptr;
+  slow::SymmetricConvolution<kRadius, WrapMirror>::Run(
+      in, rect, weights_symmetric, null_pool, &out_actual);
+
+  VerifyRelativeError(out_expected, out_actual, 1E-5f, 1E-5f);
+}
+
+// For all xsize/ysize and kernels:
+HWY_ATTR void TestSameResultsImpl() {
+  ThreadPoolInternal pool(8);
+  constexpr size_t kMinWidth = kConvolveLanes + kConvolveMaxRadius;
+  pool.Run(kMinWidth, 40, ThreadPool::SkipInit(),
+           [](const int task, int /*thread*/) {
+             const size_t xsize = task;
+             std::mt19937_64 rng(129 + 13 * xsize);
+
+             ThreadPoolInternal pool3(3);
+             for (size_t ysize = kConvolveMaxRadius; ysize < 16; ++ysize) {
+               JXL_DEBUG(JXL_DEBUG_CONVOLVE,
+                         "%zu x %zu=====================================",
+                         xsize, ysize);
+
+               JXL_DEBUG(JXL_DEBUG_CONVOLVE, "Sym3x3------------------");
+               VerifyAll<Symmetric3x3Convolution, strategy::Symmetric3>(
+                   xsize, ysize, kernel::Lowpass3(), &rng, &pool3);
+
+               JXL_DEBUG(JXL_DEBUG_CONVOLVE, "Sep5x5------------------");
+               VerifyAll<slow::SeparableConvolution, strategy::Separable5>(
+                   xsize, ysize, kernel::Lowpass5(), &rng, &pool3);
+
+               JXL_DEBUG(JXL_DEBUG_CONVOLVE, "Sep7x7------------------");
+               VerifyAll<slow::SeparableConvolution, strategy::Separable7>(
+                   xsize, ysize, kernel::Gaussian7Sigma8(), &rng, &pool3);
+
+               VerifyConvolveSymmetric3x3(xsize, ysize, &rng);
+               VerifyConvolveSymmetric5x5(xsize, ysize, &rng);
+             }
+           });
+}
+
+TEST(ConvolveTest, TestSameResults) { TestSameResultsImpl(); }
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/data_parallel_test.cc b/jxl/data_parallel_test.cc
new file mode 100644
index 000000000..256446846
--- /dev/null
+++ b/jxl/data_parallel_test.cc
@@ -0,0 +1,121 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/data_parallel.h"
+
+#include "gtest/gtest.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/test_utils.h"
+
+namespace jxl {
+namespace {
+
+class DataParallelTest : public ::testing::Test {
+ protected:
+  // A fake class to verify that DataParallel is properly calling the
+  // client-provided runner functions.
+  static int FakeRunner(void* runner_opaque, void* jpegxl_opaque,
+                        JpegxlParallelRunInit init,
+                        JpegxlParallelRunFunction func, uint32_t start_range,
+                        uint32_t end_range) {
+    DataParallelTest* self = static_cast<DataParallelTest*>(runner_opaque);
+    self->runner_called_++;
+    self->jpegxl_opaque_ = jpegxl_opaque;
+    self->init_ = init;
+    self->func_ = func;
+    self->start_range_ = start_range;
+    self->end_range_ = end_range;
+    return self->runner_return_;
+  }
+
+  ThreadPool pool_{&DataParallelTest::FakeRunner, this};
+
+  // Number of times FakeRunner() was called.
+  int runner_called_ = 0;
+
+  // Parameters passed to FakeRunner.
+  void* jpegxl_opaque_ = nullptr;
+  JpegxlParallelRunInit init_ = nullptr;
+  JpegxlParallelRunFunction func_ = nullptr;
+  uint32_t start_range_ = -1;
+  uint32_t end_range_ = -1;
+
+  // Return value that FakeRunner will return.
+  int runner_return_ = 0;
+};
+
+// JpegxlParallelRunInit interface.
+typedef int (*JpegxlParallelRunInit)();
+int TestInit(void* jpegxl_opaque, size_t num_threads) { return 0; }
+
+}  // namespace
+
+TEST_F(DataParallelTest, RunnerCalledParamenters) {
+  EXPECT_TRUE(pool_.Run(
+      1234, 5678, [](const size_t num_threads) { return true; },
+      [](const int task, const int thread) { return; }));
+  EXPECT_EQ(1, runner_called_);
+  EXPECT_NE(nullptr, init_);
+  EXPECT_NE(nullptr, func_);
+  EXPECT_NE(nullptr, jpegxl_opaque_);
+  EXPECT_EQ(1234u, start_range_);
+  EXPECT_EQ(5678u, end_range_);
+}
+
+TEST_F(DataParallelTest, RunnerFailurePropagates) {
+  runner_return_ = -1;  // FakeRunner return value.
+  EXPECT_FALSE(pool_.Run(
+      1234, 5678, [](const size_t num_threads) { return false; },
+      [](const int task, const int thread) { return; }));
+  EXPECT_FALSE(RunOnPool(
+      nullptr, 1234, 5678, [](const size_t num_threads) { return false; },
+      [](const int task, const int thread) { return; }, "Test"));
+}
+
+TEST_F(DataParallelTest, RunnerNotCalledOnEmptyRange) {
+  runner_return_ = -1;  // FakeRunner return value.
+  EXPECT_TRUE(pool_.Run(
+      123, 123, [](const size_t num_threads) { return false; },
+      [](const int task, const int thread) { return; }));
+  EXPECT_TRUE(RunOnPool(
+      nullptr, 123, 123, [](const size_t num_threads) { return false; },
+      [](const int task, const int thread) { return; }, "Test"));
+  // We don't call the external runner when the range is empty. We don't even
+  // need to call the init function.
+  EXPECT_EQ(0, runner_called_);
+}
+
+// The TestDivider is slow when compiled in debug mode.
+TEST_F(DataParallelTest, JXL_SLOW_TEST(TestDivider)) {
+  jxl::ThreadPoolInternal pool(8);
+  // 1, 2 are powers of two.
+  pool.Run(3, 2 * 1024, ThreadPool::SkipInit(),
+           [](const int d, const int thread) {
+             // powers of two are not supported.
+             if ((d & (d - 1)) == 0) return;
+
+             const Divider div(d);
+#ifdef NDEBUG
+             const int max_dividend = 4 * 1024 * 1024;
+#else
+             const int max_dividend = 2 * 1024 + 1;
+#endif
+             for (int x = 0; x < max_dividend; ++x) {
+               const int q = div(x);
+               ASSERT_EQ(x / d, q) << x << "/" << d;
+             }
+           });
+}
+
+}  // namespace jxl
diff --git a/jxl/dct.cc b/jxl/dct.cc
new file mode 100644
index 000000000..9817085ea
--- /dev/null
+++ b/jxl/dct.cc
@@ -0,0 +1,33 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dct.h"
+
+namespace jxl {
+
+// Definition of constexpr arrays.
+constexpr float DCTResampleScales<1, 8>::kScales[];
+constexpr float DCTResampleScales<2, 8>::kScales[];
+constexpr float DCTResampleScales<2, 16>::kScales[];
+constexpr float DCTResampleScales<4, 16>::kScales[];
+constexpr float DCTResampleScales<4, 32>::kScales[];
+constexpr float DCTResampleScales<8, 32>::kScales[];
+constexpr float DCTResampleScales<8, 1>::kScales[];
+constexpr float DCTResampleScales<8, 2>::kScales[];
+constexpr float DCTResampleScales<16, 2>::kScales[];
+constexpr float DCTResampleScales<16, 4>::kScales[];
+constexpr float DCTResampleScales<32, 4>::kScales[];
+constexpr float DCTResampleScales<32, 8>::kScales[];
+
+}  // namespace jxl
diff --git a/jxl/dct.h b/jxl/dct.h
new file mode 100644
index 000000000..8a81d0b34
--- /dev/null
+++ b/jxl/dct.h
@@ -0,0 +1,1077 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DCT_H_
+#define JXL_DCT_H_
+
+// Fast SIMD floating-point DCT8-32.
+
+#include <string.h>
+
+#include <cmath>
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/block.h"
+#include "jxl/dct_simd_4.h"
+#include "jxl/dct_simd_8.h"
+#include "jxl/dct_simd_any.h"
+
+namespace jxl {
+
+// Final scaling factors of outputs/inputs in the Arai, Agui, and Nakajima
+// algorithm computing the DCT/IDCT (described in the book JPEG: Still Image
+// Data Compression Standard, section 4.3.5) and the "A low multiplicative
+// complexity fast recursive DCT-2 algorithm" (Maxim Vashkevich, Alexander
+// Pertrovsky) algorithm. Note that the DCT and the IDCT scales of these two
+// algorithms are flipped. We use the first algorithm for DCT8, and the second
+// one for all other DCTs.
+/* Python snippet to produce these tables for the Arai, Agui, Nakajima
+ * algorithm:
+ *
+from mpmath import *
+N = 8
+def iscale(u):
+  eps = sqrt(mpf(0.5)) if u == 0 else mpf(1.0)
+  return sqrt(mpf(2) / mpf(N)) * eps * cos(mpf(u) * pi / mpf(2 * N))
+def scale(u):
+  return mpf(1) / (mpf(N) * iscale(i))
+mp.dps = 18
+print(", ".join([str(scale(i)) + 'f' for i in range(N)]))
+print(", ".join([str(iscale(i)) + 'f' for i in range(N)]))
+ */
+static constexpr const float kDCTScales1[1] = {1.0f};
+static constexpr const float kIDCTScales1[1] = {1.0f};
+static constexpr const float kDCTScales2[2] = {0.707106781186547524f,
+                                               0.707106781186547524f};
+static constexpr const float kIDCTScales2[2] = {0.707106781186547524f,
+                                                0.707106781186547524f};
+static constexpr const float kDCTScales4[4] = {0.5f, 0.653281482438188264f,
+                                               0.5f, 0.270598050073098492f};
+static constexpr const float kIDCTScales4[4] = {0.5f, 0.382683432365089772f,
+                                                0.5f, 0.923879532511286756f};
+static constexpr const float kDCTScales8[8] = {
+    0.353553390593273762f, 0.254897789552079584f, 0.270598050073098492f,
+    0.30067244346752264f,  0.353553390593273762f, 0.449988111568207852f,
+    0.653281482438188264f, 1.28145772387075309f};
+
+static constexpr const float kIDCTScales8[8] = {
+    0.353553390593273762f, 0.490392640201615225f, 0.461939766255643378f,
+    0.415734806151272619f, 0.353553390593273762f, 0.277785116509801112f,
+    0.191341716182544886f, 0.0975451610080641339f};
+
+static constexpr const float kIDCTScales16[16] = {0.25f,
+                                                  0.177632042131274808f,
+                                                  0.180239955501736978f,
+                                                  0.184731156892216368f,
+                                                  0.191341716182544886f,
+                                                  0.200444985785954314f,
+                                                  0.212607523691814112f,
+                                                  0.228686034616512494f,
+                                                  0.25f,
+                                                  0.278654739432954475f,
+                                                  0.318189645143208485f,
+                                                  0.375006192208515097f,
+                                                  0.461939766255643378f,
+                                                  0.608977011699708658f,
+                                                  0.906127446352887843f,
+                                                  1.80352839005774887f};
+
+static constexpr const float kDCTScales16[16] = {0.25f,
+                                                 0.351850934381595615f,
+                                                 0.346759961330536865f,
+                                                 0.33832950029358817f,
+                                                 0.326640741219094132f,
+                                                 0.311806253246667808f,
+                                                 0.293968900604839679f,
+                                                 0.273300466750439372f,
+                                                 0.25f,
+                                                 0.224291896585659071f,
+                                                 0.196423739596775545f,
+                                                 0.166663914619436624f,
+                                                 0.135299025036549246f,
+                                                 0.102631131880589345f,
+                                                 0.0689748448207357531f,
+                                                 0.0346542922997728657f};
+
+static constexpr const float kIDCTScales32[32] = {
+    0.176776695296636881f, 0.125150749558799075f, 0.125604821547038926f,
+    0.126367739974385915f, 0.127448894776039792f, 0.128861827480656137f,
+    0.13062465373492222f,  0.132760647772446044f, 0.135299025036549246f,
+    0.138275974008611132f, 0.141736008704089426f, 0.145733742051533468f,
+    0.15033622173376132f,  0.155626030758916204f, 0.161705445839997532f,
+    0.168702085363751436f, 0.176776695296636881f, 0.186134067750574612f,
+    0.197038655862812556f, 0.20983741135388176f,  0.224994055784103926f,
+    0.243142059465490173f, 0.265169421497586868f, 0.292359983358221239f,
+    0.326640741219094132f, 0.371041154078541569f, 0.430611774559583482f,
+    0.514445252488352888f, 0.640728861935376545f, 0.851902104617179697f,
+    1.27528715467229096f,  2.5475020308870142f};
+
+static constexpr const float kDCTScales32[32] = {
+    0.176776695296636881f,  0.249698864051293098f,  0.248796181668049222f,
+    0.247294127491195243f,  0.245196320100807612f,  0.242507813298635998f,
+    0.239235083933052216f,  0.235386016295755195f,  0.230969883127821689f,
+    0.225997323280860833f,  0.220480316087088757f,  0.214432152500068017f,
+    0.207867403075636309f,  0.200801882870161227f,  0.19325261334068424f,
+    0.185237781338739773f,  0.176776695296636881f,  0.1678897387117546f,
+    0.158598321040911375f,  0.148924826123108336f,  0.138892558254900556f,
+    0.128525686048305432f,  0.117849184206499412f,  0.106888773357570524f,
+    0.0956708580912724429f, 0.0842224633480550127f, 0.0725711693136155919f,
+    0.0607450449758159725f, 0.048772580504032067f,  0.0366826186138404379f,
+    0.0245042850823901505f, 0.0122669185818545036f};
+
+template <size_t N>
+constexpr const float* DCTScales() {
+  return N == 1 ? kDCTScales1
+                : (N == 2 ? kDCTScales2
+                          : (N == 4 ? kDCTScales4
+                                    : (N == 8 ? kDCTScales8
+                                              : (N == 16 ? kDCTScales16
+                                                         : kDCTScales32))));
+}
+
+template <size_t N>
+constexpr const float* IDCTScales() {
+  return N == 1 ? kIDCTScales1
+                : (N == 2 ? kIDCTScales2
+                          : (N == 4 ? kIDCTScales4
+                                    : (N == 8 ? kIDCTScales8
+                                              : (N == 16 ? kIDCTScales16
+                                                         : kIDCTScales32))));
+}
+
+// For n != 0, the n-th basis function of a N-DCT, evaluated in pixel k, has a
+// value of cos((k+1/2) n/(2N) pi). When downsampling by 2x, we average
+// the values for pixel k and k+1 to get the value for pixel (k/2), thus we get
+//
+// [cos((k+1/2) n/N pi) + cos((k+3/2) n/N pi)]/2 =
+// cos(n/(2N) pi) cos((k+1) n/N pi) =
+// cos(n/(2N) pi) cos(((k/2)+1/2) n/(N/2) pi)
+//
+// which is exactly the same as the value of pixel k/2 of a N/2-sized DCT,
+// except for the cos(n/(2N) pi) scaling factor (which does *not*
+// depend on the pixel). Thus, when using the lower-frequency coefficients of a
+// DCT-N to compute a DCT-(N/2), they should be scaled by this constant. Scaling
+// factors for a DCT-(N/4) etc can then be obtained by successive
+// multiplications. The structs below contain the above-mentioned scaling
+// factors.
+template <size_t FROM, size_t TO>
+struct DCTResampleScales;
+
+template <>
+struct DCTResampleScales<8, 1> {
+  static constexpr float kScales[1] = {
+      1.000000000000000000,
+  };
+};
+
+template <>
+struct DCTResampleScales<8, 2> {
+  static constexpr float kScales[2] = {
+      1.000000000000000000,
+      0.906127446352887778,
+  };
+};
+
+template <>
+struct DCTResampleScales<16, 2> {
+  static constexpr float kScales[2] = {
+      1.000000000000000000,
+      0.901764195028874394,
+  };
+};
+
+template <>
+struct DCTResampleScales<16, 4> {
+  static constexpr float kScales[4] = {
+      1.000000000000000000,
+      0.976062531202202877,
+      0.906127446352887778,
+      0.795666809947927156,
+  };
+};
+
+template <>
+struct DCTResampleScales<32, 4> {
+  static constexpr float kScales[4] = {
+      1.000000000000000000,
+      0.974886821136879522,
+      0.901764195028874394,
+      0.787054918159101335,
+  };
+};
+
+template <>
+struct DCTResampleScales<32, 8> {
+  static constexpr float kScales[8] = {
+      1.000000000000000000, 0.993985983084976765, 0.976062531202202877,
+      0.946582901544112176, 0.906127446352887778, 0.855491189274751540,
+      0.795666809947927156, 0.727823404688121345,
+  };
+};
+
+// Inverses of the above.
+template <>
+struct DCTResampleScales<1, 8> {
+  static constexpr float kScales[1] = {
+      1.000000000000000000,
+  };
+};
+
+template <>
+struct DCTResampleScales<2, 8> {
+  static constexpr float kScales[2] = {
+      1.000000000000000000,
+      1.103597517131772232,
+  };
+};
+
+template <>
+struct DCTResampleScales<2, 16> {
+  static constexpr float kScales[2] = {
+      1.000000000000000000,
+      1.108937353592731823,
+  };
+};
+
+template <>
+struct DCTResampleScales<4, 16> {
+  static constexpr float kScales[4] = {
+      1.000000000000000000,
+      1.024524523821556565,
+      1.103597517131772232,
+      1.256807482098500017,
+  };
+};
+
+template <>
+struct DCTResampleScales<4, 32> {
+  static constexpr float kScales[4] = {
+      1.000000000000000000,
+      1.025760096781116015,
+      1.108937353592731823,
+      1.270559368765487251,
+  };
+};
+
+template <>
+struct DCTResampleScales<8, 32> {
+  static constexpr float kScales[8] = {
+      1.000000000000000000, 1.006050404147911470, 1.024524523821556565,
+      1.056431505754806377, 1.103597517131772232, 1.168919110491081437,
+      1.256807482098500017, 1.373959663235216677,
+  };
+};
+
+// https://en.wikipedia.org/wiki/In-place_matrix_transposition#Square_matrices
+template <size_t N, class From, class To>
+HWY_ATTR JXL_INLINE void GenericTransposeBlockInplace(const From& from,
+                                                      const To& to) {
+  // This does not guarantee anything, just saves from the most stupid mistakes.
+  JXL_ASSERT(from.Address(0, 0) == to.Address(0, 0));
+  for (size_t n = 0; n < N - 1; ++n) {
+    for (size_t m = n + 1; m < N; ++m) {
+      // Swap
+      const float tmp = from.Read(m, n);
+      to.Write(from.Read(n, m), m, n);
+      to.Write(tmp, n, m);
+    }
+  }
+}
+
+template <size_t ROWS, size_t COLS, class From, class To>
+HWY_ATTR JXL_INLINE void GenericTransposeBlock(const From& from, const To& to) {
+  // This does not guarantee anything, just saves from the most stupid mistakes.
+  JXL_ASSERT(from.Address(0, 0) != to.Address(0, 0));
+  for (size_t n = 0; n < ROWS; ++n) {
+    for (size_t m = 0; m < COLS; ++m) {
+      to.Write(from.Read(n, m), m, n);
+    }
+  }
+}
+
+template <class From, class To>
+HWY_ATTR JXL_INLINE void TransposeBlock8(const From& from, const To& to) {
+#if HWY_BITS >= 256
+  TransposeBlock8_V8(from, to);
+#elif HWY_BITS == 128
+  TransposeBlock8_V4(from, to);
+#else
+  if (from.Address(0, 0) == to.Address(0, 0)) {
+    GenericTransposeBlockInplace<8>(from, to);
+  } else {
+    GenericTransposeBlock<8, 8>(from, to);
+  }
+#endif
+}
+
+template <class From, class To>
+HWY_ATTR void TransposeBlock16(const From& from, const To& to) {
+  HWY_ALIGN float tmp[8 * 8];
+  TransposeBlock8(from, to);
+  TransposeBlock8(from.View(0, 8), ToBlock<8>(tmp));
+  TransposeBlock8(from.View(8, 0), to.View(0, 8));
+  CopyBlock8(FromBlock<8>(tmp), to.View(8, 0));
+  TransposeBlock8(from.View(8, 8), to.View(8, 8));
+}
+
+template <class From, class To>
+HWY_ATTR void TransposeBlock168(const From& from, const To& to) {
+  TransposeBlock8(from, to);
+  TransposeBlock8(from.View(8, 0), to.View(0, 8));
+}
+
+template <class From, class To>
+HWY_ATTR void TransposeBlock328(const From& from, const To& to) {
+  TransposeBlock8(from, to);
+  TransposeBlock8(from.View(8, 0), to.View(0, 8));
+  TransposeBlock8(from.View(16, 0), to.View(0, 16));
+  TransposeBlock8(from.View(24, 0), to.View(0, 24));
+}
+
+template <class From, class To>
+HWY_ATTR void TransposeBlock3216(const From& from, const To& to) {
+  TransposeBlock16(from, to);
+  TransposeBlock16(from.View(16, 0), to.View(0, 16));
+}
+
+template <class From, class To>
+HWY_ATTR void TransposeBlock816(const From& from, const To& to) {
+  TransposeBlock8(from, to);
+  TransposeBlock8(from.View(0, 8), to.View(8, 0));
+}
+
+template <class From, class To>
+HWY_ATTR void TransposeBlock832(const From& from, const To& to) {
+  TransposeBlock8(from, to);
+  TransposeBlock8(from.View(0, 8), to.View(8, 0));
+  TransposeBlock8(from.View(0, 16), to.View(16, 0));
+  TransposeBlock8(from.View(0, 24), to.View(24, 0));
+}
+template <class From, class To>
+HWY_ATTR void TransposeBlock1632(const From& from, const To& to) {
+  TransposeBlock16(from, to);
+  TransposeBlock16(from.View(0, 16), to.View(16, 0));
+}
+
+template <class From, class To>
+HWY_ATTR void TransposeBlock32(const From& from, const To& to) {
+  HWY_ALIGN float tmp[8 * 8];
+  TransposeBlock8(from, to);
+  TransposeBlock8(from.View(0, 8), ToBlock<8>(tmp));
+  TransposeBlock8(from.View(8, 0), to.View(0, 8));
+  CopyBlock8(FromBlock<8>(tmp), to.View(8, 0));
+  TransposeBlock8(from.View(8, 8), to.View(8, 8));
+  TransposeBlock8(from.View(0, 16), ToBlock<8>(tmp));
+  TransposeBlock8(from.View(16, 0), to.View(0, 16));
+  CopyBlock8(FromBlock<8>(tmp), to.View(16, 0));
+  TransposeBlock8(from.View(8, 16), ToBlock<8>(tmp));
+  TransposeBlock8(from.View(16, 8), to.View(8, 16));
+  CopyBlock8(FromBlock<8>(tmp), to.View(16, 8));
+  TransposeBlock8(from.View(16, 16), to.View(16, 16));
+  TransposeBlock8(from.View(0, 24), ToBlock<8>(tmp));
+  TransposeBlock8(from.View(24, 0), to.View(0, 24));
+  CopyBlock8(FromBlock<8>(tmp), to.View(24, 0));
+  TransposeBlock8(from.View(8, 24), ToBlock<8>(tmp));
+  TransposeBlock8(from.View(24, 8), to.View(8, 24));
+  CopyBlock8(FromBlock<8>(tmp), to.View(24, 8));
+  TransposeBlock8(from.View(16, 24), ToBlock<8>(tmp));
+  TransposeBlock8(from.View(24, 16), to.View(16, 24));
+  CopyBlock8(FromBlock<8>(tmp), to.View(24, 16));
+  TransposeBlock8(from.View(24, 24), to.View(24, 24));
+}
+
+// Computes the in-place NxN transposed-scaled-DCT (tsDCT) of block.
+// Requires that block is HWY_ALIGN'ed.
+//
+// Final DCT coefficients could be obtained the following way:
+//   unscaled(f)[x, y] = f[x, y] * DCTScales<N>[x] * DCTScales<N>[y]
+//   untransposed(f)[x, y] = f[y, x]
+//   DCT(input) = unscaled(untransposed(tsDCT(input)))
+//
+// NB: DCT denotes scaled variant of DCT-II, which is orthonormal.
+//
+// See also DCTSlow, ComputeDCT
+template <size_t N>
+struct ComputeTransposedScaledDCT;
+
+template <>
+struct ComputeTransposedScaledDCT<32> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[32 * 32];
+    ColumnDCT32(from, ToBlock<32>(block));
+    TransposeBlock32(FromBlock<32>(block), ToBlock<32>(block));
+    ColumnDCT32(FromBlock<32>(block), to);
+  }
+};
+
+template <>
+struct ComputeTransposedScaledDCT<16> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[16 * 16];
+    ColumnDCT16(from, ToBlock<16>(block));
+    TransposeBlock16(FromBlock<16>(block), ToBlock<16>(block));
+    ColumnDCT16(FromBlock<16>(block), to);
+  }
+};
+
+template <>
+struct ComputeTransposedScaledDCT<8> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+#if HWY_BITS >= 256
+    ComputeTransposedScaledDCT8_V8(from, to);
+#elif HWY_BITS == 128
+    ComputeTransposedScaledDCT8_V4(from, to);
+#else
+    HWY_ALIGN float block[8 * 8];
+    ColumnDCT8(from, ToBlock<8>(block));
+    TransposeBlock8(FromBlock<8>(block), ToBlock<8>(block));
+    ColumnDCT8(FromBlock<8>(block), to);
+#endif
+  }
+};
+
+template <>
+struct ComputeTransposedScaledDCT<4> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[4 * 4];
+    ColumnDCT4(from, ToBlock<4>(block));
+    GenericTransposeBlockInplace<4>(FromBlock<4>(block), ToBlock<4>(block));
+    ColumnDCT4(FromBlock<4>(block), to);
+  }
+};
+
+template <>
+struct ComputeTransposedScaledDCT<2> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    const float a00 = from.Read(0, 0);
+    const float a01 = from.Read(0, 1);
+    const float a10 = from.Read(1, 0);
+    const float a11 = from.Read(1, 1);
+    to.Write(a00 + a01 + a10 + a11, 0, 0);
+    to.Write(a00 + a01 - a10 - a11, 0, 1);
+    to.Write(a00 - a01 + a10 - a11, 1, 0);
+    to.Write(a00 - a01 - a10 + a11, 1, 1);
+  }
+};
+
+// Computes the non-transposed, scaled DCT of a block, that needs to be
+// HWY_ALIGN'ed. Used for rectangular blocks.
+template <size_t ROWS, size_t COLS>
+struct ComputeScaledDCT;
+
+template <>
+struct ComputeScaledDCT<8, 16> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 16];
+    HWY_ALIGN float transposed_block[8 * 16];
+    using FromOriginal = FromBlock<8, 16>;
+    using FromTransposed = FromBlock<16, 8>;
+    using ToOriginal = ToBlock<8, 16>;
+    using ToTransposed = ToBlock<16, 8>;
+    ColumnDCT8<From, ToOriginal, /*COLS=*/16>(from, ToOriginal(block));
+    TransposeBlock816(FromOriginal(block), ToTransposed(transposed_block));
+    // Reusing block to reduce stack usage.
+    ColumnDCT16<FromTransposed, ToTransposed, /*COLS=*/8>(
+        FromTransposed(transposed_block), ToTransposed(block));
+    TransposeBlock168(FromTransposed(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<8, 32> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 32];
+    HWY_ALIGN float transposed_block[8 * 32];
+    using FromOriginal = FromBlock<8, 32>;
+    using FromTransposed = FromBlock<32, 8>;
+    using ToOriginal = ToBlock<8, 32>;
+    using ToTransposed = ToBlock<32, 8>;
+    ColumnDCT8<From, ToOriginal, /*COLS=*/32>(from, ToOriginal(block));
+    TransposeBlock832(FromOriginal(block), ToTransposed(transposed_block));
+    // Reusing block to reduce stack usage.
+    ColumnDCT32<FromTransposed, ToTransposed, /*COLS=*/8>(
+        FromTransposed(transposed_block), ToTransposed(block));
+    TransposeBlock328(FromTransposed(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<16, 32> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[16 * 32];
+    HWY_ALIGN float transposed_block[16 * 32];
+    using FromOriginal = FromBlock<16, 32>;
+    using FromTransposed = FromBlock<32, 16>;
+    using ToOriginal = ToBlock<16, 32>;
+    using ToTransposed = ToBlock<32, 16>;
+    ColumnDCT16<From, ToOriginal, /*COLS=*/32>(from, ToOriginal(block));
+    TransposeBlock1632(FromOriginal(block), ToTransposed(transposed_block));
+    // Reusing block to reduce stack usage.
+    ColumnDCT32<FromTransposed, ToTransposed, /*COLS=*/16>(
+        FromTransposed(transposed_block), ToTransposed(block));
+    TransposeBlock3216(FromTransposed(block), to);
+  }
+};
+
+// Blocks of the form XxY with X > Y are stored transposed.
+
+template <>
+struct ComputeScaledDCT<16, 8> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 16];
+    HWY_ALIGN float transposed_block[8 * 16];
+    using FromOriginal = FromBlock<16, 8>;
+    using FromTransposed = FromBlock<8, 16>;
+    using ToOriginal = ToBlock<16, 8>;
+    using ToTransposed = ToBlock<8, 16>;
+    ColumnDCT16<From, ToOriginal, /*COLS=*/8>(from, ToOriginal(block));
+    TransposeBlock168(FromOriginal(block), ToTransposed(transposed_block));
+    ColumnDCT8<FromTransposed, To, /*COLS=*/16>(
+        FromTransposed(transposed_block), to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<32, 8> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 32];
+    HWY_ALIGN float transposed_block[8 * 32];
+    using FromOriginal = FromBlock<32, 8>;
+    using FromTransposed = FromBlock<8, 32>;
+    using ToOriginal = ToBlock<32, 8>;
+    using ToTransposed = ToBlock<8, 32>;
+    ColumnDCT32<From, ToOriginal, /*COLS=*/8>(from, ToOriginal(block));
+    TransposeBlock328(FromOriginal(block), ToTransposed(transposed_block));
+    ColumnDCT8<FromTransposed, To, /*COLS=*/32>(
+        FromTransposed(transposed_block), to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<32, 16> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[16 * 32];
+    HWY_ALIGN float transposed_block[16 * 32];
+    using FromOriginal = FromBlock<32, 16>;
+    using FromTransposed = FromBlock<16, 32>;
+    using ToOriginal = ToBlock<32, 16>;
+    using ToTransposed = ToBlock<16, 32>;
+    ColumnDCT32<From, ToOriginal, /*COLS=*/16>(from, ToOriginal(block));
+    TransposeBlock3216(FromOriginal(block), ToTransposed(transposed_block));
+    ColumnDCT16<FromTransposed, To, /*COLS=*/32>(
+        FromTransposed(transposed_block), to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<8, 4> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[4 * 8];
+    HWY_ALIGN float transposed_block[4 * 8];
+    using FromOriginal = FromBlock<8, 4>;
+    using FromTransposed = FromBlock<4, 8>;
+    using ToOriginal = ToBlock<8, 4>;
+    using ToTransposed = ToBlock<4, 8>;
+    ColumnDCT8<From, ToOriginal, /*COLS=*/4>(from, ToOriginal(block));
+    GenericTransposeBlock<8, 4>(FromOriginal(block),
+                                ToTransposed(transposed_block));
+    ColumnDCT4<FromTransposed, To, /*COLS=*/8>(FromTransposed(transposed_block),
+                                               to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<4, 8> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 4];
+    HWY_ALIGN float transposed_block[8 * 4];
+    using FromOriginal = FromBlock<4, 8>;
+    using FromTransposed = FromBlock<8, 4>;
+    using ToOriginal = ToBlock<4, 8>;
+    using ToTransposed = ToBlock<8, 4>;
+    ColumnDCT4<From, ToOriginal, /*COLS=*/8>(from, ToOriginal(block));
+    GenericTransposeBlock<4, 8>(FromOriginal(block),
+                                ToTransposed(transposed_block));
+    // Reusing block to reduce stack usage.
+    ColumnDCT8<FromTransposed, ToTransposed, /*COLS=*/4>(
+        FromTransposed(transposed_block), ToTransposed(block));
+    GenericTransposeBlock<8, 4>(FromTransposed(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<4, 2> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[4 * 2];
+    using ToOriginal = ToBlock<4, 2>;
+    ColumnDCT4<From, ToOriginal, /*COLS=*/2>(from, ToOriginal(block));
+    for (size_t y = 0; y < 4; ++y) {
+      const float a0 = block[2 * y];
+      const float a1 = block[2 * y + 1];
+      to.Write(a0 + a1, 0, y);
+      to.Write(a0 - a1, 1, y);
+    }
+  }
+};
+
+template <>
+struct ComputeScaledDCT<2, 4> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    // `block` and `coeffs` are transposed.
+    HWY_ALIGN float block[4 * 2];
+    for (size_t y = 0; y < 4; ++y) {
+      const float a0 = from.Read(0, y);
+      const float a1 = from.Read(1, y);
+      block[2 * y] = a0 + a1;
+      block[2 * y + 1] = a0 - a1;
+    }
+    using FromTransposed = FromBlock<4, 2>;
+    using ToTransposed = ToBlock<4, 2>;
+    HWY_ALIGN float coeffs[4 * 2];
+    ColumnDCT4<FromTransposed, ToTransposed, /*COLS=*/2>(FromTransposed(block),
+                                                         ToTransposed(coeffs));
+    GenericTransposeBlock<4, 2>(FromTransposed(coeffs), to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<4, 1> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[4 * 1];
+    ColumnDCT4<From, ToBlock<4, 1>, /*COLS=*/1>(from, ToBlock<4, 1>(block));
+    GenericTransposeBlock<4, 1>(FromBlock<4, 1>(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<1, 4> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float coeffs[4 * 1];
+    using FromTransposed = FromBlock<4, 1>;
+    using ToTransposed = ToBlock<4, 1>;
+    GenericTransposeBlock<1, 4>(from, ToTransposed(coeffs));
+    HWY_ALIGN float block[4 * 1];
+    ColumnDCT4<FromTransposed, ToTransposed, /*COLS=*/1>(FromTransposed(coeffs),
+                                                         ToTransposed(block));
+    GenericTransposeBlock<4, 1>(FromTransposed(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<2, 1> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    const float a0 = from.Read(0, 0);
+    const float a1 = from.Read(1, 0);
+    to.Write(a0 + a1, 0, 0);
+    to.Write(a0 - a1, 0, 1);
+  }
+};
+
+template <>
+struct ComputeScaledDCT<1, 2> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    const float a0 = from.Read(0, 0);
+    const float a1 = from.Read(0, 1);
+    to.Write(a0 + a1, 0, 0);
+    to.Write(a0 - a1, 0, 1);
+  }
+};
+
+// Computes the in-place NxN transposed-scaled-iDCT (tsIDCT)of block.
+// Requires that block is HWY_ALIGN'ed.
+//
+// Final DCT coefficients could be obtained the following way:
+//   unscaled(f)[x, y] = f[x, y] * IDCTScales<N>[x] * IDCTScales<N>[y]
+//   untransposed(f)[x, y] = f[y, x]
+//   IDCT(input) = tsIDCT(untransposed(unscaled(input)))
+//
+// NB: IDCT denotes scaled variant of DCT-III, which is orthonormal.
+//
+// See also IDCTSlow, ComputeIDCT.
+template <size_t N>
+struct ComputeTransposedScaledIDCT;
+
+template <>
+struct ComputeTransposedScaledIDCT<32> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[32 * 32];
+    ColumnIDCT32(from, ToBlock<32>(block));
+    TransposeBlock32(FromBlock<32>(block), ToBlock<32>(block));
+    ColumnIDCT32(FromBlock<32>(block), to);
+  }
+};
+
+template <>
+struct ComputeTransposedScaledIDCT<16> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[16 * 16];
+    ColumnIDCT16(from, ToBlock<16>(block));
+    TransposeBlock16(FromBlock<16>(block), ToBlock<16>(block));
+    ColumnIDCT16(FromBlock<16>(block), to);
+  }
+};
+
+template <>
+struct ComputeTransposedScaledIDCT<8> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+#if HWY_BITS >= 256
+    ComputeTransposedScaledIDCT8_V8(from, to);
+#elif HWY_BITS == 128
+    ComputeTransposedScaledIDCT8_V4(from, to);
+#else
+    HWY_ALIGN float block[8 * 8];
+    ColumnIDCT8(from, ToBlock<8>(block));
+    TransposeBlock8(FromBlock<8>(block), ToBlock<8>(block));
+    ColumnIDCT8(FromBlock<8>(block), to);
+#endif
+  }
+};
+
+template <>
+struct ComputeTransposedScaledIDCT<4> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[4 * 4];
+    ColumnIDCT4(from, ToBlock<4>(block));
+    GenericTransposeBlockInplace<4>(FromBlock<4>(block), ToBlock<4>(block));
+    ColumnIDCT4(FromBlock<4>(block), to);
+  }
+};
+
+template <>
+struct ComputeTransposedScaledIDCT<2> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    const float a00 = from.Read(0, 0);
+    const float a01 = from.Read(0, 1);
+    const float a10 = from.Read(1, 0);
+    const float a11 = from.Read(1, 1);
+    to.Write(a00 + a01 + a10 + a11, 0, 0);
+    to.Write(a00 + a01 - a10 - a11, 0, 1);
+    to.Write(a00 - a01 + a10 - a11, 1, 0);
+    to.Write(a00 - a01 - a10 + a11, 1, 1);
+  }
+};
+
+template <>
+struct ComputeTransposedScaledIDCT<1> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    to.Write(from.Read(0, 0), 0, 0);
+  }
+};
+
+// Computes the non-transposed, scaled DCT of a block, that needs to be
+// HWY_ALIGN'ed. Used for rectangular blocks.
+template <size_t ROWS, size_t COLS>
+struct ComputeScaledIDCT;
+
+template <>
+struct ComputeScaledIDCT<8, 16> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 16];
+    HWY_ALIGN float transposed_block[8 * 16];
+    using FromOriginal = FromBlock<8, 16>;
+    using FromTransposed = FromBlock<16, 8>;
+    using ToOriginal = ToBlock<8, 16>;
+    using ToTransposed = ToBlock<16, 8>;
+    // Reverse the steps done in ComputeScaledDCT.
+    TransposeBlock816(from, ToTransposed(block));
+    ColumnIDCT16<FromTransposed, ToTransposed, /*COLS=*/8>(
+        FromTransposed(block), ToTransposed(transposed_block));
+    TransposeBlock168(FromTransposed(transposed_block), ToOriginal(block));
+    ColumnIDCT8<FromOriginal, To, /*COLS=*/16>(FromOriginal(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<8, 32> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 32];
+    HWY_ALIGN float transposed_block[8 * 32];
+    using FromOriginal = FromBlock<8, 32>;
+    using FromTransposed = FromBlock<32, 8>;
+    using ToOriginal = ToBlock<8, 32>;
+    using ToTransposed = ToBlock<32, 8>;
+    // Reverse the steps done in ComputeScaledDCT.
+    TransposeBlock832(from, ToTransposed(block));
+    ColumnIDCT32<FromTransposed, ToTransposed, /*COLS=*/8>(
+        FromTransposed(block), ToTransposed(transposed_block));
+    TransposeBlock328(FromTransposed(transposed_block), ToOriginal(block));
+    ColumnIDCT8<FromOriginal, To, /*COLS=*/32>(FromOriginal(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<16, 32> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[16 * 32];
+    HWY_ALIGN float transposed_block[16 * 32];
+    using FromOriginal = FromBlock<16, 32>;
+    using FromTransposed = FromBlock<32, 16>;
+    using ToOriginal = ToBlock<16, 32>;
+    using ToTransposed = ToBlock<32, 16>;
+    // Reverse the steps done in ComputeScaledDCT.
+    TransposeBlock1632(from, ToTransposed(block));
+    ColumnIDCT32<FromTransposed, ToTransposed, /*COLS=*/16>(
+        FromTransposed(block), ToTransposed(transposed_block));
+    TransposeBlock3216(FromTransposed(transposed_block), ToOriginal(block));
+    ColumnIDCT16<FromOriginal, To, /*COLS=*/32>(FromOriginal(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<16, 8> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 16];
+    HWY_ALIGN float transposed_block[8 * 16];
+    using FromOriginal = FromBlock<16, 8>;
+    using FromTransposed = FromBlock<8, 16>;
+    using ToOriginal = ToBlock<16, 8>;
+    using ToTransposed = ToBlock<8, 16>;
+    ColumnIDCT8<From, ToTransposed, /*COLS=*/16>(
+        from, ToTransposed(transposed_block));
+    TransposeBlock816(FromTransposed(transposed_block), ToOriginal(block));
+    ColumnIDCT16<FromOriginal, To, /*COLS=*/8>(FromOriginal(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<32, 8> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 32];
+    HWY_ALIGN float transposed_block[8 * 32];
+    using FromOriginal = FromBlock<32, 8>;
+    using FromTransposed = FromBlock<8, 32>;
+    using ToOriginal = ToBlock<32, 8>;
+    using ToTransposed = ToBlock<8, 32>;
+    ColumnIDCT8<From, ToTransposed, /*COLS=*/32>(
+        from, ToTransposed(transposed_block));
+    TransposeBlock832(FromTransposed(transposed_block), ToOriginal(block));
+    ColumnIDCT32<FromOriginal, To, /*COLS=*/8>(FromOriginal(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<32, 16> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[16 * 32];
+    HWY_ALIGN float transposed_block[16 * 32];
+    using FromOriginal = FromBlock<32, 16>;
+    using FromTransposed = FromBlock<16, 32>;
+    using ToOriginal = ToBlock<32, 16>;
+    using ToTransposed = ToBlock<16, 32>;
+    ColumnIDCT16<From, ToTransposed, /*COLS=*/32>(
+        from, ToTransposed(transposed_block));
+    TransposeBlock1632(FromTransposed(transposed_block), ToOriginal(block));
+    ColumnIDCT32<FromOriginal, To, /*COLS=*/16>(FromOriginal(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<8, 4> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[4 * 8];
+    HWY_ALIGN float transposed_block[4 * 8];
+    using FromOriginal = FromBlock<8, 4>;
+    using FromTransposed = FromBlock<4, 8>;
+    using ToOriginal = ToBlock<8, 4>;
+    using ToTransposed = ToBlock<4, 8>;
+    ColumnIDCT4<From, ToTransposed, /*COLS=*/8>(from,
+                                                ToTransposed(transposed_block));
+    GenericTransposeBlock<4, 8>(FromTransposed(transposed_block),
+                                ToOriginal(block));
+    ColumnIDCT8<FromOriginal, To, /*COLS=*/4>(FromOriginal(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<4, 8> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[8 * 4];
+    HWY_ALIGN float transposed_block[8 * 4];
+    using FromOriginal = FromBlock<4, 8>;
+    using FromTransposed = FromBlock<8, 4>;
+    using ToOriginal = ToBlock<4, 8>;
+    using ToTransposed = ToBlock<8, 4>;
+    GenericTransposeBlock<4, 8>(from, ToTransposed(block));
+    ColumnIDCT8<FromTransposed, ToTransposed, /*COLS=*/4>(
+        FromTransposed(block), ToTransposed(transposed_block));
+    GenericTransposeBlock<8, 4>(FromTransposed(transposed_block),
+                                ToOriginal(block));
+    ColumnIDCT4<FromOriginal, To, /*COLS=*/8>(FromOriginal(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<4, 2> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[4 * 2];
+    using FromOriginal = FromBlock<4, 2>;
+    for (size_t y = 0; y < 4; ++y) {
+      const float a0 = from.Read(0, y);
+      const float a1 = from.Read(1, y);
+      block[2 * y] = a0 + a1;
+      block[2 * y + 1] = a0 - a1;
+    }
+    ColumnIDCT4<FromOriginal, To, /*COLS=*/2>(FromOriginal(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<2, 4> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float coeffs[4 * 2];
+    using FromTransposed = FromBlock<4, 2>;
+    using ToTransposed = ToBlock<4, 2>;
+    GenericTransposeBlock<2, 4>(from, ToTransposed(coeffs));
+    HWY_ALIGN float block[4 * 2];
+    ColumnIDCT4<FromTransposed, ToTransposed, /*COLS=*/2>(
+        FromTransposed(coeffs), ToTransposed(block));
+    for (size_t y = 0; y < 4; ++y) {
+      const float a0 = block[2 * y];
+      const float a1 = block[2 * y + 1];
+      to.Write(a0 + a1, 0, y);
+      to.Write(a0 - a1, 1, y);
+    }
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<4, 1> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float block[4 * 1];
+    GenericTransposeBlock<1, 4>(from, ToBlock<4, 1>(block));
+    ColumnIDCT4<FromBlock<4, 1>, To, /*COLS=*/1>(FromBlock<4, 1>(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<1, 4> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    HWY_ALIGN float coeffs[4 * 1];
+    using FromTransposed = FromBlock<4, 1>;
+    using ToTransposed = ToBlock<4, 1>;
+    GenericTransposeBlock<1, 4>(from, ToTransposed(coeffs));
+    HWY_ALIGN float block[4 * 1];
+    ColumnIDCT4<FromTransposed, ToTransposed, /*COLS=*/1>(
+        FromTransposed(coeffs), ToTransposed(block));
+    GenericTransposeBlock<4, 1>(FromTransposed(block), to);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<2, 1> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    const float a0 = from.Read(0, 0);
+    const float a1 = from.Read(0, 1);
+    to.Write(a0 + a1, 0, 0);
+    to.Write(a0 - a1, 1, 0);
+  }
+};
+
+template <>
+struct ComputeScaledIDCT<1, 2> {
+  template <class From, class To>
+  HWY_ATTR JXL_INLINE void operator()(const From& from, const To& to) {
+    const float a0 = from.Read(0, 0);
+    const float a1 = from.Read(0, 1);
+    to.Write(a0 + a1, 0, 0);
+    to.Write(a0 - a1, 0, 1);
+  }
+};
+
+namespace slow_dct {
+static inline double alpha(int u) { return u == 0 ? 0.7071067811865475 : 1.0; }
+template <size_t N>
+void DCTSlow(double block[N * N]) {
+  constexpr size_t kBlockSize = N * N;
+  double g[kBlockSize];
+  memcpy(g, block, kBlockSize * sizeof(g[0]));
+  const double scale = std::sqrt(2.0 / N);
+  for (int v = 0; v < N; ++v) {
+    for (int u = 0; u < N; ++u) {
+      double val = 0.0;
+      for (int y = 0; y < N; ++y) {
+        for (int x = 0; x < N; ++x) {
+          val += (alpha(u) * cos((x + 0.5) * u * Pi(1.0 / N)) * alpha(v) *
+                  cos((y + 0.5) * v * Pi(1.0 / N)) * g[N * y + x]);
+        }
+      }
+      block[N * v + u] = val * scale * scale;
+    }
+  }
+}
+
+template <size_t N>
+void IDCTSlow(double block[N * N]) {
+  constexpr size_t kBlockSize = N * N;
+  double F[kBlockSize];
+  memcpy(F, block, kBlockSize * sizeof(F[0]));
+  const double scale = std::sqrt(2.0 / N);
+  for (int v = 0; v < N; ++v) {
+    for (int u = 0; u < N; ++u) {
+      double val = 0.0;
+      for (int y = 0; y < N; ++y) {
+        for (int x = 0; x < N; ++x) {
+          val += (alpha(x) * cos(x * (u + 0.5) * Pi(1.0 / N)) * alpha(y) *
+                  cos(y * (v + 0.5) * Pi(1.0 / N)) * F[N * y + x]);
+        }
+      }
+      block[N * v + u] = val * scale * scale;
+    }
+  }
+}
+}  // namespace slow_dct
+
+}  // namespace jxl
+
+#endif  // JXL_DCT_H_
diff --git a/jxl/dct_simd_4.h b/jxl/dct_simd_4.h
new file mode 100644
index 000000000..6b4eaacf1
--- /dev/null
+++ b/jxl/dct_simd_4.h
@@ -0,0 +1,131 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DCT_HWY_4_H_
+#define JXL_DCT_HWY_4_H_
+
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/block.h"
+#include "jxl/dct_simd_any.h"
+
+#if HWY_BITS == 128
+
+namespace jxl {
+
+// DCT building blocks that require SIMD vector length to be 4, e.g. SSE4.
+static_assert(BlockDesc<8>::N == 4, "Wrong vector size, must be 4");
+
+template <class From, class To>
+static HWY_ATTR JXL_INLINE void TransposeBlock8_V4(const From& from,
+                                                   const To& to) {
+  const auto p0L = from.LoadVec(0, 0);
+  const auto p0H = from.LoadVec(0, 4);
+  const auto p1L = from.LoadVec(1, 0);
+  const auto p1H = from.LoadVec(1, 4);
+  const auto p2L = from.LoadVec(2, 0);
+  const auto p2H = from.LoadVec(2, 4);
+  const auto p3L = from.LoadVec(3, 0);
+  const auto p3H = from.LoadVec(3, 4);
+  const auto p4L = from.LoadVec(4, 0);
+  const auto p4H = from.LoadVec(4, 4);
+  const auto p5L = from.LoadVec(5, 0);
+  const auto p5H = from.LoadVec(5, 4);
+  const auto p6L = from.LoadVec(6, 0);
+  const auto p6H = from.LoadVec(6, 4);
+  const auto p7L = from.LoadVec(7, 0);
+  const auto p7H = from.LoadVec(7, 4);
+
+  const auto q0L = InterleaveLo(p0L, p2L);
+  const auto q0H = InterleaveLo(p0H, p2H);
+  const auto q1L = InterleaveLo(p1L, p3L);
+  const auto q1H = InterleaveLo(p1H, p3H);
+  const auto q2L = InterleaveHi(p0L, p2L);
+  const auto q2H = InterleaveHi(p0H, p2H);
+  const auto q3L = InterleaveHi(p1L, p3L);
+  const auto q3H = InterleaveHi(p1H, p3H);
+  const auto q4L = InterleaveLo(p4L, p6L);
+  const auto q4H = InterleaveLo(p4H, p6H);
+  const auto q5L = InterleaveLo(p5L, p7L);
+  const auto q5H = InterleaveLo(p5H, p7H);
+  const auto q6L = InterleaveHi(p4L, p6L);
+  const auto q6H = InterleaveHi(p4H, p6H);
+  const auto q7L = InterleaveHi(p5L, p7L);
+  const auto q7H = InterleaveHi(p5H, p7H);
+
+  const auto r0L = InterleaveLo(q0L, q1L);
+  const auto r0H = InterleaveLo(q0H, q1H);
+  const auto r1L = InterleaveHi(q0L, q1L);
+  const auto r1H = InterleaveHi(q0H, q1H);
+  const auto r2L = InterleaveLo(q2L, q3L);
+  const auto r2H = InterleaveLo(q2H, q3H);
+  const auto r3L = InterleaveHi(q2L, q3L);
+  const auto r3H = InterleaveHi(q2H, q3H);
+  const auto r4L = InterleaveLo(q4L, q5L);
+  const auto r4H = InterleaveLo(q4H, q5H);
+  const auto r5L = InterleaveHi(q4L, q5L);
+  const auto r5H = InterleaveHi(q4H, q5H);
+  const auto r6L = InterleaveLo(q6L, q7L);
+  const auto r6H = InterleaveLo(q6H, q7H);
+  const auto r7L = InterleaveHi(q6L, q7L);
+  const auto r7H = InterleaveHi(q6H, q7H);
+
+  to.StoreVec(r0L, 0, 0);
+  to.StoreVec(r4L, 0, 4);
+  to.StoreVec(r1L, 1, 0);
+  to.StoreVec(r5L, 1, 4);
+  to.StoreVec(r2L, 2, 0);
+  to.StoreVec(r6L, 2, 4);
+  to.StoreVec(r3L, 3, 0);
+  to.StoreVec(r7L, 3, 4);
+  to.StoreVec(r0H, 4, 0);
+  to.StoreVec(r4H, 4, 4);
+  to.StoreVec(r1H, 5, 0);
+  to.StoreVec(r5H, 5, 4);
+  to.StoreVec(r2H, 6, 0);
+  to.StoreVec(r6H, 6, 4);
+  to.StoreVec(r3H, 7, 0);
+  to.StoreVec(r7H, 7, 4);
+}
+
+template <class From, class To>
+HWY_ATTR JXL_INLINE void ComputeTransposedScaledDCT8_V4(const From& from,
+                                                        const To& to) {
+  // TODO(user): it is possible to avoid using temporary array,
+  // after generalizing "To" to be bi-directional; all sub-transforms could
+  // be performed "in-place".
+  HWY_ALIGN float block[8 * 8];
+  ColumnDCT8(from, ToBlock<8>(block));
+  TransposeBlock8_V4(FromBlock<8>(block), ToBlock<8>(block));
+  ColumnDCT8(FromBlock<8>(block), to);
+}
+
+template <class From, class To>
+HWY_ATTR JXL_INLINE void ComputeTransposedScaledIDCT8_V4(const From& from,
+                                                         const To& to) {
+  // TODO(user): it is possible to avoid using temporary array,
+  // after generalizing "To" to be bi-directional; all sub-transforms could
+  // be performed "in-place".
+  HWY_ALIGN float block[8 * 8];
+  ColumnIDCT8(from, ToBlock<8>(block));
+  TransposeBlock8_V4(FromBlock<8>(block), ToBlock<8>(block));
+  ColumnIDCT8(FromBlock<8>(block), to);
+}
+
+}  // namespace jxl
+
+#endif
+
+#endif  // JXL_DCT_HWY_4_H_
diff --git a/jxl/dct_simd_8.h b/jxl/dct_simd_8.h
new file mode 100644
index 000000000..1a8951236
--- /dev/null
+++ b/jxl/dct_simd_8.h
@@ -0,0 +1,484 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DCT_HWY_8_H_
+#define JXL_DCT_HWY_8_H_
+
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/block.h"
+#include "jxl/dct_simd_any.h"
+
+#if HWY_BITS >= 256
+
+namespace jxl {
+
+// DCT building blocks that require SIMD vector length to be 8, e.g. AVX2.
+static_assert(BlockDesc<8>::N == 8, "Wrong vector size, must be 8");
+
+// Each vector holds one row of the input/output block.
+template <class V>
+HWY_ATTR JXL_INLINE void TransposeBlock8_V8(V& i0, V& i1, V& i2, V& i3, V& i4,
+                                            V& i5, V& i6, V& i7) {
+  // Surprisingly, this straightforward implementation (24 cycles on port5) is
+  // faster than load128+insert and LoadDup128+ConcatHiLo+blend.
+  const auto q0 = InterleaveLo(i0, i2);
+  const auto q1 = InterleaveLo(i1, i3);
+  const auto q2 = InterleaveHi(i0, i2);
+  const auto q3 = InterleaveHi(i1, i3);
+  const auto q4 = InterleaveLo(i4, i6);
+  const auto q5 = InterleaveLo(i5, i7);
+  const auto q6 = InterleaveHi(i4, i6);
+  const auto q7 = InterleaveHi(i5, i7);
+
+  const auto r0 = InterleaveLo(q0, q1);
+  const auto r1 = InterleaveHi(q0, q1);
+  const auto r2 = InterleaveLo(q2, q3);
+  const auto r3 = InterleaveHi(q2, q3);
+  const auto r4 = InterleaveLo(q4, q5);
+  const auto r5 = InterleaveHi(q4, q5);
+  const auto r6 = InterleaveLo(q6, q7);
+  const auto r7 = InterleaveHi(q6, q7);
+
+  i0 = ConcatLoLo(r4, r0);
+  i1 = ConcatLoLo(r5, r1);
+  i2 = ConcatLoLo(r6, r2);
+  i3 = ConcatLoLo(r7, r3);
+  i4 = ConcatHiHi(r4, r0);
+  i5 = ConcatHiHi(r5, r1);
+  i6 = ConcatHiHi(r6, r2);
+  i7 = ConcatHiHi(r7, r3);
+}
+
+template <class From, class To>
+HWY_ATTR JXL_INLINE void TransposeBlock8_V8(const From& from, const To& to) {
+  auto i0 = from.template LoadPart<8>(0, 0);
+  auto i1 = from.template LoadPart<8>(1, 0);
+  auto i2 = from.template LoadPart<8>(2, 0);
+  auto i3 = from.template LoadPart<8>(3, 0);
+  auto i4 = from.template LoadPart<8>(4, 0);
+  auto i5 = from.template LoadPart<8>(5, 0);
+  auto i6 = from.template LoadPart<8>(6, 0);
+  auto i7 = from.template LoadPart<8>(7, 0);
+  TransposeBlock8_V8(i0, i1, i2, i3, i4, i5, i6, i7);
+  to.template StorePart<8>(i0, 0, 0);
+  to.template StorePart<8>(i1, 1, 0);
+  to.template StorePart<8>(i2, 2, 0);
+  to.template StorePart<8>(i3, 3, 0);
+  to.template StorePart<8>(i4, 4, 0);
+  to.template StorePart<8>(i5, 5, 0);
+  to.template StorePart<8>(i6, 6, 0);
+  to.template StorePart<8>(i7, 7, 0);
+}
+
+template <class From, class To>
+HWY_ATTR JXL_INLINE void ComputeTransposedScaledDCT8_V8(const From& from,
+                                                        const To& to) {
+  const BlockDesc<8> d;
+
+  const float c1234_lanes[4] = {
+      0.707106781186548f,  // 1 / sqrt(2)
+      0.382683432365090f,  // cos(3 * pi / 8)
+      1.30656296487638f,   // 1 / (2 * cos(3 * pi / 8))
+      0.541196100146197f   // sqrt(2) * cos(3 * pi / 8)
+  };
+  const auto c1234 = LoadDup128(d, c1234_lanes);
+  const auto k1 = Set(d, 1.0f);
+
+  auto i0 = from.template LoadPart<8>(0, 0);
+  auto i7 = from.template LoadPart<8>(7, 0);
+  auto t00 = i0 + i7;                // 2 (faster than Add)
+  auto t01 = NegMulAdd(i7, k1, i0);  // 4
+  HWY_FENCE;
+
+  auto i3 = from.template LoadPart<8>(3, 0);
+  auto i4 = from.template LoadPart<8>(4, 0);
+  auto t02 = i3 + i4;
+  auto t03 = NegMulAdd(i4, k1, i3);  // 1
+  HWY_FENCE;
+
+  auto i2 = from.template LoadPart<8>(2, 0);
+  auto i5 = from.template LoadPart<8>(5, 0);
+  auto t04 = i2 + i5;  // 1
+  auto t05 = NegMulAdd(i5, k1, i2);
+  HWY_FENCE;
+
+  auto i1 = from.template LoadPart<8>(1, 0);
+  auto i6 = from.template LoadPart<8>(6, 0);
+  auto t06 = i1 + i6;  // !
+  HWY_FENCE;
+
+  auto t07 = i1 - i6;
+  auto t09 = NegMulAdd(t02, k1, t00);
+  const auto c4 = hwy::Broadcast<3>(c1234);
+
+  auto t11 = t06 - t04;             // !
+  auto t08 = MulAdd(t00, k1, t02);  // 2
+  const auto c3 = hwy::Broadcast<2>(c1234);
+
+  auto t14 = t05 + t03;
+  auto t10 = MulAdd(t06, k1, t04);  // 1; dep-1
+
+  auto t13 = t01 + t07;  // limits odd d
+  const auto c1 = hwy::Broadcast<0>(c1234);
+
+  auto t15 = t11 + t09;  // !
+  const auto c2 = hwy::Broadcast<1>(c1234);
+
+  auto t12 = t07 + t05;  // !
+  auto ct14 = c4 * t14;
+
+  auto t16 = t14 - t13;  // 1
+  auto ct13 = c3 * t13;
+
+  auto d0 = MulAdd(t08, k1, t10);
+  auto d2 = MulAdd(c1, t15, t09);
+
+  auto t21 = NegMulAdd(c1, t12, t01);  // 2
+
+  auto d6 = NegMulAdd(c1, t15, t09);
+  auto t20 = MulAdd(c1, t12, t01);  // 2
+
+  auto t23 = MulAdd(c2, t16, ct14);
+
+  auto d4 = t08 - t10;
+  auto t22 = MulAdd(c2, t16, ct13);  // !
+
+  const auto q0 = InterleaveLo(d0, d2);
+
+  const auto q2 = InterleaveHi(d0, d2);
+
+  const auto q4 = InterleaveLo(d4, d6);
+
+  auto d3 = t21 - t23;
+  const auto q6 = InterleaveHi(d4, d6);
+
+  auto d1 = t20 + t22;
+  const auto q1 = InterleaveLo(d1, d3);
+
+  const auto r0 = InterleaveLo(q0, q1);
+  const auto r1 = InterleaveHi(q0, q1);
+
+  auto d7 = t20 - t22;
+  const auto q3 = InterleaveHi(d1, d3);
+  const auto r2 = InterleaveLo(q2, q3);
+  const auto r3 = InterleaveHi(q2, q3);
+
+  auto d5 = t21 + t23;
+  const auto q5 = InterleaveLo(d5, d7);
+  const auto r4 = InterleaveLo(q4, q5);
+  const auto r5 = InterleaveHi(q4, q5);
+
+  const auto q7 = InterleaveHi(d5, d7);
+  const auto r6 = InterleaveLo(q6, q7);
+  const auto r7 = InterleaveHi(q6, q7);
+
+  // Second column-DCT after transpose
+  i0 = ConcatLoLo(r4, r0);
+  i7 = ConcatHiHi(r7, r3);
+  t01 = i0 - i7;             // 1
+  t00 = MulAdd(i0, k1, i7);  // 2
+
+  i1 = ConcatLoLo(r5, r1);
+  i6 = ConcatHiHi(r6, r2);
+  t07 = i1 - i6;             // !
+  t06 = MulAdd(i1, k1, i6);  // 2
+
+  i3 = ConcatLoLo(r7, r3);
+  i4 = ConcatHiHi(r4, r0);
+  t03 = i3 - i4;             // 1
+  t02 = MulAdd(i3, k1, i4);  // !
+
+  i2 = ConcatLoLo(r6, r2);
+  i5 = ConcatHiHi(r5, r1);
+  t05 = i2 - i5;
+
+  t13 = t01 + t07;  // 1
+
+  t04 = i2 + i5;
+
+  t14 = t05 + t03;
+  t12 = MulAdd(t07, k1, t05);  // 2
+
+  t09 = NegMulAdd(t02, k1, t00);
+  ct13 = c3 * t13;  // 1
+
+  t11 = t06 - t04;  // 1
+  t10 = MulAdd(t06, k1, t04);
+
+  t16 = t14 - t13;  // !
+  ct14 = c4 * t14;
+
+  t08 = t00 + t02;
+
+  t20 = MulAdd(c1, t12, t01);  // 1
+
+  t15 = t11 + t09;
+  t22 = MulAdd(c2, t16, ct13);
+
+  i0 = t08 + t10;
+
+  t21 = NegMulAdd(c1, t12, t01);
+  t23 = MulAdd(c2, t16, ct14);
+
+  i4 = t08 - t10;
+  i2 = MulAdd(c1, t15, t09);
+
+  i6 = NegMulAdd(c1, t15, t09);
+  to.template StorePart<8>(i0, 0, 0);
+  HWY_FENCE;
+
+  i1 = t20 + t22;
+
+  i7 = t20 - t22;
+  to.template StorePart<8>(i2, 2, 0);
+  to.template StorePart<8>(i4, 4, 0);
+  HWY_FENCE;
+
+  i3 = t21 - t23;
+  to.template StorePart<8>(i1, 1, 0);
+  HWY_FENCE;
+
+  i5 = t21 + t23;
+  to.template StorePart<8>(i6, 6, 0);
+  to.template StorePart<8>(i7, 7, 0);
+  to.template StorePart<8>(i3, 3, 0);
+  to.template StorePart<8>(i5, 5, 0);
+}
+
+template <class From, class To>
+HWY_ATTR JXL_INLINE void ComputeTransposedScaledIDCT8_V8(const From& from,
+                                                         const To& to) {
+  const BlockDesc<8> d;
+
+  const float k1_lanes[4] = {HWY_REP4(1.0f)};
+  const auto k1 = LoadDup128(d, k1_lanes);
+  const float c1234_lanes[4] = {
+      1.41421356237310f,  // sqrt(2)
+      2.61312592975275f,  // 1 / cos(3 * pi / 8)
+      0.76536686473018f,  // 2 * cos(3 * pi / 8)
+      1.08239220029239f   // 2 * sqrt(2) * cos(3 * pi / 8)
+  };
+  const auto c1234 = LoadDup128(d, c1234_lanes);
+  HWY_FENCE;
+
+  // Finish d5,d7 and d0,d2 first so we can overlap more port5 (shuffles) with
+  // other computations; they have a shorter dependency chain than d13/46.
+
+  auto i1 = from.template LoadPart<8>(1, 0);
+  auto i7 = from.template LoadPart<8>(7, 0);
+  auto t05 = i7 - i1;             // !
+  auto t04 = MulAdd(i7, k1, i1);  // 1
+
+  auto i3 = from.template LoadPart<8>(3, 0);
+  auto i5 = from.template LoadPart<8>(5, 0);
+  auto t07 = i5 - i3;             // +1
+  auto t06 = MulAdd(i5, k1, i3);  // +1
+
+  auto i2 = from.template LoadPart<8>(2, 0);
+  auto i6 = from.template LoadPart<8>(6, 0);
+  auto t02 = i6 + i2;  // 1
+  const auto c2 = hwy::Broadcast<1>(c1234);
+  HWY_FENCE;
+
+  auto i0 = from.template LoadPart<8>(0, 0);
+  auto i4 = from.template LoadPart<8>(4, 0);
+  auto t03 = i6 - i2;    // !
+  auto ct05 = c2 * t05;  // !
+  HWY_FENCE;
+
+  auto t12 = t07 - t05;                      // 1
+  const auto c1 = hwy::Broadcast<0>(c1234);  // 1
+
+  auto t00 = MulAdd(i0, k1, i4);             // +2
+  const auto c3 = hwy::Broadcast<2>(c1234);  // 2
+
+  auto t09 = NegMulAdd(t06, k1, t04);
+  auto t14 = MulAdd(c1, t03, t02);  // +3
+
+  auto t08 = MulAdd(t04, k1, t06);           // 1
+  const auto c4 = hwy::Broadcast<3>(c1234);  // 2
+
+  auto t01 = i0 - i4;                // +1
+  auto t17 = MulAdd(c3, t12, ct05);  // !
+  HWY_FENCE;
+
+  //
+
+  auto t10 = MulAdd(t00, k1, t02);
+  auto ct07 = c4 * t07;  // !
+
+  auto t15 = NegMulAdd(t14, k1, t01);  // 1
+  auto ct09 = c1 * t09;
+
+  auto t11 = NegMulAdd(t02, k1, t00);  // 6
+
+  auto t19 = t08 + t17;  // !
+
+  auto t16 = MulAdd(t01, k1, t14);
+
+  auto d0 = MulAdd(t10, k1, t08);    // dep-3; 4
+  auto t18 = MulAdd(c3, t12, ct07);  // !
+
+  auto t20 = ct09 + t19;              // !
+  auto d7 = NegMulAdd(t08, k1, t10);  // 1
+
+  auto d1 = NegMulAdd(t19, k1, t15);  // 5
+
+  //
+
+  auto d5 = t16 - t20;  // !
+  auto d2 = MulAdd(t16, k1, t20);
+
+  auto t21 = t18 - t20;  // !
+
+  //
+
+  // Begin transposing finished d#
+
+  auto d6 = t15 + t19;  // 1
+  const auto q5 = InterleaveLo(d5, d7);
+
+  auto d4 = t11 - t21;                   // !
+  const auto q7 = InterleaveHi(d5, d7);  // 8
+
+  auto d3 = t11 + t21;  // !
+  const auto q0 = InterleaveLo(d0, d2);
+
+  const auto q2 = InterleaveHi(d0, d2);  // 8
+
+  const auto q4 = InterleaveLo(d4, d6);
+
+  const auto q1 = InterleaveLo(d1, d3);
+
+  const auto r4 = InterleaveLo(q4, q5);
+
+  const auto r0 = InterleaveLo(q0, q1);
+
+  i0 = ConcatLoLo(r4, r0);
+
+  i4 = ConcatHiHi(r4, r0);
+  const auto _c1234 = LoadDup128(d, c1234_lanes);
+
+  const auto q3 = InterleaveHi(d1, d3);
+
+  // Begin second column-IDCT for transposed r#
+
+  const auto q6 = InterleaveHi(d4, d6);
+
+  t00 = MulAdd(i0, k1, i4);
+  const auto r2 = InterleaveLo(q2, q3);
+
+  t01 = NegMulAdd(i4, k1, i0);
+  const auto r6 = InterleaveLo(q6, q7);
+
+  i2 = ConcatLoLo(r6, r2);
+
+  i6 = ConcatHiHi(r6, r2);
+
+  const auto r7 = InterleaveHi(q6, q7);
+
+  const auto r3 = InterleaveHi(q2, q3);
+
+  t03 = i6 - i2;
+  i7 = ConcatHiHi(r7, r3);
+
+  t02 = i6 + i2;
+  const auto r5 = InterleaveHi(q4, q5);
+
+  const auto r1 = InterleaveHi(q0, q1);
+  const auto _c1 = hwy::Broadcast<0>(_c1234);
+
+  i1 = ConcatLoLo(r5, r1);
+  auto ct03 = _c1 * t03;
+
+  t10 = MulAdd(t00, k1, t02);  // 5
+  i5 = ConcatHiHi(r5, r1);
+
+  i3 = ConcatLoLo(r7, r3);
+
+  t05 = i7 - i1;  // !
+  const auto _c2 = hwy::Broadcast<1>(_c1234);
+
+  t04 = MulAdd(i7, k1, i1);  // 1
+
+  t07 = i5 - i3;
+
+  t06 = i5 + i3;
+  ct05 = _c2 * t05;  // !
+
+  t14 = ct03 + t02;  // 1
+
+  t12 = t07 - t05;
+
+  t08 = t04 + t06;
+
+  t09 = t04 - t06;
+
+  t15 = NegMulAdd(t14, k1, t01);  // 3
+  t17 = MulAdd(c3, t12, ct05);    // !
+
+  d0 = t10 + t08;
+
+  d7 = t10 - t08;
+
+  ct09 = _c1 * t09;
+
+  const auto _c4 = hwy::Broadcast<3>(_c1234);
+  to.StoreVec(d0, 0, 0);
+  HWY_FENCE;
+
+  t19 = t08 + t17;   // !
+  ct07 = _c4 * t07;  // !
+  to.StoreVec(d7, 7, 0);
+  HWY_FENCE;
+
+  t11 = t00 - t02;  // 8
+
+  t16 = t01 + t14;  // 3
+
+  d1 = t15 - t19;
+  t20 = ct09 + t19;  // !
+
+  d6 = t15 + t19;
+  const auto _c3 = hwy::Broadcast<2>(_c1234);
+
+  t18 = MulAdd(_c3, t12, ct07);  // !
+
+  d2 = t16 + t20;
+  to.StoreVec(d1, 1, 0);
+  HWY_FENCE;
+
+  d5 = t16 - t20;
+  to.StoreVec(d6, 6, 0);
+  HWY_FENCE;
+
+  t21 = t18 - t20;  // !
+
+  d4 = t11 - t21;
+  to.StoreVec(d2, 2, 0);
+
+  d3 = t11 + t21;
+  to.StoreVec(d5, 5, 0);
+
+  to.StoreVec(d4, 4, 0);
+  to.StoreVec(d3, 3, 0);
+}
+
+}  // namespace jxl
+
+#endif
+
+#endif  // JXL_DCT_HWY_8_H_
diff --git a/jxl/dct_simd_any.h b/jxl/dct_simd_any.h
new file mode 100644
index 000000000..fa8e375d0
--- /dev/null
+++ b/jxl/dct_simd_any.h
@@ -0,0 +1,1353 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DCT_HWY_ANY_H_
+#define JXL_DCT_HWY_ANY_H_
+
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/block.h"
+
+namespace jxl {
+
+// DCT building blocks that does not require specific SIMD vector length.
+
+template <class From, class To>
+HWY_ATTR JXL_INLINE void CopyBlock8(const From& from, const To& to) {
+  const BlockDesc<8> d;
+  for (size_t i = 0; i < 8; i += d.N) {
+    const auto i0 = from.template LoadPart<8>(0, i);
+    const auto i1 = from.template LoadPart<8>(1, i);
+    const auto i2 = from.template LoadPart<8>(2, i);
+    const auto i3 = from.template LoadPart<8>(3, i);
+    const auto i4 = from.template LoadPart<8>(4, i);
+    const auto i5 = from.template LoadPart<8>(5, i);
+    const auto i6 = from.template LoadPart<8>(6, i);
+    const auto i7 = from.template LoadPart<8>(7, i);
+    to.template StorePart<8>(i0, 0, i);
+    to.template StorePart<8>(i1, 1, i);
+    to.template StorePart<8>(i2, 2, i);
+    to.template StorePart<8>(i3, 3, i);
+    to.template StorePart<8>(i4, 4, i);
+    to.template StorePart<8>(i5, 5, i);
+    to.template StorePart<8>(i6, 6, i);
+    to.template StorePart<8>(i7, 7, i);
+  }
+}
+
+template <size_t SZ, class V>
+HWY_ATTR JXL_INLINE void ColumnDCT8(V& i0, V& i1, V& i2, V& i3, V& i4, V& i5,
+                                     V& i6, V& i7) {
+  const BlockDesc<SZ> d;
+
+  const auto c1 = Set(d, 0.707106781186548f);  // 1 / sqrt(2)
+  const auto c2 = Set(d, 0.382683432365090f);  // cos(3 * pi / 8)
+  const auto c3 = Set(d, 1.30656296487638f);   // 1 / (2 * cos(3 * pi / 8))
+  const auto c4 = Set(d, 0.541196100146197f);  // sqrt(2) * cos(3 * pi / 8)
+
+  const auto t00 = i0 + i7;
+  const auto t01 = i0 - i7;
+  const auto t02 = i3 + i4;
+  const auto t03 = i3 - i4;
+  const auto t04 = i2 + i5;
+  const auto t05 = i2 - i5;
+  const auto t06 = i1 + i6;
+  const auto t07 = i1 - i6;
+  const auto t08 = t00 + t02;
+  const auto t09 = t00 - t02;
+  const auto t10 = t06 + t04;
+  const auto t11 = t06 - t04;
+  const auto t12 = t07 + t05;
+  const auto t13 = t01 + t07;
+  const auto t14 = t05 + t03;
+  const auto t15 = t11 + t09;
+  const auto t16 = t14 - t13;
+  const auto t17 = c1 * t15;
+  const auto t18 = c1 * t12;
+  const auto t19 = c2 * t16;
+  const auto t20 = t01 + t18;
+  const auto t21 = t01 - t18;
+  const auto t22 = MulAdd(c3, t13, t19);
+  const auto t23 = MulAdd(c4, t14, t19);
+  i0 = t08 + t10;
+  i1 = t20 + t22;
+  i2 = t09 + t17;
+  i3 = t21 - t23;
+  i4 = t08 - t10;
+  i5 = t21 + t23;
+  i6 = t09 - t17;
+  i7 = t20 - t22;
+}
+
+// "A low multiplicative complexity fast recursive DCT-2 algorithm"
+// Maxim Vashkevich, Alexander Pertrovsky, 27 Jul 2012
+template <size_t SZ, class V>
+HWY_ATTR JXL_INLINE void ColumnDCT16(V& i00, V& i01, V& i02, V& i03, V& i04,
+                                      V& i05, V& i06, V& i07, V& i08, V& i09,
+                                      V& i10, V& i11, V& i12, V& i13, V& i14,
+                                      V& i15) {
+  const BlockDesc<SZ> d;
+
+  const auto c1_16 = Set(d, 1.9615705608064609f);   // 2 * cos(1 * pi / 16)
+  const auto c2_16 = Set(d, 1.8477590650225735f);   // 2 * cos(2 * pi / 16)
+  const auto c3_16 = Set(d, 1.6629392246050905f);   // 2 * cos(3 * pi / 16)
+  const auto c4_16 = Set(d, 1.4142135623730951f);   // 2 * cos(4 * pi / 16)
+  const auto c5_16 = Set(d, 1.1111404660392046f);   // 2 * cos(5 * pi / 16)
+  const auto c6_16 = Set(d, 0.7653668647301797f);   // 2 * cos(6 * pi / 16)
+  const auto c7_16 = Set(d, 0.39018064403225666f);  // 2 * cos(7 * pi / 16)
+
+  const auto t00 = i00 + i15;
+  const auto t01 = i01 + i14;
+  const auto t02 = i02 + i13;
+  const auto t03 = i03 + i12;
+  const auto t04 = i04 + i11;
+  const auto t05 = i05 + i10;
+  const auto t06 = i06 + i09;
+  const auto t07 = i07 + i08;
+  const auto t08 = i00 - i15;
+  const auto t09 = i01 - i14;
+  const auto t10 = i02 - i13;
+  const auto t11 = i03 - i12;
+  const auto t12 = i04 - i11;
+  const auto t13 = i05 - i10;
+  const auto t14 = i06 - i09;
+  const auto t15 = i07 - i08;
+  const auto t16 = t00 + t07;
+  const auto t17 = t01 + t06;
+  const auto t18 = t02 + t05;
+  const auto t19 = t03 + t04;
+  const auto t20 = t00 - t07;
+  const auto t21 = t01 - t06;
+  const auto t22 = t02 - t05;
+  const auto t23 = t03 - t04;
+  const auto t24 = t16 + t19;
+  const auto t25 = t17 + t18;
+  const auto t26 = t16 - t19;
+  const auto t27 = t17 - t18;
+  i00 = t24 + t25;
+  i08 = t24 - t25;
+  const auto t30 = t26 - t27;
+  const auto t31 = t27 * c4_16;
+  i04 = t30 + t31;
+  i12 = t30 - t31;
+  const auto t34 = t20 - t23;
+  const auto t35 = t21 - t22;
+  const auto t36 = t22 * c4_16;
+  const auto t37 = t23 * c4_16;
+  const auto t38 = t34 + t36;
+  const auto t39 = t35 + t37;
+  const auto t40 = t34 - t36;
+  const auto t41 = t35 - t37;
+  const auto t42 = t38 - t39;
+  const auto t43 = t39 * c2_16;
+  i02 = t42 + t43;
+  i14 = t42 - t43;
+  const auto t46 = t40 - t41;
+  const auto t47 = t41 * c6_16;
+  i06 = t46 + t47;
+  i10 = t46 - t47;
+  const auto t50 = t08 - t15;
+  const auto t51 = t09 - t14;
+  const auto t52 = t10 - t13;
+  const auto t53 = t11 - t12;
+  const auto t54 = t12 * c4_16;
+  const auto t55 = t13 * c4_16;
+  const auto t56 = t14 * c4_16;
+  const auto t57 = t15 * c4_16;
+  const auto t58 = t50 + t54;
+  const auto t59 = t51 + t55;
+  const auto t60 = t52 + t56;
+  const auto t61 = t53 + t57;
+  const auto t62 = t50 - t54;
+  const auto t63 = t51 - t55;
+  const auto t64 = t52 - t56;
+  const auto t65 = t53 - t57;
+  const auto t66 = t58 - t61;
+  const auto t67 = t59 - t60;
+  const auto t68 = t60 * c2_16;
+  const auto t69 = t61 * c2_16;
+  const auto t70 = t66 + t68;
+  const auto t71 = t67 + t69;
+  const auto t72 = t66 - t68;
+  const auto t73 = t67 - t69;
+  const auto t74 = t70 - t71;
+  const auto t75 = t71 * c1_16;
+  i01 = t74 + t75;
+  i15 = t74 - t75;
+  const auto t78 = t72 - t73;
+  const auto t79 = t73 * c7_16;
+  i07 = t78 + t79;
+  i09 = t78 - t79;
+  const auto t82 = t62 - t65;
+  const auto t83 = t63 - t64;
+  const auto t84 = t64 * c6_16;
+  const auto t85 = t65 * c6_16;
+  const auto t86 = t82 + t84;
+  const auto t87 = t83 + t85;
+  const auto t88 = t82 - t84;
+  const auto t89 = t83 - t85;
+  const auto t90 = t86 - t87;
+  const auto t91 = t87 * c3_16;
+  i03 = t90 + t91;
+  i13 = t90 - t91;
+  const auto t94 = t88 - t89;
+  const auto t95 = t89 * c5_16;
+  i05 = t94 + t95;
+  i11 = t94 - t95;
+}
+
+template <class From, class To, size_t COLS = 8>
+HWY_ATTR JXL_INLINE void ColumnDCT8(const From& from, const To& to) {
+  const BlockDesc<8> d;
+  constexpr size_t kSize = COLS < d.N ? COLS : d.N;
+
+  for (size_t i = 0; i < COLS; i += d.N) {
+    auto i0 = from.template LoadPart<kSize>(0, i);
+    auto i1 = from.template LoadPart<kSize>(1, i);
+    auto i2 = from.template LoadPart<kSize>(2, i);
+    auto i3 = from.template LoadPart<kSize>(3, i);
+    auto i4 = from.template LoadPart<kSize>(4, i);
+    auto i5 = from.template LoadPart<kSize>(5, i);
+    auto i6 = from.template LoadPart<kSize>(6, i);
+    auto i7 = from.template LoadPart<kSize>(7, i);
+    ColumnDCT8<kSize>(i0, i1, i2, i3, i4, i5, i6, i7);
+    to.template StorePart<kSize>(i0, 0, i);
+    to.template StorePart<kSize>(i1, 1, i);
+    to.template StorePart<kSize>(i2, 2, i);
+    to.template StorePart<kSize>(i3, 3, i);
+    to.template StorePart<kSize>(i4, 4, i);
+    to.template StorePart<kSize>(i5, 5, i);
+    to.template StorePart<kSize>(i6, 6, i);
+    to.template StorePart<kSize>(i7, 7, i);
+  }
+}
+
+template <class From, class To, size_t COLS = 16>
+HWY_ATTR void ColumnDCT16(const From& from, const To& to) {
+  const BlockDesc<COLS> d;
+
+  for (size_t i = 0; i < COLS; i += d.N) {
+    auto i00 = from.template LoadPart<COLS>(0, i);
+    auto i01 = from.template LoadPart<COLS>(1, i);
+    auto i02 = from.template LoadPart<COLS>(2, i);
+    auto i03 = from.template LoadPart<COLS>(3, i);
+    auto i04 = from.template LoadPart<COLS>(4, i);
+    auto i05 = from.template LoadPart<COLS>(5, i);
+    auto i06 = from.template LoadPart<COLS>(6, i);
+    auto i07 = from.template LoadPart<COLS>(7, i);
+    auto i08 = from.template LoadPart<COLS>(8, i);
+    auto i09 = from.template LoadPart<COLS>(9, i);
+    auto i10 = from.template LoadPart<COLS>(10, i);
+    auto i11 = from.template LoadPart<COLS>(11, i);
+    auto i12 = from.template LoadPart<COLS>(12, i);
+    auto i13 = from.template LoadPart<COLS>(13, i);
+    auto i14 = from.template LoadPart<COLS>(14, i);
+    auto i15 = from.template LoadPart<COLS>(15, i);
+    ColumnDCT16<COLS>(i00, i01, i02, i03, i04, i05, i06, i07, i08, i09, i10,
+                      i11, i12, i13, i14, i15);
+    to.template StorePart<COLS>(i00, 0, i);
+    to.template StorePart<COLS>(i01, 1, i);
+    to.template StorePart<COLS>(i02, 2, i);
+    to.template StorePart<COLS>(i03, 3, i);
+    to.template StorePart<COLS>(i04, 4, i);
+    to.template StorePart<COLS>(i05, 5, i);
+    to.template StorePart<COLS>(i06, 6, i);
+    to.template StorePart<COLS>(i07, 7, i);
+    to.template StorePart<COLS>(i08, 8, i);
+    to.template StorePart<COLS>(i09, 9, i);
+    to.template StorePart<COLS>(i10, 10, i);
+    to.template StorePart<COLS>(i11, 11, i);
+    to.template StorePart<COLS>(i12, 12, i);
+    to.template StorePart<COLS>(i13, 13, i);
+    to.template StorePart<COLS>(i14, 14, i);
+    to.template StorePart<COLS>(i15, 15, i);
+  }
+}
+
+// NB: ColumnIDCT8(ColumnDCT8(I)) = 8.0 * I
+template <size_t SZ, class V>
+HWY_ATTR JXL_INLINE void ColumnIDCT8(V& i0, V& i1, V& i2, V& i3, V& i4, V& i5,
+                                      V& i6, V& i7) {
+  const BlockDesc<SZ> d;
+
+  const auto c1 = Set(d, 1.41421356237310f);  // sqrt(2)
+  const auto c2 = Set(d, 2.61312592975275f);  // 1 / cos(3 * pi / 8)
+  const auto c3 = Set(d, 0.76536686473018f);  // 2 * cos(3 * pi / 8)
+  const auto c4 = Set(d, 1.08239220029239f);  // 2 * sqrt(2) * cos(3 * pi / 8)
+
+  const auto t00 = i0 + i4;
+  const auto t01 = i0 - i4;
+  const auto t02 = i6 + i2;
+  const auto t03 = i6 - i2;
+  const auto t04 = i7 + i1;
+  const auto t05 = i7 - i1;
+  const auto t06 = i5 + i3;
+  const auto t07 = i5 - i3;
+  const auto t08 = t04 + t06;
+  const auto t09 = t04 - t06;
+  const auto t10 = t00 + t02;
+  const auto t11 = t00 - t02;
+  const auto t12 = t07 - t05;
+  const auto t13 = c3 * t12;
+  const auto t14 = MulAdd(c1, t03, t02);
+  const auto t15 = t01 - t14;
+  const auto t16 = t01 + t14;
+  const auto t17 = MulAdd(c2, t05, t13);
+  const auto t18 = MulAdd(c4, t07, t13);
+  const auto t19 = t08 + t17;
+  const auto t20 = MulAdd(c1, t09, t19);
+  const auto t21 = t18 - t20;
+  i0 = t10 + t08;
+  i1 = t15 - t19;
+  i2 = t16 + t20;
+  i3 = t11 + t21;
+  i4 = t11 - t21;
+  i5 = t16 - t20;
+  i6 = t15 + t19;
+  i7 = t10 - t08;
+}
+
+template <class From, class To, size_t COLS = 8>
+HWY_ATTR JXL_INLINE void ColumnIDCT8(const From& from, const To& to) {
+  const BlockDesc<COLS> d;
+
+  for (size_t i = 0; i < COLS; i += d.N) {
+    auto i0 = from.template LoadPart<COLS>(0, i);
+    auto i1 = from.template LoadPart<COLS>(1, i);
+    auto i2 = from.template LoadPart<COLS>(2, i);
+    auto i3 = from.template LoadPart<COLS>(3, i);
+    auto i4 = from.template LoadPart<COLS>(4, i);
+    auto i5 = from.template LoadPart<COLS>(5, i);
+    auto i6 = from.template LoadPart<COLS>(6, i);
+    auto i7 = from.template LoadPart<COLS>(7, i);
+    ColumnIDCT8<COLS>(i0, i1, i2, i3, i4, i5, i6, i7);
+    to.template StorePart<COLS>(i0, 0, i);
+    to.template StorePart<COLS>(i1, 1, i);
+    to.template StorePart<COLS>(i2, 2, i);
+    to.template StorePart<COLS>(i3, 3, i);
+    to.template StorePart<COLS>(i4, 4, i);
+    to.template StorePart<COLS>(i5, 5, i);
+    to.template StorePart<COLS>(i6, 6, i);
+    to.template StorePart<COLS>(i7, 7, i);
+  }
+}
+
+// "A low multiplicative complexity fast recursive DCT-2 algorithm"
+// Maxim Vashkevich, Alexander Pertrovsky, 27 Jul 2012
+template <class From, class To, size_t COLS = 16>
+HWY_ATTR void ColumnIDCT16(const From& from, const To& to) {
+  const BlockDesc<COLS> d;
+
+  // Odd constants are only used once, so broadcast as needed.
+  HWY_ALIGN constexpr float k1357[4] = {
+      0.5097955791041592f,  // 0.5 / cos(1 * pi / 16)
+      0.6013448869350453f,  // 0.5 / cos(3 * pi / 16)
+      0.8999762231364156f,  // 0.5 / cos(5 * pi / 16)
+      2.5629154477415055f,  // 0.5 / cos(7 * pi / 16)
+  };
+
+  HWY_ALIGN constexpr float c2_lanes[4] = {
+      HWY_REP4(0.541196100146197f)};  // 0.5 / cos(2 * pi / 16)
+  HWY_ALIGN constexpr float c4_lanes[4] = {
+      HWY_REP4(0.7071067811865475f)};  // 0.5 / cos(4 * pi / 16)
+  HWY_ALIGN constexpr float c6_lanes[4] = {
+      HWY_REP4(1.3065629648763764f)};  // 0.5 / cos(6 * pi / 16)
+
+  // Unrolling does not help.
+  for (size_t i = 0; i < COLS; i += d.N) {
+    auto i00 = from.template LoadPart<COLS>(0, i);
+    auto i08 = from.template LoadPart<COLS>(8, i);
+    auto i04 = from.template LoadPart<COLS>(4, i);
+    auto i12 = from.template LoadPart<COLS>(12, i);
+    const auto t00 = i00 + i08;
+    const auto t01 = i00 - i08;
+    const auto t02 = i04 + i12;
+    const auto t03 = i04 - i12;
+    HWY_FENCE;
+
+    auto i02 = from.template LoadPart<COLS>(2, i);
+    auto i06 = from.template LoadPart<COLS>(6, i);
+    auto i10 = from.template LoadPart<COLS>(10, i);
+    auto i14 = from.template LoadPart<COLS>(14, i);
+    const auto t10 = i02 + i14;
+    const auto t11 = i02 - i14;
+    const auto t14 = i06 + i10;
+    const auto t15 = i06 - i10;
+    HWY_FENCE;
+
+    auto i01 = from.template LoadPart<COLS>(1, i);
+    auto i07 = from.template LoadPart<COLS>(7, i);
+    auto i09 = from.template LoadPart<COLS>(9, i);
+    auto i15 = from.template LoadPart<COLS>(15, i);
+
+    auto i03 = from.template LoadPart<COLS>(3, i);
+    auto i05 = from.template LoadPart<COLS>(5, i);
+    auto i11 = from.template LoadPart<COLS>(11, i);
+    auto i13 = from.template LoadPart<COLS>(13, i);
+
+#if HWY_BITS == 0
+    const auto c1_16 = Set(d, k1357[0]);
+    const auto c3_16 = Set(d, k1357[1]);
+    const auto c5_16 = Set(d, k1357[2]);
+    const auto c7_16 = Set(d, k1357[3]);
+#else
+    const auto c7531 = LoadDup128(d, k1357);  // c1 is in lane0.
+    const auto c1_16 = hwy::Broadcast<0>(c7531);
+    const auto c3_16 = hwy::Broadcast<1>(c7531);
+    const auto c5_16 = hwy::Broadcast<2>(c7531);
+    const auto c7_16 = hwy::Broadcast<3>(c7531);
+#endif
+    const auto c2_16 = LoadDup128(d, c2_lanes);
+    const auto c4_16 = LoadDup128(d, c4_lanes);
+    const auto c6_16 = LoadDup128(d, c6_lanes);
+
+    const auto t05 = MulAdd(t03, c4_16, t02);
+    const auto t07 = MulAdd(t03, c4_16, t01);
+    const auto t06 = t00 + t05;
+    const auto t08 = t00 - t05;
+    const auto t09 = NegMulAdd(t03, c4_16, t01);
+    const auto t13 = MulAdd(t11, c2_16, t10);
+    const auto t16 = t15 * c6_16;  // used in subsequent FMA
+    const auto t17 = t14 + t16;
+    const auto t18 = t13 + t17;
+    const auto t19 = MulAdd(t11, c2_16, t16);
+    const auto t20 = t13 - t17;
+    const auto t21 = hwy::ext::MulSub(t11, c2_16, t16);
+    const auto t23 = t21 * c4_16;
+    const auto t24 = t18 + t23;
+    const auto t25 = MulAdd(t20, c4_16, t19);
+    const auto t26 = t06 + t24;
+    const auto t27 = t07 + t25;
+    const auto t28 = MulAdd(t20, c4_16, t09);
+    const auto t29 = t08 + t23;
+    const auto t30 = t06 - t24;
+    const auto t31 = t07 - t25;
+    const auto t32 = NegMulAdd(t20, c4_16, t09);
+    const auto t33 = t08 - t23;
+    const auto t34 = i01 + i15;
+    const auto t35 = i01 - i15;
+    const auto t37 = MulAdd(t35, c1_16, t34);
+    const auto t38 = i07 + i09;
+    const auto t39 = i07 - i09;
+    const auto t40 = t39 * c7_16;  // used in subsequent FMA
+    const auto t41 = t38 + t40;
+    const auto t42 = t37 + t41;
+    const auto t43 = MulAdd(t35, c1_16, t40);
+    const auto t44 = t37 - t41;
+    const auto t45 = hwy::ext::MulSub(t35, c1_16, t40);
+    const auto t48 = MulAdd(t45, c2_16, t42);
+    const auto t49 = MulAdd(t44, c2_16, t43);
+    const auto t50 = i03 + i13;
+    const auto t51 = i03 - i13;
+    const auto t52 = t51 * c3_16;
+    const auto t53 = t50 + t52;
+    const auto t54 = i05 + i11;
+    const auto t55 = i05 - i11;
+    const auto t57 = MulAdd(t55, c5_16, t54);
+    const auto t58 = t53 + t57;
+    const auto t60 = t53 - t57;
+    const auto t59 = MulAdd(t55, c5_16, t52);
+    const auto t61 = NegMulAdd(t55, c5_16, t52);
+    const auto t62 = t60 * c6_16;  // used in subsequent FMA
+    const auto t63 = t61 * c6_16;  // used in subsequent FMA
+    const auto t64 = t58 + t63;
+    const auto t65 = t59 + t62;
+    const auto t66 = t48 + t64;
+    const auto t67 = t49 + t65;
+    const auto t68 = MulAdd(t44, c2_16, t62);
+    const auto t69 = MulAdd(t45, c2_16, t63);
+    const auto t70 = t48 - t64;
+    const auto t71 = t49 - t65;
+    const auto t72 = hwy::ext::MulSub(t44, c2_16, t62);
+    const auto t73 = hwy::ext::MulSub(t45, c2_16, t63);
+    const auto t78 = MulAdd(t73, c4_16, t66);
+    const auto t79 = MulAdd(t72, c4_16, t67);
+    const auto t80 = MulAdd(t71, c4_16, t68);
+    const auto t81 = MulAdd(t70, c4_16, t69);
+
+    to.template StorePart<COLS>(t26 + t78, 0, i);
+    to.template StorePart<COLS>(t27 + t79, 1, i);
+    to.template StorePart<COLS>(t28 + t80, 2, i);
+    to.template StorePart<COLS>(t29 + t81, 3, i);
+    to.template StorePart<COLS>(MulAdd(t70, c4_16, t33), 4, i);
+    to.template StorePart<COLS>(MulAdd(t71, c4_16, t32), 5, i);
+    to.template StorePart<COLS>(MulAdd(t72, c4_16, t31), 6, i);
+    to.template StorePart<COLS>(MulAdd(t73, c4_16, t30), 7, i);
+    to.template StorePart<COLS>(NegMulAdd(t73, c4_16, t30), 8, i);
+    to.template StorePart<COLS>(NegMulAdd(t72, c4_16, t31), 9, i);
+    to.template StorePart<COLS>(NegMulAdd(t71, c4_16, t32), 10, i);
+    to.template StorePart<COLS>(NegMulAdd(t70, c4_16, t33), 11, i);
+    to.template StorePart<COLS>(t29 - t81, 12, i);
+    to.template StorePart<COLS>(t28 - t80, 13, i);
+    to.template StorePart<COLS>(t27 - t79, 14, i);
+    to.template StorePart<COLS>(t26 - t78, 15, i);
+  }
+}
+
+template <size_t SZ, class V>
+HWY_ATTR JXL_INLINE void ColumnDCT4(V& i0, V& i1, V& i2, V& i3) {
+  const BlockDesc<SZ> d;
+  const auto c2_8 = Set(d, 1.414213562373095048f);  // 2 * cos(2 * pi / 8)
+  auto t0 = i0 + i3;
+  auto t1 = i1 + i2;
+  auto t2 = i0 - i3;
+  auto t3 = i1 - i2;
+  i0 = t0 + t1;
+  i2 = t0 - t1;
+  auto t6 = t2 - t3;
+  auto t7 = t3 * c2_8;
+  i1 = t6 + t7;
+  i3 = t6 - t7;
+}
+
+template <class From, class To, size_t COLS = 4>
+HWY_ATTR JXL_INLINE void ColumnDCT4(const From& from, const To& to) {
+  const BlockDesc<4> d;
+  constexpr size_t kSize = COLS < d.N ? COLS : d.N;
+  for (size_t i = 0; i < COLS; i += d.N) {
+    auto i0 = from.template LoadPart<kSize>(0, i);
+    auto i1 = from.template LoadPart<kSize>(1, i);
+    auto i2 = from.template LoadPart<kSize>(2, i);
+    auto i3 = from.template LoadPart<kSize>(3, i);
+    ColumnDCT4<kSize>(i0, i1, i2, i3);
+    to.template StorePart<kSize>(i0, 0, i);
+    to.template StorePart<kSize>(i1, 1, i);
+    to.template StorePart<kSize>(i2, 2, i);
+    to.template StorePart<kSize>(i3, 3, i);
+  }
+}
+
+template <size_t SZ, class V>
+HWY_ATTR JXL_INLINE void ColumnIDCT4(V& i0, V& i1, V& i2, V& i3) {
+  const BlockDesc<SZ> d;
+  const auto c2_8 = Set(d, 0.7071067811865475244f);  // 0.5 / cos(2 * pi / 8)
+  auto t0 = i0 + i2;
+  auto t1 = i0 - i2;
+  auto t2 = i1 + i3;
+  auto t3 = i1 - i3;
+  auto t4 = t3 * c2_8;
+  auto t5 = t2 + t4;
+  i0 = t0 + t5;
+  i1 = t1 + t4;
+  i3 = t0 - t5;
+  i2 = t1 - t4;
+}
+
+template <class From, class To, size_t COLS = 4>
+HWY_ATTR JXL_INLINE void ColumnIDCT4(const From& from, const To& to) {
+  const BlockDesc<4> d;
+  constexpr size_t kSize = COLS < d.N ? COLS : d.N;
+  for (size_t i = 0; i < COLS; i += d.N) {
+    auto i0 = from.template LoadPart<kSize>(0, i);
+    auto i1 = from.template LoadPart<kSize>(1, i);
+    auto i2 = from.template LoadPart<kSize>(2, i);
+    auto i3 = from.template LoadPart<kSize>(3, i);
+    ColumnIDCT4<kSize>(i0, i1, i2, i3);
+    to.template StorePart<kSize>(i0, 0, i);
+    to.template StorePart<kSize>(i1, 1, i);
+    to.template StorePart<kSize>(i2, 2, i);
+    to.template StorePart<kSize>(i3, 3, i);
+  }
+}
+
+template <size_t SZ, class V>
+HWY_ATTR JXL_INLINE void ColumnDCT32(V& i00, V& i01, V& i02, V& i03, V& i04,
+                                      V& i05, V& i06, V& i07, V& i08, V& i09,
+                                      V& i10, V& i11, V& i12, V& i13, V& i14,
+                                      V& i15, V& i16, V& i17, V& i18, V& i19,
+                                      V& i20, V& i21, V& i22, V& i23, V& i24,
+                                      V& i25, V& i26, V& i27, V& i28, V& i29,
+                                      V& i30, V& i31) {
+  const BlockDesc<SZ> d;
+  const auto c2_64 = Set(d, 1.990369453344393857f);   // 2 * cos(2 * pi / 64)
+  const auto c4_64 = Set(d, 1.961570560806460861f);   // 2 * cos(4 * pi / 64)
+  const auto c6_64 = Set(d, 1.913880671464417649f);   // 2 * cos(6 * pi / 64)
+  const auto c8_64 = Set(d, 1.847759065022573477f);   // 2 * cos(8 * pi / 64)
+  const auto c10_64 = Set(d, 1.763842528696710099f);  // 2 * cos(10 * pi / 64)
+  const auto c12_64 = Set(d, 1.662939224605090471f);  // 2 * cos(12 * pi / 64)
+  const auto c14_64 = Set(d, 1.546020906725473987f);  // 2 * cos(14 * pi / 64)
+  const auto c16_64 = Set(d, 1.414213562373095145f);  // 2 * cos(16 * pi / 64)
+  const auto c18_64 = Set(d, 1.268786568327290976f);  // 2 * cos(18 * pi / 64)
+  const auto c20_64 = Set(d, 1.111140466039204577f);  // 2 * cos(20 * pi / 64)
+  const auto c22_64 = Set(d, 0.942793473651995617f);  // 2 * cos(22 * pi / 64)
+  const auto c24_64 = Set(d, 0.765366864730179675f);  // 2 * cos(24 * pi / 64)
+  const auto c26_64 = Set(d, 0.580569354508924662f);  // 2 * cos(26 * pi / 64)
+  const auto c28_64 = Set(d, 0.390180644032256663f);  // 2 * cos(28 * pi / 64)
+  const auto c30_64 = Set(d, 0.196034280659121540f);  // 2 * cos(30 * pi / 64)
+
+  const auto t00 = i00 + i31;
+  const auto t01 = i01 + i30;
+  const auto t02 = i02 + i29;
+  const auto t03 = i03 + i28;
+  const auto t04 = i04 + i27;
+  const auto t05 = i05 + i26;
+  const auto t06 = i06 + i25;
+  const auto t07 = i07 + i24;
+  const auto t08 = i08 + i23;
+  const auto t09 = i09 + i22;
+  const auto t10 = i10 + i21;
+  const auto t11 = i11 + i20;
+  const auto t12 = i12 + i19;
+  const auto t13 = i13 + i18;
+  const auto t14 = i14 + i17;
+  const auto t15 = i15 + i16;
+  const auto t16 = i00 - i31;
+  const auto t17 = i01 - i30;
+  const auto t18 = i02 - i29;
+  const auto t19 = i03 - i28;
+  const auto t20 = i04 - i27;
+  const auto t21 = i05 - i26;
+  const auto t22 = i06 - i25;
+  const auto t23 = i07 - i24;
+  const auto t24 = i08 - i23;
+  const auto t25 = i09 - i22;
+  const auto t26 = i10 - i21;
+  const auto t27 = i11 - i20;
+  const auto t28 = i12 - i19;
+  const auto t29 = i13 - i18;
+  const auto t30 = i14 - i17;
+  const auto t31 = i15 - i16;
+  const auto t32 = t00 + t15;
+  const auto t33 = t01 + t14;
+  const auto t34 = t02 + t13;
+  const auto t35 = t03 + t12;
+  const auto t36 = t04 + t11;
+  const auto t37 = t05 + t10;
+  const auto t38 = t06 + t09;
+  const auto t39 = t07 + t08;
+  const auto t40 = t00 - t15;
+  const auto t41 = t01 - t14;
+  const auto t42 = t02 - t13;
+  const auto t43 = t03 - t12;
+  const auto t44 = t04 - t11;
+  const auto t45 = t05 - t10;
+  const auto t46 = t06 - t09;
+  const auto t47 = t07 - t08;
+  const auto t48 = t32 + t39;
+  const auto t49 = t33 + t38;
+  const auto t50 = t34 + t37;
+  const auto t51 = t35 + t36;
+  const auto t52 = t32 - t39;
+  const auto t53 = t33 - t38;
+  const auto t54 = t34 - t37;
+  const auto t55 = t35 - t36;
+  const auto t56 = t48 + t51;
+  const auto t57 = t49 + t50;
+  const auto t58 = t48 - t51;
+  const auto t59 = t49 - t50;
+  const auto t60 = t56 + t57;
+  const auto t61 = t56 - t57;
+  const auto t62 = t58 - t59;
+  const auto t63 = t59 * c16_64;
+  const auto t64 = t62 + t63;
+  const auto t65 = t62 - t63;
+  const auto t66 = t52 - t55;
+  const auto t67 = t53 - t54;
+  const auto t68 = t54 * c16_64;
+  const auto t69 = t55 * c16_64;
+  const auto t70 = t66 + t68;
+  const auto t71 = t67 + t69;
+  const auto t72 = t66 - t68;
+  const auto t73 = t67 - t69;
+  const auto t74 = t70 - t71;
+  const auto t75 = t71 * c8_64;
+  const auto t76 = t74 + t75;
+  const auto t77 = t74 - t75;
+  const auto t78 = t72 - t73;
+  const auto t79 = t73 * c24_64;
+  const auto t80 = t78 + t79;
+  const auto t81 = t78 - t79;
+  const auto t82 = t40 - t47;
+  const auto t83 = t41 - t46;
+  const auto t84 = t42 - t45;
+  const auto t85 = t43 - t44;
+  const auto t86 = t44 * c16_64;
+  const auto t87 = t45 * c16_64;
+  const auto t88 = t46 * c16_64;
+  const auto t89 = t47 * c16_64;
+  const auto t90 = t82 + t86;
+  const auto t91 = t83 + t87;
+  const auto t92 = t84 + t88;
+  const auto t93 = t85 + t89;
+  const auto t94 = t82 - t86;
+  const auto t95 = t83 - t87;
+  const auto t96 = t84 - t88;
+  const auto t97 = t85 - t89;
+  const auto t98 = t90 - t93;
+  const auto t99 = t91 - t92;
+  const auto t100 = t92 * c8_64;
+  const auto t101 = t93 * c8_64;
+  const auto t102 = t98 + t100;
+  const auto t103 = t99 + t101;
+  const auto t104 = t98 - t100;
+  const auto t105 = t99 - t101;
+  const auto t106 = t102 - t103;
+  const auto t107 = t103 * c4_64;
+  const auto t108 = t106 + t107;
+  const auto t109 = t106 - t107;
+  const auto t110 = t104 - t105;
+  const auto t111 = t105 * c28_64;
+  const auto t112 = t110 + t111;
+  const auto t113 = t110 - t111;
+  const auto t114 = t94 - t97;
+  const auto t115 = t95 - t96;
+  const auto t116 = t96 * c24_64;
+  const auto t117 = t97 * c24_64;
+  const auto t118 = t114 + t116;
+  const auto t119 = t115 + t117;
+  const auto t120 = t114 - t116;
+  const auto t121 = t115 - t117;
+  const auto t122 = t118 - t119;
+  const auto t123 = t119 * c12_64;
+  const auto t124 = t122 + t123;
+  const auto t125 = t122 - t123;
+  const auto t126 = t120 - t121;
+  const auto t127 = t121 * c20_64;
+  const auto t128 = t126 + t127;
+  const auto t129 = t126 - t127;
+  const auto t130 = t16 - t31;
+  const auto t131 = t17 - t30;
+  const auto t132 = t18 - t29;
+  const auto t133 = t19 - t28;
+  const auto t134 = t20 - t27;
+  const auto t135 = t21 - t26;
+  const auto t136 = t22 - t25;
+  const auto t137 = t23 - t24;
+  const auto t138 = t24 * c16_64;
+  const auto t139 = t25 * c16_64;
+  const auto t140 = t26 * c16_64;
+  const auto t141 = t27 * c16_64;
+  const auto t142 = t28 * c16_64;
+  const auto t143 = t29 * c16_64;
+  const auto t144 = t30 * c16_64;
+  const auto t145 = t31 * c16_64;
+  const auto t146 = t130 + t138;
+  const auto t147 = t131 + t139;
+  const auto t148 = t132 + t140;
+  const auto t149 = t133 + t141;
+  const auto t150 = t134 + t142;
+  const auto t151 = t135 + t143;
+  const auto t152 = t136 + t144;
+  const auto t153 = t137 + t145;
+  const auto t154 = t130 - t138;
+  const auto t155 = t131 - t139;
+  const auto t156 = t132 - t140;
+  const auto t157 = t133 - t141;
+  const auto t158 = t134 - t142;
+  const auto t159 = t135 - t143;
+  const auto t160 = t136 - t144;
+  const auto t161 = t137 - t145;
+  const auto t162 = t146 - t153;
+  const auto t163 = t147 - t152;
+  const auto t164 = t148 - t151;
+  const auto t165 = t149 - t150;
+  const auto t166 = t150 * c8_64;
+  const auto t167 = t151 * c8_64;
+  const auto t168 = t152 * c8_64;
+  const auto t169 = t153 * c8_64;
+  const auto t170 = t162 + t166;
+  const auto t171 = t163 + t167;
+  const auto t172 = t164 + t168;
+  const auto t173 = t165 + t169;
+  const auto t174 = t162 - t166;
+  const auto t175 = t163 - t167;
+  const auto t176 = t164 - t168;
+  const auto t177 = t165 - t169;
+  const auto t178 = t170 - t173;
+  const auto t179 = t171 - t172;
+  const auto t180 = t172 * c4_64;
+  const auto t181 = t173 * c4_64;
+  const auto t182 = t178 + t180;
+  const auto t183 = t179 + t181;
+  const auto t184 = t178 - t180;
+  const auto t185 = t179 - t181;
+  const auto t186 = t182 - t183;
+  const auto t187 = t183 * c2_64;
+  const auto t188 = t186 + t187;
+  const auto t189 = t186 - t187;
+  const auto t190 = t184 - t185;
+  const auto t191 = t185 * c30_64;
+  const auto t192 = t190 + t191;
+  const auto t193 = t190 - t191;
+  const auto t194 = t174 - t177;
+  const auto t195 = t175 - t176;
+  const auto t196 = t176 * c28_64;
+  const auto t197 = t177 * c28_64;
+  const auto t198 = t194 + t196;
+  const auto t199 = t195 + t197;
+  const auto t200 = t194 - t196;
+  const auto t201 = t195 - t197;
+  const auto t202 = t198 - t199;
+  const auto t203 = t199 * c14_64;
+  const auto t204 = t202 + t203;
+  const auto t205 = t202 - t203;
+  const auto t206 = t200 - t201;
+  const auto t207 = t201 * c18_64;
+  const auto t208 = t206 + t207;
+  const auto t209 = t206 - t207;
+  const auto t210 = t154 - t161;
+  const auto t211 = t155 - t160;
+  const auto t212 = t156 - t159;
+  const auto t213 = t157 - t158;
+  const auto t214 = t158 * c24_64;
+  const auto t215 = t159 * c24_64;
+  const auto t216 = t160 * c24_64;
+  const auto t217 = t161 * c24_64;
+  const auto t218 = t210 + t214;
+  const auto t219 = t211 + t215;
+  const auto t220 = t212 + t216;
+  const auto t221 = t213 + t217;
+  const auto t222 = t210 - t214;
+  const auto t223 = t211 - t215;
+  const auto t224 = t212 - t216;
+  const auto t225 = t213 - t217;
+  const auto t226 = t218 - t221;
+  const auto t227 = t219 - t220;
+  const auto t228 = t220 * c12_64;
+  const auto t229 = t221 * c12_64;
+  const auto t230 = t226 + t228;
+  const auto t231 = t227 + t229;
+  const auto t232 = t226 - t228;
+  const auto t233 = t227 - t229;
+  const auto t234 = t230 - t231;
+  const auto t235 = t231 * c6_64;
+  const auto t236 = t234 + t235;
+  const auto t237 = t234 - t235;
+  const auto t238 = t232 - t233;
+  const auto t239 = t233 * c26_64;
+  const auto t240 = t238 + t239;
+  const auto t241 = t238 - t239;
+  const auto t242 = t222 - t225;
+  const auto t243 = t223 - t224;
+  const auto t244 = t224 * c20_64;
+  const auto t245 = t225 * c20_64;
+  const auto t246 = t242 + t244;
+  const auto t247 = t243 + t245;
+  const auto t248 = t242 - t244;
+  const auto t249 = t243 - t245;
+  const auto t250 = t246 - t247;
+  const auto t251 = t247 * c10_64;
+  const auto t252 = t250 + t251;
+  const auto t253 = t250 - t251;
+  const auto t254 = t248 - t249;
+  const auto t255 = t249 * c22_64;
+  const auto t256 = t254 + t255;
+  const auto t257 = t254 - t255;
+
+  i00 = t60;
+  i01 = t188;
+  i02 = t108;
+  i03 = t236;
+  i04 = t76;
+  i05 = t252;
+  i06 = t124;
+  i07 = t204;
+  i08 = t64;
+  i09 = t208;
+  i10 = t128;
+  i11 = t256;
+  i12 = t80;
+  i13 = t240;
+  i14 = t112;
+  i15 = t192;
+  i16 = t61;
+  i17 = t193;
+  i18 = t113;
+  i19 = t241;
+  i20 = t81;
+  i21 = t257;
+  i22 = t129;
+  i23 = t209;
+  i24 = t65;
+  i25 = t205;
+  i26 = t125;
+  i27 = t253;
+  i28 = t77;
+  i29 = t237;
+  i30 = t109;
+  i31 = t189;
+}
+
+template <size_t SZ, class V>
+HWY_ATTR JXL_INLINE void ColumnIDCT32(V& i00, V& i01, V& i02, V& i03, V& i04,
+                                       V& i05, V& i06, V& i07, V& i08, V& i09,
+                                       V& i10, V& i11, V& i12, V& i13, V& i14,
+                                       V& i15, V& i16, V& i17, V& i18, V& i19,
+                                       V& i20, V& i21, V& i22, V& i23, V& i24,
+                                       V& i25, V& i26, V& i27, V& i28, V& i29,
+                                       V& i30, V& i31) {
+  const BlockDesc<SZ> d;
+  const auto c2_64 = Set(d, 0.502419286188155678f);   // 0.5 / cos(2 * pi / 64)
+  const auto c4_64 = Set(d, 0.509795579104159180f);   // 0.5 / cos(4 * pi / 64)
+  const auto c6_64 = Set(d, 0.522498614939688855f);   // 0.5 / cos(6 * pi / 64)
+  const auto c8_64 = Set(d, 0.541196100146197012f);   // 0.5 / cos(8 * pi / 64)
+  const auto c10_64 = Set(d, 0.566944034816357689f);  // 0.5 / cos(10 * pi / 64)
+  const auto c12_64 = Set(d, 0.601344886935045286f);  // 0.5 / cos(12 * pi / 64)
+  const auto c14_64 = Set(d, 0.646821783359990077f);  // 0.5 / cos(14 * pi / 64)
+  const auto c16_64 = Set(d, 0.707106781186547462f);  // 0.5 / cos(16 * pi / 64)
+  const auto c18_64 = Set(d, 0.788154623451250202f);  // 0.5 / cos(18 * pi / 64)
+  const auto c20_64 = Set(d, 0.899976223136415565f);  // 0.5 / cos(20 * pi / 64)
+  const auto c22_64 = Set(d, 1.060677685990347063f);  // 0.5 / cos(22 * pi / 64)
+  const auto c24_64 = Set(d, 1.306562964876376354f);  // 0.5 / cos(24 * pi / 64)
+  const auto c26_64 = Set(d, 1.722447098238334195f);  // 0.5 / cos(26 * pi / 64)
+  const auto c28_64 = Set(d, 2.562915447741505481f);  // 0.5 / cos(28 * pi / 64)
+  const auto c30_64 = Set(d, 5.101148618689155256f);  // 0.5 / cos(30 * pi / 64)
+
+  const auto t00 = i00 + i16;
+  const auto t01 = i00 - i16;
+  const auto t02 = i08 + i24;
+  const auto t03 = i08 - i24;
+  const auto t04 = t03 * c16_64;
+  const auto t05 = t02 + t04;
+  const auto t06 = t00 + t05;
+  const auto t07 = t01 + t04;
+  const auto t08 = t00 - t05;
+  const auto t09 = t01 - t04;
+  const auto t10 = i04 + i28;
+  const auto t11 = i04 - i28;
+  const auto t12 = t11 * c8_64;
+  const auto t13 = t10 + t12;
+  const auto t14 = i12 + i20;
+  const auto t15 = i12 - i20;
+  const auto t16 = t15 * c24_64;
+  const auto t17 = t14 + t16;
+  const auto t18 = t13 + t17;
+  const auto t19 = t12 + t16;
+  const auto t20 = t13 - t17;
+  const auto t21 = t12 - t16;
+  const auto t22 = t20 * c16_64;
+  const auto t23 = t21 * c16_64;
+  const auto t24 = t18 + t23;
+  const auto t25 = t19 + t22;
+  const auto t26 = t06 + t24;
+  const auto t27 = t07 + t25;
+  const auto t28 = t09 + t22;
+  const auto t29 = t08 + t23;
+  const auto t30 = t06 - t24;
+  const auto t31 = t07 - t25;
+  const auto t32 = t09 - t22;
+  const auto t33 = t08 - t23;
+  const auto t34 = i02 + i30;
+  const auto t35 = i02 - i30;
+  const auto t36 = t35 * c4_64;
+  const auto t37 = t34 + t36;
+  const auto t38 = i14 + i18;
+  const auto t39 = i14 - i18;
+  const auto t40 = t39 * c28_64;
+  const auto t41 = t38 + t40;
+  const auto t42 = t37 + t41;
+  const auto t43 = t36 + t40;
+  const auto t44 = t37 - t41;
+  const auto t45 = t36 - t40;
+  const auto t46 = t44 * c8_64;
+  const auto t47 = t45 * c8_64;
+  const auto t48 = t42 + t47;
+  const auto t49 = t43 + t46;
+  const auto t50 = i06 + i26;
+  const auto t51 = i06 - i26;
+  const auto t52 = t51 * c12_64;
+  const auto t53 = t50 + t52;
+  const auto t54 = i10 + i22;
+  const auto t55 = i10 - i22;
+  const auto t56 = t55 * c20_64;
+  const auto t57 = t54 + t56;
+  const auto t58 = t53 + t57;
+  const auto t59 = t52 + t56;
+  const auto t60 = t53 - t57;
+  const auto t61 = t52 - t56;
+  const auto t62 = t60 * c24_64;
+  const auto t63 = t61 * c24_64;
+  const auto t64 = t58 + t63;
+  const auto t65 = t59 + t62;
+  const auto t66 = t48 + t64;
+  const auto t67 = t49 + t65;
+  const auto t68 = t46 + t62;
+  const auto t69 = t47 + t63;
+  const auto t70 = t48 - t64;
+  const auto t71 = t49 - t65;
+  const auto t72 = t46 - t62;
+  const auto t73 = t47 - t63;
+  const auto t74 = t70 * c16_64;
+  const auto t75 = t71 * c16_64;
+  const auto t76 = t72 * c16_64;
+  const auto t77 = t73 * c16_64;
+  const auto t78 = t66 + t77;
+  const auto t79 = t67 + t76;
+  const auto t80 = t68 + t75;
+  const auto t81 = t69 + t74;
+  const auto t82 = t26 + t78;
+  const auto t83 = t27 + t79;
+  const auto t84 = t28 + t80;
+  const auto t85 = t29 + t81;
+  const auto t86 = t33 + t74;
+  const auto t87 = t32 + t75;
+  const auto t88 = t31 + t76;
+  const auto t89 = t30 + t77;
+  const auto t90 = t26 - t78;
+  const auto t91 = t27 - t79;
+  const auto t92 = t28 - t80;
+  const auto t93 = t29 - t81;
+  const auto t94 = t33 - t74;
+  const auto t95 = t32 - t75;
+  const auto t96 = t31 - t76;
+  const auto t97 = t30 - t77;
+  const auto t98 = i01 + i31;
+  const auto t99 = i01 - i31;
+  const auto t100 = t99 * c2_64;
+  const auto t101 = t98 + t100;
+  const auto t102 = i15 + i17;
+  const auto t103 = i15 - i17;
+  const auto t104 = t103 * c30_64;
+  const auto t105 = t102 + t104;
+  const auto t106 = t101 + t105;
+  const auto t107 = t100 + t104;
+  const auto t108 = t101 - t105;
+  const auto t109 = t100 - t104;
+  const auto t110 = t108 * c4_64;
+  const auto t111 = t109 * c4_64;
+  const auto t112 = t106 + t111;
+  const auto t113 = t107 + t110;
+  const auto t114 = i07 + i25;
+  const auto t115 = i07 - i25;
+  const auto t116 = t115 * c14_64;
+  const auto t117 = t114 + t116;
+  const auto t118 = i09 + i23;
+  const auto t119 = i09 - i23;
+  const auto t120 = t119 * c18_64;
+  const auto t121 = t118 + t120;
+  const auto t122 = t117 + t121;
+  const auto t123 = t116 + t120;
+  const auto t124 = t117 - t121;
+  const auto t125 = t116 - t120;
+  const auto t126 = t124 * c28_64;
+  const auto t127 = t125 * c28_64;
+  const auto t128 = t122 + t127;
+  const auto t129 = t123 + t126;
+  const auto t130 = t112 + t128;
+  const auto t131 = t113 + t129;
+  const auto t132 = t110 + t126;
+  const auto t133 = t111 + t127;
+  const auto t134 = t112 - t128;
+  const auto t135 = t113 - t129;
+  const auto t136 = t110 - t126;
+  const auto t137 = t111 - t127;
+  const auto t138 = t134 * c8_64;
+  const auto t139 = t135 * c8_64;
+  const auto t140 = t136 * c8_64;
+  const auto t141 = t137 * c8_64;
+  const auto t142 = t130 + t141;
+  const auto t143 = t131 + t140;
+  const auto t144 = t132 + t139;
+  const auto t145 = t133 + t138;
+  const auto t146 = i03 + i29;
+  const auto t147 = i03 - i29;
+  const auto t148 = t147 * c6_64;
+  const auto t149 = t146 + t148;
+  const auto t150 = i13 + i19;
+  const auto t151 = i13 - i19;
+  const auto t152 = t151 * c26_64;
+  const auto t153 = t150 + t152;
+  const auto t154 = t149 + t153;
+  const auto t155 = t148 + t152;
+  const auto t156 = t149 - t153;
+  const auto t157 = t148 - t152;
+  const auto t158 = t156 * c12_64;
+  const auto t159 = t157 * c12_64;
+  const auto t160 = t154 + t159;
+  const auto t161 = t155 + t158;
+  const auto t162 = i05 + i27;
+  const auto t163 = i05 - i27;
+  const auto t164 = t163 * c10_64;
+  const auto t165 = t162 + t164;
+  const auto t166 = i11 + i21;
+  const auto t167 = i11 - i21;
+  const auto t168 = t167 * c22_64;
+  const auto t169 = t166 + t168;
+  const auto t170 = t165 + t169;
+  const auto t171 = t164 + t168;
+  const auto t172 = t165 - t169;
+  const auto t173 = t164 - t168;
+  const auto t174 = t172 * c20_64;
+  const auto t175 = t173 * c20_64;
+  const auto t176 = t170 + t175;
+  const auto t177 = t171 + t174;
+  const auto t178 = t160 + t176;
+  const auto t179 = t161 + t177;
+  const auto t180 = t158 + t174;
+  const auto t181 = t159 + t175;
+  const auto t182 = t160 - t176;
+  const auto t183 = t161 - t177;
+  const auto t184 = t158 - t174;
+  const auto t185 = t159 - t175;
+  const auto t186 = t182 * c24_64;
+  const auto t187 = t183 * c24_64;
+  const auto t188 = t184 * c24_64;
+  const auto t189 = t185 * c24_64;
+  const auto t190 = t178 + t189;
+  const auto t191 = t179 + t188;
+  const auto t192 = t180 + t187;
+  const auto t193 = t181 + t186;
+  const auto t194 = t142 + t190;
+  const auto t195 = t143 + t191;
+  const auto t196 = t144 + t192;
+  const auto t197 = t145 + t193;
+  const auto t198 = t138 + t186;
+  const auto t199 = t139 + t187;
+  const auto t200 = t140 + t188;
+  const auto t201 = t141 + t189;
+  const auto t202 = t142 - t190;
+  const auto t203 = t143 - t191;
+  const auto t204 = t144 - t192;
+  const auto t205 = t145 - t193;
+  const auto t206 = t138 - t186;
+  const auto t207 = t139 - t187;
+  const auto t208 = t140 - t188;
+  const auto t209 = t141 - t189;
+  const auto t210 = t202 * c16_64;
+  const auto t211 = t203 * c16_64;
+  const auto t212 = t204 * c16_64;
+  const auto t213 = t205 * c16_64;
+  const auto t214 = t206 * c16_64;
+  const auto t215 = t207 * c16_64;
+  const auto t216 = t208 * c16_64;
+  const auto t217 = t209 * c16_64;
+  const auto t218 = t194 + t217;
+  const auto t219 = t195 + t216;
+  const auto t220 = t196 + t215;
+  const auto t221 = t197 + t214;
+  const auto t222 = t198 + t213;
+  const auto t223 = t199 + t212;
+  const auto t224 = t200 + t211;
+  const auto t225 = t201 + t210;
+  const auto t226 = t82 + t218;
+  const auto t227 = t83 + t219;
+  const auto t228 = t84 + t220;
+  const auto t229 = t85 + t221;
+  const auto t230 = t86 + t222;
+  const auto t231 = t87 + t223;
+  const auto t232 = t88 + t224;
+  const auto t233 = t89 + t225;
+  const auto t234 = t97 + t210;
+  const auto t235 = t96 + t211;
+  const auto t236 = t95 + t212;
+  const auto t237 = t94 + t213;
+  const auto t238 = t93 + t214;
+  const auto t239 = t92 + t215;
+  const auto t240 = t91 + t216;
+  const auto t241 = t90 + t217;
+  const auto t242 = t82 - t218;
+  const auto t243 = t83 - t219;
+  const auto t244 = t84 - t220;
+  const auto t245 = t85 - t221;
+  const auto t246 = t86 - t222;
+  const auto t247 = t87 - t223;
+  const auto t248 = t88 - t224;
+  const auto t249 = t89 - t225;
+  const auto t250 = t97 - t210;
+  const auto t251 = t96 - t211;
+  const auto t252 = t95 - t212;
+  const auto t253 = t94 - t213;
+  const auto t254 = t93 - t214;
+  const auto t255 = t92 - t215;
+  const auto t256 = t91 - t216;
+  const auto t257 = t90 - t217;
+
+  i00 = t226;
+  i01 = t227;
+  i02 = t228;
+  i03 = t229;
+  i04 = t230;
+  i05 = t231;
+  i06 = t232;
+  i07 = t233;
+  i08 = t234;
+  i09 = t235;
+  i10 = t236;
+  i11 = t237;
+  i12 = t238;
+  i13 = t239;
+  i14 = t240;
+  i15 = t241;
+  i16 = t257;
+  i17 = t256;
+  i18 = t255;
+  i19 = t254;
+  i20 = t253;
+  i21 = t252;
+  i22 = t251;
+  i23 = t250;
+  i24 = t249;
+  i25 = t248;
+  i26 = t247;
+  i27 = t246;
+  i28 = t245;
+  i29 = t244;
+  i30 = t243;
+  i31 = t242;
+}
+
+template <class From, class To, size_t COLS = 32>
+HWY_ATTR void ColumnDCT32(const From& from, const To& to) {
+  const BlockDesc<COLS> d;
+
+  for (size_t i = 0; i < COLS; i += d.N) {
+    auto i00 = from.template LoadPart<COLS>(0, i);
+    auto i01 = from.template LoadPart<COLS>(1, i);
+    auto i02 = from.template LoadPart<COLS>(2, i);
+    auto i03 = from.template LoadPart<COLS>(3, i);
+    auto i04 = from.template LoadPart<COLS>(4, i);
+    auto i05 = from.template LoadPart<COLS>(5, i);
+    auto i06 = from.template LoadPart<COLS>(6, i);
+    auto i07 = from.template LoadPart<COLS>(7, i);
+    auto i08 = from.template LoadPart<COLS>(8, i);
+    auto i09 = from.template LoadPart<COLS>(9, i);
+    auto i10 = from.template LoadPart<COLS>(10, i);
+    auto i11 = from.template LoadPart<COLS>(11, i);
+    auto i12 = from.template LoadPart<COLS>(12, i);
+    auto i13 = from.template LoadPart<COLS>(13, i);
+    auto i14 = from.template LoadPart<COLS>(14, i);
+    auto i15 = from.template LoadPart<COLS>(15, i);
+    auto i16 = from.template LoadPart<COLS>(16, i);
+    auto i17 = from.template LoadPart<COLS>(17, i);
+    auto i18 = from.template LoadPart<COLS>(18, i);
+    auto i19 = from.template LoadPart<COLS>(19, i);
+    auto i20 = from.template LoadPart<COLS>(20, i);
+    auto i21 = from.template LoadPart<COLS>(21, i);
+    auto i22 = from.template LoadPart<COLS>(22, i);
+    auto i23 = from.template LoadPart<COLS>(23, i);
+    auto i24 = from.template LoadPart<COLS>(24, i);
+    auto i25 = from.template LoadPart<COLS>(25, i);
+    auto i26 = from.template LoadPart<COLS>(26, i);
+    auto i27 = from.template LoadPart<COLS>(27, i);
+    auto i28 = from.template LoadPart<COLS>(28, i);
+    auto i29 = from.template LoadPart<COLS>(29, i);
+    auto i30 = from.template LoadPart<COLS>(30, i);
+    auto i31 = from.template LoadPart<COLS>(31, i);
+    ColumnDCT32<COLS>(i00, i01, i02, i03, i04, i05, i06, i07, i08, i09, i10,
+                      i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21,
+                      i22, i23, i24, i25, i26, i27, i28, i29, i30, i31);
+    to.template StorePart<COLS>(i00, 0, i);
+    to.template StorePart<COLS>(i01, 1, i);
+    to.template StorePart<COLS>(i02, 2, i);
+    to.template StorePart<COLS>(i03, 3, i);
+    to.template StorePart<COLS>(i04, 4, i);
+    to.template StorePart<COLS>(i05, 5, i);
+    to.template StorePart<COLS>(i06, 6, i);
+    to.template StorePart<COLS>(i07, 7, i);
+    to.template StorePart<COLS>(i08, 8, i);
+    to.template StorePart<COLS>(i09, 9, i);
+    to.template StorePart<COLS>(i10, 10, i);
+    to.template StorePart<COLS>(i11, 11, i);
+    to.template StorePart<COLS>(i12, 12, i);
+    to.template StorePart<COLS>(i13, 13, i);
+    to.template StorePart<COLS>(i14, 14, i);
+    to.template StorePart<COLS>(i15, 15, i);
+    to.template StorePart<COLS>(i16, 16, i);
+    to.template StorePart<COLS>(i17, 17, i);
+    to.template StorePart<COLS>(i18, 18, i);
+    to.template StorePart<COLS>(i19, 19, i);
+    to.template StorePart<COLS>(i20, 20, i);
+    to.template StorePart<COLS>(i21, 21, i);
+    to.template StorePart<COLS>(i22, 22, i);
+    to.template StorePart<COLS>(i23, 23, i);
+    to.template StorePart<COLS>(i24, 24, i);
+    to.template StorePart<COLS>(i25, 25, i);
+    to.template StorePart<COLS>(i26, 26, i);
+    to.template StorePart<COLS>(i27, 27, i);
+    to.template StorePart<COLS>(i28, 28, i);
+    to.template StorePart<COLS>(i29, 29, i);
+    to.template StorePart<COLS>(i30, 30, i);
+    to.template StorePart<COLS>(i31, 31, i);
+  }
+}
+
+template <class From, class To, size_t COLS = 32>
+HWY_ATTR void ColumnIDCT32(const From& from, const To& to) {
+  const BlockDesc<COLS> d;
+
+  for (size_t i = 0; i < COLS; i += d.N) {
+    auto i00 = from.template LoadPart<COLS>(0, i);
+    auto i01 = from.template LoadPart<COLS>(1, i);
+    auto i02 = from.template LoadPart<COLS>(2, i);
+    auto i03 = from.template LoadPart<COLS>(3, i);
+    auto i04 = from.template LoadPart<COLS>(4, i);
+    auto i05 = from.template LoadPart<COLS>(5, i);
+    auto i06 = from.template LoadPart<COLS>(6, i);
+    auto i07 = from.template LoadPart<COLS>(7, i);
+    auto i08 = from.template LoadPart<COLS>(8, i);
+    auto i09 = from.template LoadPart<COLS>(9, i);
+    auto i10 = from.template LoadPart<COLS>(10, i);
+    auto i11 = from.template LoadPart<COLS>(11, i);
+    auto i12 = from.template LoadPart<COLS>(12, i);
+    auto i13 = from.template LoadPart<COLS>(13, i);
+    auto i14 = from.template LoadPart<COLS>(14, i);
+    auto i15 = from.template LoadPart<COLS>(15, i);
+    auto i16 = from.template LoadPart<COLS>(16, i);
+    auto i17 = from.template LoadPart<COLS>(17, i);
+    auto i18 = from.template LoadPart<COLS>(18, i);
+    auto i19 = from.template LoadPart<COLS>(19, i);
+    auto i20 = from.template LoadPart<COLS>(20, i);
+    auto i21 = from.template LoadPart<COLS>(21, i);
+    auto i22 = from.template LoadPart<COLS>(22, i);
+    auto i23 = from.template LoadPart<COLS>(23, i);
+    auto i24 = from.template LoadPart<COLS>(24, i);
+    auto i25 = from.template LoadPart<COLS>(25, i);
+    auto i26 = from.template LoadPart<COLS>(26, i);
+    auto i27 = from.template LoadPart<COLS>(27, i);
+    auto i28 = from.template LoadPart<COLS>(28, i);
+    auto i29 = from.template LoadPart<COLS>(29, i);
+    auto i30 = from.template LoadPart<COLS>(30, i);
+    auto i31 = from.template LoadPart<COLS>(31, i);
+    ColumnIDCT32<COLS>(i00, i01, i02, i03, i04, i05, i06, i07, i08, i09, i10,
+                       i11, i12, i13, i14, i15, i16, i17, i18, i19, i20, i21,
+                       i22, i23, i24, i25, i26, i27, i28, i29, i30, i31);
+    to.template StorePart<COLS>(i00, 0, i);
+    to.template StorePart<COLS>(i01, 1, i);
+    to.template StorePart<COLS>(i02, 2, i);
+    to.template StorePart<COLS>(i03, 3, i);
+    to.template StorePart<COLS>(i04, 4, i);
+    to.template StorePart<COLS>(i05, 5, i);
+    to.template StorePart<COLS>(i06, 6, i);
+    to.template StorePart<COLS>(i07, 7, i);
+    to.template StorePart<COLS>(i08, 8, i);
+    to.template StorePart<COLS>(i09, 9, i);
+    to.template StorePart<COLS>(i10, 10, i);
+    to.template StorePart<COLS>(i11, 11, i);
+    to.template StorePart<COLS>(i12, 12, i);
+    to.template StorePart<COLS>(i13, 13, i);
+    to.template StorePart<COLS>(i14, 14, i);
+    to.template StorePart<COLS>(i15, 15, i);
+    to.template StorePart<COLS>(i16, 16, i);
+    to.template StorePart<COLS>(i17, 17, i);
+    to.template StorePart<COLS>(i18, 18, i);
+    to.template StorePart<COLS>(i19, 19, i);
+    to.template StorePart<COLS>(i20, 20, i);
+    to.template StorePart<COLS>(i21, 21, i);
+    to.template StorePart<COLS>(i22, 22, i);
+    to.template StorePart<COLS>(i23, 23, i);
+    to.template StorePart<COLS>(i24, 24, i);
+    to.template StorePart<COLS>(i25, 25, i);
+    to.template StorePart<COLS>(i26, 26, i);
+    to.template StorePart<COLS>(i27, 27, i);
+    to.template StorePart<COLS>(i28, 28, i);
+    to.template StorePart<COLS>(i29, 29, i);
+    to.template StorePart<COLS>(i30, 30, i);
+    to.template StorePart<COLS>(i31, 31, i);
+  }
+}
+
+}  // namespace jxl
+
+#endif  // JXL_DCT_HWY_ANY_H_
diff --git a/jxl/dct_test.cc b/jxl/dct_test.cc
new file mode 100644
index 000000000..1b5ae2dac
--- /dev/null
+++ b/jxl/dct_test.cc
@@ -0,0 +1,647 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dct.h"
+
+#include <string.h>
+
+#include <cmath>
+
+#include "gtest/gtest.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/block.h"
+#include "jxl/common.h"
+#include "jxl/image.h"
+#include "jxl/test_utils.h"
+
+namespace jxl {
+namespace {
+
+using jxl::slow_dct::DCTSlow;
+using jxl::slow_dct::IDCTSlow;
+
+// Tests in the DctShardedTest class are sharded in 32 cases.
+class DctShardedTest : public ::testing::TestWithParam<uint32_t> {};
+std::vector<uint32_t> ShardRange(uint32_t n) {
+#ifdef JXL_DISABLE_SLOW_TESTS
+  JXL_ASSERT(n > 6);
+  std::vector<uint32_t> ret = {0, 1, 3, 5, n - 1};
+#else
+  std::vector<uint32_t> ret(n);
+  std::iota(ret.begin(), ret.end(), 0);
+#endif  // JXL_DISABLE_SLOW_TESTS
+  return ret;
+}
+INSTANTIATE_TEST_CASE_P(SlowDctTestInstantiation, DctShardedTest,
+                        ::testing::ValuesIn(ShardRange(32)));
+
+namespace slow_dct {
+
+// These functions should be equivalent to ComputeTransposedScaledDCT in the jxl
+// namespace (but slower and implemented for more transform sizes).
+template <size_t N, class From, class To>
+HWY_ATTR static void ComputeTransposedScaledDCT(const From& from,
+                                                const To& to) {
+  double blockd[N * N] = {};
+  for (size_t y = 0; y < N; y++) {
+    for (size_t x = 0; x < N; x++) {
+      blockd[y * N + x] = from.Read(y, x);
+    }
+  }
+  DCTSlow<N>(blockd);
+
+  // Scale.
+  for (size_t y = 0; y < N; ++y) {
+    for (size_t x = 0; x < N; ++x) {
+      to.Write(
+          blockd[N * x + y] * N * N * IDCTScales<N>()[x] * IDCTScales<N>()[y],
+          y, x);
+    }
+  }
+}
+
+template <size_t N, class From, class To>
+HWY_ATTR static void ComputeTransposedScaledIDCT(const From& from,
+                                                 const To& to) {
+  // Scale.
+  double blockd[N * N] = {};
+  for (size_t y = 0; y < N; y++) {
+    for (size_t x = 0; x < N; x++) {
+      blockd[x * N + y] =
+          from.Read(y, x) * N * N * DCTScales<N>()[x] * DCTScales<N>()[y];
+    }
+  }
+  IDCTSlow<N>(blockd);
+
+  for (size_t y = 0; y < N; ++y) {
+    for (size_t x = 0; x < N; ++x) {
+      to.Write(blockd[N * y + x], y, x);
+    }
+  }
+}
+}  // namespace slow_dct
+
+// Computes the in-place NxN DCT of block.
+// Requires that block is HWY_ALIGN'ed.
+//
+// Performs ComputeTransposedScaledDCT and then transposes and scales it to
+// obtain "vanilla" DCT.
+template <size_t N>
+HWY_ATTR void ComputeDCT(float block[N * N]) {
+  ComputeTransposedScaledDCT<N>()(FromBlock<N>(block), ToBlock<N>(block));
+
+  // Untranspose.
+  GenericTransposeBlockInplace<N>(FromBlock<N>(block), ToBlock<N>(block));
+
+  // Unscale.
+  for (size_t y = 0; y < N; ++y) {
+    for (size_t x = 0; x < N; ++x) {
+      block[N * y + x] *= DCTScales<N>()[x] * DCTScales<N>()[y];
+    }
+  }
+}
+
+// Computes the in-place 8x8 iDCT of block.
+// Requires that block is HWY_ALIGN'ed.
+template <int N>
+HWY_ATTR void ComputeIDCT(float block[N * N]) {
+  // Unscale.
+  for (int y = 0; y < N; ++y) {
+    for (int x = 0; x < N; ++x) {
+      block[N * y + x] *= IDCTScales<N>()[x] * IDCTScales<N>()[y];
+    }
+  }
+
+  // Untranspose.
+  GenericTransposeBlockInplace<N>(FromBlock<N>(block), ToBlock<N>(block));
+
+  ComputeTransposedScaledIDCT<N>()(FromBlock<N>(block), ToBlock<N>(block));
+}
+
+template <size_t N, class From, class To>
+HWY_ATTR JXL_INLINE void TransposeBlock(const From& from, const To& to) {
+  if (N == 8) {
+    TransposeBlock8(from, to);
+  } else if (N == 16) {
+    TransposeBlock16(from, to);
+  } else {
+    TransposeBlock32(from, to);
+  }
+}
+
+// Cannot just use `if N == 8 / else` because the generated code will be invalid
+// for the other codepath.
+template <size_t N>
+struct SpecializedColumn {};  // primary
+
+template <>
+struct SpecializedColumn<8> {
+  template <class From, class To>
+  static HWY_ATTR JXL_INLINE void DCT(const From& from, const To& to) {
+    ColumnDCT8(from, to);
+  }
+
+  template <class From, class To>
+  static HWY_ATTR JXL_INLINE void IDCT(const From& from, const To& to) {
+    ColumnIDCT8(from, to);
+  }
+};
+
+template <>
+struct SpecializedColumn<16> {
+  template <class From, class To>
+  static HWY_ATTR JXL_INLINE void DCT(const From& from, const To& to) {
+    ColumnDCT16(from, to);
+  }
+
+  template <class From, class To>
+  static HWY_ATTR JXL_INLINE void IDCT(const From& from, const To& to) {
+    ColumnIDCT16(from, to);
+  }
+};
+
+template <>
+struct SpecializedColumn<32> {
+  template <class From, class To>
+  static HWY_ATTR JXL_INLINE void DCT(const From& from, const To& to) {
+    ColumnDCT32(from, to);
+  }
+
+  template <class From, class To>
+  static HWY_ATTR JXL_INLINE void IDCT(const From& from, const To& to) {
+    ColumnIDCT32(from, to);
+  }
+};
+
+template <size_t N>
+HWY_ATTR void TransposeTest(float accuracy) {
+  constexpr size_t kBlockSize = N * N;
+  HWY_ALIGN float src[kBlockSize];
+  ToBlock<N> to_src(src);
+  for (int y = 0; y < N; ++y) {
+    for (int x = 0; x < N; ++x) {
+      to_src.Write(y * N + x, y, x);
+    }
+  }
+  HWY_ALIGN float dst[kBlockSize];
+  TransposeBlock<N>(FromBlock<N>(src), ToBlock<N>(dst));
+  FromBlock<N> from_dst(dst);
+  for (int y = 0; y < N; ++y) {
+    for (int x = 0; x < N; ++x) {
+      float expected = x * N + y;
+      float actual = from_dst.Read(y, x);
+      EXPECT_NEAR(expected, actual, accuracy) << "x = " << x << ", y = " << y;
+    }
+  }
+}
+
+TEST(TransposeTest, Transpose8) { TransposeTest<8>(1e-7); }
+
+TEST(TransposeTest, Transpose16) { TransposeTest<16>(1e-7); }
+
+TEST(TransposeTest, Transpose32) { TransposeTest<32>(1e-7); }
+
+template <size_t N>
+HWY_ATTR void TestDctAccuracy(float accuracy, size_t start = 0,
+                              size_t end = N * N) {
+  constexpr size_t kBlockSize = N * N;
+  for (size_t i = start; i < end; i++) {
+    HWY_ALIGN float fast[kBlockSize] = {0.0f};
+    double slow[kBlockSize] = {0.0};
+    fast[i] = 1.0;
+    slow[i] = 1.0;
+    DCTSlow<N>(slow);
+    ComputeDCT<N>(fast);
+    for (size_t k = 0; k < kBlockSize; ++k) {
+      EXPECT_NEAR(fast[k], slow[k], accuracy) << "i = " << i << ", k = " << k;
+    }
+  }
+}
+
+template <size_t N>
+HWY_ATTR void TestDctAccuracyShard(float accuracy, size_t shard) {
+  TestDctAccuracy<N>(accuracy, N * shard, N * (shard + 1));
+}
+
+TEST(DctTest, Accuracy8) { TestDctAccuracy<8>(1.1e-7); }
+TEST(DctTest, Accuracy16) { TestDctAccuracy<16>(1.1e-7); }
+TEST_P(DctShardedTest, DctTest_Accuracy32) {
+  TestDctAccuracyShard<32>(1.1e-7, GetParam());
+}
+
+template <size_t N>
+HWY_ATTR void TestIdctAccuracy(float accuracy, size_t start = 0,
+                               size_t end = N * N) {
+  constexpr size_t kBlockSize = N * N;
+  for (size_t i = start; i < end; i++) {
+    HWY_ALIGN float fast[kBlockSize] = {0.0f};
+    double slow[kBlockSize] = {0.0};
+    fast[i] = 1.0;
+    slow[i] = 1.0;
+    IDCTSlow<N>(slow);
+    ComputeIDCT<N>(fast);
+    for (size_t k = 0; k < kBlockSize; ++k) {
+      EXPECT_NEAR(fast[k], slow[k], accuracy) << "i = " << i << ", k = " << k;
+    }
+  }
+}
+
+template <size_t N>
+HWY_ATTR void TestIdctAccuracyShard(float accuracy, size_t shard) {
+  TestIdctAccuracy<N>(accuracy, N * shard, N * (shard + 1));
+}
+
+TEST(IdctTest, Accuracy8) { TestIdctAccuracy<8>(1e-7); }
+TEST(IdctTest, Accuracy16) { TestIdctAccuracy<16>(1e-7); }
+TEST_P(DctShardedTest, IdctTest_Accuracy32) {
+  TestIdctAccuracyShard<32>(1e-7, GetParam());
+}
+
+template <size_t N>
+HWY_ATTR void TestInverse(float accuracy) {
+  ThreadPoolInternal pool(N < 32 ? 0 : 8);
+  enum { kBlockSize = N * N };
+  RunOnPool(
+      &pool, 0, kBlockSize, ThreadPool::SkipInit(),
+      [accuracy](const int task, int /*thread*/) {
+        const size_t i = static_cast<size_t>(task);
+        HWY_ALIGN float x[kBlockSize] = {0.0f};
+        x[i] = 1.0;
+
+        ComputeIDCT<N>(x);
+        ComputeDCT<N>(x);
+
+        for (size_t k = 0; k < kBlockSize; ++k) {
+          EXPECT_NEAR(x[k], (k == i) ? 1.0f : 0.0f, accuracy)
+              << "i = " << i << ", k = " << k;
+        }
+      },
+      "TestInverse");
+}
+
+TEST(IdctTest, Inverse8) { TestInverse<8>(1e-6f); }
+
+TEST(IdctTest, Inverse16) { TestInverse<16>(1e-6f); }
+
+TEST(IdctTest, Inverse32) { TestInverse<32>(3e-6f); }
+
+template <size_t N>
+HWY_ATTR void TestIdctOrthonormal(float accuracy) {
+  constexpr size_t kBlockSize = N * N;
+  ImageF xs(kBlockSize, kBlockSize);
+  for (int i = 0; i < kBlockSize; ++i) {
+    float* x = xs.Row(i);
+    for (int j = 0; j < kBlockSize; ++j) x[j] = (i == j) ? 1.0f : 0.0f;
+    ComputeIDCT<N>(x);
+  }
+  for (int i = 0; i < kBlockSize; ++i) {
+    for (int j = 0; j < kBlockSize; ++j) {
+      float product = 0.0f;
+      for (int k = 0; k < kBlockSize; ++k) {
+        product += xs.Row(i)[k] * xs.Row(j)[k];
+      }
+      EXPECT_NEAR(product, (i == j) ? 1.0f : 0.0f, accuracy)
+          << "i = " << i << ", j = " << j;
+    }
+  }
+}
+
+TEST(IdctTest, IDCTOrthonormal8) { TestIdctOrthonormal<8>(1e-6f); }
+
+TEST(IdctTest, IDCTOrthonormal16) { TestIdctOrthonormal<16>(1.2e-6f); }
+
+TEST(IdctTest, IDCTOrthonormal32) { TestIdctOrthonormal<32>(5e-6f); }
+
+template <size_t N>
+HWY_ATTR void TestDctTranspose(float accuracy, size_t start = 0,
+                               size_t end = N * N) {
+  constexpr size_t kBlockSize = N * N;
+  for (size_t i = start; i < end; i++) {
+    for (size_t j = 0; j < kBlockSize; ++j) {
+      // We check that <e_i, Me_j> = <M^\dagger{}e_i, e_j>.
+      // That means (Me_j)_i = (M^\dagger{}e_i)_j
+
+      // x := Me_j
+      HWY_ALIGN float x[kBlockSize] = {0.0f};
+      x[j] = 1.0;
+      ComputeIDCT<N>(x);
+      // y := M^\dagger{}e_i
+      HWY_ALIGN float y[kBlockSize] = {0.0f};
+      y[i] = 1.0;
+      ComputeDCT<N>(y);
+
+      EXPECT_NEAR(x[i], y[j], accuracy) << "i = " << i << ", j = " << j;
+    }
+  }
+}
+
+TEST(IdctTest, Transpose8) { TestDctTranspose<8>(1e-6); }
+
+TEST(IdctTest, Transpose16) { TestDctTranspose<16>(1e-6); }
+
+template <size_t N>
+HWY_ATTR void TestDctTransposeShard(float accuracy, size_t shard) {
+  TestDctTranspose<N>(accuracy, N * shard, N * (shard + 1));
+}
+TEST_P(DctShardedTest, IdctTest_Transpose32) {
+  TestDctTransposeShard<32>(3e-6, GetParam());
+}
+
+template <size_t N>
+HWY_ATTR void TestDctOrthonormal(float accuracy) {
+  constexpr size_t kBlockSize = N * N;
+  ImageF xs(kBlockSize, kBlockSize);
+  for (int i = 0; i < kBlockSize; ++i) {
+    float* x = xs.Row(i);
+    for (int j = 0; j < kBlockSize; ++j) x[j] = (i == j) ? 1.0f : 0.0f;
+    ComputeDCT<N>(x);
+  }
+  for (int i = 0; i < kBlockSize; ++i) {
+    for (int j = 0; j < kBlockSize; ++j) {
+      float product = 0.0f;
+      for (int k = 0; k < kBlockSize; ++k) {
+        product += xs.Row(i)[k] * xs.Row(j)[k];
+      }
+      EXPECT_NEAR(product, (i == j) ? 1.0f : 0.0f, accuracy)
+          << "i = " << i << ", j = " << j;
+    }
+  }
+}
+
+TEST(IdctTest, DCTOrthonormal8) { TestDctOrthonormal<8>(1e-6f); }
+
+TEST(IdctTest, DCTOrthonormal16) { TestDctOrthonormal<16>(1e-6f); }
+
+TEST(IdctTest, DCTOrthonormal32) { TestDctOrthonormal<32>(1e-6f); }
+
+template <size_t N>
+HWY_ATTR void ColumnDctRoundtrip(float accuracy) {
+  constexpr size_t kBlockSize = N * N;
+  // Though we are only interested in single column result, dct.h has built-in
+  // limit on minimal number of columns processed. So, to be safe, we do
+  // regular 8x8 block transformation. On the bright side - we could check all
+  // 8 basis vectors at once.
+  HWY_ALIGN float block[kBlockSize];
+  ToBlock<N> to(block);
+  FromBlock<N> from(block);
+  for (size_t i = 0; i < N; ++i) {
+    for (size_t j = 0; j < N; ++j) {
+      to.Write((i == j) ? 1.0f : 0.0f, i, j);
+    }
+  }
+
+  // Running (I)DCT on the same memory block seems to trigger a compiler bug on
+  // ARMv7 with clang6.
+  HWY_ALIGN float tmp[kBlockSize];
+  ToBlock<N> to_tmp(tmp);
+  FromBlock<N> from_tmp(tmp);
+
+  SpecializedColumn<N>::DCT(from, to_tmp);
+  SpecializedColumn<N>::IDCT(from_tmp, to);
+  constexpr float scale = 1.0f / N;
+
+  for (size_t i = 0; i < N; ++i) {
+    for (size_t j = 0; j < N; ++j) {
+      float expected = (i == j) ? 1.0f : 0.0f;
+      float actual = from.Read(i, j) * scale;
+      EXPECT_NEAR(expected, actual, accuracy) << " i=" << i << ", j=" << j;
+    }
+  }
+}
+
+TEST(IdctTest, ColumnDctRoundtrip8) { ColumnDctRoundtrip<8>(1e-6f); }
+
+TEST(IdctTest, ColumnDctRoundtrip16) { ColumnDctRoundtrip<16>(1e-6f); }
+
+TEST(IdctTest, ColumnDctRoundtrip32) { ColumnDctRoundtrip<32>(1e-6f); }
+
+// See "Steerable Discrete Cosine Transform", Fracastoro G., Fosson S., Magli
+// E., https://arxiv.org/pdf/1610.09152.pdf
+template <int N>
+void RotateDCT(float angle, float block[N * N]) {
+  float a2a = std::cos(angle);
+  float a2b = -std::sin(angle);
+  float b2a = std::sin(angle);
+  float b2b = std::cos(angle);
+  for (int y = 0; y < N; y++) {
+    for (int x = 0; x < y; x++) {
+      if (x >= 2 || y >= 2) continue;
+      float a = block[N * y + x];
+      float b = block[N * x + y];
+      block[N * y + x] = a2a * a + b2a * b;
+      block[N * x + y] = a2b * a + b2b * b;
+    }
+  }
+}
+
+TEST(RotateTest, ZeroIdentity) {
+  for (int i = 0; i < 64; i++) {
+    float x[64] = {0.0f};
+    x[i] = 1.0;
+    RotateDCT<8>(0.0f, x);
+    for (int j = 0; j < 64; j++) {
+      EXPECT_NEAR(x[j], (i == j) ? 1.0f : 0.0f, 1e-6f);
+    }
+  }
+}
+
+TEST(RotateTest, InverseRotation) {
+  const float angle = 0.1f;
+  for (int i = 0; i < 64; i++) {
+    float x[64] = {0.0f};
+    x[i] = 1.0;
+    RotateDCT<8>(angle, x);
+    RotateDCT<8>(-angle, x);
+    for (int j = 0; j < 64; j++) {
+      EXPECT_NEAR(x[j], (i == j) ? 1.0f : 0.0f, 1e-6f);
+    }
+  }
+}
+
+template <size_t N>
+HWY_ATTR void TestSlowIsSameDCT(float accuracy, size_t start = 0,
+                                size_t end = N * N) {
+  for (size_t i = start; i < end; i++) {
+    HWY_ALIGN float block1[N * N] = {};
+    HWY_ALIGN float block2[N * N] = {};
+    block1[i] = 1.0;
+    block2[i] = 1.0;
+    ComputeTransposedScaledDCT<N>()(FromBlock<N>(block1), ToBlock<N>(block1));
+    slow_dct::ComputeTransposedScaledDCT<N>(FromBlock<N>(block2),
+                                            ToBlock<N>(block2));
+    for (int j = 0; j < N * N; j++) {
+      EXPECT_NEAR(block1[j], block2[j], accuracy);
+    }
+  }
+}
+
+template <size_t N>
+HWY_ATTR void TestSlowIsSameDCTShard(float accuracy, int shard) {
+  TestSlowIsSameDCT<N>(accuracy, N * shard, N * (shard + 1));
+}
+
+TEST(SlowDctTest, DCTIsSame2) { TestSlowIsSameDCT<2>(1e-5); }
+TEST(SlowDctTest, DCTIsSame4) { TestSlowIsSameDCT<4>(1e-5); }
+TEST(SlowDctTest, DCTIsSame8) { TestSlowIsSameDCT<8>(1e-5); }
+TEST(SlowDctTest, DCTIsSame16) { TestSlowIsSameDCT<16>(3e-5); }
+TEST_P(DctShardedTest, DCTIsSame32) {
+  TestSlowIsSameDCTShard<32>(1e-4, GetParam());
+}
+
+template <size_t N>
+HWY_ATTR void TestSlowIsSameIDCT(float accuracy, size_t start = 0,
+                                 size_t end = N * N) {
+  for (size_t i = start; i < end; i++) {
+    HWY_ALIGN float block1[N * N] = {};
+    HWY_ALIGN float block2[N * N] = {};
+    block1[i] = 1.0;
+    block2[i] = 1.0;
+    ComputeTransposedScaledIDCT<N>()(FromBlock<N>(block1), ToBlock<N>(block1));
+    slow_dct::ComputeTransposedScaledIDCT<N>(FromBlock<N>(block2),
+                                             ToBlock<N>(block2));
+    for (int j = 0; j < N * N; j++) {
+      EXPECT_NEAR(block1[j], block2[j], accuracy);
+    }
+  }
+}
+
+template <size_t N>
+HWY_ATTR void TestSlowIsSameIDCTShard(float accuracy, int shard) {
+  TestSlowIsSameIDCT<N>(accuracy, N * shard, N * (shard + 1));
+}
+
+TEST(SlowDctTest, IDCTIsSame2) { TestSlowIsSameIDCT<2>(1e-5); }
+TEST(SlowDctTest, IDCTIsSame4) { TestSlowIsSameIDCT<4>(1e-5); }
+TEST(SlowDctTest, IDCTIsSame8) { TestSlowIsSameIDCT<8>(1e-5); }
+TEST(SlowDctTest, IDCTIsSame16) { TestSlowIsSameIDCT<16>(2e-5); }
+TEST_P(DctShardedTest, IDCTIsSame32) {
+  TestSlowIsSameIDCTShard<32>(1e-4, GetParam());
+}
+
+template <size_t N>
+HWY_ATTR void TestSlowInverse(float accuracy, size_t start = 0,
+                              size_t end = N * N) {
+  constexpr size_t kBlockSize = N * N;
+  for (size_t i = start; i < end; i++) {
+    float x[kBlockSize] = {0.0f};
+    x[i] = 1.0;
+
+    slow_dct::ComputeTransposedScaledDCT<N>(FromBlock<N>(x),
+                                            ScaleToBlock<N>(x));
+    slow_dct::ComputeTransposedScaledIDCT<N>(FromBlock<N>(x), ToBlock<N>(x));
+
+    for (int k = 0; k < kBlockSize; ++k) {
+      EXPECT_NEAR(x[k], (k == i) ? 1.0f : 0.0f, accuracy)
+          << "i = " << i << ", k = " << k;
+    }
+  }
+}
+
+template <size_t N>
+HWY_ATTR void TestSlowInverseShard(float accuracy, int shard) {
+  TestSlowInverse<N>(accuracy, N * shard, N * (shard + 1));
+}
+
+TEST(SlowDctTest, SlowInverse2) { TestSlowInverse<2>(1e-5); }
+TEST(SlowDctTest, SlowInverse4) { TestSlowInverse<4>(1e-5); }
+TEST(SlowDctTest, SlowInverse8) { TestSlowInverse<8>(1e-5); }
+TEST(SlowDctTest, SlowInverse16) { TestSlowInverse<16>(1e-5); }
+TEST_P(DctShardedTest, SlowInverse32) {
+  TestSlowInverseShard<32>(1e-5, GetParam());
+}
+
+template <size_t ROWS, size_t COLS>
+HWY_ATTR void TestRectInverse(float accuracy) {
+  constexpr size_t kBlockSize = ROWS * COLS;
+  for (int i = 0; i < kBlockSize; ++i) {
+    HWY_ALIGN float x[kBlockSize] = {0.0f};
+    HWY_ALIGN float out[kBlockSize] = {0.0f};
+    x[i] = 1.0;
+    HWY_ALIGN float coeffs[kBlockSize] = {0.0f};
+
+    constexpr size_t OUT_ROWS = ROWS < COLS ? ROWS : COLS;
+    constexpr size_t OUT_COLS = ROWS < COLS ? COLS : ROWS;
+
+    ComputeScaledDCT<ROWS, COLS>()(FromBlock<ROWS, COLS>(x),
+                                   ScaleToBlock<OUT_ROWS, OUT_COLS>(coeffs));
+    ComputeScaledIDCT<ROWS, COLS>()(FromBlock<OUT_ROWS, OUT_COLS>(coeffs),
+                                    ToBlock<ROWS, COLS>(out));
+
+    for (int k = 0; k < kBlockSize; ++k) {
+      EXPECT_NEAR(out[k], (k == i) ? 1.0f : 0.0f, accuracy)
+          << "i = " << i << ", k = " << k;
+    }
+  }
+}
+
+TEST(RectDctTest, Inverse1632) { TestRectInverse<16, 32>(1e-6f); }
+TEST(RectDctTest, Inverse832) { TestRectInverse<8, 32>(1e-6f); }
+TEST(RectDctTest, Inverse816) { TestRectInverse<8, 16>(1e-6f); }
+TEST(RectDctTest, Inverse48) { TestRectInverse<4, 8>(1e-6f); }
+TEST(RectDctTest, Inverse24) { TestRectInverse<2, 4>(1e-6f); }
+TEST(RectDctTest, Inverse14) { TestRectInverse<1, 4>(1e-6f); }
+TEST(RectDctTest, Inverse12) { TestRectInverse<1, 2>(1e-6f); }
+
+TEST(RectDctTest, Inverse3216) { TestRectInverse<32, 16>(1e-6f); }
+TEST(RectDctTest, Inverse328) { TestRectInverse<32, 8>(1e-6f); }
+TEST(RectDctTest, Inverse168) { TestRectInverse<16, 8>(1e-6f); }
+TEST(RectDctTest, Inverse84) { TestRectInverse<8, 4>(1e-6f); }
+TEST(RectDctTest, Inverse42) { TestRectInverse<4, 2>(1e-6f); }
+TEST(RectDctTest, Inverse41) { TestRectInverse<4, 1>(1e-6f); }
+TEST(RectDctTest, Inverse21) { TestRectInverse<2, 1>(1e-6f); }
+
+template <size_t ROWS, size_t COLS>
+HWY_ATTR void TestRectTranspose(float accuracy) {
+  constexpr size_t kBlockSize = ROWS * COLS;
+  for (int px = 0; px < COLS; ++px) {
+    for (int py = 0; py < ROWS; ++py) {
+      HWY_ALIGN float x1[kBlockSize] = {0.0f};
+      HWY_ALIGN float x2[kBlockSize] = {0.0f};
+      HWY_ALIGN float coeffs1[kBlockSize] = {0.0f};
+      HWY_ALIGN float coeffs2[kBlockSize] = {0.0f};
+      x1[py * COLS + px] = 1;
+      x2[px * ROWS + py] = 1;
+
+      constexpr size_t OUT_ROWS = ROWS < COLS ? ROWS : COLS;
+      constexpr size_t OUT_COLS = ROWS < COLS ? COLS : ROWS;
+
+      ComputeScaledDCT<ROWS, COLS>()(FromBlock<ROWS, COLS>(x1),
+                                     ScaleToBlock<OUT_ROWS, OUT_COLS>(coeffs1));
+      ComputeScaledDCT<COLS, ROWS>()(FromBlock<COLS, ROWS>(x2),
+                                     ScaleToBlock<OUT_ROWS, OUT_COLS>(coeffs2));
+
+      for (int x = 0; x < OUT_COLS; ++x) {
+        for (int y = 0; y < OUT_ROWS; ++y) {
+          EXPECT_NEAR(coeffs1[y * OUT_COLS + x], coeffs2[y * OUT_COLS + x],
+                      accuracy)
+              << " px = " << px << ", py = " << py << ", x = " << x
+              << ", y = " << y;
+        }
+      }
+    }
+  }
+}
+
+TEST(RectDctTest, TransposeTest1632) { TestRectTranspose<16, 32>(1e-6f); }
+TEST(RectDctTest, TransposeTest832) { TestRectTranspose<8, 32>(1e-6f); }
+TEST(RectDctTest, TransposeTest816) { TestRectTranspose<8, 16>(1e-6f); }
+TEST(RectDctTest, TransposeTest48) { TestRectTranspose<4, 8>(1e-6f); }
+TEST(RectDctTest, TransposeTest24) { TestRectTranspose<2, 4>(1e-6f); }
+TEST(RectDctTest, TransposeTest14) { TestRectTranspose<1, 4>(1e-6f); }
+TEST(RectDctTest, TransposeTest12) { TestRectTranspose<1, 2>(1e-6f); }
+
+// This would be identical to TransposeTest816.
+// TEST(RectDctTest, TransposeTest168) { TestRectTranspose<16, 8>(1e-6f); }
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/dct_util.cc b/jxl/dct_util.cc
new file mode 100644
index 000000000..62dae0887
--- /dev/null
+++ b/jxl/dct_util.cc
@@ -0,0 +1,74 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dct_util.h"
+
+#include <hwy/static_targets.h>
+
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/block.h"
+#include "jxl/common.h"
+#include "jxl/dct.h"
+
+namespace jxl {
+
+HWY_ATTR void TransposedScaledDCT(const Image3F& image,
+                                  Image3F* JXL_RESTRICT dct) {
+  PROFILER_ZONE("TransposedScaledDCT facade");
+  JXL_ASSERT(image.xsize() % kBlockDim == 0);
+  JXL_ASSERT(image.ysize() % kBlockDim == 0);
+  const size_t xsize_blocks = image.xsize() / kBlockDim;
+  const size_t ysize_blocks = image.ysize() / kBlockDim;
+  JXL_ASSERT(dct->xsize() == xsize_blocks * kDCTBlockSize);
+  JXL_ASSERT(dct->ysize() == ysize_blocks);
+
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t by = 0; by < ysize_blocks; ++by) {
+      const float* JXL_RESTRICT row_in = image.ConstPlaneRow(c, by * kBlockDim);
+      float* JXL_RESTRICT row_dct = dct->PlaneRow(c, by);
+
+      for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+        ComputeTransposedScaledDCT<kBlockDim>()(
+            FromLines<kBlockDim>(row_in + bx * kBlockDim, image.PixelsPerRow()),
+            ScaleToBlock<kBlockDim>(row_dct + bx * kDCTBlockSize));
+      }
+    }
+  }
+}
+
+HWY_ATTR void TransposedScaledIDCT(const Image3F& dct,
+                                   Image3F* JXL_RESTRICT idct) {
+  PROFILER_ZONE("IDCT facade");
+  JXL_ASSERT(dct.xsize() % kDCTBlockSize == 0);
+  const size_t xsize_blocks = dct.xsize() / kDCTBlockSize;
+  const size_t ysize_blocks = dct.ysize();
+  JXL_ASSERT(idct->xsize() == xsize_blocks * kBlockDim);
+  JXL_ASSERT(idct->ysize() == ysize_blocks * kBlockDim);
+
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t by = 0; by < ysize_blocks; ++by) {
+      const float* JXL_RESTRICT row_dct = dct.ConstPlaneRow(c, by);
+      float* JXL_RESTRICT row_idct = idct->PlaneRow(c, by * kBlockDim);
+      for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+        ComputeTransposedScaledIDCT<kBlockDim>()(
+            FromBlock<kBlockDim>(row_dct + bx * kDCTBlockSize),
+            ToLines<kBlockDim>(row_idct + bx * kBlockDim,
+                               idct->PixelsPerRow()));
+      }
+    }
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/dct_util.h b/jxl/dct_util.h
new file mode 100644
index 000000000..3b96a54f4
--- /dev/null
+++ b/jxl/dct_util.h
@@ -0,0 +1,91 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DCT_UTIL_H_
+#define JXL_DCT_UTIL_H_
+
+#include <stddef.h>
+
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/image.h"
+
+namespace jxl {
+using ac_qcoeff_t = float;
+using ACImage = Plane<ac_qcoeff_t>;
+using ACImage3 = Image3<ac_qcoeff_t>;
+
+// Fills a preallocated (N*N)*W x H `dct` with (N*N)x1 blocks produced by
+// ComputeTransposedScaledDCT() from the corresponding NxN block of
+// `image`. Note that `dct` coefficients are scaled by 1 / (N*N), so that
+// ComputeTransposedScaledIDCT applied to each block of TransposedScaledIDCT
+// will return the original input.
+// REQUIRES: image.xsize() == N*W, image.ysize() == N*H
+HWY_ATTR void TransposedScaledDCT(const Image3F& image,
+                                  Image3F* JXL_RESTRICT dct);
+
+// Fills a preallocated N*W x N*H `idct` with NxN blocks produced by
+// ComputeTransposedScaledIDCT() from the (N*N)x1 blocks of `dct`.
+// REQUIRES: dct.xsize() == N*N*W, dct.ysize() == H
+HWY_ATTR void TransposedScaledIDCT(const Image3F& dct,
+                                   Image3F* JXL_RESTRICT idct);
+
+// Returns an N x M image by taking the DC coefficient from each 64x1 block.
+// REQUIRES: coeffs.xsize() == 64*N, coeffs.ysize() == M
+template <typename T>
+Plane<T> DCImage(const Plane<T>& coeffs) {
+  JXL_ASSERT(coeffs.xsize() % kDCTBlockSize == 0);
+  Plane<T> out(coeffs.xsize() / kDCTBlockSize, coeffs.ysize());
+  for (size_t y = 0; y < out.ysize(); ++y) {
+    const T* JXL_RESTRICT row_in = coeffs.ConstRow(y);
+    T* JXL_RESTRICT row_out = out.Row(y);
+    for (size_t x = 0; x < out.xsize(); ++x) {
+      row_out[x] = row_in[x * kDCTBlockSize];
+    }
+  }
+  return out;
+}
+
+template <typename T>
+Image3<T> DCImage(const Image3<T>& coeffs) {
+  return Image3<T>(DCImage(coeffs.Plane(0)), DCImage(coeffs.Plane(1)),
+                   DCImage(coeffs.Plane(2)));
+}
+
+// Scatters dc into "coeffs" at offset 0 within 1x64 blocks.
+template <typename T>
+void FillDC(const Plane<T>& dc, T* JXL_RESTRICT dst, size_t dst_stride) {
+  for (size_t y = 0; y < dc.ysize(); y++) {
+    const T* JXL_RESTRICT row_dc = dc.ConstRow(y);
+    T* JXL_RESTRICT row_out = dst + dst_stride * y;
+    for (size_t x = 0; x < dc.xsize(); ++x) {
+      row_out[kDCTBlockSize * x] = row_dc[x];
+    }
+  }
+}
+
+template <typename T>
+void FillDC(const Image3<T>& dc, Image3<T>* JXL_RESTRICT coeffs) {
+  for (size_t c = 0; c < 3; ++c) {
+    FillDC(dc.Plane(c), coeffs->PlaneRow(c, 0), coeffs->PixelsPerRow());
+  }
+}
+
+}  // namespace jxl
+
+#endif  // JXL_DCT_UTIL_H_
diff --git a/jxl/dct_util_test.cc b/jxl/dct_util_test.cc
new file mode 100644
index 000000000..38c303b6b
--- /dev/null
+++ b/jxl/dct_util_test.cc
@@ -0,0 +1,247 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dct_util.h"
+
+#include <stdint.h>
+
+#include <string>
+
+#include "gtest/gtest.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/block.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/dct.h"
+#include "jxl/enc_xyb.h"
+#include "jxl/extras/codec.h"
+#include "jxl/image_ops.h"
+#include "jxl/image_test_utils.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+
+// Zeroes out the top-left 2x2 corner of each DCT block.
+// REQUIRES: coeffs.xsize() == 64*N, coeffs.ysize() == M
+static void ZeroOut2x2(Image3F* coeffs) {
+  JXL_ASSERT(coeffs->xsize() % kDCTBlockSize == 0);
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < coeffs->ysize(); ++y) {
+      float* JXL_RESTRICT row = coeffs->PlaneRow(c, y);
+      for (size_t x = 0; x < coeffs->xsize(); x += kDCTBlockSize) {
+        row[x] = row[x + 1] = row[x + kBlockDim] = row[x + kBlockDim + 1] =
+            0.0f;
+      }
+    }
+  }
+}
+
+Image3F KeepOnly2x2Corners(const Image3F& coeffs) {
+  Image3F copy = CopyImage(coeffs);
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < coeffs.ysize(); ++y) {
+      float* JXL_RESTRICT row = copy.PlaneRow(c, y);
+      // TODO(user): might be better to copy 4 values, zero-out, place-back.
+      for (size_t x = 0; x < coeffs.xsize(); x += kDCTBlockSize) {
+        for (size_t k = 0; k < kDCTBlockSize; ++k) {
+          if ((k >= (2 * kBlockDim)) || ((k % kBlockDim) >= 2)) {
+            row[x + k] = 0.0f;
+          }
+        }
+      }
+    }
+  }
+  return copy;
+}
+
+// Returns a 2*N x 2*M image which is defined by the following 3 transforms:
+//  1) zero out every coefficient that is outside the top 2x2 corner
+//  2) apply ComputeTransposedScaledIDCT() to every block
+//  3) subsample the result 4x4 by taking simple averages
+// REQUIRES: coeffs.xsize() == kBlockSize*N, coeffs.ysize() == M
+static Image3F GetPixelSpaceImageFrom0HVD_64(const Image3F& coeffs) {
+  constexpr size_t N = kBlockDim;
+  JXL_ASSERT(coeffs.xsize() % kDCTBlockSize == 0);
+  const size_t block_xsize = coeffs.xsize() / kDCTBlockSize;
+  const size_t block_ysize = coeffs.ysize();
+  Image3F out(block_xsize * 2, block_ysize * 2);
+  const float kScale01 = N * DCTResampleScales<8, 2>::kScales[1] *
+                         DCTScales<N>()[0] * DCTScales<N>()[1];
+  const float kScale11 = N * DCTResampleScales<8, 2>::kScales[1] *
+                         DCTResampleScales<8, 2>::kScales[1] *
+                         DCTScales<N>()[1] * DCTScales<N>()[1];
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t by = 0; by < block_ysize; ++by) {
+      const float* JXL_RESTRICT row_coeffs = coeffs.PlaneRow(c, by);
+      float* JXL_RESTRICT row_out0 = out.PlaneRow(c, 2 * by + 0);
+      float* JXL_RESTRICT row_out1 = out.PlaneRow(c, 2 * by + 1);
+      for (size_t bx = 0; bx < block_xsize; ++bx) {
+        const float* block = row_coeffs + bx * kDCTBlockSize;
+        const float a00 = block[0];
+        const float a01 = block[N] * kScale01;
+        const float a10 = block[1] * kScale01;
+        const float a11 = block[N + 1] * kScale11;
+        row_out0[2 * bx + 0] = a00 + a01 + a10 + a11;
+        row_out0[2 * bx + 1] = a00 - a01 + a10 - a11;
+        row_out1[2 * bx + 0] = a00 + a01 - a10 - a11;
+        row_out1[2 * bx + 1] = a00 - a01 - a10 + a11;
+      }
+    }
+  }
+  return out;
+}
+
+// Puts back the top 2x2 corner of each 8x8 block of *coeffs from the
+// transformed pixel space image img.
+// REQUIRES: coeffs->xsize() == 64*N, coeffs->ysize() == M
+static void Add2x2CornersFromPixelSpaceImage(const Image3F& img,
+                                             Image3F* coeffs) {
+  constexpr size_t N = kBlockDim;
+  JXL_ASSERT(coeffs->xsize() % kDCTBlockSize == 0);
+  const size_t block_xsize = coeffs->xsize() / kDCTBlockSize;
+  const size_t block_ysize = coeffs->ysize();
+  JXL_ASSERT(block_xsize * 2 <= img.xsize());
+  JXL_ASSERT(block_ysize * 2 <= img.ysize());
+  const float kScale01 = N * DCTResampleScales<8, 2>::kScales[1] *
+                         DCTScales<N>()[0] * DCTScales<N>()[1];
+  const float kScale11 = N * DCTResampleScales<8, 2>::kScales[1] *
+                         DCTResampleScales<8, 2>::kScales[1] *
+                         DCTScales<N>()[1] * DCTScales<N>()[1];
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t by = 0; by < block_ysize; ++by) {
+      const float* JXL_RESTRICT row0 = img.PlaneRow(c, 2 * by + 0);
+      const float* JXL_RESTRICT row1 = img.PlaneRow(c, 2 * by + 1);
+      float* row_out = coeffs->PlaneRow(c, by);
+      for (size_t bx = 0; bx < block_xsize; ++bx) {
+        const float b00 = row0[2 * bx + 0];
+        const float b01 = row0[2 * bx + 1];
+        const float b10 = row1[2 * bx + 0];
+        const float b11 = row1[2 * bx + 1];
+        const float a00 = 0.25f * (b00 + b01 + b10 + b11);
+        const float a01 = 0.25f * (b00 - b01 + b10 - b11);
+        const float a10 = 0.25f * (b00 + b01 - b10 - b11);
+        const float a11 = 0.25f * (b00 - b01 - b10 + b11);
+        float* JXL_RESTRICT block = &row_out[bx * kDCTBlockSize];
+        block[0] = a00;
+        block[1] = a10 / kScale01;
+        block[N] = a01 / kScale01;
+        block[N + 1] = a11 / kScale11;
+      }
+    }
+  }
+}
+
+// Returns an N x M image where each pixel is the average of the corresponding
+// f x f block in the original.
+// REQUIRES: image.xsize() == f*N, image.ysize() == f *M
+static Image3F Subsample(const Image3F& image, int f) {
+  JXL_CHECK(image.xsize() % f == 0);
+  JXL_CHECK(image.ysize() % f == 0);
+  const size_t shift = CeilLog2Nonzero(static_cast<uint32_t>(f));
+  JXL_CHECK(f == (1 << shift));
+  const size_t nxs = image.xsize() >> shift;
+  const size_t nys = image.ysize() >> shift;
+  Image3F retval(nxs, nys);
+  ZeroFillImage(&retval);
+  const float mul = 1.0f / (f * f);
+  for (size_t y = 0; y < image.ysize(); ++y) {
+    const float* row_in[3] = {image.PlaneRow(0, y), image.PlaneRow(1, y),
+                              image.PlaneRow(2, y)};
+    const size_t ny = y >> shift;
+    float* row_out[3] = {retval.PlaneRow(0, ny), retval.PlaneRow(1, ny),
+                         retval.PlaneRow(2, ny)};
+    for (size_t c = 0; c < 3; ++c) {
+      for (size_t x = 0; x < image.xsize(); ++x) {
+        size_t nx = x >> shift;
+        row_out[c][nx] += mul * row_in[c][x];
+      }
+    }
+  }
+  return retval;
+}
+
+static Image3F OpsinTestImage() {
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  JXL_CHECK(SetFromFile(pathname, &io, /*pool=*/nullptr));
+  ThreadPool* null_pool = nullptr;
+  Image3F opsin(io.xsize(), io.ysize());
+  ToXYB(io.Main(), 1.0f, null_pool, &opsin);
+  opsin.ShrinkTo(opsin.ysize() & ~7, opsin.xsize() & ~7);
+  return opsin;
+}
+
+static HWY_ATTR Image3F TransposedScaledIDCT(const Image3F& coeffs) {
+  constexpr size_t N = kBlockDim;
+  const size_t xsize_blocks = coeffs.xsize() / kDCTBlockSize;
+  const size_t ysize_blocks = coeffs.ysize();
+  Image3F img(xsize_blocks * N, ysize_blocks * N);
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t by = 0; by < ysize_blocks; ++by) {
+      for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+        const size_t stride = img.PlaneRow(c, 1) - img.PlaneRow(c, 0);
+        const float* JXL_RESTRICT row_in = coeffs.PlaneRow(c, by);
+        float* JXL_RESTRICT row_out = img.PlaneRow(c, by * N);
+        ComputeTransposedScaledIDCT<N>()(
+            FromBlock<N>(row_in + bx * kDCTBlockSize),
+            ToLines<N>(row_out + bx * N, stride));
+      }
+    }
+  }
+  return img;
+}
+
+TEST(DctUtilTest, DCTRoundtrip) {
+  Image3F opsin = OpsinTestImage();
+  const size_t xsize_blocks = opsin.xsize() / kBlockDim;
+  const size_t ysize_blocks = opsin.ysize() / kBlockDim;
+
+  Image3F coeffs(xsize_blocks * kDCTBlockSize, ysize_blocks);
+  TransposedScaledDCT(opsin, &coeffs);
+  Image3F recon = TransposedScaledIDCT(coeffs);
+
+  VerifyRelativeError(opsin, recon, 1e-6, 1e-6);
+}
+
+TEST(DctUtilTest, Transform2x2Corners) {
+  Image3F opsin = OpsinTestImage();
+  const size_t xsize_blocks = opsin.xsize() / kBlockDim;
+  const size_t ysize_blocks = opsin.ysize() / kBlockDim;
+
+  Image3F coeffs(xsize_blocks * kDCTBlockSize, ysize_blocks);
+  TransposedScaledDCT(opsin, &coeffs);
+  Image3F t1 = GetPixelSpaceImageFrom0HVD_64(coeffs);
+  Image3F t2 = Subsample(TransposedScaledIDCT(KeepOnly2x2Corners(coeffs)), 4);
+  VerifyRelativeError(t1, t2, 1e-6, 1e-6);
+}
+
+TEST(DctUtilTest, Roundtrip2x2Corners) {
+  Image3F opsin = OpsinTestImage();
+  const size_t xsize_blocks = opsin.xsize() / kBlockDim;
+  const size_t ysize_blocks = opsin.ysize() / kBlockDim;
+
+  Image3F coeffs(xsize_blocks * kDCTBlockSize, ysize_blocks);
+  TransposedScaledDCT(opsin, &coeffs);
+  Image3F tmp = GetPixelSpaceImageFrom0HVD_64(coeffs);
+  Image3F coeffs_out = CopyImage(coeffs);
+  ZeroOut2x2(&coeffs_out);
+  Add2x2CornersFromPixelSpaceImage(tmp, &coeffs_out);
+  VerifyRelativeError(coeffs, coeffs_out, 1e-6, 1e-6);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/dec_ans.cc b/jxl/dec_ans.cc
new file mode 100644
index 000000000..7bbc91ab8
--- /dev/null
+++ b/jxl/dec_ans.cc
@@ -0,0 +1,262 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dec_ans.h"
+
+#include <stdint.h>
+
+#include <vector>
+
+#include "c/dec/bit_reader.h"
+#include "jxl/ans_common.h"
+#include "jxl/ans_params.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/dec_context_map.h"
+
+namespace jxl {
+namespace {
+
+// Decodes a number in the range [0..255], by reading 1 - 11 bits.
+HWY_ATTR inline int DecodeVarLenUint8(BitReader* input) {
+  if (input->ReadFixedBits<1>()) {
+    int nbits = static_cast<int>(input->ReadFixedBits<3>());
+    if (nbits == 0) {
+      return 1;
+    } else {
+      return static_cast<int>(input->ReadBits(nbits)) + (1 << nbits);
+    }
+  }
+  return 0;
+}
+
+HWY_ATTR Status ReadHistogram(int precision_bits, std::vector<int>* counts,
+                              BitReader* input) {
+  int simple_code = input->ReadBits(1);
+  if (simple_code == 1) {
+    int i;
+    int symbols[2] = {0};
+    int max_symbol = 0;
+    const int num_symbols = input->ReadBits(1) + 1;
+    for (i = 0; i < num_symbols; ++i) {
+      symbols[i] = DecodeVarLenUint8(input);
+      if (symbols[i] > max_symbol) max_symbol = symbols[i];
+    }
+    counts->resize(max_symbol + 1);
+    if (num_symbols == 1) {
+      (*counts)[symbols[0]] = 1 << precision_bits;
+    } else {
+      if (symbols[0] == symbols[1]) {  // corrupt data
+        return false;
+      }
+      (*counts)[symbols[0]] = input->ReadBits(precision_bits);
+      (*counts)[symbols[1]] = (1 << precision_bits) - (*counts)[symbols[0]];
+    }
+  } else {
+    int is_flat = input->ReadBits(1);
+    if (is_flat == 1) {
+      int alphabet_size = DecodeVarLenUint8(input) + 1;
+      if (alphabet_size == 0) {
+        return JXL_FAILURE("Invalid alphabet size for flat histogram.");
+      }
+      *counts = CreateFlatHistogram(alphabet_size, 1 << precision_bits);
+      return true;
+    }
+
+    uint32_t shift;
+    {
+      // TODO(veluca): speed up reading with table lookups.
+      int upper_bound_log = FloorLog2Nonzero(ANS_LOG_TAB_SIZE + 1);
+      int log = 0;
+      for (; log < upper_bound_log; log++) {
+        if (input->ReadFixedBits<1>() == 0) break;
+      }
+      shift = (input->ReadBits(log) | (1 << log)) - 1;
+      if (shift > ANS_LOG_TAB_SIZE + 1) {
+        return JXL_FAILURE("Invalid shift value");
+      }
+    }
+
+    int length = DecodeVarLenUint8(input) + 3;
+    counts->resize(length);
+    int total_count = 0;
+
+    static const uint8_t huff[128][2] = {
+        {3, 10}, {7, 12}, {3, 7}, {4, 3}, {3, 6}, {3, 8}, {3, 9}, {4, 5},
+        {3, 10}, {4, 4},  {3, 7}, {4, 1}, {3, 6}, {3, 8}, {3, 9}, {4, 2},
+        {3, 10}, {5, 0},  {3, 7}, {4, 3}, {3, 6}, {3, 8}, {3, 9}, {4, 5},
+        {3, 10}, {4, 4},  {3, 7}, {4, 1}, {3, 6}, {3, 8}, {3, 9}, {4, 2},
+        {3, 10}, {6, 11}, {3, 7}, {4, 3}, {3, 6}, {3, 8}, {3, 9}, {4, 5},
+        {3, 10}, {4, 4},  {3, 7}, {4, 1}, {3, 6}, {3, 8}, {3, 9}, {4, 2},
+        {3, 10}, {5, 0},  {3, 7}, {4, 3}, {3, 6}, {3, 8}, {3, 9}, {4, 5},
+        {3, 10}, {4, 4},  {3, 7}, {4, 1}, {3, 6}, {3, 8}, {3, 9}, {4, 2},
+        {3, 10}, {7, 13}, {3, 7}, {4, 3}, {3, 6}, {3, 8}, {3, 9}, {4, 5},
+        {3, 10}, {4, 4},  {3, 7}, {4, 1}, {3, 6}, {3, 8}, {3, 9}, {4, 2},
+        {3, 10}, {5, 0},  {3, 7}, {4, 3}, {3, 6}, {3, 8}, {3, 9}, {4, 5},
+        {3, 10}, {4, 4},  {3, 7}, {4, 1}, {3, 6}, {3, 8}, {3, 9}, {4, 2},
+        {3, 10}, {6, 11}, {3, 7}, {4, 3}, {3, 6}, {3, 8}, {3, 9}, {4, 5},
+        {3, 10}, {4, 4},  {3, 7}, {4, 1}, {3, 6}, {3, 8}, {3, 9}, {4, 2},
+        {3, 10}, {5, 0},  {3, 7}, {4, 3}, {3, 6}, {3, 8}, {3, 9}, {4, 5},
+        {3, 10}, {4, 4},  {3, 7}, {4, 1}, {3, 6}, {3, 8}, {3, 9}, {4, 2},
+    };
+
+    std::vector<int> logcounts(counts->size());
+    int omit_log = -1;
+    int omit_pos = -1;
+    // This array remembers which symbols have an RLE length.
+    std::vector<int> same(counts->size(), 0);
+    for (size_t i = 0; i < logcounts.size(); ++i) {
+      input->Refill();  // for PeekFixedBits + Advance
+      int idx = input->PeekFixedBits<7>();
+      input->Consume(huff[idx][0]);
+      logcounts[i] = huff[idx][1];
+      // The RLE symbol.
+      if (logcounts[i] == ANS_LOG_TAB_SIZE + 1) {
+        int rle_length = DecodeVarLenUint8(input);
+        same[i] = rle_length + 5;
+        i += rle_length + 3;
+        continue;
+      }
+      if (logcounts[i] > omit_log) {
+        omit_log = logcounts[i];
+        omit_pos = i;
+      }
+    }
+    // Invalid input, e.g. due to invalid usage of RLE.
+    if (omit_pos < 0) return JXL_FAILURE("Invalid histogram.");
+    if (static_cast<size_t>(omit_pos) + 1 < logcounts.size() &&
+        logcounts[omit_pos + 1] == ANS_TAB_SIZE + 1) {
+      return JXL_FAILURE("Invalid histogram.");
+    }
+    int prev = 0;
+    int numsame = 0;
+    for (size_t i = 0; i < logcounts.size(); ++i) {
+      if (same[i]) {
+        // RLE sequence, let this loop output the same count for the next
+        // iterations.
+        numsame = same[i] - 1;
+        prev = i > 0 ? (*counts)[i - 1] : 0;
+      }
+      if (numsame > 0) {
+        (*counts)[i] = prev;
+        numsame--;
+      } else {
+        int code = logcounts[i];
+        // omit_pos may not be negative at this point (checked before).
+        if (i == static_cast<size_t>(omit_pos)) {
+          continue;
+        } else if (code == 0) {
+          continue;
+        } else if (code == 1) {
+          (*counts)[i] = 1;
+        } else {
+          int bitcount = GetPopulationCountPrecision(code - 1, shift);
+          (*counts)[i] = (1 << (code - 1)) +
+                         (input->ReadBits(bitcount) << (code - 1 - bitcount));
+        }
+      }
+      total_count += (*counts)[i];
+    }
+    (*counts)[omit_pos] = (1 << precision_bits) - total_count;
+    if ((*counts)[omit_pos] <= 0) {
+      // The histogram we've read sums to more than total_count (including at
+      // least 1 for the omitted value).
+      return JXL_FAILURE("Invalid histogram count.");
+    }
+  }
+  return true;
+}
+
+}  // namespace
+
+bool DecodeANSCodes(const size_t num_histograms, const size_t max_alphabet_size,
+                    BitReader* in, ANSCode* result) {
+  JXL_ASSERT(max_alphabet_size <= ANS_MAX_ALPHA_SIZE);
+  if (result->use_prefix_code) {
+    result->huffman_data.resize(num_histograms);
+    std::vector<uint16_t> alphabet_sizes(num_histograms);
+    for (size_t c = 0; c < num_histograms; c++) {
+      alphabet_sizes[c] = DecodeVarLenUint8(in) + 1;
+      if (alphabet_sizes[c] > max_alphabet_size)
+        return JXL_FAILURE("Alphabet size is too long");
+    }
+    size_t pos = in->TotalBitsConsumed();
+    size_t size = in->TotalBytes();
+    const uint8_t* data = in->FirstByte();
+    if (pos > size * kBitsPerByte) return JXL_FAILURE("Truncated bitstream");
+    data += pos / kBitsPerByte;
+    size -= pos / kBitsPerByte;
+    pos %= kBitsPerByte;
+    size_t orig_size = size;
+    brunsli::BrunsliBitReader br;
+    brunsli::BrunsliBitReaderInit(&br, data, size);
+    // Skip already consumed bits.
+    (void)brunsli::BrunsliBitReaderRead(&br, pos);
+    for (size_t c = 0; c < num_histograms; c++) {
+      if (alphabet_sizes[c] > 1) {
+        if (!result->huffman_data[c].ReadFromBitStream(alphabet_sizes[c],
+                                                       &br)) {
+          return JXL_FAILURE(
+              "Invalid huffman tree number %zu, alphabet size %u", c,
+              alphabet_sizes[c]);
+        }
+      } else {
+        result->huffman_data[c].table_.resize(brunsli::kMaxHuffmanTableSize);
+      }
+    }
+    if (!brunsli::BrunsliBitReaderIsHealthy(&br)) {
+      return JXL_FAILURE("Invalid huffman code bitstream.");
+    }
+    size_t num_unused_bits = br.num_bits_;
+    size_t consumed_bytes = orig_size - brunsli::BrunsliBitReaderFinish(&br);
+    in->SkipBits(consumed_bytes * kBitsPerByte - num_unused_bits - pos);
+  } else {
+    result->alias_tables = AllocateArray(num_histograms * ANS_MAX_ALPHA_SIZE *
+                                         sizeof(AliasTable::Entry));
+    AliasTable::Entry* alias_tables =
+        reinterpret_cast<AliasTable::Entry*>(result->alias_tables.get());
+    for (size_t c = 0; c < num_histograms; ++c) {
+      std::vector<int> counts;
+      if (!ReadHistogram(ANS_LOG_TAB_SIZE, &counts, in)) {
+        return JXL_FAILURE("Invalid histogram bitstream.");
+      }
+      if (counts.size() > max_alphabet_size) {
+        return JXL_FAILURE("Alphabet size is too long.");
+      }
+      InitAliasTable(counts, ANS_TAB_SIZE,
+                     alias_tables + c * ANS_MAX_ALPHA_SIZE);
+    }
+  }
+  return true;
+}
+
+HWY_ATTR bool DecodeHistograms(BitReader* br, const size_t num_contexts,
+                               const size_t max_alphabet_size, ANSCode* code,
+                               std::vector<uint8_t>* context_map) {
+  PROFILER_FUNC;
+  size_t num_histograms = 1;
+  context_map->resize(num_contexts);
+  if (num_contexts > 1) {
+    JXL_RETURN_IF_ERROR(DecodeContextMap(context_map, &num_histograms, br));
+  }
+  code->use_prefix_code = br->ReadFixedBits<1>();
+  if (!DecodeANSCodes(num_histograms, max_alphabet_size, br, code)) {
+    return JXL_FAILURE("Histo DecodeANSCodes");
+  }
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/dec_ans.h b/jxl/dec_ans.h
new file mode 100644
index 000000000..61b3a9c38
--- /dev/null
+++ b/jxl/dec_ans.h
@@ -0,0 +1,133 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_ANS_H_
+#define JXL_DEC_ANS_H_
+
+// Library to decode the ANS population counts from the bit-stream and build a
+// decoding table from them.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/static_targets.h>
+#include <vector>
+
+#include "c/dec/huffman_decode.h"
+#include "jxl/ans_common.h"
+#include "jxl/ans_params.h"
+#include "jxl/base/byte_order.h"
+#include "jxl/base/cache_aligned.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/dec_bit_reader.h"
+
+namespace jxl {
+
+struct ANSCode {
+  CacheAlignedUniquePtr alias_tables;
+  std::vector<brunsli::HuffmanDecodingData> huffman_data;
+  bool use_prefix_code;
+};
+
+class ANSSymbolReader {
+ public:
+  // Invalid symbol reader, to be overwritten.
+  ANSSymbolReader() = default;
+  HWY_ATTR ANSSymbolReader(const ANSCode* code, BitReader* JXL_RESTRICT br)
+      : alias_tables_(
+            reinterpret_cast<AliasTable::Entry*>(code->alias_tables.get())),
+        huffman_data_(&code->huffman_data),
+        use_prefix_code_(code->use_prefix_code) {
+    if (!use_prefix_code_) {
+      state_ = static_cast<uint32_t>(br->ReadFixedBits<32>());
+    } else {
+      state_ = (ANS_SIGNATURE << 16u);
+    }
+  }
+
+  HWY_ATTR JXL_INLINE size_t ReadSymbolANSWithoutRefill(
+      const size_t histo_idx, BitReader* JXL_RESTRICT br) {
+    const uint32_t res = state_ & (ANS_TAB_SIZE - 1u);
+
+    const AliasTable::Entry* table =
+        &alias_tables_[histo_idx * ANS_MAX_ALPHA_SIZE];
+    const AliasTable::Symbol symbol = AliasTable::Lookup(table, res);
+    state_ = symbol.freq * (state_ >> ANS_LOG_TAB_SIZE) + symbol.offset;
+
+#if 1
+    // Branchless version is about equally fast on SKX.
+    const uint32_t new_state =
+        (state_ << 16u) | static_cast<uint32_t>(br->PeekFixedBits<16>());
+    const bool normalize = state_ < (1u << 16u);
+    state_ = normalize ? new_state : state_;
+    br->Consume(normalize ? 16 : 0);
+#else
+    if (JXL_UNLIKELY(state_ < (1u << 16u))) {
+      state_ = (state_ << 16u) | br->PeekFixedBits<16>();
+      br->Consume(16);
+    }
+#endif
+    const uint32_t next_res = state_ & (ANS_TAB_SIZE - 1u);
+    AliasTable::Prefetch(table, next_res);
+
+    return symbol.value;
+  }
+
+  HWY_ATTR JXL_INLINE size_t ReadSymbolHuffWithoutRefill(
+      const size_t histo_idx, BitReader* JXL_RESTRICT br) {
+    // Adapted from brunsli.
+    const brunsli::HuffmanCode* table = &(*huffman_data_)[histo_idx].table_[0];
+    table += br->PeekFixedBits<8>();
+    size_t nbits = table->bits;
+    if (nbits > 8) {
+      nbits -= 8;
+      br->Consume(8);
+      table += table->value;
+      table += br->PeekBits(nbits);
+    }
+    br->Consume(table->bits);
+    return table->value;
+  }
+
+  HWY_ATTR JXL_INLINE size_t
+  ReadSymbolWithoutRefill(const size_t histo_idx, BitReader* JXL_RESTRICT br) {
+    // TODO(veluca): hoist if in hotter loops.
+    if (JXL_UNLIKELY(use_prefix_code_)) {
+      return ReadSymbolHuffWithoutRefill(histo_idx, br);
+    }
+    return ReadSymbolANSWithoutRefill(histo_idx, br);
+  }
+
+  HWY_ATTR JXL_INLINE size_t ReadSymbol(const size_t histo_idx,
+                                        BitReader* JXL_RESTRICT br) {
+    br->Refill();
+    return ReadSymbolWithoutRefill(histo_idx, br);
+  }
+
+  bool CheckANSFinalState() { return state_ == (ANS_SIGNATURE << 16u); }
+
+ private:
+  const AliasTable::Entry* JXL_RESTRICT alias_tables_;  // not owned
+  const std::vector<brunsli::HuffmanDecodingData>* huffman_data_;
+  bool use_prefix_code_;
+  uint32_t state_ = ANS_SIGNATURE << 16u;
+};
+
+bool DecodeHistograms(BitReader* br, size_t num_contexts,
+                      size_t max_alphabet_size, ANSCode* code,
+                      std::vector<uint8_t>* context_map);
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_ANS_H_
diff --git a/jxl/dec_bit_reader.h b/jxl/dec_bit_reader.h
new file mode 100644
index 000000000..e51a4c110
--- /dev/null
+++ b/jxl/dec_bit_reader.h
@@ -0,0 +1,315 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_BIT_READER_H_
+#define JXL_DEC_BIT_READER_H_
+
+// Bounds-checked bit reader; 64-bit buffer with support for deferred refills.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>  // memcpy
+
+#include <algorithm>
+#include <hwy/static_targets.h>
+
+#include "jxl/base/byte_order.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+
+namespace jxl {
+
+// Reads bits previously written to memory by BitWriter. Uses unaligned 8-byte
+// little-endian loads.
+class BitReader {
+ public:
+  static constexpr size_t kMaxBitsPerCall = 56;
+
+  // Constructs an invalid BitReader, to be overwritten before usage.
+  BitReader() : first_byte_(nullptr) {}
+  BitReader(const BitReader&) = delete;
+
+  // bytes need not be aligned nor padded!
+  template <class ArrayLike>
+  HWY_ATTR explicit BitReader(const ArrayLike& bytes)
+      : buf_(0),
+        bits_in_buf_(0),
+        next_byte_(bytes.data()),
+        // Assumes first_byte_ >= 8.
+        end_minus_8_(bytes.data() - 8 + bytes.size()),
+        first_byte_(bytes.data()) {
+    Refill();
+  }
+  ~BitReader() {
+    // Close() must be called before destroying an initialized bit reader.
+    // Invalid bit readers will have a nullptr in first_byte_.
+    JXL_ASSERT(close_called_ || !first_byte_);
+  }
+
+  // Move operator needs to invalidate the other BitReader such that it is
+  // irrelevant if we call Close() on it or not.
+  BitReader& operator=(BitReader&& other) {
+    // Ensure the current instance was already closed, before we overwrite it
+    // with other.
+    JXL_ASSERT(close_called_ || !first_byte_);
+
+    JXL_DASSERT(!other.close_called_);
+    buf_ = other.buf_;
+    bits_in_buf_ = other.bits_in_buf_;
+    next_byte_ = other.next_byte_;
+    end_minus_8_ = other.end_minus_8_;
+    first_byte_ = other.first_byte_;
+    overread_bytes_ = other.overread_bytes_;
+    close_called_ = other.close_called_;
+
+    other.first_byte_ = nullptr;
+    other.next_byte_ = nullptr;
+    return *this;
+  }
+  BitReader& operator=(const BitReader& other) = delete;
+
+  // For time-critical reads, refills can be shared by multiple reads.
+  // Based on variant 4 (plus bounds-checking), see
+  // fgiesen.wordpress.com/2018/02/20/reading-bits-in-far-too-many-ways-part-2/
+  HWY_ATTR JXL_INLINE void Refill() {
+    if (JXL_UNLIKELY(next_byte_ > end_minus_8_)) {
+      BoundsCheckedRefill();
+    } else {
+      // It's safe to load 64 bits; insert valid (possibly nonzero) bits above
+      // bits_in_buf_. The shift requires bits_in_buf_ < 64.
+      buf_ |= LoadLE64(next_byte_) << bits_in_buf_;
+
+      // Advance by bytes fully absorbed into the buffer.
+      next_byte_ += (63 - bits_in_buf_) >> 3;
+
+      // We absorbed a multiple of 8 bits, so the lower 3 bits of bits_in_buf_
+      // must remain unchanged, otherwise the next refill's shifted bits will
+      // not align with buf_. Set the three upper bits so the result >= 56.
+      bits_in_buf_ |= 56;
+      JXL_DASSERT(56 <= bits_in_buf_ && bits_in_buf_ < 64);
+    }
+  }
+
+  // Returns the bits that would be returned by Read without calling Advance().
+  // It is legal to PEEK at more bits than present in the bitstream (required
+  // by Huffman), and those bits will be zero.
+  template <size_t N>
+  HWY_ATTR HWY_INLINE uint64_t PeekFixedBits() const {
+    static_assert(N <= kMaxBitsPerCall, "Reading too many bits in one call.");
+    JXL_DASSERT(!close_called_);
+    return buf_ & ((1ULL << N) - 1);
+  }
+
+  HWY_ATTR HWY_INLINE uint64_t PeekBits(size_t nbits) const {
+    JXL_DASSERT(nbits <= kMaxBitsPerCall);
+    JXL_DASSERT(!close_called_);
+    return ZeroBitsAbove(buf_, nbits);
+  }
+
+  // Removes bits from the buffer. Need not match the previous Peek size, but
+  // the buffer must contain at least num_bits (this prevents consuming more
+  // than the total number of bits).
+  HWY_INLINE void Consume(size_t num_bits) {
+    JXL_DASSERT(!close_called_);
+    JXL_DASSERT(bits_in_buf_ >= num_bits);
+#ifdef JXL_CRASH_ON_ERROR
+    // When JXL_CRASH_ON_ERROR is defined, it is a fatal error to read more bits
+    // than available in the stream. A non-zero overread_bytes_ implies that
+    // next_byte_ is already at the end of the stream, so we don't need to
+    // check that.
+    JXL_ASSERT(bits_in_buf_ >= num_bits + overread_bytes_ * kBitsPerByte);
+#endif
+    bits_in_buf_ -= num_bits;
+    buf_ >>= num_bits;
+  }
+
+  HWY_ATTR JXL_INLINE uint64_t ReadBits(size_t nbits) {
+    JXL_DASSERT(!close_called_);
+    Refill();
+    const uint64_t bits = PeekBits(nbits);
+    Consume(nbits);
+    return bits;
+  }
+
+  template <size_t N>
+  HWY_ATTR JXL_INLINE uint64_t ReadFixedBits() {
+    JXL_DASSERT(!close_called_);
+    Refill();
+    const uint64_t bits = PeekFixedBits<N>();
+    Consume(N);
+    return bits;
+  }
+
+  // Equivalent to calling ReadFixedBits(1) `skip` times, but much faster.
+  // `skip` is typically large.
+  HWY_ATTR void SkipBits(size_t skip) {
+    JXL_DASSERT(!close_called_);
+    // Buffer is large enough - don't zero buf_ below.
+    if (JXL_UNLIKELY(skip <= bits_in_buf_)) {
+      Consume(skip);
+      return;
+    }
+
+    // First deduct what we can satisfy from the buffer
+    skip -= bits_in_buf_;
+    bits_in_buf_ = 0;
+    // Not enough to call Advance - that may leave some bits in the buffer
+    // which were previously ABOVE bits_in_buf.
+    buf_ = 0;
+
+    // Skip whole bytes
+    const size_t whole_bytes = skip / kBitsPerByte;
+    next_byte_ += whole_bytes;
+    skip %= kBitsPerByte;
+
+    Refill();
+    Consume(skip);
+  }
+
+  size_t TotalBitsConsumed() const {
+    const size_t bytes_read = static_cast<size_t>(next_byte_ - first_byte_);
+    return (bytes_read + overread_bytes_) * kBitsPerByte - bits_in_buf_;
+  }
+
+  HWY_ATTR Status JumpToByteBoundary() {
+    const size_t remainder = TotalBitsConsumed() % kBitsPerByte;
+    if (remainder == 0) return true;
+    if (JXL_UNLIKELY(ReadBits(kBitsPerByte - remainder) != 0)) {
+      return JXL_FAILURE("Non-zero padding bits");
+    }
+    return true;
+  }
+
+  // For interoperability with other bitreaders (for resuming at
+  // non-byte-aligned positions).
+  const uint8_t* FirstByte() const { return first_byte_; }
+  size_t TotalBytes() const {
+    return static_cast<size_t>(end_minus_8_ + 8 - first_byte_);
+  }
+
+  // Returns span of the remaining (unconsumed) bytes, e.g. for passing to
+  // external decoders such as Brotli.
+  Span<const uint8_t> GetSpan() const {
+    JXL_DASSERT(first_byte_ != nullptr);
+    JXL_ASSERT(TotalBitsConsumed() % kBitsPerByte == 0);
+    const size_t offset = TotalBitsConsumed() / kBitsPerByte;  // no remainder
+    JXL_ASSERT(offset <= TotalBytes());
+    return Span<const uint8_t>(first_byte_ + offset, TotalBytes() - offset);
+  }
+
+  // Returns whether all the bits read so far have been within the input bounds.
+  // When reading past the EOF, the Read*() and Consume() functions return zeros
+  // but flag a failure when calling close or this function.
+  Status AllReadsWithinBounds() const {
+    if (TotalBitsConsumed() > TotalBytes() * kBitsPerByte) {
+      return JXL_FAILURE("Read more bits than available in the bit_reader");
+    }
+    return true;
+  }
+
+  // Close the bit reader and return whether all the previous reads were
+  // successful. Close must be called once.
+  Status Close() {
+    JXL_DASSERT(!close_called_);
+    close_called_ = true;
+    if (!first_byte_) return true;
+    JXL_RETURN_IF_ERROR(AllReadsWithinBounds());
+    return true;
+  }
+
+ private:
+  // Separate function avoids inlining this relatively cold code into callers.
+  JXL_NOINLINE void BoundsCheckedRefill() {
+    PROFILER_FUNC;
+    const uint8_t* end = end_minus_8_ + 8;
+
+    // Read whole bytes until we have [56, 64) bits (same as LoadLE64)
+    for (; bits_in_buf_ < 64 - kBitsPerByte; bits_in_buf_ += kBitsPerByte) {
+      if (next_byte_ >= end) break;
+      buf_ |= static_cast<uint64_t>(*next_byte_++) << bits_in_buf_;
+    }
+    JXL_DASSERT(bits_in_buf_ < 64);
+
+    // Add extra bytes as 0 at the end of the stream in the bit_buffer_. If
+    // these bits are read, Close() will return a failure.
+    size_t extra_bytes = (63 - bits_in_buf_) / kBitsPerByte;
+    overread_bytes_ += extra_bytes;
+    bits_in_buf_ += extra_bytes * kBitsPerByte;
+
+    JXL_DASSERT(bits_in_buf_ < 64);
+    JXL_DASSERT(bits_in_buf_ >= 56);
+  }
+
+  static HWY_ATTR HWY_INLINE uint64_t ZeroBitsAbove(uint64_t bits,
+                                                    size_t index) {
+#if (HWY_STATIC_TARGETS & HWY_AVX2) && defined(__x86_64__)
+    // BMI2: Clears bits with index >= nbits
+    return _bzhi_u64(bits, index);
+#else
+    const uint64_t mask = (1ULL << index) - 1;
+    return bits & mask;
+#endif
+  }
+
+  uint64_t buf_;
+  size_t bits_in_buf_;  // [0, 64)
+  const uint8_t* JXL_RESTRICT next_byte_;
+  const uint8_t* end_minus_8_;  // for refill bounds check
+  const uint8_t* first_byte_;   // for GetSpan
+
+  // Number of bytes past the end that were loaded into the buf_. These bytes
+  // are not read from memory, but instead assumed 0. It is an error (likely due
+  // to an invalid stream) to Consume() more bits than specified in the range
+  // passed to the constructor.
+  uint64_t overread_bytes_{0};
+  bool close_called_{false};
+};
+
+// Closes a BitReader when the BitReaderScopedCloser goes out of scope. When
+// closing the bit reader, if the status result was failure it sets this failure
+// to the passed variable pointer. Typical usage.
+//
+// Status ret = true;
+// {
+//   BitReader reader(...);
+//   BitReaderScopedCloser reader_closer(&reader, &ret);
+//
+//   // ... code that can return errors here ...
+// }
+// // ... more code that doesn't use the BitReader.
+// return ret;
+
+class BitReaderScopedCloser {
+ public:
+  BitReaderScopedCloser(BitReader* reader, Status* status)
+      : reader_(reader), status_(status) {
+    JXL_DASSERT(reader_ != nullptr);
+    JXL_DASSERT(status_ != nullptr);
+  }
+  ~BitReaderScopedCloser() {
+    Status close_ret = reader_->Close();
+    if (!close_ret) *status_ = close_ret;
+  }
+
+ private:
+  BitReader* reader_;
+  Status* status_;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_BIT_READER_H_
diff --git a/jxl/dec_cache.h b/jxl/dec_cache.h
new file mode 100644
index 000000000..110c42894
--- /dev/null
+++ b/jxl/dec_cache.h
@@ -0,0 +1,145 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_CACHE_H_
+#define JXL_DEC_CACHE_H_
+
+#include <stdint.h>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/coeff_order.h"
+#include "jxl/common.h"
+#include "jxl/image.h"
+#include "jxl/passes_state.h"
+#include "jxl/quant_weights.h"
+
+namespace jxl {
+
+// Line-based EPF only needs to keep in cache 13 lines of the image, so 256 is
+// sufficient for everything to fit in the L2 cache.
+constexpr size_t kApplyImageFeaturesTileDim = 256;
+
+constexpr size_t kEpf1InputRows = 7;
+constexpr size_t kEpf2InputRows = 3;
+
+// Per-frame decoder state. All the images here should be accessed through a
+// group rect (either with block units or pixel units).
+struct PassesDecoderState {
+  PassesSharedState shared_storage;
+  // Allows avoiding copies for encoder loop.
+  const PassesSharedState* JXL_RESTRICT shared = &shared_storage;
+
+  // Storage for RNG output for noise synthesis.
+  Image3F noise;
+
+  // Pointer to previous/next frame, to be added to the current one, if any.
+  // Gets updated by adding the currently decoded frame to it.
+  Image3F* JXL_RESTRICT frame_storage = nullptr;
+
+  // For ANS decoding.
+  std::vector<ANSCode> code;
+  std::vector<std::vector<uint8_t>> context_map;
+
+  bool keep_dct = false;
+
+  // Multiplier to be applied to the quant matrices of the x channel.
+  float x_dm_multiplier;
+
+  // Decoded image, with padding.
+  Image3F decoded;
+
+  // Sigma values for EPF, if enabled.
+  // Note that, for speed reasons, this is actually kInvSigmaNum / sigma.
+  ImageF sigma;
+
+  // Tile storage for ApplyImageFeatures steps. Storage1 has 2 blocks of padding
+  // per side, storage2 has 1.
+  std::vector<Image3F> storage1;
+  std::vector<Image3F> storage2;
+
+  void EnsureStorage(size_t num_threads) {
+    for (size_t i = storage1.size(); i < num_threads; i++) {
+      // We allocate twice what is needed since the last rects are larger in one
+      // dimension.
+      // Since we use row-based processing and cyclic addressing, we only need 7
+      // rows in storage1 and 3 in storage2.
+      storage1.push_back(
+          Image3F(kApplyImageFeaturesTileDim + 4 * kBlockDim, kEpf1InputRows));
+      storage2.push_back(
+          Image3F(kApplyImageFeaturesTileDim + 2 * kBlockDim, kEpf2InputRows));
+    }
+  }
+
+  // Initializes decoder-specific structures using information from *shared.
+  void Init(ThreadPool* pool) {
+    frame_storage = shared->multiframe->FrameStorage(
+        shared->frame_dim.xsize_padded, shared->frame_dim.ysize_padded);
+
+    if (shared->frame_header.color_transform == ColorTransform::kXYB) {
+      x_dm_multiplier =
+          std::pow(0.5f, 0.5f * shared->frame_header.x_qm_scale - 0.5f);
+    } else {
+      x_dm_multiplier = 1.0f;  // don't scale X quantization in YCbCr
+    }
+
+    if (shared->frame_header.flags & FrameHeader::kNoise) {
+      noise = Image3F(shared->frame_dim.xsize_padded,
+                      shared->frame_dim.ysize_padded);
+      PROFILER_ZONE("GenerateNoise");
+      auto generate_noise = [&](int group_index, int _) {
+        RandomImage3(shared->PaddedGroupRect(group_index), &noise);
+      };
+      RunOnPool(pool, 0, shared->frame_dim.num_groups, ThreadPool::SkipInit(),
+                generate_noise, "Generate noise");
+    }
+
+    const LoopFilter& lf = shared->image_features.loop_filter;
+    if (lf.epf || lf.gab) {
+      decoded = Image3F(shared->frame_dim.xsize_padded + 4 * kBlockDim,
+                        shared->frame_dim.ysize_padded + 4 * kBlockDim);
+#if MEMORY_SANITIZER
+      // Avoid errors due to loading vectors on the outermost padding.
+      ZeroFillImage(&decoded);
+#endif
+    }
+    if (lf.epf) {
+      sigma = ImageF(shared->frame_dim.xsize_blocks + 4,
+                     shared->frame_dim.ysize_blocks + 4);
+    }
+  }
+};
+
+// Temp images required for decoding a single group. Reduces memory allocations
+// for large images because we only initialize min(#threads, #groups) instances.
+struct GroupDecCache {
+  void InitOnce(size_t num_passes) {
+    PROFILER_FUNC;
+
+    if (num_passes != 0 && num_nzeroes[0].xsize() == 0) {
+      // Allocate enough for a whole group - partial groups on the right/bottom
+      // border just use a subset. The valid size is passed via Rect.
+
+      for (size_t i = 0; i < num_passes; i++) {
+        num_nzeroes[i] = Image3I(kGroupDimInBlocks, kGroupDimInBlocks);
+      }
+    }
+  }
+
+  // AC decoding
+  Image3I num_nzeroes[kMaxNumPasses];
+};
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_CACHE_H_
diff --git a/jxl/dec_context_map.cc b/jxl/dec_context_map.cc
new file mode 100644
index 000000000..3258e2750
--- /dev/null
+++ b/jxl/dec_context_map.cc
@@ -0,0 +1,123 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dec_context_map.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "jxl/ans_params.h"
+#include "jxl/base/status.h"
+#include "jxl/dec_ans.h"
+#include "jxl/enc_context_map.h"
+#include "jxl/entropy_coder.h"
+
+namespace jxl {
+
+namespace {
+
+void MoveToFront(uint8_t* v, uint8_t index) {
+  uint8_t value = v[index];
+  uint8_t i = index;
+  for (; i; --i) v[i] = v[i - 1];
+  v[0] = value;
+}
+
+void InverseMoveToFrontTransform(uint8_t* v, int v_len) {
+  uint8_t mtf[256];
+  int i;
+  for (i = 0; i < 256; ++i) {
+    mtf[i] = static_cast<uint8_t>(i);
+  }
+  for (i = 0; i < v_len; ++i) {
+    uint8_t index = v[i];
+    v[i] = mtf[index];
+    if (index) MoveToFront(mtf, index);
+  }
+}
+
+bool VerifyContextMap(const std::vector<uint8_t>& context_map,
+                      const size_t num_htrees) {
+  std::vector<bool> have_htree(num_htrees);
+  size_t num_found = 0;
+  for (const uint8_t htree : context_map) {
+    if (htree >= num_htrees) {
+      return JXL_FAILURE("Invalid histogram index in context map.");
+    }
+    if (!have_htree[htree]) {
+      have_htree[htree] = true;
+      ++num_found;
+    }
+  }
+  if (num_found != num_htrees) {
+    return JXL_FAILURE("Incomplete context map.");
+  }
+  return true;
+}
+
+}  // namespace
+
+HWY_ATTR bool DecodeContextMap(std::vector<uint8_t>* context_map,
+                               size_t* num_htrees, BitReader* input) {
+  bool is_simple = input->ReadFixedBits<1>();
+  if (is_simple) {
+    int bits_per_entry = input->ReadFixedBits<2>();
+    if (bits_per_entry != 0) {
+      for (size_t i = 0; i < context_map->size(); i++) {
+        (*context_map)[i] = input->ReadBits(bits_per_entry);
+      }
+    } else {
+      std::fill(context_map->begin(), context_map->end(), 0);
+    }
+  } else {
+    ANSCode code;
+    std::vector<uint8_t> dummy_ctx_map;
+    JXL_RETURN_IF_ERROR(
+        DecodeHistograms(input, 1, ANS_MAX_ALPHA_SIZE, &code, &dummy_ctx_map));
+    ANSSymbolReader reader(&code, input);
+    size_t i = 0;
+    while (i < context_map->size()) {
+      input->Refill();  // covers ReadSymbolWithoutRefill + PeekBits
+      uint32_t sym = reader.ReadSymbolWithoutRefill(0, input);
+      if (sym >= 8) {
+        uint32_t nbits = sym - 8;
+        // Max 32 bits for ReadBits, but max 31 bits for left shift.
+        if (nbits > 31) {
+          return JXL_FAILURE("Too many bits");
+        }
+        uint32_t bits = input->PeekBits(nbits);
+        input->Consume(nbits);
+        i += DecodeVarLenUint(nbits, bits) + 2;
+      } else {
+        uint32_t nbits = sym;
+        uint32_t bits = input->PeekBits(nbits);
+        input->Consume(nbits);
+        uint32_t cluster = DecodeVarLenUint(sym, bits);
+        if (cluster >= kMaxClusters) {
+          return JXL_FAILURE("Invalid cluster ID");
+        }
+        (*context_map)[i] = cluster;
+        i++;
+      }
+    }
+    if (!reader.CheckANSFinalState()) {
+      return JXL_FAILURE("Invalid context map");
+    }
+    InverseMoveToFrontTransform(context_map->data(), context_map->size());
+  }
+  *num_htrees = *std::max_element(context_map->begin(), context_map->end()) + 1;
+  return VerifyContextMap(*context_map, *num_htrees);
+}
+
+}  // namespace jxl
diff --git a/jxl/dec_context_map.h b/jxl/dec_context_map.h
new file mode 100644
index 000000000..edad87ea2
--- /dev/null
+++ b/jxl/dec_context_map.h
@@ -0,0 +1,36 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_CONTEXT_MAP_H_
+#define JXL_DEC_CONTEXT_MAP_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/dec_bit_reader.h"
+
+namespace jxl {
+
+// Reads the context map from the bit stream. On calling this function,
+// context_map->size() must be the number of possible context ids.
+// Sets *num_htrees to the number of different histogram ids in
+// *context_map.
+bool DecodeContextMap(std::vector<uint8_t>* context_map, size_t* num_htrees,
+                      BitReader* input);
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_CONTEXT_MAP_H_
diff --git a/jxl/dec_file.cc b/jxl/dec_file.cc
new file mode 100644
index 000000000..cf5066ebd
--- /dev/null
+++ b/jxl/dec_file.cc
@@ -0,0 +1,196 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dec_file.h"
+
+#include <stddef.h>
+
+#include <utility>
+#include <vector>
+
+#include "jpegxl/decode.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/override.h"
+#include "jxl/base/profiler.h"
+#include "jxl/brunsli.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_frame.h"
+#include "jxl/frame_header.h"
+#include "jxl/headers.h"
+#include "jxl/icc_codec.h"
+#include "jxl/image_bundle.h"
+#include "jxl/multiframe.h"
+
+namespace jxl {
+namespace {
+
+Status DecodePreview(const DecompressParams& dparams,
+                     BitReader* JXL_RESTRICT reader,
+                     const Span<const uint8_t> file, AuxOut* aux_out,
+                     ThreadPool* pool, CodecInOut* JXL_RESTRICT io) {
+  // No preview present in file.
+  if (!io->metadata.m2.have_preview) {
+    if (dparams.preview == Override::kOn) {
+      return JXL_FAILURE("preview == kOn but no preview present");
+    }
+    return true;
+  }
+
+  // Have preview; prepare to skip or read it.
+  JXL_RETURN_IF_ERROR(reader->JumpToByteBoundary());
+  FrameDimensions frame_dim;
+  frame_dim.Set(io->preview.xsize(), io->preview.ysize());
+
+  const AnimationHeader* animation = nullptr;
+  if (dparams.preview == Override::kOff) {
+    JXL_RETURN_IF_ERROR(SkipFrame(file, animation, &frame_dim, reader));
+    return true;
+  }
+
+  // Else: default or kOn => decode preview.
+  Multiframe multiframe;
+  JXL_RETURN_IF_ERROR(DecodeFrame(dparams, file, animation, &frame_dim,
+                                  &multiframe, pool, reader, aux_out,
+                                  &io->preview_frame));
+  io->dec_pixels += frame_dim.xsize * frame_dim.ysize;
+  return true;
+}
+
+Status DecodeHeaders(BitReader* reader, size_t* JXL_RESTRICT xsize,
+                     size_t* JXL_RESTRICT ysize, CodecInOut* io) {
+  SizeHeader size;
+  JXL_RETURN_IF_ERROR(ReadSizeHeader(reader, &size));
+  *xsize = size.xsize();
+  *ysize = size.ysize();
+
+  JXL_RETURN_IF_ERROR(ReadImageMetadata(reader, &io->metadata));
+
+  if (io->metadata.m2.have_preview) {
+    JXL_RETURN_IF_ERROR(ReadPreviewHeader(reader, &io->preview));
+  }
+
+  if (io->metadata.m2.have_animation) {
+    JXL_RETURN_IF_ERROR(ReadAnimationHeader(reader, &io->animation));
+  }
+
+  return true;
+}
+
+}  // namespace
+
+// To avoid the complexity of file I/O and buffering, we assume the bitstream
+// is loaded (or for large images/sequences: mapped into) memory.
+HWY_ATTR Status DecodeFile(const DecompressParams& dparams,
+                           const Span<const uint8_t> file,
+                           CodecInOut* JXL_RESTRICT io, AuxOut* aux_out,
+                           ThreadPool* pool) {
+  PROFILER_ZONE("DecodeFile uninstrumented");
+
+  io->enc_size = file.size();
+
+  // Marker
+  JpegxlSignature signature = JpegxlSignatureCheck(file.data(), file.size());
+
+  if (signature == JPEGXL_SIG_BRUNSLI) {
+    // Brunsli mode.
+    BrunsliDecoderMeta meta;
+    JXL_RETURN_IF_ERROR(
+        BrunsliToPixels(file, io, dparams.brunsli, &meta, pool));
+    io->CheckMetadata();
+    return true;
+  }
+
+  if (signature != JPEGXL_SIG_JPEGXL) {
+    return JXL_FAILURE("File does not start with JPEG XL marker");
+  }
+  if (file.data()[1] == kMarkerFlexible) {
+    return JXL_FAILURE("Flexible mode not yet supported");
+  }
+  Status ret = true;
+  {
+    BitReader reader(file);
+    BitReaderScopedCloser reader_closer(&reader, &ret);
+    (void)reader.ReadFixedBits<16>();  // skip marker
+
+    FrameDimensions main_frame_dim;
+    {
+      size_t xsize, ysize;
+      JXL_RETURN_IF_ERROR(DecodeHeaders(&reader, &xsize, &ysize, io));
+      JXL_RETURN_IF_ERROR(io->VerifyDimensions(xsize, ysize));
+      main_frame_dim.Set(xsize, ysize);
+    }
+
+    if (io->metadata.have_icc) {
+      PaddedBytes icc;
+      JXL_RETURN_IF_ERROR(ReadICC(&reader, &icc));
+      JXL_RETURN_IF_ERROR(ColorManagement::SetProfile(
+          std::move(icc), &io->metadata.color_encoding));
+    } else {
+      JXL_RETURN_IF_ERROR(
+          ColorManagement::CreateProfile(&io->metadata.color_encoding));
+    }
+
+    JXL_RETURN_IF_ERROR(
+        DecodePreview(dparams, &reader, file, aux_out, pool, io));
+
+    // Only necessary if no ICC and no preview.
+    JXL_RETURN_IF_ERROR(reader.JumpToByteBoundary());
+
+    Multiframe multiframe;
+
+    io->frames.clear();
+    io->animation_frames.clear();
+    if (io->metadata.m2.have_animation) {
+      do {
+        io->animation_frames.emplace_back();
+        io->frames.emplace_back(&io->metadata);
+        FrameDimensions frame_dim;
+        // Skip frames that are not displayed.
+        do {
+          frame_dim = main_frame_dim;
+          JXL_RETURN_IF_ERROR(DecodeFrame(dparams, file, &io->animation,
+                                          &frame_dim, &multiframe, pool,
+                                          &reader, aux_out, &io->frames.back(),
+                                          &io->animation_frames.back()));
+        } while (!multiframe.IsDisplayed());
+        io->dec_pixels += frame_dim.xsize * frame_dim.ysize;
+      } while (!io->animation_frames.back().is_last);
+    } else {
+      io->frames.emplace_back(&io->metadata);
+      FrameDimensions frame_dim;
+      // Skip frames that are not displayed.
+      do {
+        frame_dim = main_frame_dim;
+        JXL_RETURN_IF_ERROR(DecodeFrame(
+            dparams, file, /*animation=*/nullptr, &frame_dim, &multiframe, pool,
+            &reader, aux_out, &io->frames.back(), /*animation_frame=*/nullptr));
+      } while (!multiframe.IsDisplayed());
+      io->dec_pixels += frame_dim.xsize * frame_dim.ysize;
+    }
+
+    if (dparams.check_decompressed_size && dparams.max_downsampling == 1) {
+      if (reader.TotalBitsConsumed() != file.size() * kBitsPerByte) {
+        return JXL_FAILURE("DecodeFile reader position not at EOF.");
+      }
+    }
+
+    io->CheckMetadata();
+    // reader is closed here.
+  }
+  return ret;
+}
+
+}  // namespace jxl
diff --git a/jxl/dec_file.h b/jxl/dec_file.h
new file mode 100644
index 000000000..498c0e1ee
--- /dev/null
+++ b/jxl/dec_file.h
@@ -0,0 +1,51 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_FILE_H_
+#define JXL_DEC_FILE_H_
+
+// Top-level interface for JXL decoding.
+
+#include <stdint.h>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/dec_params.h"
+
+namespace jxl {
+
+// Implementation detail: currently decodes to linear sRGB. The contract is:
+// `io` appears 'identical' (modulo compression artifacts) to the encoder input
+// in a color-aware viewer. Note that `io->metadata.color_encoding`
+// identifies the color space that was passed to the encoder; clients that want
+// that same encoding must call `io->TransformTo` afterwards.
+Status DecodeFile(const DecompressParams& params,
+                  const Span<const uint8_t> file, CodecInOut* io,
+                  AuxOut* aux_out = nullptr, ThreadPool* pool = nullptr);
+
+static inline Status DecodeFile(const DecompressParams& params,
+                                const PaddedBytes& file, CodecInOut* io,
+                                AuxOut* aux_out = nullptr,
+                                ThreadPool* pool = nullptr) {
+  return DecodeFile(params, Span<const uint8_t>(file), io, aux_out, pool);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_FILE_H_
diff --git a/jxl/dec_frame.cc b/jxl/dec_frame.cc
new file mode 100644
index 000000000..f24a43ad3
--- /dev/null
+++ b/jxl/dec_frame.cc
@@ -0,0 +1,692 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dec_frame.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <algorithm>
+#include <atomic>
+#include <utility>
+#include <vector>
+
+#include "jxl/ac_context.h"
+#include "jxl/ac_strategy.h"
+#include "jxl/ans_params.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/brunsli.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/coeff_order.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/compressed_dc.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_cache.h"
+#include "jxl/dec_group.h"
+#include "jxl/dec_modular.h"
+#include "jxl/dec_params.h"
+#include "jxl/dec_reconstruct.h"
+#include "jxl/dec_xyb.h"
+#include "jxl/dot_dictionary.h"
+#include "jxl/fields.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/loop_filter.h"
+#include "jxl/multiframe.h"
+#include "jxl/noise.h"
+#include "jxl/passes_state.h"
+#include "jxl/patch_dictionary.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+#include "jxl/splines.h"
+#include "jxl/toc.h"
+
+namespace jxl {
+
+// NOLINTNEXTLINE(clang-analyzer-optin.performance.Padding)
+class LossyFrameDecoder {
+ public:
+  Status Init(const FrameHeader& frame_header, const LoopFilter& loop_filter,
+              const ImageMetadata& image_metadata,
+              const FrameDimensions& frame_dim, Multiframe* multiframe,
+              size_t downsampling, ThreadPool* pool, AuxOut* aux_out) {
+    downsampling_ = downsampling;
+    pool_ = pool;
+    aux_out_ = aux_out;
+    if (frame_header.IsLossy() &&
+        frame_header.chroma_subsampling != YCbCrChromaSubsampling::k444) {
+      return JXL_FAILURE("Chroma subsampling is not allowed in Lossy mode");
+    }
+    return InitializePassesSharedState(frame_header, loop_filter,
+                                       image_metadata, frame_dim, multiframe,
+                                       &dec_state_.shared_storage);
+  }
+
+  // Sets the number of threads that will be used. The value of the "thread"
+  // parameter passed to DecodeDCGroup and DecodeACGroup must be smaller than
+  // the "num_threads" passed here.
+  void SetNumThreads(size_t num_threads) {
+    group_dec_caches_.resize(num_threads);
+    dec_state_.EnsureStorage(num_threads);
+  }
+
+  Status DecodeGlobalDCInfo(BitReader* reader) {
+    PROFILER_FUNC;
+    JXL_RETURN_IF_ERROR(dec_state_.shared_storage.matrices.DecodeDC(reader));
+    JXL_RETURN_IF_ERROR(dec_state_.shared_storage.quantizer.Decode(reader));
+
+    JXL_RETURN_IF_ERROR(dec_state_.shared_storage.cmap.DecodeDC(reader));
+
+    // Pre-compute info for decoding a group.
+    if (dec_state_.keep_dct) {
+      dec_state_.shared_storage.quantizer.ClearDCMul();  // Don't dequant DC
+    }
+
+    for (size_t c = 0; c < 3; c++) {
+      FillImage(dec_state_.shared_storage.quantizer.GetDcStep(c),
+                const_cast<ImageF*>(
+                    &dec_state_.shared_storage.dc_quant_field.Plane(c)));
+    }
+    if (downsampling_ != 16) {
+      dec_state_.shared_storage.ac_strategy.FillInvalid();
+    } else {
+      dec_state_.shared_storage.ac_strategy.FillDCT8();
+    }
+    dec_state_.Init(pool_);
+    return true;
+  }
+  Status DecodeModularGlobalInfo(BitReader* reader) {
+    // currently just initializes the dec_state
+    // TODO: add patches and maybe other features
+    dec_state_.Init(pool_);
+    return true;
+  }
+
+  Status DecodeDCGroup(size_t group_index, size_t thread, const Rect& rect,
+                       BitReader* reader, const LoopFilter& loop_filter,
+                       AuxOut* local_aux_out) {
+    return jxl::DecodeDCGroup(reader, group_index, &dec_state_, local_aux_out);
+  }
+
+  HWY_ATTR Status DecodeGlobalACInfo(BitReader* reader) {
+    uint64_t flags = dec_state_.shared_storage.frame_header.flags;
+    if (!(flags & FrameHeader::kSkipAdaptiveDCSmoothing) &&
+        !(flags & FrameHeader::kUseDcFrame)) {
+      AdaptiveDCSmoothing(dec_state_.shared_storage.dc_quant_field,
+                          &dec_state_.shared_storage.dc_storage, pool_);
+    }
+
+    if (aux_out_ && aux_out_->testing_aux.dc) {
+      *aux_out_->testing_aux.dc = CopyImage(*dec_state_.shared_storage.dc);
+    }
+
+    if (dec_state_.shared_storage.experiments.use_new_cmap) {
+      DecodeFullColorMap(reader, &dec_state_.shared_storage.cmap,
+                         dec_state_.shared_storage.experiments.use_new_cmap);
+    }
+
+    JXL_RETURN_IF_ERROR(dec_state_.shared_storage.matrices.Decode(reader));
+
+    size_t num_histo_bits =
+        dec_state_.shared->frame_dim.num_groups == 1
+            ? 0
+            : CeilLog2Nonzero(dec_state_.shared->frame_dim.num_groups - 1);
+    dec_state_.shared_storage.num_histograms =
+        1 + reader->ReadBits(num_histo_bits);
+    dec_state_.code.resize(kMaxNumPasses * dec_state_.shared->num_histograms);
+    dec_state_.context_map.resize(kMaxNumPasses *
+                                  dec_state_.shared->num_histograms);
+    // Read coefficient orders and histograms.
+    for (size_t i = 0;
+         i < dec_state_.shared_storage.frame_header.passes.num_passes; i++) {
+      for (size_t histo = 0; histo < dec_state_.shared->num_histograms;
+           histo++) {
+        uint16_t used_orders = U32Coder::Read(kOrderEnc, reader);
+        size_t idx =
+            histo * dec_state_.shared_storage.frame_header.passes.num_passes +
+            i;
+        JXL_RETURN_IF_ERROR(DecodeCoeffOrders(
+            used_orders,
+            &dec_state_.shared_storage.coeff_orders[idx * kCoeffOrderSize],
+            reader));
+      }
+      for (size_t histo = 0; histo < dec_state_.shared->num_histograms;
+           histo++) {
+        size_t idx =
+            histo * dec_state_.shared_storage.frame_header.passes.num_passes +
+            i;
+        JXL_RETURN_IF_ERROR(DecodeHistograms(
+            reader, kNumContexts, ANS_MAX_ALPHA_SIZE, &dec_state_.code[idx],
+            &dec_state_.context_map[idx]));
+      }
+    }
+    return true;
+  }
+
+  Status DecodeACGroup(size_t group_index, size_t thread,
+                       BitReader* JXL_RESTRICT* JXL_RESTRICT readers,
+                       size_t num_passes, Image3F* JXL_RESTRICT opsin,
+                       AuxOut* local_aux_out) {
+    return DecodeGroup(readers, num_passes, group_index, &dec_state_,
+                       &group_dec_caches_[thread], thread, opsin,
+                       local_aux_out);
+  }
+
+  Status FinalizeJPEG(Image3F* JXL_RESTRICT opsin, ImageBundle* decoded) {
+    decoded->SetFromImage(std::move(*opsin),
+                          decoded->metadata()->color_encoding);
+    decoded->color_transform =
+        dec_state_.shared_storage.frame_header.color_transform;
+    decoded->chroma_subsampling =
+        dec_state_.shared_storage.frame_header.chroma_subsampling;
+    const std::vector<QuantEncoding>& qe =
+        dec_state_.shared_storage.matrices.encodings();
+    if (qe.size() == 0 || qe[0].mode != QuantEncoding::Mode::kQuantModeRAW ||
+        qe[0].qraw.qtable_den_shift != 0) {
+      return JXL_FAILURE(
+          "Quantization table is not a JPEG quantization table.");
+    }
+    decoded->jpeg_quant_table.resize(192);
+    JXL_DASSERT(qe[0].qraw.qtable);
+    JXL_DASSERT(qe[0].qraw.qtable->size() >= 3 * 8 * 8);
+    for (size_t c = 0; c < 3; c++) {
+      for (size_t y = 0; y < 8; y++) {
+        for (size_t x = 0; x < 8; x++) {
+          // JPEG XL transposes the DCT, JPEG doesn't.
+          decoded->jpeg_quant_table[c * 64 + 8 * x + y] =
+              (*qe[0].qraw.qtable)[c * 64 + 8 * y + x];
+        }
+      }
+    }
+    return true;
+  }
+
+  Status FinalizeDecoding(Image3F* JXL_RESTRICT opsin,
+                          const LoopFilter& loop_filter, ImageBundle* decoded) {
+    if (dec_state_.keep_dct) {
+      JXL_RETURN_IF_ERROR(FinalizeJPEG(opsin, decoded));
+      return true;
+    }
+    JXL_RETURN_IF_ERROR(
+        FinalizeFrameDecoding(opsin, &dec_state_, pool_, aux_out_,
+                              /*save_decompressed=*/true,
+                              /*apply_color_transform=*/
+                              dec_state_.shared->multiframe->IsDisplayed()));
+
+    const float intensity_multiplier =
+        decoded->metadata()->IntensityTarget() * kIntensityMultiplier;
+    if (intensity_multiplier != 1.f) {
+      ScaleImage(1.f / intensity_multiplier, opsin);
+    }
+
+    if (dec_state_.shared_storage.frame_header.color_transform ==
+        ColorTransform::kXYB) {
+      // Do not use decoded->IsGray() - c_current is not yet valid.
+      const bool is_gray = decoded->metadata()->color_encoding.IsGray();
+      decoded->SetFromImage(std::move(*opsin),
+                            ColorManagement::LinearSRGB(is_gray));
+    } else {
+      decoded->SetFromImage(std::move(*opsin),
+                            decoded->metadata()->color_encoding);
+    }
+    return true;
+  }
+
+  PassesDecoderState* State() { return &dec_state_; }
+
+  void KeepDCT() { dec_state_.keep_dct = true; }
+
+ private:
+  PassesDecoderState dec_state_;
+  size_t downsampling_;
+
+  ThreadPool* pool_;
+  AuxOut* aux_out_;
+
+  std::vector<GroupDecCache> group_dec_caches_;
+};
+
+namespace {
+
+Status DecodeFrameHeader(const AnimationHeader* animation_or_null,
+                         BitReader* JXL_RESTRICT reader,
+                         FrameHeader* JXL_RESTRICT frame_header,
+                         FrameDimensions* frame_dim,
+                         LoopFilter* JXL_RESTRICT loop_filter) {
+  frame_header->animation_frame.nonserialized_have_timecode =
+      animation_or_null ? animation_or_null->have_timecodes : false;
+  JXL_RETURN_IF_ERROR(ReadFrameHeader(reader, frame_header));
+
+  if (frame_header->animation_frame.have_crop) {
+    frame_dim->Set(frame_header->animation_frame.xsize,
+                   frame_header->animation_frame.ysize);
+  }
+
+  if (frame_header->dc_level != 0) {
+    frame_dim->Set(
+        DivCeil(frame_dim->xsize, 1 << (3 * frame_header->dc_level)),
+        DivCeil(frame_dim->ysize, 1 << (3 * frame_header->dc_level)));
+  }
+
+  if (frame_header->IsLossy()) {
+    JXL_RETURN_IF_ERROR(ReadLoopFilter(reader, loop_filter));
+  }
+
+  return true;
+}
+
+}  // namespace
+
+Status SkipFrame(const Span<const uint8_t> file,
+                 const AnimationHeader* animation_or_null,
+                 FrameDimensions* JXL_RESTRICT frame_dim,
+                 BitReader* JXL_RESTRICT reader) {
+  FrameHeader frame_header;
+  LoopFilter loop_filter;
+  JXL_RETURN_IF_ERROR(DecodeFrameHeader(
+      animation_or_null, reader, &frame_header, frame_dim, &loop_filter));
+
+  // Read TOC.
+  std::vector<uint64_t> group_offsets;
+  const bool has_ac_global = !frame_header.IsJpeg();
+  const size_t toc_entries =
+      NumTocEntries(frame_dim->num_groups, frame_dim->num_dc_groups,
+                    frame_header.passes.num_passes, has_ac_global);
+  JXL_RETURN_IF_ERROR(ReadGroupOffsets(toc_entries, reader, &group_offsets));
+
+  // Pretend all groups are read.
+  reader->SkipBits(group_offsets.back() * kBitsPerByte);
+  if (reader->TotalBitsConsumed() > file.size() * kBitsPerByte) {
+    return JXL_FAILURE("Group code extends after stream end");
+  }
+
+  return true;
+}
+
+Status DecodeFrame(const DecompressParams& dparams,
+                   const Span<const uint8_t> file,
+                   const AnimationHeader* animation_or_null,
+                   FrameDimensions* frame_dim,
+                   Multiframe* JXL_RESTRICT multiframe,
+                   ThreadPool* JXL_RESTRICT pool,
+                   BitReader* JXL_RESTRICT reader, AuxOut* JXL_RESTRICT aux_out,
+                   ImageBundle* decoded, AnimationFrame* animation) {
+  PROFILER_ZONE("DecodeFrame uninstrumented");
+
+  FrameHeader frame_header;
+  LoopFilter loop_filter;
+  JXL_RETURN_IF_ERROR(DecodeFrameHeader(
+      animation_or_null, reader, &frame_header, frame_dim, &loop_filter));
+  const size_t num_passes = frame_header.passes.num_passes;
+  const size_t xsize = frame_dim->xsize;
+  const size_t ysize = frame_dim->ysize;
+  const size_t num_groups = frame_dim->num_groups;
+
+  if (animation != nullptr) {
+    *animation = frame_header.animation_frame;
+  }
+
+  // If the previous frame was not displayed, `decoded` may have different
+  // dimensions; must reset to avoid error when setting alpha.
+  decoded->RemoveColor();
+  if (frame_header.HasAlpha()) {
+    decoded->SetAlpha(ImageU(xsize, ysize));
+  }
+
+  if (decoded->metadata()->m2.HasDepth())
+    decoded->SetDepth(
+        ImageU(decoded->DepthSize(xsize), decoded->DepthSize(ysize)));
+  if (decoded->metadata()->m2.num_extra_channels > 0 &&
+      frame_header.IsDisplayed()) {
+    std::vector<ImageU> ecv;
+    for (size_t i = 0; i < decoded->metadata()->m2.num_extra_channels; i++)
+      ecv.push_back(ImageU(xsize, ysize));
+    decoded->SetExtraChannels(std::move(ecv));
+  }
+  if (frame_header.encoding == FrameEncoding::kModularGroup) {
+    decoded->SetFromImage(Image3F(xsize, ysize),
+                          decoded->metadata()->color_encoding);
+    loop_filter.gab = false;
+    loop_filter.epf = false;
+  }
+
+  // Handling of progressive decoding for kPasses mode.
+  size_t downsampling;
+  size_t max_passes = dparams.max_passes;
+  // TODO(veluca): deal with downsamplings >= 8.
+  if (dparams.max_downsampling >= 8) {
+    downsampling = 8;
+    max_passes = 0;
+  } else {
+    downsampling = 1;
+    for (uint32_t i = 0; i < frame_header.passes.num_downsample; ++i) {
+      if (dparams.max_downsampling >= frame_header.passes.downsample[i] &&
+          max_passes > frame_header.passes.last_pass[i]) {
+        downsampling = frame_header.passes.downsample[i];
+        max_passes = frame_header.passes.last_pass[i] + 1;
+      }
+    }
+  }
+  // Do not use downsampling for animation frames or non-displayed frames.
+  if (!multiframe->IsDisplayed() || !frame_header.animation_frame.is_last) {
+    downsampling = 1;
+    max_passes = frame_header.passes.num_passes;
+  }
+  if (aux_out != nullptr) {
+    aux_out->downsampling = downsampling;
+  }
+
+  multiframe->StartFrame(frame_header);
+
+  LossyFrameDecoder lossy_frame_decoder;
+  JXL_RETURN_IF_ERROR(lossy_frame_decoder.Init(
+      frame_header, loop_filter, *decoded->metadata(), *frame_dim, multiframe,
+      downsampling, pool, aux_out));
+  BrunsliFrameDecoder jpeg_frame_decoder(pool);
+  ModularFrameDecoder modular_frame_decoder;
+
+  if (dparams.keep_dct) lossy_frame_decoder.KeepDCT();
+
+  // Read TOC.
+  std::vector<uint64_t> group_offsets;
+  const bool has_ac_global = !frame_header.IsJpeg();
+  const size_t toc_entries = NumTocEntries(num_groups, frame_dim->num_dc_groups,
+                                           num_passes, has_ac_global);
+  JXL_RETURN_IF_ERROR(ReadGroupOffsets(toc_entries, reader, &group_offsets));
+  const size_t global_ac_index = frame_dim->num_dc_groups + 1;
+
+  JXL_DASSERT((reader->TotalBitsConsumed() % kBitsPerByte) == 0);
+  const size_t group_codes_begin = reader->TotalBitsConsumed() / kBitsPerByte;
+
+  auto get_reader = [num_groups, num_passes, group_codes_begin, &group_offsets,
+                     &file,
+                     &reader](BitReader* JXL_RESTRICT store, size_t index) {
+    if (num_groups == 1 && num_passes == 1) return reader;
+    const size_t group_offset = group_codes_begin + group_offsets[index];
+    const size_t group_size =
+        group_codes_begin + group_offsets[index + 1] - group_offset;
+    const size_t remaining_size = file.size() - group_offset;
+    const size_t size = std::min(group_size + 8, remaining_size);
+    *store = BitReader(Span<const uint8_t>(file.data() + group_offset, size));
+    return store;
+  };
+
+  BitReader global_store;
+  BitReader* global_reader = get_reader(&global_store, 0);
+
+  {
+    PassesSharedState& shared = lossy_frame_decoder.State()->shared_storage;
+
+    if (shared.frame_header.flags & FrameHeader::kPatches) {
+      JXL_RETURN_IF_ERROR(shared.image_features.patches.Decode(
+          global_reader, shared.frame_dim.xsize_padded,
+          shared.frame_dim.ysize_padded,
+          shared.frame_header.save_as_reference));
+    }
+    if (shared.frame_header.flags & FrameHeader::kSplines) {
+      JXL_RETURN_IF_ERROR(shared.image_features.splines.Decode(global_reader));
+    }
+    if (shared.frame_header.flags & FrameHeader::kNoise) {
+      JXL_RETURN_IF_ERROR(
+          DecodeNoise(global_reader, &shared.image_features.noise_params));
+    }
+  }
+
+  if (frame_header.IsJpeg()) {
+    JXL_RETURN_IF_ERROR(jpeg_frame_decoder.ReadHeader(
+        frame_dim, global_reader, frame_header.chroma_subsampling));
+  } else if (frame_header.IsLossy()) {
+    JXL_RETURN_IF_ERROR(lossy_frame_decoder.DecodeGlobalDCInfo(global_reader));
+  } else if (frame_header.encoding == FrameEncoding::kModularGroup) {
+    JXL_RETURN_IF_ERROR(
+        lossy_frame_decoder.DecodeModularGlobalInfo(global_reader));
+  }
+  JXL_RETURN_IF_ERROR(modular_frame_decoder.DecodeGlobalInfo(
+      global_reader, frame_header, decoded,
+      /*decode_color = */
+      (frame_header.encoding == FrameEncoding::kModularGroup), xsize, ysize));
+  // global_store is either never used (in which case Close() is optional
+  // and a no-op) or the real deal and we must call Close().
+  JXL_RETURN_IF_ERROR(global_store.Close());
+
+  // Decode DC groups.
+  std::atomic<int> num_errors{0};
+  std::vector<AuxOut> aux_outs;
+  const auto resize_aux_outs = [&](size_t num_threads) {
+    // Updates aux_outs size Assimilating its elements if the size decreases.
+    if (aux_out != nullptr) {
+      for (size_t i = num_threads; i < aux_outs.size(); i++) {
+        aux_out->Assimilate(aux_outs[i]);
+      }
+      aux_outs.resize(num_threads);
+    }
+  };
+
+  JXL_RETURN_IF_ERROR(num_errors.load(std::memory_order_relaxed) == 0);
+  decoded->SetDecodedBytes(group_offsets[1] + group_codes_begin);
+
+  const auto process_dc_group_init = [&](size_t num_threads) {
+    lossy_frame_decoder.SetNumThreads(num_threads);
+    resize_aux_outs(num_threads);
+    return true;
+  };
+  const auto process_dc_group = [&](const int group_index, const int thread) {
+    PROFILER_ZONE("DC group");
+    BitReader group_store;
+    BitReader* group_reader = get_reader(&group_store, group_index + 1);
+    const size_t gx = group_index % frame_dim->xsize_dc_groups;
+    const size_t gy = group_index / frame_dim->xsize_dc_groups;
+    AuxOut* my_aux_out = aux_out ? &aux_outs[thread] : nullptr;
+    bool ok = true;
+    const Rect mrect(gx * kDcGroupDim, gy * kDcGroupDim, kDcGroupDim,
+                     kDcGroupDim);
+    ok = modular_frame_decoder.DecodeGroup(dparams, mrect, group_reader,
+                                           my_aux_out, 3, 1000);
+    if (frame_header.IsJpeg()) {
+      ok &= jpeg_frame_decoder.DecodeDcGroup(group_index, group_reader);
+    } else if (frame_header.IsLossy()) {
+      const Rect rect(gx * kDcGroupDimInBlocks, gy * kDcGroupDimInBlocks,
+                      kDcGroupDimInBlocks, kDcGroupDimInBlocks,
+                      frame_dim->xsize_blocks, frame_dim->ysize_blocks);
+      ok &= lossy_frame_decoder.DecodeDCGroup(
+          group_index, thread, rect, group_reader, loop_filter, my_aux_out);
+    }
+    if (!ok || !group_store.Close()) {
+      num_errors.fetch_add(1, std::memory_order_relaxed);
+    }
+  };
+  RunOnPool(pool, 0, frame_dim->num_dc_groups, process_dc_group_init,
+            process_dc_group, "DecodeDCGroup");
+  JXL_RETURN_IF_ERROR(num_errors.load(std::memory_order_relaxed) == 0);
+  if (downsampling < 16 && frame_dim->num_groups > 1) {
+    decoded->SetDecodedBytes(group_offsets[frame_dim->num_dc_groups + 1] +
+                             group_codes_begin);
+  }
+
+  Image3F opsin(frame_dim->xsize_blocks * kBlockDim,
+                frame_dim->ysize_blocks * kBlockDim);
+
+  // Read global AC info.
+  {
+    PROFILER_ZONE("Global AC");
+    BitReader ac_info_store;
+    BitReader* ac_info_reader = get_reader(&ac_info_store, global_ac_index);
+
+    if (frame_header.IsLossy()) {
+      JXL_RETURN_IF_ERROR(
+          lossy_frame_decoder.DecodeGlobalACInfo(ac_info_reader));
+    }
+
+    // ac_info_store is either never used (in which case Close() is optional
+    // and a no-op) or the real deal and we must call Close().
+    JXL_RETURN_IF_ERROR(ac_info_store.Close());
+  }
+
+  max_passes = std::min<size_t>(max_passes, frame_header.passes.num_passes);
+
+  // Decode groups.
+  const auto process_group_init = [&](size_t num_threads) {
+    // The number of threads here might be different from the previous run, so
+    // we need to re-update them.
+    lossy_frame_decoder.SetNumThreads(num_threads);
+    resize_aux_outs(num_threads);
+    return true;
+  };
+
+  const auto process_group = [&](const int task, const int thread) {
+    PROFILER_ZONE("process_group");
+    const size_t group_index = static_cast<size_t>(task);
+    const size_t gx = group_index % frame_dim->xsize_groups;
+    const size_t gy = group_index / frame_dim->xsize_groups;
+    const size_t x = gx * kGroupDim;
+    const size_t y = gy * kGroupDim;
+    Rect rect(x, y, kGroupDim, kGroupDim, xsize, ysize);
+
+    AuxOut* my_aux_out = aux_out ? &aux_outs[thread] : nullptr;
+    BitReader pass0_reader_store;
+    BitReader* pass0_group_reader =
+        get_reader(&pass0_reader_store,
+                   AcGroupIndex(0, group_index, num_groups,
+                                frame_dim->num_dc_groups, has_ac_global));
+
+    // don't limit to image dimensions here (is done in DecodeGroup)
+    const Rect mrect(x, y, kGroupDim, kGroupDim);
+
+    // Read passes.
+    BitReader storage[kMaxNumPasses];
+    BitReader* JXL_RESTRICT readers[kMaxNumPasses];
+    readers[0] = pass0_group_reader;
+    for (size_t i = 1; i < max_passes; i++) {
+      readers[i] =
+          get_reader(&storage[i - 1],
+                     AcGroupIndex(i, group_index, num_groups,
+                                  frame_dim->num_dc_groups, has_ac_global));
+    }
+    int minShift = 0;
+    int maxShift = 2;
+    for (size_t i = 0; i < max_passes; i++) {
+      for (uint32_t j = 0; j < frame_header.passes.num_downsample; ++j) {
+        if (i <= frame_header.passes.last_pass[j]) {
+          if (frame_header.passes.downsample[j] == 8) minShift = 3;
+          if (frame_header.passes.downsample[j] == 4) minShift = 2;
+          if (frame_header.passes.downsample[j] == 2) minShift = 1;
+          if (frame_header.passes.downsample[j] == 1) minShift = 0;
+        }
+      }
+      if (!modular_frame_decoder.DecodeGroup(dparams, mrect, readers[i],
+                                             my_aux_out, minShift, maxShift)) {
+        num_errors.fetch_add(1, std::memory_order_relaxed);
+      }
+      maxShift = minShift - 1;
+      minShift = 0;
+    }
+    if (frame_header.IsLossy()) {
+      if (!lossy_frame_decoder.DecodeACGroup(group_index, thread, readers,
+                                             max_passes, &opsin, my_aux_out)) {
+        num_errors.fetch_add(1, std::memory_order_relaxed);
+      }
+    }
+    for (size_t i = 1; i < max_passes; i++) {
+      if (!storage[i - 1].Close()) {
+        num_errors.fetch_add(1, std::memory_order_relaxed);
+      }
+    }
+
+    bool ok = true;
+    if (frame_header.IsJpeg()) {
+      ok = jpeg_frame_decoder.DecodeAcGroup(group_index, pass0_group_reader,
+                                            &opsin, rect);
+    }
+
+    if (!ok) {
+      num_errors.fetch_add(1, std::memory_order_relaxed);
+      return;
+    }
+
+    if (!pass0_reader_store.Close()) {
+      num_errors.fetch_add(1, std::memory_order_relaxed);
+    }
+  };
+
+  {
+    PROFILER_ZONE("DecodeFrame pool");
+    RunOnPool(pool, 0, num_groups, process_group_init, process_group,
+              "DecodeFrame");
+    JXL_RETURN_IF_ERROR(num_errors.load(std::memory_order_relaxed) == 0);
+  }
+
+  if (downsampling < 8 && frame_dim->num_groups > 1) {
+    decoded->SetDecodedBytes(
+        group_offsets[AcGroupIndex(max_passes, 0, num_groups,
+                                   frame_dim->num_dc_groups, has_ac_global)] +
+        group_codes_begin);
+  }
+
+  // Resizing to 0 assimilates all the results when needed.
+  resize_aux_outs(0);
+  // undo global modular transforms and copy int pixel buffers to float ones
+  JXL_RETURN_IF_ERROR(modular_frame_decoder.FinalizeDecoding(
+      &opsin, decoded, pool, frame_header));
+  if (frame_header.IsJpeg()) {
+    jpeg_frame_decoder.FinalizeDecoding(&opsin);
+    decoded->SetFromImage(std::move(opsin),
+                          decoded->metadata()->color_encoding);
+  } else {
+    JXL_RETURN_IF_ERROR(
+        lossy_frame_decoder.FinalizeDecoding(&opsin, loop_filter, decoded));
+  }
+
+  if (num_groups == 1 && num_passes == 1) {
+    // get_reader used reader, so AC groups have already been consumed by
+    // reader - unless we did not decode all passes.
+    if (max_passes < num_passes) {
+      const size_t frame_end =
+          group_codes_begin + group_offsets.back() * kBitsPerByte;
+      reader->SkipBits(frame_end - reader->TotalBitsConsumed());
+    }
+  } else {
+    // Used per-group readers, need to skip groups in main reader.
+    reader->SkipBits(group_offsets.back() * kBitsPerByte);  // aligned
+  }
+
+  if (!frame_header.IsLossy() && dparams.keep_dct) {
+    return JXL_FAILURE("Cannot decode this frame mode to JPEG.\n");
+  }
+
+  JXL_RETURN_IF_ERROR(reader->JumpToByteBoundary());
+  if ((reader->TotalBitsConsumed()) > file.size() * kBitsPerByte) {
+    return JXL_FAILURE("Read past stream end");
+  }
+
+  if (!dparams.keep_dct) {
+    decoded->ShrinkTo(xsize, ysize);
+  } else {
+    decoded->jpeg_xsize = xsize;
+    decoded->jpeg_ysize = ysize;
+  }
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/dec_frame.h b/jxl/dec_frame.h
new file mode 100644
index 000000000..4f2a06119
--- /dev/null
+++ b/jxl/dec_frame.h
@@ -0,0 +1,62 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_FRAME_H_
+#define JXL_DEC_FRAME_H_
+
+#include <stdint.h>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_cache.h"
+#include "jxl/dec_params.h"
+#include "jxl/frame_header.h"
+#include "jxl/headers.h"
+#include "jxl/image_bundle.h"
+#include "jxl/multiframe.h"
+
+namespace jxl {
+
+// Decodes a frame, either a single image or animation frame (depending on
+// `multiframe`). Groups may be processed in parallel by `pool`.
+// `frame_dim` must already be set from SizeHeader and may be overridden if
+// animation_frame.have_crop.
+// See DecodeFile for explanation of c_decoded.
+// decoded->metadata must already be set!
+Status DecodeFrame(const DecompressParams& dparams,
+                   const Span<const uint8_t> file,
+                   const AnimationHeader* animation_or_null,
+                   FrameDimensions* JXL_RESTRICT frame_dim,
+                   Multiframe* JXL_RESTRICT multiframe,
+                   ThreadPool* JXL_RESTRICT pool,
+                   BitReader* JXL_RESTRICT reader, AuxOut* JXL_RESTRICT aux_out,
+                   ImageBundle* decoded, AnimationFrame* animation = nullptr);
+
+// Leaves reader in the same state as DecodeFrame would. Used to skip preview.
+// `frame_dim` must already be set from SizeHeader and may be overridden if
+// animation_frame.have_crop.
+Status SkipFrame(const Span<const uint8_t> file,
+                 const AnimationHeader* animation_or_null,
+                 FrameDimensions* JXL_RESTRICT frame_dim,
+                 BitReader* JXL_RESTRICT reader);
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_FRAME_H_
diff --git a/jxl/dec_group.cc b/jxl/dec_group.cc
new file mode 100644
index 000000000..a9a1ad8eb
--- /dev/null
+++ b/jxl/dec_group.cc
@@ -0,0 +1,678 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dec_group.h"
+
+#include <stdint.h>
+#include <string.h>
+
+#include <algorithm>
+#include <hwy/static_targets.h>
+#include <utility>
+
+#include "jxl/ac_context.h"
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/coeff_order.h"
+#include "jxl/common.h"
+#include "jxl/convolve.h"
+#include "jxl/dec_cache.h"
+#include "jxl/dec_reconstruct.h"
+#include "jxl/dec_xyb.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/epf.h"
+#include "jxl/opsin_params.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+namespace {
+using D = HWY_FULL(float);
+using DU = HWY_FULL(uint32_t);
+constexpr D d;
+
+template <class V>
+HWY_ATTR void DequantLane(V scaled_dequant, V x_dm_multiplier,
+                          const float* JXL_RESTRICT dequant_matrices,
+                          size_t dq_ofs, size_t size, size_t k, V x_cc_mul,
+                          V b_cc_mul, const float* JXL_RESTRICT biases,
+                          float* JXL_RESTRICT block) {
+  const auto x_mul =
+      Load(d, dequant_matrices + dq_ofs + k) * scaled_dequant * x_dm_multiplier;
+  const auto y_mul =
+      Load(d, dequant_matrices + dq_ofs + size + k) * scaled_dequant;
+  const auto b_mul =
+      Load(d, dequant_matrices + dq_ofs + 2 * size + k) * scaled_dequant;
+
+  const auto quantized_x = Load(d, block + k);
+  const auto quantized_y = Load(d, block + size + k);
+  const auto quantized_b = Load(d, block + 2 * size + k);
+
+  const auto dequant_x_cc = AdjustQuantBias(d, 0, quantized_x, biases) * x_mul;
+  const auto dequant_y = AdjustQuantBias(d, 1, quantized_y, biases) * y_mul;
+  const auto dequant_b_cc = AdjustQuantBias(d, 2, quantized_b, biases) * b_mul;
+
+  const auto dequant_x = MulAdd(x_cc_mul, dequant_y, dequant_x_cc);
+  const auto dequant_b = MulAdd(b_cc_mul, dequant_y, dequant_b_cc);
+  Store(dequant_x, d, block + k);
+  Store(dequant_y, d, block + size + k);
+  Store(dequant_b, d, block + 2 * size + k);
+}
+
+template <class V>
+HWY_ATTR void DequantBlock(const AcStrategy& acs, float inv_global_scale,
+                           int quant, float x_dm_multiplier, V x_cc_mul,
+                           V b_cc_mul, size_t kind, size_t size,
+                           const Quantizer& quantizer,
+                           const float* JXL_RESTRICT dequant_matrices,
+                           size_t covered_blocks, size_t bx,
+                           const float* JXL_RESTRICT* JXL_RESTRICT dc_row,
+                           size_t dc_stride, const float* JXL_RESTRICT biases,
+                           float* JXL_RESTRICT block) {
+  PROFILER_FUNC;
+
+  const auto scaled_dequant = Set(d, inv_global_scale / quant);
+  const auto x_dm_multiplier_v = Set(d, x_dm_multiplier);
+
+  const size_t dq_ofs = quantizer.DequantMatrixOffset(kind, 0);
+
+  for (size_t k = 0; k < covered_blocks * kDCTBlockSize; k += D::N) {
+    DequantLane(scaled_dequant, x_dm_multiplier_v, dequant_matrices, dq_ofs,
+                size, k, x_cc_mul, b_cc_mul, biases, block);
+  }
+  for (size_t c = 0; c < 3; c++) {
+    acs.LowestFrequenciesFromDC(dc_row[c] + bx, dc_stride, block + c * size);
+  }
+}
+
+template <class GetBlock>
+HWY_ATTR Status DecodeGroupImpl(GetBlock* JXL_RESTRICT get_block,
+                                PassesDecoderState* JXL_RESTRICT dec_state,
+                                size_t thread, const Rect& block_rect,
+                                bool save_decompressed,
+                                bool apply_color_transform, AuxOut* aux_out,
+                                Image3F* JXL_RESTRICT idct) {
+  PROFILER_FUNC;
+
+  const LoopFilter& lf = dec_state->shared->image_features.loop_filter;
+  const AcStrategyImage& ac_strategy = dec_state->shared->ac_strategy;
+
+  const size_t xsize_blocks = block_rect.xsize();
+  const size_t ysize_blocks = block_rect.ysize();
+
+  const size_t dc_stride = dec_state->shared->dc->PixelsPerRow();
+
+  const float inv_global_scale = dec_state->shared->quantizer.InvGlobalScale();
+  const float* JXL_RESTRICT dequant_matrices =
+      dec_state->shared->quantizer.DequantMatrix(0, 0);
+
+  const size_t sigma_stride = dec_state->sigma.PixelsPerRow();
+  const size_t sharpness_stride =
+      dec_state->shared->epf_sharpness.PixelsPerRow();
+
+  bool save_to_decoded = !dec_state->keep_dct && (lf.epf || lf.gab);
+  size_t padding = save_to_decoded ? 2 : 0;
+
+  const size_t idct_stride =
+      (!save_to_decoded ? *idct : dec_state->decoded).PixelsPerRow();
+
+  const float quant_scale = dec_state->shared->quantizer.Scale();
+
+  for (size_t by = 0; by < ysize_blocks; ++by) {
+    get_block->StartRow(by);
+
+    const int32_t* JXL_RESTRICT row_quant =
+        block_rect.ConstRow(dec_state->shared->raw_quant_field, by);
+
+    const float* JXL_RESTRICT dc_rows[3] = {
+        block_rect.ConstPlaneRow(*dec_state->shared->dc, 0, by),
+        block_rect.ConstPlaneRow(*dec_state->shared->dc, 1, by),
+        block_rect.ConstPlaneRow(*dec_state->shared->dc, 2, by),
+    };
+
+    const size_t ty = (block_rect.y0() + by) / kColorTileDimInBlocks;
+    AcStrategyRow acs_row = ac_strategy.ConstRow(block_rect, by);
+
+    const uint8_t* JXL_RESTRICT row_cmap[3] = {
+        dec_state->shared->cmap.ytox_map.ConstRow(ty),
+        nullptr,
+        dec_state->shared->cmap.ytob_map.ConstRow(ty),
+    };
+
+    float* JXL_RESTRICT sigma_row =
+        lf.epf ? block_rect.Row(&dec_state->sigma, by) : nullptr;
+    const uint8_t* JXL_RESTRICT sharpness_row =
+        block_rect.ConstRow(dec_state->shared->epf_sharpness, by);
+    float* JXL_RESTRICT idct_row[3];
+    for (size_t c = 0; c < 3; c++) {
+      idct_row[c] = (!save_to_decoded ? *idct : dec_state->decoded)
+                        .PlaneRow(c, (block_rect.y0() + by) * kBlockDim) +
+                    (block_rect.x0()) * kBlockDim;
+    }
+
+    for (size_t tx = 0; tx < DivCeil(xsize_blocks, kColorTileDimInBlocks);
+         tx++) {
+      size_t abs_tx = tx + block_rect.x0() / kColorTileDimInBlocks;
+      auto x_cc_mul =
+          Set(d, dec_state->shared->cmap.YtoXRatio(row_cmap[0][abs_tx]));
+      auto b_cc_mul =
+          Set(d, dec_state->shared->cmap.YtoBRatio(row_cmap[2][abs_tx]));
+      // Increment bx by llf_x because those iterations would otherwise
+      // immediately continue (!IsFirstBlock). Reduces mispredictions.
+      for (size_t bx = tx * kColorTileDimInBlocks;
+           bx < xsize_blocks && bx < (tx + 1) * kColorTileDimInBlocks;) {
+        AcStrategy acs = acs_row[bx];
+        const size_t llf_x = acs.covered_blocks_x();
+
+        // Can only happen in the second or lower rows of a varblock.
+        if (JXL_UNLIKELY(!acs.IsFirstBlock())) {
+          bx += llf_x;
+          continue;
+        }
+        PROFILER_ZONE("DecodeGroupImpl inner");
+        const size_t log2_covered_blocks = acs.log2_covered_blocks();
+
+        const size_t covered_blocks = 1 << log2_covered_blocks;
+        const size_t size = covered_blocks * kDCTBlockSize;
+
+        HWY_ALIGN float block[3 * AcStrategy::kMaxCoeffArea];
+        JXL_RETURN_IF_ERROR(
+            get_block->GetBlock(bx, by, acs, size, log2_covered_blocks, block));
+
+        if (JXL_UNLIKELY(dec_state->keep_dct)) {
+          if (acs.Strategy() != AcStrategy::Type::DCT)
+            return JXL_FAILURE(
+                "Can only decode to JPEG if only DCT-8 is used.");
+          const std::vector<QuantEncoding>& qe =
+              dec_state->shared->matrices.encodings();
+          if (qe.size() == 0 ||
+              qe[0].mode != QuantEncoding::Mode::kQuantModeRAW ||
+              qe[0].qraw.qtable_den_shift != 0)
+            return JXL_FAILURE(
+                "Quantization table is not a JPEG quantization table.");
+
+          for (size_t c : {1, 0, 2}) {
+            float* JXL_RESTRICT idct_pos = idct_row[c] + bx * kBlockDim;
+            idct_pos[0] = dc_rows[c][bx];
+            if (c == 1) {
+              for (int i = 1; i < 64; i++) {
+                idct_pos[(i % 8) * idct_stride + (i / 8)] = block[c * size + i];
+              }
+            } else {
+              float scale =
+                  (c == 0
+                       ? dec_state->shared->cmap.YtoXRatio(row_cmap[c][abs_tx])
+                       : dec_state->shared->cmap.YtoBRatio(
+                             row_cmap[c][abs_tx]));
+              for (int i = 1; i < 64; i++) {
+                size_t x = i % 8;
+                size_t y = i / 8;
+                // JPEG XL is transposed, JPEG is not.
+                idct_pos[x * idct_stride + y] =
+                    block[c * size + i] +
+                    (int)(scale * block[size + i] *
+                          (*qe[0].qraw.qtable)[64 + x * 8 + y] /
+                          (*qe[0].qraw.qtable)[c * 64 + x * 8 + y]);
+              }
+            }
+          }
+          bx += llf_x;
+          continue;
+        }
+
+        // Dequantize and add predictions.
+        {
+          DequantBlock(
+              acs, inv_global_scale, row_quant[bx], dec_state->x_dm_multiplier,
+              x_cc_mul, b_cc_mul, acs.RawStrategy(), size,
+              dec_state->shared->quantizer, dequant_matrices,
+              acs.covered_blocks_y() * acs.covered_blocks_x(), bx, dc_rows,
+              dc_stride, dec_state->shared->opsin_params.quant_biases, block);
+        }
+
+        for (size_t c : {1, 0, 2}) {
+          // IDCT
+          float* JXL_RESTRICT idct_pos =
+              idct_row[c] + (padding + bx + padding * idct_stride) * kBlockDim;
+          acs.TransformToPixels(block + c * size, idct_pos, idct_stride);
+        }
+
+        if (dec_state->shared->image_features.loop_filter.epf) {
+          size_t sbx = block_rect.x0() + bx;
+          size_t sby = block_rect.y0() + by;
+          size_t xbl = dec_state->shared->frame_dim.xsize_blocks;
+          size_t ybl = dec_state->shared->frame_dim.ysize_blocks;
+          float quant = 1.0f / (row_quant[bx] * quant_scale);
+          float sigma_quant = quant * lf.epf_quant_mul;
+          for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+            for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+              // Increase sigma near edges.
+              float dc_range = 0;
+              for (size_t c = 0; c < 3; c++) {
+                const float* JXL_RESTRICT base_dc_ptr =
+                    dc_rows[c] + bx + ix + iy * dc_stride;
+                // UBSAN complains about overflowing unsigned addition here,
+                // hence we use a slightly more convoluted syntax than simple
+                // array access to ensure we only ever add or subtract positive
+                // quantities.
+                float dc_ref = *base_dc_ptr;
+                float dc_top = *(base_dc_ptr - (sby + iy == 0 ? 0 : dc_stride));
+                float dc_bottom =
+                    base_dc_ptr[sby + iy + 1 == ybl ? 0 : dc_stride];
+                float dc_left = *(base_dc_ptr - (sbx + ix == 0 ? 0 : 1));
+                float dc_right = base_dc_ptr[sbx + ix + 1 == xbl ? 0 : 1];
+
+                float dc_range_c = std::abs(dc_top - dc_ref);
+                dc_range_c = std::max(dc_range_c, std::abs(dc_bottom - dc_ref));
+                dc_range_c = std::max(dc_range_c, std::abs(dc_left - dc_ref));
+                dc_range_c = std::max(dc_range_c, std::abs(dc_right - dc_ref));
+                dc_range =
+                    std::max(dc_range_c * lf.epf_channel_scale[c], dc_range);
+              }
+              float sigma =
+                  sigma_quant *
+                  (2.0f - 1.0f / (1.0f + lf.epf_dc_range_mul * dc_range));
+              sigma *= lf.epf_sharp_lut[sharpness_row[bx + ix +
+                                                      iy * sharpness_stride]];
+              // Avoid infinities.
+              sigma = std::max(1e-4f, sigma);
+              sigma_row[bx + ix + 2 + (iy + 2) * sigma_stride] =
+                  kInvSigmaNum / sigma;
+            }
+          }
+          // Left padding with mirroring.
+          if (bx + block_rect.x0() == 0) {
+            for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+              sigma_row[1 + (iy + 2) * sigma_stride] =
+                  sigma_row[2 + (iy + 2) * sigma_stride];
+            }
+          }
+          // Right padding with mirroring.
+          if (bx + block_rect.x0() + llf_x ==
+              dec_state->shared->frame_dim.xsize_blocks) {
+            for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+              sigma_row[(iy + 2) * sigma_stride + bx + llf_x + 2] =
+                  sigma_row[(iy + 2) * sigma_stride + bx + llf_x + 1];
+            }
+          }
+          // Offsets for row copying, in blocks.
+          size_t offset_before = bx + block_rect.x0() == 0 ? 1 : bx + 2;
+          size_t offset_after =
+              bx + block_rect.x0() + llf_x ==
+                      dec_state->shared->frame_dim.xsize_blocks
+                  ? llf_x + bx + 3
+                  : llf_x + bx + 2;
+          size_t num = offset_after - offset_before;
+          // Above
+          if (by + block_rect.y0() == 0) {
+            memcpy(sigma_row + offset_before + sigma_stride,
+                   sigma_row + offset_before + 2 * sigma_stride,
+                   num * sizeof(*sigma_row));
+          }
+          // Below
+          if (by + block_rect.y0() + acs.covered_blocks_y() ==
+              dec_state->shared->frame_dim.ysize_blocks) {
+            memcpy(sigma_row + offset_before +
+                       sigma_stride * (acs.covered_blocks_y() + 2),
+                   sigma_row + offset_before +
+                       sigma_stride * (acs.covered_blocks_y() + 1),
+                   num * sizeof(*sigma_row));
+          }
+        }
+
+        if (save_to_decoded) {
+          for (size_t c : {1, 0, 2}) {
+            // Left padding with mirroring.
+            if (bx + block_rect.x0() == 0) {
+              for (size_t iy = 0; iy < kBlockDim * acs.covered_blocks_y();
+                   iy++) {
+                size_t row_offset =
+                    idct_stride * (2 * kBlockDim + iy) + kBlockDim;
+                for (size_t ix = 0; ix < kBlockDim; ix++) {
+                  idct_row[c][kBlockDim - ix - 1 + row_offset] =
+                      idct_row[c][kBlockDim + ix + row_offset];
+                }
+              }
+            }
+            // Right padding with mirroring.
+            if (bx + block_rect.x0() + llf_x ==
+                dec_state->shared->frame_dim.xsize_blocks) {
+              for (size_t iy = 0; iy < kBlockDim * acs.covered_blocks_y();
+                   iy++) {
+                size_t row_offset = idct_stride * (2 * kBlockDim + iy) +
+                                    (bx + llf_x + 1) * kBlockDim;
+                for (size_t ix = 0; ix < kBlockDim; ix++) {
+                  idct_row[c][kBlockDim + ix + row_offset] =
+                      idct_row[c][kBlockDim - ix - 1 + row_offset];
+                }
+              }
+            }
+            // Offsets for row copying, in blocks.
+            size_t offset_before = bx + block_rect.x0() == 0 ? 1 : bx + 2;
+            size_t offset_after =
+                bx + block_rect.x0() + llf_x ==
+                        dec_state->shared->frame_dim.xsize_blocks
+                    ? llf_x + bx + 3
+                    : llf_x + bx + 2;
+            size_t num = offset_after - offset_before;
+            // Above
+            if (by + block_rect.y0() == 0) {
+              for (size_t iy = 0; iy < kBlockDim; iy++) {
+                float* JXL_RESTRICT row =
+                    idct_row[c] + offset_before * kBlockDim;
+                memcpy(row + (2 * kBlockDim - iy - 1) * idct_stride,
+                       row + (2 * kBlockDim + iy) * idct_stride,
+                       num * sizeof(**idct_row) * kBlockDim);
+              }
+            }
+            // Below
+            if (by + block_rect.y0() + acs.covered_blocks_y() ==
+                dec_state->shared->frame_dim.ysize_blocks) {
+              for (size_t iy = 0; iy < kBlockDim; iy++) {
+                float* JXL_RESTRICT row =
+                    idct_row[c] +
+                    (acs.covered_blocks_y() + 1) * kBlockDim * idct_stride +
+                    offset_before * kBlockDim;
+                memcpy(row + (kBlockDim + iy) * idct_stride,
+                       row + (kBlockDim - iy - 1) * idct_stride,
+                       num * sizeof(**idct_row) * kBlockDim);
+              }
+            }
+          }
+        }
+        bx += llf_x;
+      }
+    }
+  }
+  // No ApplyImageFeatures in keep_dct mode.
+  if (JXL_UNLIKELY(dec_state->keep_dct)) return true;
+
+  // Apply image features to
+  // - the whole AC group, if no loop filtering is enabled, or
+  // - only the interior part of the group, skipping 1 block of border
+  // ... unless this is the first or the last group in a row, in which case we
+  // process the corresponding border too.
+  size_t xsize = xsize_blocks * kBlockDim;
+  size_t ysize = ysize_blocks * kBlockDim;
+  size_t xstart = block_rect.x0() != 0 && (lf.epf || lf.gab) ? kBlockDim : 0;
+  size_t ystart = block_rect.y0() != 0 && (lf.epf || lf.gab) ? kBlockDim : 0;
+  bool is_last_block_x = dec_state->shared->frame_dim.xsize_blocks ==
+                         block_rect.x0() + block_rect.xsize();
+  bool is_last_block_y = dec_state->shared->frame_dim.ysize_blocks ==
+                         block_rect.y0() + block_rect.ysize();
+  size_t xend =
+      !is_last_block_x && (lf.epf || lf.gab) ? xsize - kBlockDim : xsize;
+  size_t yend =
+      !is_last_block_y && (lf.epf || lf.gab) ? ysize - kBlockDim : ysize;
+  if (xstart >= xend) return true;
+  if (ystart >= yend) return true;
+
+  for (size_t ycur = ystart; ycur < yend; ycur += kApplyImageFeaturesTileDim) {
+    size_t cur_size_y =
+        std::min(yend, ycur + kApplyImageFeaturesTileDim) - ycur;
+    for (size_t xcur = xstart; xcur < xend;
+         xcur += kApplyImageFeaturesTileDim) {
+      size_t cur_size_x =
+          std::min(xend, xcur + kApplyImageFeaturesTileDim) - xcur;
+      Rect rect(block_rect.x0() * kBlockDim + xcur,
+                block_rect.y0() * kBlockDim + ycur, cur_size_x, cur_size_y);
+      ApplyImageFeatures(idct, rect, dec_state, thread, aux_out,
+                         save_decompressed, apply_color_transform);
+    }
+  }
+
+  return true;
+}
+
+// Decode quantized AC coefficients of DCT blocks.
+// LLF components in the output block will not be modified.
+HWY_ATTR Status DecodeACVarBlock(
+    size_t log2_covered_blocks, int32_t* JXL_RESTRICT row_nzeros,
+    const int32_t* JXL_RESTRICT row_nzeros_top, size_t nzeros_stride, size_t c,
+    size_t bx, size_t by, AcStrategy acs,
+    const coeff_order_t* JXL_RESTRICT coeff_order, BitReader* JXL_RESTRICT br,
+    ANSSymbolReader* JXL_RESTRICT decoder,
+    const std::vector<uint8_t>& context_map, ac_qcoeff_t* JXL_RESTRICT block,
+    size_t shift = 0) {
+  PROFILER_FUNC;
+  size_t c_ctx = c == 1 ? 0 : 1;
+  // Equal to number of LLF coefficients.
+  const size_t covered_blocks = 1 << log2_covered_blocks;
+  const size_t size = covered_blocks * kDCTBlockSize;
+  int32_t predicted_nzeros =
+      PredictFromTopAndLeft(row_nzeros_top, row_nzeros, bx, 32);
+  size_t nzeros = 0;
+
+  size_t ord = kStrategyOrder[acs.RawStrategy()];
+  const coeff_order_t* JXL_RESTRICT order =
+      &coeff_order[(ord * 3 + c) * AcStrategy::kMaxCoeffArea];
+  ord = ord > 2 ? ord / 2 + 1 : ord;
+  // ord is in [0, 5), so we multiply c_ctx by 5.
+  size_t block_ctx_id = c_ctx * 5 + ord;
+
+  const size_t nzero_ctx = NonZeroContext(predicted_nzeros, block_ctx_id);
+  nzeros = ReadHybridUint(nzero_ctx, br, decoder, context_map);
+  if (nzeros > size) {
+    return JXL_FAILURE("Invalid AC: nzeros too large");
+  }
+  for (size_t y = 0; y < acs.covered_blocks_y(); y++) {
+    for (size_t x = 0; x < acs.covered_blocks_x(); x++) {
+      row_nzeros[bx + x + y * nzeros_stride] =
+          (nzeros + covered_blocks - 1) >> log2_covered_blocks;
+    }
+  }
+
+  const size_t histo_offset = ZeroDensityContextsOffset(block_ctx_id);
+
+  // Skip LLF
+  {
+    PROFILER_ZONE("AcDecSkipLLF, reader");
+    size_t prev = (nzeros > size / 16 ? 0 : 1);
+    for (size_t k = covered_blocks; k < size && nzeros != 0; ++k) {
+      const size_t ctx =
+          histo_offset + ZeroDensityContext(nzeros, k, covered_blocks,
+                                            log2_covered_blocks, prev);
+      const size_t u_coeff = ReadHybridUint(ctx, br, decoder, context_map);
+      // Hand-rolled version of UnpackSigned, shifting before the conversion to
+      // signed integer to avoid undefined behavior of shifting negative
+      // numbers.
+      const size_t magnitude = u_coeff >> 1;
+      const size_t neg_sign = (~u_coeff) & 1;
+      const intptr_t coeff =
+          static_cast<intptr_t>((magnitude ^ (neg_sign - 1)) << shift);
+      block[order[k]] += static_cast<ac_qcoeff_t>(coeff);
+      prev = u_coeff != 0;
+      nzeros -= prev;
+    }
+    if (JXL_UNLIKELY(nzeros != 0)) {
+      return JXL_FAILURE("Invalid AC: nzeros not 0.");
+    }
+  }
+  return true;
+}
+
+// Structs used by DecodeGroupImpl to get a quantized block.
+// GetBlockFromBitstream uses ANS decoding (and thus keeps track of row
+// pointers in row_nzeros), GetBlockFromEncoder simply reads the coefficient
+// image provided by the encoder.
+
+struct GetBlockFromBitstream {
+  void StartRow(size_t by) {
+    for (size_t i = 0; i < num_passes; i++) {
+      for (size_t c = 0; c < 3; c++) {
+        row_nzeros[i][c] = group_dec_cache->num_nzeroes[i].PlaneRow(c, by);
+        row_nzeros_top[i][c] =
+            by == 0 ? nullptr
+                    : group_dec_cache->num_nzeroes[i].ConstPlaneRow(c, by - 1);
+      }
+    }
+  }
+
+  Status GetBlock(size_t bx, size_t by, const AcStrategy& acs, size_t size,
+                  size_t log2_covered_blocks, float* JXL_RESTRICT block) {
+    memset(block, 0, sizeof(float) * size * 3);
+    for (size_t c : {1, 0, 2}) {
+      float* JXL_RESTRICT block_c = block + c * size;
+
+      for (size_t pass = 0; JXL_UNLIKELY(pass < num_passes); pass++) {
+        JXL_RETURN_IF_ERROR(DecodeACVarBlock(
+            log2_covered_blocks, row_nzeros[pass][c], row_nzeros_top[pass][c],
+            nzeros_stride, c, bx, by, acs,
+            &coeff_orders[idx[pass] * kCoeffOrderSize], readers[pass],
+            &decoders[pass], context_map[idx[pass]], block_c,
+            shift_for_pass[pass]));
+      }
+    }
+    return true;
+  }
+
+  HWY_ATTR Status Init(BitReader* JXL_RESTRICT* JXL_RESTRICT readers,
+                       size_t num_passes, size_t group_idx,
+                       size_t histo_selector_bits,
+                       GroupDecCache* JXL_RESTRICT group_dec_cache,
+                       PassesDecoderState* dec_state) {
+    this->coeff_orders = dec_state->shared->coeff_orders.data();
+    this->context_map = dec_state->context_map.data();
+    this->readers = readers;
+    this->num_passes = num_passes;
+    this->shift_for_pass = dec_state->shared->frame_header.passes.shift;
+    this->group_dec_cache = group_dec_cache;
+    for (size_t pass = 0; pass < num_passes; pass++) {
+      // Select which histogram to use among those of the current pass.
+      size_t cur_histogram = 0;
+      if (histo_selector_bits != 0) {
+        cur_histogram = readers[pass]->ReadBits(histo_selector_bits);
+      }
+      if (cur_histogram >= dec_state->shared->num_histograms) {
+        return JXL_FAILURE("Invalid histogram selector");
+      }
+      // GetNumPasses() is *not* the same as num_passes!
+      idx[pass] =
+          cur_histogram * dec_state->shared->frame_header.passes.num_passes +
+          pass;
+
+      decoders[pass] =
+          ANSSymbolReader(&dec_state->code[idx[pass]], readers[pass]);
+    }
+    nzeros_stride = group_dec_cache->num_nzeroes[0].PixelsPerRow();
+    for (size_t i = 0; i < num_passes; i++) {
+      JXL_ASSERT(
+          nzeros_stride ==
+          static_cast<size_t>(group_dec_cache->num_nzeroes[i].PixelsPerRow()));
+    }
+    return true;
+  }
+
+  const uint32_t* shift_for_pass = nullptr;  // not owned
+  size_t idx[kMaxNumPasses];
+  const coeff_order_t* JXL_RESTRICT coeff_orders;
+  const std::vector<uint8_t>* JXL_RESTRICT context_map;
+  ANSSymbolReader decoders[kMaxNumPasses];
+  BitReader* JXL_RESTRICT* JXL_RESTRICT readers;
+  size_t num_passes;
+  size_t nzeros_stride;
+  int32_t* JXL_RESTRICT row_nzeros[kMaxNumPasses][3];
+  const int32_t* JXL_RESTRICT row_nzeros_top[kMaxNumPasses][3];
+  GroupDecCache* JXL_RESTRICT group_dec_cache;
+};
+
+struct GetBlockFromEncoder {
+  void StartRow(size_t by) {}
+
+  Status GetBlock(size_t bx, size_t by, const AcStrategy& acs, size_t size,
+                  size_t log2_covered_blocks, float* JXL_RESTRICT block) {
+    memset(block, 0, size * 3 * sizeof(float));
+    for (size_t c = 0; c < 3; c++) {
+      // for each pass
+      for (size_t i = 0; i < quantized_ac->size(); i++) {
+        for (size_t k = 0; k < size; k++) {
+          // TODO(veluca): SIMD.
+          block[c * size + k] += rows[i][c][offset + k];
+        }
+      }
+    }
+    offset += size;
+    return true;
+  }
+
+  GetBlockFromEncoder(const std::vector<ACImage3>& ac, size_t group_idx)
+      : quantized_ac(&ac) {
+    for (size_t i = 0; i < quantized_ac->size(); i++) {
+      for (size_t c = 0; c < 3; c++) {
+        rows[i][c] = (*quantized_ac)[i].ConstPlaneRow(c, group_idx);
+      }
+    }
+  }
+
+  const std::vector<ACImage3>* JXL_RESTRICT quantized_ac;
+  size_t offset = 0;
+  const float* JXL_RESTRICT rows[kMaxNumPasses][3];
+};
+
+}  // namespace
+
+Status DecodeGroup(BitReader* JXL_RESTRICT* JXL_RESTRICT readers,
+                   size_t num_passes, size_t group_idx,
+                   PassesDecoderState* JXL_RESTRICT dec_state,
+                   GroupDecCache* JXL_RESTRICT group_dec_cache, size_t thread,
+                   Image3F* opsin, AuxOut* aux_out) {
+  PROFILER_FUNC;
+  size_t histo_selector_bits =
+      dec_state->shared->num_histograms == 1
+          ? 0
+          : CeilLog2Nonzero(dec_state->shared->num_histograms - 1);
+
+  const Rect block_group_rect = dec_state->shared->BlockGroupRect(group_idx);
+
+  group_dec_cache->InitOnce(num_passes);
+
+  GetBlockFromBitstream get_block;
+  JXL_RETURN_IF_ERROR(get_block.Init(readers, num_passes, group_idx,
+                                     histo_selector_bits, group_dec_cache,
+                                     dec_state));
+
+  JXL_RETURN_IF_ERROR(
+      DecodeGroupImpl(&get_block, dec_state, thread, block_group_rect,
+                      /*save_decompressed=*/true,
+                      /*apply_color_transform=*/true, aux_out, opsin));
+
+  for (size_t pass = 0; pass < num_passes; pass++) {
+    if (!get_block.decoders[pass].CheckANSFinalState()) {
+      return JXL_FAILURE("ANS checksum failure.");
+    }
+  }
+  return true;
+}
+
+Status DecodeGroupForRoundtrip(const std::vector<ACImage3>& ac,
+                               size_t group_idx,
+                               PassesDecoderState* JXL_RESTRICT dec_state,
+                               size_t thread, Image3F* JXL_RESTRICT opsin,
+                               AuxOut* aux_out, bool save_decompressed,
+                               bool apply_color_transform) {
+  PROFILER_FUNC;
+
+  const Rect block_group_rect = dec_state->shared->BlockGroupRect(group_idx);
+
+  GetBlockFromEncoder get_block(ac, group_idx);
+
+  return DecodeGroupImpl(&get_block, dec_state, thread, block_group_rect,
+                         save_decompressed, apply_color_transform, aux_out,
+                         opsin);
+}
+}  // namespace jxl
diff --git a/jxl/dec_group.h b/jxl/dec_group.h
new file mode 100644
index 000000000..74701a5a5
--- /dev/null
+++ b/jxl/dec_group.h
@@ -0,0 +1,56 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_GROUP_H_
+#define JXL_DEC_GROUP_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/dct_util.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_cache.h"
+#include "jxl/dec_params.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/multiframe.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+Status DecodeGroup(BitReader* JXL_RESTRICT* JXL_RESTRICT readers,
+                   size_t num_passes, size_t group_idx,
+                   PassesDecoderState* JXL_RESTRICT dec_state,
+                   GroupDecCache* JXL_RESTRICT group_dec_cache, size_t thread,
+                   Image3F* opsin, AuxOut* aux_out);
+
+Status DecodeGroupForRoundtrip(const std::vector<ACImage3>& ac,
+                               size_t group_idx,
+                               PassesDecoderState* JXL_RESTRICT dec_state,
+                               size_t thread, Image3F* JXL_RESTRICT opsin,
+                               AuxOut* aux_out, bool save_decompressed,
+                               bool apply_color_transform);
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_GROUP_H_
diff --git a/jxl/dec_modular.cc b/jxl/dec_modular.cc
new file mode 100644
index 000000000..4fd3a5971
--- /dev/null
+++ b/jxl/dec_modular.cc
@@ -0,0 +1,242 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dec_modular.h"
+
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/modular/encoding/encoding.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+// TODO: signal these multipliers (need larger ones for encoding Patch reference
+// frames in kPasses)
+static const float kDecoderMul2[3] = {1. / 32768., 1. / 2048., 1. / 2048.};
+
+Status ModularFrameDecoder::DecodeGlobalInfo(BitReader* reader,
+                                             const FrameHeader& frame_header,
+                                             ImageBundle* decoded,
+                                             bool decode_color, size_t xsize,
+                                             size_t ysize) {
+  int nb_chans = 3, depth_chan = 3;
+  if (decoded->IsGray() &&
+      frame_header.color_transform == ColorTransform::kNone) {
+    nb_chans = 1;
+    depth_chan = 1;
+  }
+  do_color = decode_color;
+  if (!do_color) nb_chans = depth_chan = 0;
+  if (frame_header.HasAlpha()) {
+    nb_chans++;
+    depth_chan++;
+  }
+  if (decoded->HasDepth()) nb_chans++;
+  if (decoded->HasExtraChannels() && frame_header.IsDisplayed()) {
+    nb_chans += decoded->extra_channels().size();
+  }
+  int maxval = (1 << decoded->metadata()->bits_per_sample) - 1;
+  Image gi(xsize, ysize, maxval, nb_chans);
+  if (decoded->HasDepth()) {
+    gi.channel[depth_chan].resize(decoded->depth().xsize(),
+                                  decoded->depth().ysize());
+    gi.channel[depth_chan].hshift = decoded->metadata()->m2.depth_shift;
+    gi.channel[depth_chan].vshift = decoded->metadata()->m2.depth_shift;
+  }
+  if (!reader->JumpToByteBoundary()) return false;
+  const Span<const uint8_t> span = reader->GetSpan();
+  size_t pos = 0;
+  modular_options options;
+  set_default_modular_options(options);
+  options.max_chan_size = kGroupDim;
+  if (!modular_generic_decompress(span, &pos, gi, options, 0, -2))
+    return JXL_FAILURE("Failed to decode global modular info");
+
+  // ensure all the channel buffers are allocated
+  have_something = false;
+  for (int c = 0; c < gi.channel.size(); c++) {
+    Channel& gic = gi.channel[c];
+    if (c >= gi.nb_meta_channels && gic.w < kGroupDim && gic.h < kGroupDim)
+      have_something = true;
+    gic.resize();
+  }
+  full_image = std::move(gi);
+  reader->SkipBits(8 * pos);
+  return true;
+}
+
+Status ModularFrameDecoder::DecodeGroup(const DecompressParams& dparams,
+                                        const Rect& rect, BitReader* reader,
+                                        AuxOut* aux_out, size_t minShift,
+                                        size_t maxShift) {
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+  int maxval = full_image.maxval;
+  Image gi(xsize, ysize, maxval, 0);
+  // start at the first bigger-than-groupsize non-metachannel
+  int c = full_image.nb_meta_channels;
+  for (; c < full_image.channel.size(); c++) {
+    Channel& fc = full_image.channel[c];
+    if (fc.w > kGroupDim || fc.h > kGroupDim) break;
+  }
+  int beginc = c;
+  for (; c < full_image.channel.size(); c++) {
+    Channel& fc = full_image.channel[c];
+    int shift = std::min(fc.hshift, fc.vshift);
+    if (shift > maxShift) continue;
+    if (shift < minShift) continue;
+    Rect r(rect.x0() >> fc.hshift, rect.y0() >> fc.vshift,
+           rect.xsize() >> fc.hshift, rect.ysize() >> fc.vshift, fc.w, fc.h);
+    if (r.xsize() == 0 || r.ysize() == 0) continue;
+    Channel gc(r.xsize(), r.ysize(), 0, maxval);
+    gc.hshift = fc.hshift;
+    gc.vshift = fc.vshift;
+    gi.channel.emplace_back(std::move(gc));
+  }
+  gi.nb_channels = gi.channel.size();
+  gi.real_nb_channels = gi.nb_channels;
+  if (!reader->JumpToByteBoundary()) return false;
+  const Span<const uint8_t> span = reader->GetSpan();
+  size_t pos = 0;
+  modular_options options;
+  set_default_modular_options(options);
+  if (!modular_generic_decompress(span, &pos, gi, options))
+    return JXL_FAILURE("Failed to decode modular group");
+  reader->SkipBits(8 * pos);
+  int gic = 0;
+  for (c = beginc; c < full_image.channel.size(); c++) {
+    Channel& fc = full_image.channel[c];
+    int shift = std::min(fc.hshift, fc.vshift);
+    if (shift > maxShift) continue;
+    if (shift < minShift) continue;
+    Rect r(rect.x0() >> fc.hshift, rect.y0() >> fc.vshift,
+           rect.xsize() >> fc.hshift, rect.ysize() >> fc.vshift, fc.w, fc.h);
+    if (r.xsize() == 0 || r.ysize() == 0) continue;
+    for (size_t y = 0; y < r.ysize(); ++y) {
+      pixel_type* const JXL_RESTRICT row_out = r.MutableRow(&fc.plane, y);
+      const pixel_type* const JXL_RESTRICT row_in = gi.channel[gic].Row(y);
+      for (size_t x = 0; x < r.xsize(); ++x) {
+        row_out[x] = row_in[x];
+      }
+    }
+    gic++;
+  }
+  return true;
+}
+
+Status ModularFrameDecoder::FinalizeDecoding(Image3F* color,
+                                             ImageBundle* decoded,
+                                             jxl::ThreadPool* pool,
+                                             const FrameHeader& frame_header) {
+  Image& gi = full_image;
+  size_t xsize = gi.w;
+  size_t ysize = gi.h;
+
+  // Don't use threads if total image size is smaller than a group
+  if (xsize * ysize < kGroupDim * kGroupDim) pool = nullptr;
+
+  // Undo the global transforms
+  gi.undo_transforms(-1, pool);
+
+  int c = 0;
+  if (do_color) {
+    for (; c < 3; c++) {
+      float factor = 255.f / (float)full_image.maxval;
+      int c_in = c;
+      if (frame_header.color_transform == ColorTransform::kXYB) {
+        factor = 1.0f;
+        // XYB is encoded as YX(B-Y)
+        if (c < 2) c_in = 1 - c;
+        factor *= kDecoderMul2[c];
+      }
+      if (frame_header.color_transform == ColorTransform::kXYB && c == 2) {
+        RunOnPool(
+            pool, 0, ysize, jxl::ThreadPool::SkipInit(),
+            [&](const int task, const int thread) {
+              const size_t y = task;
+              const pixel_type* const JXL_RESTRICT row_in =
+                  gi.channel[c_in].Row(y);
+              const pixel_type* const JXL_RESTRICT row_in_Y =
+                  gi.channel[0].Row(y);
+              float* const JXL_RESTRICT row_out = color->PlaneRow(c, y);
+              for (size_t x = 0; x < xsize; ++x) {
+                row_out[x] = factor * (row_in[x] + row_in_Y[x]);
+              }
+            },
+            "ModularIntToFloat");
+      } else
+        RunOnPool(
+            pool, 0, ysize, jxl::ThreadPool::SkipInit(),
+            [&](const int task, const int thread) {
+              const size_t y = task;
+              const pixel_type* const JXL_RESTRICT row_in =
+                  gi.channel[decoded->IsGray() ? 0 : c_in].Row(y);
+              float* const JXL_RESTRICT row_out = color->PlaneRow(c, y);
+              for (size_t x = 0; x < xsize; ++x) {
+                row_out[x] = factor * row_in[x];
+              }
+            },
+            "ModularIntToFloat");
+    }
+    if (decoded->IsGray() &&
+        frame_header.color_transform == ColorTransform::kNone) {
+      c = 1;
+    }
+  }
+  if (frame_header.HasAlpha()) {
+    pixel_type max_alpha = (1 << decoded->metadata()->alpha_bits) - 1;
+    for (size_t y = 0; y < ysize; ++y) {
+      uint16_t* const JXL_RESTRICT row_out = decoded->alpha().MutableRow(y);
+      const pixel_type* const JXL_RESTRICT row_in = gi.channel[c].Row(y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row_out[x] = CLAMP(row_in[x], 0, max_alpha);
+      }
+    }
+    c++;
+  }
+  if (decoded->HasDepth()) {
+    pixel_type max_depth = (1 << decoded->metadata()->m2.depth_bits) - 1;
+    for (size_t y = 0; y < decoded->depth().ysize(); ++y) {
+      uint16_t* const JXL_RESTRICT row_out = decoded->depth().MutableRow(y);
+      const pixel_type* const JXL_RESTRICT row_in = gi.channel[c].Row(y);
+      for (size_t x = 0; x < decoded->depth().xsize(); ++x) {
+        row_out[x] = CLAMP(row_in[x], 0, max_depth);
+      }
+    }
+    c++;
+  }
+  if (decoded->HasExtraChannels() && frame_header.IsDisplayed()) {
+    pixel_type max_extra =
+        (1 << decoded->metadata()->m2.extra_channel_bits) - 1;
+    for (size_t ec = 0; ec < decoded->extra_channels().size(); ec++, c++) {
+      for (size_t y = 0; y < ysize; ++y) {
+        uint16_t* const JXL_RESTRICT row_out =
+            decoded->extra_channels()[ec].MutableRow(y);
+        const pixel_type* const JXL_RESTRICT row_in = gi.channel[c].Row(y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row_out[x] = CLAMP(row_in[x], 0, max_extra);
+        }
+      }
+    }
+  }
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/dec_modular.h b/jxl/dec_modular.h
new file mode 100644
index 000000000..7b12a1cc6
--- /dev/null
+++ b/jxl/dec_modular.h
@@ -0,0 +1,60 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_MODULAR_H_
+#define JXL_DEC_MODULAR_H_
+
+#include <stddef.h>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_params.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+Status DecodeModularRect(const DecompressParams& dparams,
+                         size_t responsive_preview, ImageBundle* decoded,
+                         const Rect& rect, BitReader* reader, AuxOut* aux_out,
+                         size_t bytes_to_read, const FrameHeader& frame_header);
+
+class ModularFrameDecoder {
+ public:
+  ModularFrameDecoder() {}
+  Status DecodeGlobalInfo(BitReader* reader, const FrameHeader& frame_header,
+                          ImageBundle* decoded, bool decode_color, size_t xsize,
+                          size_t ysize);
+  Status DecodeGroup(const DecompressParams& dparams, const Rect& rect,
+                     BitReader* reader, AuxOut* aux_out, size_t minShift,
+                     size_t maxShift);
+  Status FinalizeDecoding(Image3F* color, ImageBundle* decoded,
+                          jxl::ThreadPool* pool,
+                          const FrameHeader& frame_header);
+  bool have_dc() { return have_something; };
+
+ private:
+  Image full_image;
+  bool do_color;
+  bool have_something;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_MODULAR_H_
diff --git a/jxl/dec_params.h b/jxl/dec_params.h
new file mode 100644
index 000000000..2761a8c2f
--- /dev/null
+++ b/jxl/dec_params.h
@@ -0,0 +1,59 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_PARAMS_H_
+#define JXL_DEC_PARAMS_H_
+
+// Parameters and flags that govern JXL decompression.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <limits>
+
+#include "jxl/base/override.h"
+
+namespace jxl {
+
+struct BrunsliDecoderOptions {
+  bool fix_dc_staircase = false;
+  bool gaborish = false;
+};
+
+struct DecompressParams {
+  // If true, checks at the end of decoding that all of the compressed data
+  // was consumed by the decoder.
+  bool check_decompressed_size = true;
+
+  // If true, skip dequant and iDCT and decode to JPEG (only if possible)
+  bool keep_dct = false;
+
+  // These cannot be kOn because they need encoder support.
+  Override preview = Override::kDefault;
+  Override noise = Override::kDefault;
+  Override adaptive_reconstruction = Override::kDefault;
+
+  // How many passes to decode at most. By default, decode everything.
+  uint32_t max_passes = std::numeric_limits<uint32_t>::max();
+  // Alternatively, one can specify the maximum tolerable downscaling factor
+  // with respect to the full size of the image. By default, nothing less than
+  // the full size is requested.
+  size_t max_downsampling = 1;
+
+  BrunsliDecoderOptions brunsli;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_PARAMS_H_
diff --git a/jxl/dec_reconstruct.cc b/jxl/dec_reconstruct.cc
new file mode 100644
index 000000000..0be0a111c
--- /dev/null
+++ b/jxl/dec_reconstruct.cc
@@ -0,0 +1,214 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dec_reconstruct.h"
+
+#include <utility>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/profiler.h"
+#include "jxl/common.h"
+#include "jxl/dec_xyb.h"
+#include "jxl/epf.h"
+#include "jxl/frame_header.h"
+#include "jxl/gaborish.h"
+#include "jxl/loop_filter.h"
+#include "jxl/multiframe.h"
+#include "jxl/passes_state.h"
+
+namespace jxl {
+using DF = HWY_CAPPED(float, kBlockDim);
+
+// TODO(veluca): implement dynamic dispatch.
+HWY_ATTR void ApplyImageFeatures(Image3F* JXL_RESTRICT idct, const Rect& rect,
+                                 PassesDecoderState* dec_state, size_t thread,
+                                 AuxOut* aux_out, bool save_decompressed,
+                                 bool apply_color_transform) {
+  JXL_CHECK(rect.x0() % kBlockDim == 0);
+  JXL_CHECK(rect.y0() % kBlockDim == 0);
+  JXL_CHECK(rect.xsize() % kBlockDim == 0);
+  JXL_CHECK(rect.ysize() % kBlockDim == 0);
+
+  const Rect block_rect(rect.x0() / kBlockDim, rect.y0() / kBlockDim,
+                        rect.xsize() / kBlockDim, rect.ysize() / kBlockDim);
+
+  const ImageFeatures& image_features = dec_state->shared->image_features;
+  const FrameHeader& frame_header = dec_state->shared->frame_header;
+  const LoopFilter& lf = image_features.loop_filter;
+  const OpsinParams& opsin_params = dec_state->shared->opsin_params;
+
+  if (lf.epf || lf.gab) {
+    PROFILER_ZONE("EPF");
+    EdgePreservingFilter(
+        lf, rect, dec_state->decoded, block_rect, dec_state->sigma, rect, idct,
+        &dec_state->storage1[thread], &dec_state->storage2[thread]);
+  }
+
+  // At this point, `idct:rect` holds the decoded pixels, independently of epf
+  // or gaborish having been applied.
+
+  // TODO(veluca): switch the rest of the function to row-based processing.
+  image_features.patches.AddTo(idct, rect, rect);
+  image_features.splines.AddTo(idct, rect, rect, dec_state->shared->cmap);
+
+  if (dec_state->shared->multiframe->NeedsRestoring()) {
+    PROFILER_ZONE("MultiframeRestore");
+    for (size_t c = 0; c < 3; c++) {
+      AddTo(rect, dec_state->frame_storage->Plane(c), rect, &idct->Plane(c));
+    }
+  }
+
+  if (dec_state->shared->multiframe->NeedsSaving() && save_decompressed) {
+    PROFILER_ZONE("MultiframeSave");
+    CopyImageTo(rect, *idct, rect, dec_state->frame_storage);
+  }
+
+  if (frame_header.flags & FrameHeader::kNoise) {
+    PROFILER_ZONE("AddNoise");
+    AddNoise(image_features.noise_params, rect, dec_state->noise, rect,
+             dec_state->shared->cmap, idct);
+  }
+
+  if (apply_color_transform &&
+      frame_header.color_transform == ColorTransform::kXYB) {
+    PROFILER_ZONE("ToXYB");
+    for (size_t y = 0; y < rect.ysize(); y++) {
+      float* JXL_RESTRICT row0 = rect.PlaneRow(idct, 0, y);
+      float* JXL_RESTRICT row1 = rect.PlaneRow(idct, 1, y);
+      float* JXL_RESTRICT row2 = rect.PlaneRow(idct, 2, y);
+
+      const HWY_FULL(float) d;
+
+      for (size_t x = 0; x < rect.xsize(); x += d.N) {
+        const auto in_opsin_x = Load(d, row0 + x);
+        const auto in_opsin_y = Load(d, row1 + x);
+        const auto in_opsin_b = Load(d, row2 + x);
+        JXL_COMPILER_FENCE;
+        auto linear_r = Undefined(d);
+        auto linear_g = Undefined(d);
+        auto linear_b = Undefined(d);
+        XybToRgb(d, in_opsin_x, in_opsin_y, in_opsin_b, opsin_params, &linear_r,
+                 &linear_g, &linear_b);
+
+        Store(linear_r, d, row0 + x);
+        Store(linear_g, d, row1 + x);
+        Store(linear_b, d, row2 + x);
+      }
+    }
+  }
+}
+
+Status FinalizeFrameDecoding(Image3F* JXL_RESTRICT idct,
+                             PassesDecoderState* dec_state, ThreadPool* pool,
+                             AuxOut* aux_out, bool save_decompressed,
+                             bool apply_color_transform) {
+  std::vector<Rect> rects_to_process;
+
+  const LoopFilter& lf = dec_state->shared->image_features.loop_filter;
+  const FrameHeader& frame_header = dec_state->shared->frame_header;
+
+  if (lf.epf || lf.gab) {
+    // We can process larger tiles here, as they are at most 16 pixel wide.
+    // Adding padding, this is 48 pixels, hence we can process up to
+    // 96*96/48-32=160 pixels in the other dimension without exhausting L2
+    // cache. We instead go with 128 pixels to keep rectangle sizes mostly
+    // uniform (vertical rectangles sum up to 240 pixels as they should not
+    // overlap with the horizontal rectangles, hence one rectangle of 160
+    // pixels would imply another rectangle of 80).
+    size_t xsize = dec_state->shared->frame_dim.xsize_padded;
+    size_t ysize = dec_state->shared->frame_dim.ysize_padded;
+    size_t xsize_groups = dec_state->shared->frame_dim.xsize_groups;
+    size_t ysize_groups = dec_state->shared->frame_dim.ysize_groups;
+    // For every gap between groups, vertically, enqueue bottom gap with next
+    // group ...
+    for (size_t ygroup = 0; ygroup < ysize_groups - 1; ygroup++) {
+      size_t gystart = ygroup * kGroupDim;
+      size_t gyend = std::min(ysize, kGroupDim * (ygroup + 1));
+      // Group is processed together with another group.
+      if (gyend <= gystart + kBlockDim) continue;
+      for (size_t xstart = 0; xstart < xsize;
+           xstart += kApplyImageFeaturesTileDim) {
+        rects_to_process.emplace_back(xstart, gyend - kBlockDim,
+                                      kApplyImageFeaturesTileDim, 2 * kBlockDim,
+                                      xsize, ysize);
+      }
+    }
+    // For every gap between groups, horizontally, enqueue right gap with next
+    // group, carefully avoiding overlaps with the horizontal gaps enqueued
+    // before...
+    for (size_t xgroup = 0; xgroup < xsize_groups - 1; xgroup++) {
+      size_t gxstart = xgroup == 0 ? kBlockDim : xgroup * kGroupDim;
+      size_t gxend = std::min(xsize, kGroupDim * (xgroup + 1));
+      // Group is processed together with another group.
+      if (gxend <= gxstart + kBlockDim) continue;
+      for (size_t ygroup = 0; ygroup < ysize_groups; ygroup++) {
+        size_t gystart = ygroup == 0 ? 0 : ygroup * kGroupDim + kBlockDim;
+        size_t gyend = ygroup == ysize_groups - 1
+                           ? ysize
+                           : kGroupDim * (ygroup + 1) - kBlockDim;
+        if (gyend <= gystart) continue;
+        for (size_t ystart = gystart; ystart < gyend;
+             ystart += kApplyImageFeaturesTileDim) {
+          rects_to_process.emplace_back(
+              gxend - kBlockDim, ystart, 2 * kBlockDim,
+              kApplyImageFeaturesTileDim, xsize, gyend);
+        }
+      }
+    }
+  }
+  if (frame_header.encoding == FrameEncoding::kModularGroup) {
+    for (size_t y = 0; y < idct->ysize(); y += kGroupDim) {
+      for (size_t x = 0; x < idct->xsize(); x += kGroupDim) {
+        rects_to_process.emplace_back(x, y, kGroupDim, kGroupDim, idct->xsize(),
+                                      idct->ysize());
+      }
+    }
+  }
+  const auto allocate_storage = [&](size_t num_threads) {
+    dec_state->EnsureStorage(num_threads);
+    return true;
+  };
+
+  auto apply_features = [&](size_t rect_id, size_t thread) {
+    ApplyImageFeatures(idct, rects_to_process[rect_id], dec_state, thread,
+                       aux_out, save_decompressed, apply_color_transform);
+  };
+
+  RunOnPool(pool, 0, rects_to_process.size(), allocate_storage, apply_features,
+            "ApplyFeatures");
+
+  if (dec_state->shared->multiframe->NeedsSaving() && save_decompressed) {
+    dec_state->shared->multiframe->SetDecodedFrame();
+  }
+
+  const size_t xsize = dec_state->shared->frame_dim.xsize;
+  const size_t ysize = dec_state->shared->frame_dim.ysize;
+
+  idct->ShrinkTo(xsize, ysize);
+
+  if (apply_color_transform &&
+      frame_header.color_transform == ColorTransform::kYCbCr) {
+    // TODO(veluca): create per-pixel version of YcbcrToRgb for line-based
+    // decoding in ApplyImageFeatures.
+    YcbcrToRgb(idct->Plane(1), idct->Plane(0), idct->Plane(2),
+               const_cast<ImageF*>(&idct->Plane(0)),
+               const_cast<ImageF*>(&idct->Plane(1)),
+               const_cast<ImageF*>(&idct->Plane(2)), pool);
+  }  // otherwise no color transform needed
+
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/dec_reconstruct.h b/jxl/dec_reconstruct.h
new file mode 100644
index 000000000..ebc956faa
--- /dev/null
+++ b/jxl/dec_reconstruct.h
@@ -0,0 +1,55 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_RECONSTRUCT_H_
+#define JXL_DEC_RECONSTRUCT_H_
+
+#include <stddef.h>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/dec_cache.h"
+#include "jxl/dot_dictionary.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/loop_filter.h"
+#include "jxl/multiframe.h"
+#include "jxl/noise.h"
+#include "jxl/patch_dictionary.h"
+#include "jxl/quantizer.h"
+#include "jxl/splines.h"
+
+namespace jxl {
+
+// Finalizes the decoding of a pass by running per-pass post processing:
+// smoothing and adaptive reconstruction. Writes linear sRGB to `idct` and
+// shrinks it to `x/ysize` to undo prior padding.
+Status FinalizeFrameDecoding(Image3F* JXL_RESTRICT idct,
+                             PassesDecoderState* dec_state, ThreadPool* pool,
+                             AuxOut* aux_out, bool save_decompressed,
+                             bool apply_color_transform);
+
+// Applies image features on the given `idct_rect` of `idct`, interpreted as the
+// `image_rect` region of the full image.
+HWY_ATTR void ApplyImageFeatures(Image3F* JXL_RESTRICT idct, const Rect& rect,
+                                 PassesDecoderState* dec_state, size_t thread,
+                                 AuxOut* aux_out, bool save_decompressed,
+                                 bool apply_color_transform);
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_RECONSTRUCT_H_
diff --git a/jxl/dec_xyb.cc b/jxl/dec_xyb.cc
new file mode 100644
index 000000000..17b3a6bb3
--- /dev/null
+++ b/jxl/dec_xyb.cc
@@ -0,0 +1,330 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dec_xyb.h"
+
+#include <string.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/fields.h"
+#include "jxl/image.h"
+#include "jxl/opsin_params.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+void OpsinParams::Init() {
+  InitSIMDInverseMatrix(GetOpsinAbsorbanceInverseMatrix(),
+                        inverse_opsin_matrix);
+  memcpy(opsin_biases, kNegOpsinAbsorbanceBiasRGB,
+         sizeof(kNegOpsinAbsorbanceBiasRGB));
+  memcpy(quant_biases, kDefaultQuantBias, sizeof(kDefaultQuantBias));
+  for (size_t c = 0; c < 4; c++) {
+    opsin_biases_cbrt[c] = std::cbrt(opsin_biases[c]);
+  }
+}
+
+// In-place.
+HWY_ATTR void OpsinToLinear(Image3F* JXL_RESTRICT inout, ThreadPool* pool,
+                            const OpsinParams& opsin_params) {
+  PROFILER_FUNC;
+
+  const size_t xsize = inout->xsize();  // not padded
+  RunOnPool(
+      pool, 0, inout->ysize(), ThreadPool::SkipInit(),
+      [&](const int task, const int thread) HWY_ATTR {
+        const size_t y = task;
+
+        // Faster than adding via ByteOffset at end of loop.
+        float* JXL_RESTRICT row0 = inout->PlaneRow(0, y);
+        float* JXL_RESTRICT row1 = inout->PlaneRow(1, y);
+        float* JXL_RESTRICT row2 = inout->PlaneRow(2, y);
+
+        const HWY_FULL(float) d;
+
+        for (size_t x = 0; x < xsize; x += d.N) {
+          const auto in_opsin_x = Load(d, row0 + x);
+          const auto in_opsin_y = Load(d, row1 + x);
+          const auto in_opsin_b = Load(d, row2 + x);
+          JXL_COMPILER_FENCE;
+          auto linear_r = Undefined(d);
+          auto linear_g = Undefined(d);
+          auto linear_b = Undefined(d);
+          XybToRgb(d, in_opsin_x, in_opsin_y, in_opsin_b, opsin_params,
+                   &linear_r, &linear_g, &linear_b);
+
+          Store(linear_r, d, row0 + x);
+          Store(linear_g, d, row1 + x);
+          Store(linear_b, d, row2 + x);
+        }
+      },
+      "OpsinToLinear");
+}
+
+// Same, but not in-place.
+HWY_ATTR void OpsinToLinear(const Image3F& opsin, const Rect& rect,
+                            ThreadPool* pool, Image3F* JXL_RESTRICT linear,
+                            const OpsinParams& opsin_params) {
+  PROFILER_FUNC;
+
+  JXL_ASSERT(SameSize(rect, *linear));
+
+  RunOnPool(
+      pool, 0, static_cast<int>(rect.ysize()), ThreadPool::SkipInit(),
+      [&](const int task, int /*thread*/) HWY_ATTR {
+        const size_t y = static_cast<size_t>(task);
+
+        // Faster than adding via ByteOffset at end of loop.
+        const float* JXL_RESTRICT row_opsin_0 = rect.ConstPlaneRow(opsin, 0, y);
+        const float* JXL_RESTRICT row_opsin_1 = rect.ConstPlaneRow(opsin, 1, y);
+        const float* JXL_RESTRICT row_opsin_2 = rect.ConstPlaneRow(opsin, 2, y);
+        float* JXL_RESTRICT row_linear_0 = linear->PlaneRow(0, y);
+        float* JXL_RESTRICT row_linear_1 = linear->PlaneRow(1, y);
+        float* JXL_RESTRICT row_linear_2 = linear->PlaneRow(2, y);
+
+        const HWY_FULL(float) d;
+
+        for (size_t x = 0; x < rect.xsize(); x += d.N) {
+          const auto in_opsin_x = Load(d, row_opsin_0 + x);
+          const auto in_opsin_y = Load(d, row_opsin_1 + x);
+          const auto in_opsin_b = Load(d, row_opsin_2 + x);
+          JXL_COMPILER_FENCE;
+          auto linear_r = Undefined(d);
+          auto linear_g = Undefined(d);
+          auto linear_b = Undefined(d);
+          XybToRgb(d, in_opsin_x, in_opsin_y, in_opsin_b, opsin_params,
+                   &linear_r, &linear_g, &linear_b);
+
+          Store(linear_r, d, row_linear_0 + x);
+          Store(linear_g, d, row_linear_1 + x);
+          Store(linear_b, d, row_linear_2 + x);
+        }
+      },
+      "OpsinToLinear(Rect)");
+}
+
+// Transform YCbCr to RGB.
+// Could be performed in-place (i.e. Y, Cb and Cr could alias R, B and B).
+HWY_ATTR void YcbcrToRgb(const ImageF& y_plane, const ImageF& cb_plane,
+                         const ImageF& cr_plane, ImageF* r_plane,
+                         ImageF* g_plane, ImageF* b_plane, ThreadPool* pool) {
+  const HWY_FULL(float) df;
+  constexpr size_t S = df.N;  // Step.
+
+  const size_t xsize = y_plane.xsize();
+  const size_t ysize = y_plane.ysize();
+
+  if ((xsize == 0) || (ysize == 0)) return;
+
+  const auto c128 = Set(df, 128.0f);
+  const auto crcr = Set(df, 1.402f);
+  const auto cgcb = Set(df, -0.34414f);
+  const auto cgcr = Set(df, -0.71414f);
+  const auto cbcb = Set(df, 1.772f);
+
+  constexpr size_t kGroupArea = kGroupDim * kGroupDim;
+  const size_t lines_per_group = DivCeil(kGroupArea, xsize);
+  const size_t num_stripes = DivCeil(ysize, lines_per_group);
+  const auto transform = [&](int idx, int /* thread*/) HWY_ATTR {
+    const size_t y0 = idx * lines_per_group;
+    const size_t y1 = std::min<size_t>(y0 + lines_per_group, ysize);
+    for (size_t y = y0; y < y1; ++y) {
+      const float* y_row = y_plane.ConstRow(y);
+      const float* cb_row = cb_plane.ConstRow(y);
+      const float* cr_row = cr_plane.ConstRow(y);
+      float* r_row = r_plane->Row(y);
+      float* g_row = g_plane->Row(y);
+      float* b_row = b_plane->Row(y);
+      for (size_t x = 0; x < xsize; x += S) {
+        const auto y_vec = Load(df, y_row + x) + c128;
+        const auto cb_vec = Load(df, cb_row + x);
+        const auto cr_vec = Load(df, cr_row + x);
+        const auto r_vec = MulAdd(crcr, cr_vec, y_vec);
+        const auto g_vec = MulAdd(cgcr, cr_vec, MulAdd(cgcb, cb_vec, y_vec));
+        const auto b_vec = MulAdd(cbcb, cb_vec, y_vec);
+        Store(r_vec, df, r_row + x);
+        Store(g_vec, df, g_row + x);
+        Store(b_vec, df, b_row + x);
+      }
+    }
+  };
+  RunOnPool(pool, 0, static_cast<int>(num_stripes), ThreadPool::SkipInit(),
+            transform, "YcbcrToRgb");
+}
+
+/* Vertical upsampling:
+ *  input:
+ *   (a, b, c) := |a1 a2 a3 a4|
+ *                |b1 b2 b3 b4| <- current line
+ *                |c1 c2 c3 c4|
+ *  intermediate:
+ *   u := a + 3 * b
+ *   d := c + 3 * b
+ *  output:
+ *  |u1 u2 u3 u4| =: (u, d)
+ *  |d1 d2 d3 d4|
+ */
+HWY_ATTR ImageF UpsampleV2(const ImageF& src, ThreadPool* pool) {
+  const HWY_FULL(float) df;
+  constexpr size_t S = df.N;
+  const auto c14 = Set(df, 0.25f);
+  const auto c34 = Set(df, 0.75f);
+
+  const size_t xsize = src.xsize();
+  const size_t ysize = src.ysize();
+  JXL_ASSERT(xsize != 0);
+  JXL_ASSERT(ysize != 0);
+  ImageF dst(xsize, ysize * 2);
+  if (ysize == 1) {
+    memcpy(dst.Row(0), src.Row(0), xsize * sizeof(*src.Row(0)));
+    memcpy(dst.Row(1), src.Row(0), xsize * sizeof(*src.Row(0)));
+  } else {
+    constexpr size_t kGroupArea = kGroupDim * kGroupDim;
+    const size_t lines_per_group = DivCeil(kGroupArea, xsize);
+    const size_t num_stripes = DivCeil(ysize, lines_per_group);
+    const auto upsample = [&](int idx, int /* thread*/) HWY_ATTR {
+      const size_t y0 = idx * lines_per_group;
+      const size_t y1 = std::min<size_t>(y0 + lines_per_group, ysize);
+      for (size_t y = y0; y < y1; ++y) {
+        const float* JXL_RESTRICT prev_row = src.ConstRow(y == 0 ? 1 : y - 1);
+        const float* JXL_RESTRICT current_row = src.ConstRow(y);
+        const float* JXL_RESTRICT next_row =
+            src.ConstRow(y == ysize - 1 ? ysize - 2 : y + 1);
+        float* JXL_RESTRICT dst1_row = dst.Row(2 * y);
+        float* JXL_RESTRICT dst2_row = dst.Row(2 * y + 1);
+        for (size_t x = 0; x < xsize; x += S) {
+          const auto current34 = Load(df, current_row + x) * c34;
+          const auto prev = Load(df, prev_row + x);
+          const auto next = Load(df, next_row + x);
+          Store(MulAdd(prev, c14, current34), df, dst1_row + x);
+          Store(MulAdd(next, c14, current34), df, dst2_row + x);
+        }
+      }
+    };
+    RunOnPool(pool, 0, static_cast<int>(num_stripes), ThreadPool::SkipInit(),
+              upsample, "UpsampleV2");
+  }
+  return dst;
+}
+
+/* Horizontal upsampling:
+ *  input:
+ *   (a, b, c) := |a1 a2 a3 a4 b1 b2 b3 b4 c1 c2 c3 c4|
+ *                             ^^^^^^^^^^^
+ *                            current block
+ *  intermediate:
+ *   l := (a << 3) {0001} (b >> 1) = [a4 b1 b2 b3]
+ *   r := (c >> 3) {1000} (b << 1) = [b2 b3 b4 c1]
+ *   o := 3 * b + l
+ *   e := 3 * b + r
+ *  output:
+ *   |o1 e1 o2 e2 o3 e3 o4 e4| =: (o, e)
+ */
+HWY_ATTR ImageF UpsampleH2(const ImageF& src, ThreadPool* pool) {
+  const size_t xsize = src.xsize();
+  const size_t ysize = src.ysize();
+  JXL_ASSERT(xsize != 0);
+  JXL_ASSERT(ysize != 0);
+  ImageF dst(xsize * 2, ysize);
+
+  constexpr size_t kGroupArea = kGroupDim * kGroupDim;
+  const size_t lines_per_group = DivCeil(kGroupArea, xsize);
+  const size_t num_stripes = DivCeil(ysize, lines_per_group);
+
+#if HWY_BITS == 0
+  const auto upsample = [&](int idx, int /* thread*/) {
+    const size_t y0 = idx * lines_per_group;
+    const size_t y1 = std::min<size_t>(y0 + lines_per_group, ysize);
+    for (size_t y = y0; y < y1; ++y) {
+      const float* JXL_RESTRICT current_row = src.ConstRow(y);
+      float* JXL_RESTRICT dst_row = dst.Row(y);
+      // TODO(eustas): roll prev <- current <- next?
+      for (size_t x = 1; x < xsize - 1; ++x) {
+        const float current34 = current_row[x] * 0.75f;
+        const float prev = current_row[x - 1];
+        const float next = current_row[x + 1];
+        dst_row[x * 2] = current34 + prev * 0.25f;
+        dst_row[x * 2 + 1] = current34 + next * 0.25f;
+      }
+      if (xsize == 1) {
+        dst_row[0] = dst_row[1] = current_row[0];
+      } else {
+        const float leftmost = current_row[0] * 0.75f + current_row[1] * 0.25f;
+        dst_row[0] = dst_row[1] = leftmost;
+        const float rightmost =
+            current_row[xsize - 1] * 0.75f + current_row[xsize - 2] * 0.25f;
+        dst_row[xsize * 2 - 2] = dst_row[xsize * 2 - 1] = rightmost;
+      }
+    }
+  };
+#else
+  // TODO(eustas): Neighbors::(L|R)1 from convolve.h allows full-vector shift
+  //               even for AVX2; make those helpers more independent and use
+  //               both in convolve.h and here.
+  constexpr size_t S = 4;  // Half of AVX2, until TODO is resolved.
+  const HWY_CAPPED(float, S) df;
+  const HWY_CAPPED(uint32_t, S) du;
+
+  HWY_ALIGN static const uint32_t k1000[S] = {0u, 0u, 0u, ~0u};
+  HWY_ALIGN static const uint32_t k0001[S] = {~0u, 0u, 0u, 0u};
+  const auto c1000 = MaskFromVec(BitCast(df, Load(du, k1000)));
+  const auto c0001 = MaskFromVec(BitCast(df, Load(du, k0001)));
+
+  const auto upsample = [&](int idx, int /* thread*/) HWY_ATTR {
+    const size_t y0 = idx * lines_per_group;
+    const size_t y1 = std::min<size_t>(y0 + lines_per_group, ysize);
+    for (size_t y = y0; y < y1; ++y) {
+      const float* JXL_RESTRICT current_row = src.ConstRow(y);
+      float* JXL_RESTRICT dst_row = dst.Row(y);
+      const auto c14 = Set(df, 0.25f);
+      const auto c34 = Set(df, 0.75f);
+      auto prev = Undefined(df);
+      auto current = Load(df, current_row);
+      for (size_t x = 0; x < xsize; x += S) {
+        // Image provides 2x vector size of extra space after the row.
+        // So, it is valid to read one extra vector after the end.
+        const auto next = Load(df, current_row + x + S);
+        const auto current34 = current * c34;
+        const auto l =
+            IfThenElse(c0001, hwy::Broadcast<3>(prev), Shuffle2103(current));
+        const auto r =
+            IfThenElse(c1000, hwy::Broadcast<0>(next), Shuffle0321(current));
+        const auto o = MulAdd(l, c14, current34);
+        const auto e = MulAdd(r, c14, current34);
+        Store(InterleaveLo(o, e), df, dst_row + x * 2);
+        Store(InterleaveHi(o, e), df, dst_row + x * 2 + S);
+        prev = current;
+        current = next;
+      }
+      // Fix border values.
+      if (xsize == 1) {
+        dst_row[0] = dst_row[1] = current_row[0];
+      } else {
+        const float leftmost = current_row[0] * 0.75f + current_row[1] * 0.25f;
+        dst_row[0] = dst_row[1] = leftmost;
+        const float rightmost =
+            current_row[xsize - 1] * 0.75f + current_row[xsize - 2] * 0.25f;
+        dst_row[xsize * 2 - 2] = dst_row[xsize * 2 - 1] = rightmost;
+      }
+    }
+  };
+#endif
+  RunOnPool(pool, 0, static_cast<int>(num_stripes), ThreadPool::SkipInit(),
+            upsample, "UpsampleH2");
+  return dst;
+}
+
+}  // namespace jxl
diff --git a/jxl/dec_xyb.h b/jxl/dec_xyb.h
new file mode 100644
index 000000000..6693662a6
--- /dev/null
+++ b/jxl/dec_xyb.h
@@ -0,0 +1,121 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DEC_XYB_H_
+#define JXL_DEC_XYB_H_
+
+// XYB -> linear sRGB.
+
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/image.h"
+#include "jxl/opsin_params.h"
+
+namespace jxl {
+
+// Parameters for XYB->sRGB conversion.
+struct OpsinParams {
+  HWY_ALIGN float inverse_opsin_matrix[9 * 4];
+  HWY_ALIGN float opsin_biases[4];
+  HWY_ALIGN float opsin_biases_cbrt[4];
+  float quant_biases[4];
+  void Init();
+};
+
+// Inverts the pixel-wise RGB->XYB conversion in OpsinDynamicsImage() (including
+// the gamma mixing and simple gamma). Avoids clamping to [0, 255] - out of
+// (sRGB) gamut values may be in-gamut after transforming to a wider space.
+// "inverse_matrix" points to 9 broadcasted vectors, which are the 3x3 entries
+// of the (row-major) opsin absorbance matrix inverse. Pre-multiplying its
+// entries by c is equivalent to multiplying linear_* by c afterwards.
+template <class D, class V>
+HWY_ATTR JXL_INLINE void XybToRgb(D d, const V opsin_x, const V opsin_y,
+                                  const V opsin_b,
+                                  const OpsinParams& opsin_params,
+                                  V* const JXL_RESTRICT linear_r,
+                                  V* const JXL_RESTRICT linear_g,
+                                  V* const JXL_RESTRICT linear_b) {
+#if HWY_BITS == 0
+  const auto inv_scale_x = Set(d, kInvScaleR);
+  const auto inv_scale_y = Set(d, kInvScaleG);
+  const auto neg_bias_r = Set(d, opsin_params.opsin_biases[0]);
+  const auto neg_bias_g = Set(d, opsin_params.opsin_biases[1]);
+  const auto neg_bias_b = Set(d, opsin_params.opsin_biases[2]);
+#else
+  const auto neg_bias_rgb = LoadDup128(d, opsin_params.opsin_biases);
+  HWY_ALIGN const float inv_scale_lanes[4] = {kInvScaleR, kInvScaleG};
+  const auto inv_scale = LoadDup128(d, inv_scale_lanes);
+  const auto inv_scale_x = hwy::Broadcast<0>(inv_scale);
+  const auto inv_scale_y = hwy::Broadcast<1>(inv_scale);
+  const auto neg_bias_r = hwy::Broadcast<0>(neg_bias_rgb);
+  const auto neg_bias_g = hwy::Broadcast<1>(neg_bias_rgb);
+  const auto neg_bias_b = hwy::Broadcast<2>(neg_bias_rgb);
+#endif
+
+  // Color space: XYB -> RGB
+  auto gamma_r = inv_scale_x * (opsin_y + opsin_x);
+  auto gamma_g = inv_scale_y * (opsin_y - opsin_x);
+  auto gamma_b = opsin_b;
+
+  gamma_r -= Set(d, opsin_params.opsin_biases_cbrt[0]);
+  gamma_g -= Set(d, opsin_params.opsin_biases_cbrt[1]);
+  gamma_b -= Set(d, opsin_params.opsin_biases_cbrt[2]);
+
+  // Undo gamma compression: linear = gamma^3 for efficiency.
+  const auto gamma_r2 = gamma_r * gamma_r;
+  const auto gamma_g2 = gamma_g * gamma_g;
+  const auto gamma_b2 = gamma_b * gamma_b;
+  const auto mixed_r = MulAdd(gamma_r2, gamma_r, neg_bias_r);
+  const auto mixed_g = MulAdd(gamma_g2, gamma_g, neg_bias_g);
+  const auto mixed_b = MulAdd(gamma_b2, gamma_b, neg_bias_b);
+
+  const float* JXL_RESTRICT inverse_matrix = opsin_params.inverse_opsin_matrix;
+
+  // Unmix (multiply by 3x3 inverse_matrix)
+  *linear_r = LoadDup128(d, &inverse_matrix[0 * 4]) * mixed_r;
+  *linear_g = LoadDup128(d, &inverse_matrix[3 * 4]) * mixed_r;
+  *linear_b = LoadDup128(d, &inverse_matrix[6 * 4]) * mixed_r;
+  *linear_r = MulAdd(LoadDup128(d, &inverse_matrix[1 * 4]), mixed_g, *linear_r);
+  *linear_g = MulAdd(LoadDup128(d, &inverse_matrix[4 * 4]), mixed_g, *linear_g);
+  *linear_b = MulAdd(LoadDup128(d, &inverse_matrix[7 * 4]), mixed_g, *linear_b);
+  *linear_r = MulAdd(LoadDup128(d, &inverse_matrix[2 * 4]), mixed_b, *linear_r);
+  *linear_g = MulAdd(LoadDup128(d, &inverse_matrix[5 * 4]), mixed_b, *linear_g);
+  *linear_b = MulAdd(LoadDup128(d, &inverse_matrix[8 * 4]), mixed_b, *linear_b);
+}
+
+// Converts `inout` (not padded) from opsin to linear sRGB in-place. Called from
+// per-pass postprocessing, hence parallelized.
+HWY_ATTR void OpsinToLinear(Image3F* JXL_RESTRICT inout, ThreadPool* pool,
+                            const OpsinParams& opsin_params);
+
+// Converts `opsin:rect` (opsin may be padded, rect.x0 must be vector-aligned)
+// to linear sRGB. Called from whole-frame encoder, hence parallelized.
+HWY_ATTR void OpsinToLinear(const Image3F& opsin, const Rect& rect,
+                            ThreadPool* pool, Image3F* JXL_RESTRICT linear,
+                            const OpsinParams& opsin_params);
+
+HWY_ATTR void YcbcrToRgb(const ImageF& y_plane, const ImageF& cb_plane,
+                         const ImageF& cr_plane, ImageF* r_plane,
+                         ImageF* g_plane, ImageF* b_plane, ThreadPool* pool);
+
+HWY_ATTR ImageF UpsampleV2(const ImageF& src, ThreadPool* pool);
+HWY_ATTR ImageF UpsampleH2(const ImageF& src, ThreadPool* pool);
+
+}  // namespace jxl
+
+#endif  // JXL_DEC_XYB_H_
diff --git a/jxl/descriptive_statistics_test.cc b/jxl/descriptive_statistics_test.cc
new file mode 100644
index 000000000..6423717ea
--- /dev/null
+++ b/jxl/descriptive_statistics_test.cc
@@ -0,0 +1,161 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/descriptive_statistics.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <random>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/noise_distributions.h"
+
+namespace jxl {
+namespace {
+
+// Assigns x to one of two streams so we can later test Assimilate.
+template <typename Random>
+void NotifyEither(float x, Random* rng, Stats* JXL_RESTRICT stats1,
+                  Stats* JXL_RESTRICT stats2) {
+  if ((*rng)() & 128) {
+    stats1->Notify(x);
+  } else {
+    stats2->Notify(x);
+  }
+}
+
+TEST(StatsTest, TestGaussian) {
+  Stats stats;
+  Stats stats1, stats2;
+  const float mean = 5.0f;
+  const float stddev = 4.0f;
+  NoiseGaussian noise(stddev);
+  std::mt19937 rng(129);
+  for (size_t i = 0; i < 1000 * 1000; ++i) {
+    const float x = noise(mean, &rng);
+    stats.Notify(x);
+    NotifyEither(x, &rng, &stats1, &stats2);
+  }
+  EXPECT_NEAR(mean, stats.Mean(), 0.01);
+  EXPECT_NEAR(stddev, stats.StandardDeviation(), 0.02);
+  EXPECT_NEAR(0.0, stats.Skewness(), 0.02);
+  EXPECT_NEAR(0.0, stats.Kurtosis() - 3, 0.02);
+  printf("%s\n", stats.ToString().c_str());
+
+  // Same results after merging both accumulators.
+  stats1.Assimilate(stats2);
+  EXPECT_NEAR(mean, stats1.Mean(), 0.01);
+  EXPECT_NEAR(stddev, stats1.StandardDeviation(), 0.02);
+  EXPECT_NEAR(0.0, stats1.Skewness(), 0.02);
+  EXPECT_NEAR(0.0, stats1.Kurtosis() - 3, 0.02);
+}
+
+TEST(StatsTest, TestUniform) {
+  Stats stats;
+  Stats stats1, stats2;
+  NoiseUniform noise(0, 256);
+  std::mt19937 rng(129), rng_split(65537);
+  for (size_t i = 0; i < 1000 * 1000; ++i) {
+    const float x = noise(0.0f, &rng);
+    stats.Notify(x);
+    NotifyEither(x, &rng_split, &stats1, &stats2);
+  }
+  EXPECT_NEAR(128.0, stats.Mean(), 0.05);
+  EXPECT_NEAR(0.0, stats.Min(), 0.01);
+  EXPECT_NEAR(256.0, stats.Max(), 0.01);
+  EXPECT_NEAR(70, stats.StandardDeviation(), 10);
+  // No outliers.
+  EXPECT_NEAR(-1.2, stats.Kurtosis() - 3, 0.1);
+  printf("%s\n", stats.ToString().c_str());
+
+  // Same results after merging both accumulators.
+  stats1.Assimilate(stats2);
+  EXPECT_NEAR(128.0, stats1.Mean(), 0.05);
+  EXPECT_NEAR(0.0, stats1.Min(), 0.01);
+  EXPECT_NEAR(256.0, stats1.Max(), 0.01);
+  EXPECT_NEAR(70, stats1.StandardDeviation(), 10);
+}
+
+TEST(StatsTest, CompareCentralMomentsAgainstTwoPass) {
+  // Vary seed so the thresholds are not specific to one distribution.
+  for (int rep = 0; rep < 200; ++rep) {
+    // Uniform avoids outliers.
+    NoiseUniform noise(0, 256);
+    std::mt19937 rng(129 + 13 * rep), rng_split(65537);
+
+    // Small count so bias (population vs sample) is visible.
+    const size_t kSamples = 20;
+
+    // First pass: compute mean
+    std::vector<float> samples;
+    samples.reserve(kSamples);
+    double sum = 0.0;
+    for (size_t i = 0; i < kSamples; ++i) {
+      const float x = noise(0.0f, &rng);
+      samples.push_back(x);
+      sum += x;
+    }
+    const double mean = sum / kSamples;
+
+    // Second pass: compute stats and moments
+    Stats stats;
+    Stats stats1, stats2;
+    double sum2 = 0.0;
+    double sum3 = 0.0;
+    double sum4 = 0.0;
+    for (const double x : samples) {
+      const double d = x - mean;
+      sum2 += d * d;
+      sum3 += d * d * d;
+      sum4 += d * d * d * d;
+
+      stats.Notify(x);
+      NotifyEither(x, &rng_split, &stats1, &stats2);
+    }
+    const double mu1 = mean;
+    const double mu2 = sum2 / kSamples;
+    const double mu3 = sum3 / kSamples;
+    const double mu4 = sum4 / kSamples;
+
+    // Raw central moments (note: Mu1 is zero by definition)
+    EXPECT_NEAR(mu1, stats.Mu1(), 1E-13);
+    EXPECT_NEAR(mu2, stats.Mu2(), 1E-11);
+    EXPECT_NEAR(mu3, stats.Mu3(), 1E-9);
+    EXPECT_NEAR(mu4, stats.Mu4(), 1E-6);
+
+    // Same results after merging both accumulators.
+    stats1.Assimilate(stats2);
+    EXPECT_NEAR(mu1, stats1.Mu1(), 1E-13);
+    EXPECT_NEAR(mu2, stats1.Mu2(), 1E-11);
+    EXPECT_NEAR(mu3, stats1.Mu3(), 1E-9);
+    EXPECT_NEAR(mu4, stats1.Mu4(), 1E-6);
+
+    const double sample_variance = mu2;
+    // Scaling factor for sampling bias
+    const double r = (kSamples - 1.0) / kSamples;
+    const double skewness = mu3 * pow(r / mu2, 1.5);
+    const double kurtosis = mu4 * pow(r / mu2, 2.0);
+
+    EXPECT_NEAR(sample_variance, stats.SampleVariance(),
+                sample_variance * 1E-12);
+    EXPECT_NEAR(skewness, stats.Skewness(), std::abs(skewness * 1E-11));
+    EXPECT_NEAR(kurtosis, stats.Kurtosis(), kurtosis * 1E-12);
+  }
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/detect_dots.cc b/jxl/detect_dots.cc
new file mode 100644
index 000000000..a8be46a69
--- /dev/null
+++ b/jxl/detect_dots.cc
@@ -0,0 +1,667 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/detect_dots.h"
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <cstdio>
+#include <hwy/static_targets.h>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/common.h"
+#include "jxl/convolve.h"
+#include "jxl/image.h"
+#include "jxl/image_ops.h"
+#include "jxl/linalg.h"
+#include "jxl/optimize.h"
+
+// Set JXL_DEBUG_DOT_DETECT to 1 to enable debugging.
+#ifndef JXL_DEBUG_DOT_DETECT
+#define JXL_DEBUG_DOT_DETECT 0
+#endif
+
+#if JXL_DEBUG_DOT_DETECT
+#include "jxl/aux_out.h"
+#endif
+
+namespace jxl {
+const int kEllipseWindowSize = 5;
+
+namespace {
+struct GaussianEllipse {
+  double x;                         // position in x
+  double y;                         // position in y
+  double sigma_x;                   // scale in x
+  double sigma_y;                   // scale in y
+  double angle;                     // ellipse rotation in radians
+  std::array<double, 3> intensity;  // intensity in each channel
+
+  // The following variables do not need to be encoded
+  double l2_loss;  // error after the Gaussian was fit
+  double l1_loss;
+  double ridge_loss;              // the l2_loss plus regularization term
+  double custom_loss;             // experimental custom loss
+  std::array<double, 3> bgColor;  // best background color
+  size_t neg_pixels;  // number of negative pixels when subtracting dot
+  std::array<double, 3> neg_value;  // debt due to channel truncation
+};
+double DotGaussianModel(double dx, double dy, double ct, double st,
+                        double sigma_x, double sigma_y, double intensity) {
+  double rx = ct * dx + st * dy;
+  double ry = -st * dx + ct * dy;
+  double md = (rx * rx / sigma_x) + (ry * ry / sigma_y);
+  double value = intensity * exp(-0.5 * md);
+  return value;
+}
+
+constexpr bool kOptimizeBackground = true;
+constexpr std::array<float, 3> colorCoefXyb{0.0, 10.0, 0.0};
+
+// Gaussian that smooths noise but preserves dots
+struct Gaussian5Sigma0_65 {
+  JXL_INLINE const WeightsSeparable5& Weights() const {
+    constexpr float w0 = 0.558311f;
+    constexpr float w1 = 0.210395f;
+    constexpr float w2 = 0.010449f;
+    static constexpr WeightsSeparable5 weights = {
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)},
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}};
+    return weights;
+  }
+};
+
+// (Iterated) Gaussian that removes dots.
+struct Gaussian5Sigma3 {
+  JXL_INLINE const WeightsSeparable5& Weights() const {
+    constexpr float w0 = 0.222338f;
+    constexpr float w1 = 0.210431f;
+    constexpr float w2 = 0.1784f;
+    static constexpr WeightsSeparable5 weights = {
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)},
+        {HWY_REP4(w0), HWY_REP4(w1), HWY_REP4(w2)}};
+    return weights;
+  }
+};
+
+HWY_ATTR ImageF ComputeEnergyImage(const Image3F& orig, Image3F* smooth,
+                                   ThreadPool* pool) {
+  PROFILER_FUNC;
+
+  std::array<float, 3> colorCoef{1.0f, 1.0f, 1.0f};
+  colorCoef = colorCoefXyb;
+  const HWY_FULL(float) d;
+  const auto color_coef0 = Set(d, colorCoef[0]);
+  const auto color_coef1 = Set(d, colorCoef[1]);
+  const auto color_coef2 = Set(d, colorCoef[2]);
+
+  // Prepare guidance images for dot selection.
+  Image3F forig(orig.xsize(), orig.ysize());
+  Image3F tmp(orig.xsize(), orig.ysize());
+  *smooth = Image3F(orig.xsize(), orig.ysize());
+
+  if (orig.xsize() < kConvolveMinWidth) {
+    using Conv = slow::SeparableConvolution<2, WrapMirror>;
+    Conv::Run(orig, Rect(orig), Gaussian5Sigma0_65(), &forig);
+
+    Conv::Run(orig, Rect(orig), Gaussian5Sigma3(), &tmp);
+    Conv::Run(tmp, Rect(tmp), Gaussian5Sigma3(), smooth);
+
+  } else {
+    using Conv = ConvolveT<strategy::Separable5>;
+    Conv::Run(orig, Rect(orig), Gaussian5Sigma0_65(), pool, &forig);
+
+    Conv::Run(orig, Rect(orig), Gaussian5Sigma3(), pool, &tmp);
+    Conv::Run(tmp, Rect(tmp), Gaussian5Sigma3(), pool, smooth);
+  }
+
+#if JXL_DEBUG_DOT_DETECT
+  AuxOut aux;
+  aux.debug_prefix = "/tmp/sebastian/";
+  aux.DumpImage("filtered", forig);
+  aux.DumpImage("sm", *smooth);
+#endif
+
+  ImageF sum_of_squares(orig.xsize(), orig.ysize());
+  RunOnPool(
+      pool, 0, orig.ysize(), ThreadPool::SkipInit(),
+      [&](const int task, const int thread) HWY_ATTR {
+        const size_t y = static_cast<size_t>(task);
+        const float* JXL_RESTRICT orig_row0 = forig.Plane(0).ConstRow(y);
+        const float* JXL_RESTRICT orig_row1 = forig.Plane(1).ConstRow(y);
+        const float* JXL_RESTRICT orig_row2 = forig.Plane(2).ConstRow(y);
+        const float* JXL_RESTRICT smooth_row0 = smooth->Plane(0).ConstRow(y);
+        const float* JXL_RESTRICT smooth_row1 = smooth->Plane(1).ConstRow(y);
+        const float* JXL_RESTRICT smooth_row2 = smooth->Plane(2).ConstRow(y);
+        float* JXL_RESTRICT sos_row = sum_of_squares.Row(y);
+
+        for (size_t x = 0; x < orig.xsize(); x += d.N) {
+          auto v0 = Load(d, orig_row0 + x) - Load(d, smooth_row0 + x);
+          auto v1 = Load(d, orig_row1 + x) - Load(d, smooth_row1 + x);
+          auto v2 = Load(d, orig_row2 + x) - Load(d, smooth_row2 + x);
+          v0 *= v0;
+          v1 *= v1;
+          v2 *= v2;
+          v0 *= color_coef0;  // FMA doesn't help here.
+          v1 *= color_coef1;
+          v2 *= color_coef2;
+          const auto sos = v0 + v1 + v2;  // weighted sum of square diffs
+          Store(sos, d, sos_row + x);
+        }
+      },
+      "ComputeEnergyImage");
+  return sum_of_squares;
+}
+
+struct Pixel {
+  int x;
+  int y;
+};
+
+Pixel operator+(const Pixel& a, const Pixel& b) {
+  return Pixel{a.x + b.x, a.y + b.y};
+}
+
+// Maximum area in pixels of a ellipse
+const size_t kMaxCCSize = 1000;
+
+// Extracts a connected component from a Binary image where seed is part
+// of the component
+bool ExtractComponent(ImageF* img, std::vector<Pixel>* pixels,
+                      const Pixel& seed, double threshold) {
+  PROFILER_FUNC;
+  static const std::vector<Pixel> neighbors{{1, -1}, {1, 0},   {1, 1},  {0, -1},
+                                            {0, 1},  {-1, -1}, {-1, 1}, {1, 0}};
+  std::vector<Pixel> q{seed};
+  while (!q.empty()) {
+    Pixel current = q.back();
+    q.pop_back();
+    pixels->push_back(current);
+    if (pixels->size() > kMaxCCSize) return false;
+    for (const Pixel& delta : neighbors) {
+      Pixel child = current + delta;
+      if (child.x >= 0 && static_cast<size_t>(child.x) < img->xsize() &&
+          child.y >= 0 && static_cast<size_t>(child.y) < img->ysize()) {
+        float* value = &img->Row(child.y)[child.x];
+        if (*value > threshold) {
+          *value = 0.0;
+          q.push_back(child);
+        }
+      }
+    }
+  }
+  return true;
+}
+
+inline bool PointInRect(const Rect& r, const Pixel& p) {
+  return (static_cast<size_t>(p.x) >= r.x0() &&
+          static_cast<size_t>(p.x) < (r.x0() + r.xsize()) &&
+          static_cast<size_t>(p.y) >= r.y0() &&
+          static_cast<size_t>(p.y) < (r.y0() + r.ysize()));
+}
+
+struct ConnectedComponent {
+  ConnectedComponent(const Rect& bounds, const std::vector<Pixel>&& pixels)
+      : bounds(bounds), pixels(pixels) {}
+  Rect bounds;
+  std::vector<Pixel> pixels;
+  float maxEnergy;
+  float meanEnergy;
+  float varEnergy;
+  float meanBg;
+  float varBg;
+  float score;
+  Pixel mode;
+
+  void CompStats(const ImageF& energy, int extra) {
+    PROFILER_FUNC;
+    maxEnergy = 0.0;
+    meanEnergy = 0.0;
+    varEnergy = 0.0;
+    meanBg = 0.0;
+    varBg = 0.0;
+    int nIn = 0;
+    int nOut = 0;
+    mode.x = 0;
+    mode.y = 0;
+    for (int sy = -extra; sy < (static_cast<int>(bounds.ysize()) + extra);
+         sy++) {
+      int y = sy + static_cast<int>(bounds.y0());
+      if (y < 0 || static_cast<size_t>(y) >= energy.ysize()) continue;
+      const float* JXL_RESTRICT erow = energy.ConstRow(y);
+      for (int sx = -extra; sx < (static_cast<int>(bounds.xsize()) + extra);
+           sx++) {
+        int x = sx + static_cast<int>(bounds.x0());
+        if (x < 0 || static_cast<size_t>(x) >= energy.xsize()) continue;
+        if (erow[x] > maxEnergy) {
+          maxEnergy = erow[x];
+          mode.x = x;
+          mode.y = y;
+        }
+        if (PointInRect(bounds, Pixel{x, y})) {
+          meanEnergy += erow[x];
+          varEnergy += erow[x] * erow[x];
+          nIn++;
+        } else {
+          meanBg += erow[x];
+          varBg += erow[x] * erow[x];
+          nOut++;
+        }
+      }
+    }
+    meanEnergy = meanEnergy / nIn;
+    meanBg = meanBg / nOut;
+    varEnergy = (varEnergy / nIn) - meanEnergy * meanEnergy;
+    varBg = (varBg / nOut) - meanBg * meanBg;
+    score = (meanEnergy - meanBg) / std::sqrt(varBg);
+  }
+};
+
+Rect BoundingRectangle(const std::vector<Pixel>& pixels) {
+  PROFILER_FUNC;
+  JXL_ASSERT(pixels.size() > 0);
+  int low_x, high_x, low_y, high_y;
+  low_x = high_x = pixels[0].x;
+  low_y = high_y = pixels[0].y;
+  for (const Pixel& p : pixels) {
+    low_x = std::min(low_x, p.x);
+    high_x = std::max(high_x, p.x);
+    low_y = std::min(low_y, p.y);
+    high_y = std::max(high_y, p.y);
+  }
+  return Rect(low_x, low_y, high_x - low_x + 1, high_y - low_y + 1);
+}
+
+std::vector<ConnectedComponent> FindCC(const ImageF& energy, double t_low,
+                                       double t_high, uint32_t maxWindow,
+                                       double minScore) {
+  PROFILER_FUNC;
+  const int kExtraRect = 4;
+  ImageF img = CopyImage(energy);
+  std::vector<ConnectedComponent> ans;
+  for (size_t y = 0; y < img.ysize(); y++) {
+    float* JXL_RESTRICT row = img.Row(y);
+    for (size_t x = 0; x < img.xsize(); x++) {
+      if (row[x] > t_high) {
+        std::vector<Pixel> pixels;
+        row[x] = 0.0;
+        bool success = ExtractComponent(
+            &img, &pixels, Pixel{static_cast<int>(x), static_cast<int>(y)},
+            t_low);
+        if (!success) continue;
+#if JXL_DEBUG_DOT_DETECT
+        for (size_t i = 0; i < pixels.size(); i++) {
+          fprintf(stderr, "(%d,%d) ", pixels[i].x, pixels[i].y);
+        }
+        fprintf(stderr, "\n");
+#endif  // JXL_DEBUG_DOT_DETECT
+        Rect bounds = BoundingRectangle(pixels);
+        if (bounds.xsize() < maxWindow && bounds.ysize() < maxWindow) {
+          ConnectedComponent cc{bounds, std::move(pixels)};
+          cc.CompStats(energy, kExtraRect);
+          if (cc.score < minScore) continue;
+          JXL_DEBUG(JXL_DEBUG_DOT_DETECT,
+                    "cc mode: (%d,%d), max: %f, bgMean: %f bgVar: "
+                    "%f bound:(%zu,%zu,%zu,%zu)\n",
+                    cc.mode.x, cc.mode.y, cc.maxEnergy, cc.meanEnergy,
+                    cc.varEnergy, cc.bounds.x0(), cc.bounds.y0(),
+                    cc.bounds.xsize(), cc.bounds.ysize());
+          ans.push_back(cc);
+        }
+      }
+    }
+  }
+  return ans;
+}
+
+// TODO (sggonzalez): Adapt this function for the different color spaces or
+// remove it if the color space with the best performance does not need it
+void ComputeDotLosses(GaussianEllipse* ellipse, const ConnectedComponent& cc,
+                      const Image3F& img, const Image3F& background) {
+  PROFILER_FUNC;
+  const int rectBounds = 2;
+  const double kIntensityR = 0.0;   // 0.015;
+  const double kSigmaR = 0.0;       // 0.01;
+  const double kZeroEpsilon = 0.1;  // Tolerance to consider a value negative
+  double ct = cos(ellipse->angle), st = sin(ellipse->angle);
+  const std::array<double, 3> channelGains{1.0, 1.0, 1.0};
+  int N = 0;
+  ellipse->l1_loss = 0.0;
+  ellipse->l2_loss = 0.0;
+  ellipse->neg_pixels = 0;
+  ellipse->neg_value.fill(0.0);
+  double distMeanModeSq = (cc.mode.x - ellipse->x) * (cc.mode.x - ellipse->x) +
+                          (cc.mode.y - ellipse->y) * (cc.mode.y - ellipse->y);
+  ellipse->custom_loss = 0.0;
+  for (int c = 0; c < 3; c++) {
+    for (int sy = -rectBounds;
+         sy < (static_cast<int>(cc.bounds.ysize()) + rectBounds); sy++) {
+      int y = sy + cc.bounds.y0();
+      if (y < 0 || static_cast<size_t>(y) >= img.ysize()) continue;
+      const float* JXL_RESTRICT row = img.ConstPlaneRow(c, y);
+      // bgrow is only used if kOptimizeBackground is false.
+      // NOLINTNEXTLINE(clang-analyzer-deadcode.DeadStores)
+      const float* JXL_RESTRICT bgrow = background.ConstPlaneRow(c, y);
+      for (int sx = -rectBounds;
+           sx < (static_cast<int>(cc.bounds.xsize()) + rectBounds); sx++) {
+        int x = sx + cc.bounds.x0();
+        if (x < 0 || static_cast<size_t>(x) >= img.xsize()) continue;
+        double target = row[x];
+        double dotDelta = DotGaussianModel(
+            x - ellipse->x, y - ellipse->y, ct, st, ellipse->sigma_x,
+            ellipse->sigma_y, ellipse->intensity[c]);
+        if (dotDelta > target + kZeroEpsilon) {
+          ellipse->neg_pixels++;
+          ellipse->neg_value[c] += dotDelta - target;
+        }
+        double bkg = kOptimizeBackground ? ellipse->bgColor[c] : bgrow[x];
+        double pred = bkg + dotDelta;
+        double diff = target - pred;
+        double l2 = channelGains[c] * diff * diff;
+        double l1 = channelGains[c] * std::fabs(diff);
+        ellipse->l2_loss += l2;
+        ellipse->l1_loss += l1;
+        double w = DotGaussianModel(x - cc.mode.x, y - cc.mode.y, 1.0, 0.0,
+                                    1.0 + ellipse->sigma_x,
+                                    1.0 + ellipse->sigma_y, 1.0);
+        ellipse->custom_loss += w * l2;
+        N++;
+      }
+    }
+  }
+  ellipse->l2_loss /= N;
+  ellipse->custom_loss /= N;
+  ellipse->custom_loss += 20.0 * distMeanModeSq + ellipse->neg_value[1];
+  ellipse->l1_loss /= N;
+  double ridgeTerm = kSigmaR * ellipse->sigma_x + kSigmaR * ellipse->sigma_y;
+  for (int c = 0; c < 3; c++) {
+    ridgeTerm += kIntensityR * ellipse->intensity[c] * ellipse->intensity[c];
+  }
+  ellipse->ridge_loss = ellipse->l2_loss + ridgeTerm;
+}
+
+GaussianEllipse FitGaussianFast(const ConnectedComponent& cc,
+                                const ImageF& energy, const Image3F& img,
+                                const Image3F& background) {
+  PROFILER_FUNC;
+  constexpr bool leastSqIntensity = true;
+  constexpr double kEpsilon = 1e-6;
+  GaussianEllipse ans;
+  constexpr int kRectBounds = (kEllipseWindowSize >> 1);
+
+  // Compute the 1st and 2nd moments of the CC
+  double sum = 0.0;
+  int N = 0;
+  std::array<double, 3> m1{0.0, 0.0, 0.0};
+  std::array<double, 3> m2{0.0, 0.0, 0.0};
+  std::array<double, 3> color{0.0, 0.0, 0.0};
+  std::array<double, 3> bgColor{0.0, 0.0, 0.0};
+
+  JXL_DEBUG(JXL_DEBUG_DOT_DETECT, "%zu %zu %zu %zu\n", cc.bounds.x0(),
+            cc.bounds.y0(), cc.bounds.xsize(), cc.bounds.ysize());
+  for (int c = 0; c < 3; c++) {
+    color[c] = img.ConstPlaneRow(c, cc.mode.y)[cc.mode.x] -
+               background.ConstPlaneRow(c, cc.mode.y)[cc.mode.x];
+  }
+  double sign = (color[1] > 0) ? 1 : -1;
+  for (int sy = -kRectBounds; sy <= kRectBounds; sy++) {
+    int y = sy + cc.mode.y;
+    if (y < 0 || static_cast<size_t>(y) >= energy.ysize()) continue;
+    const float* JXL_RESTRICT row = img.ConstPlaneRow(1, y);
+    const float* JXL_RESTRICT bgrow = background.ConstPlaneRow(1, y);
+    for (int sx = -kRectBounds; sx <= kRectBounds; sx++) {
+      int x = sx + cc.mode.x;
+      if (x < 0 || static_cast<size_t>(x) >= energy.xsize()) continue;
+      double w = std::max(kEpsilon, sign * (row[x] - bgrow[x]));
+      sum += w;
+
+      m1[0] += w * x;
+      m1[1] += w * y;
+      m2[0] += w * x * x;
+      m2[1] += w * x * y;
+      m2[2] += w * y * y;
+      for (int c = 0; c < 3; c++) {
+        bgColor[c] += background.ConstPlaneRow(c, y)[x];
+      }
+      N++;
+    }
+  }
+  JXL_CHECK(N > 0);
+
+  for (int i = 0; i < 3; i++) {
+    m1[i] /= sum;
+    m2[i] /= sum;
+    bgColor[i] /= N;
+  }
+
+  // Some magic constants
+  constexpr double kSigmaMult = 1.0;
+  constexpr std::array<double, 3> kScaleMult{1.1, 1.1, 1.1};
+
+  // Now set the parameters of the Gaussian
+  ans.x = m1[0];
+  ans.y = m1[1];
+  for (int j = 0; j < 3; j++) {
+    ans.intensity[j] = kScaleMult[j] * color[j];
+  }
+
+  ImageD Sigma(2, 2), D(1, 2), U(2, 2);
+  Sigma.Row(0)[0] = m2[0] - m1[0] * m1[0];
+  Sigma.Row(1)[1] = m2[2] - m1[1] * m1[1];
+  Sigma.Row(0)[1] = Sigma.Row(1)[0] = m2[1] - m1[0] * m1[1];
+  ConvertToDiagonal(Sigma, &D, &U);
+  const double* JXL_RESTRICT d = D.ConstRow(0);
+  const double* JXL_RESTRICT u = U.ConstRow(1);
+  int p1 = 0, p2 = 1;
+  if (d[0] < d[1]) std::swap(p1, p2);
+  ans.sigma_x = kSigmaMult * d[p1];
+  ans.sigma_y = kSigmaMult * d[p2];
+  ans.angle = std::atan2(u[p1], u[p2]);
+  ans.l2_loss = 0.0;
+  ans.bgColor = bgColor;
+  if (leastSqIntensity) {
+    GaussianEllipse* ellipse = &ans;
+    double ct = cos(ans.angle), st = sin(ans.angle);
+    // Estimate intensity with least squares (fixed background)
+    for (int c = 0; c < 3; c++) {
+      double gg = 0.0;
+      double gd = 0.0;
+      int yc = static_cast<int>(cc.mode.y);
+      int xc = static_cast<int>(cc.mode.x);
+      for (int y = yc - kRectBounds; y <= yc + kRectBounds; y++) {
+        if (y < 0 || static_cast<size_t>(y) >= img.ysize()) continue;
+        const float* JXL_RESTRICT row = img.ConstPlaneRow(c, y);
+        const float* JXL_RESTRICT bgrow = background.ConstPlaneRow(c, y);
+        for (int x = xc - kRectBounds; x <= xc + kRectBounds; x++) {
+          if (x < 0 || static_cast<size_t>(x) >= img.xsize()) continue;
+          double target = row[x] - bgrow[x];
+          double gaussian =
+              DotGaussianModel(x - ellipse->x, y - ellipse->y, ct, st,
+                               ellipse->sigma_x, ellipse->sigma_y, 1.0);
+          gg += gaussian * gaussian;
+          gd += gaussian * target;
+        }
+      }
+      ans.intensity[c] = gd / (gg + 1e-6);  // Regularized least squares
+    }
+  }
+  ComputeDotLosses(&ans, cc, img, background);
+  return ans;
+}
+
+// Probably slower, but optimizes the right thing
+GaussianEllipse FitGaussianOpt(const ConnectedComponent& cc,
+                               const ImageF& energy, const Image3F& img,
+                               const Image3F& background) {
+  const std::array<float, 3> colorFactor{1.0, 1.0, 1.0};
+  GaussianEllipse ans = FitGaussianFast(cc, energy, img, background);
+  auto l2_loss = [&img, &cc, &background,
+                  &colorFactor](const std::vector<double>& params) -> double {
+    GaussianEllipse ellipse;
+    ellipse.x = img.xsize() * params[0];
+    ellipse.y = img.ysize() * params[1];
+    ellipse.sigma_x = exp(params[2]);  // strictly positive
+    ellipse.sigma_y = exp(params[3]);  // strictly positive
+    ellipse.angle = params[4];
+    for (int c = 0; c < 3; c++) {
+      ellipse.intensity[c] = colorFactor[c] * params[5 + c];
+      ellipse.bgColor[c] = colorFactor[c] * params[8 + c];
+    }
+    ComputeDotLosses(&ellipse, cc, img, background);
+    return ellipse.l2_loss;
+  };
+  std::vector<double> init{ans.x / img.xsize(),
+                           ans.y / img.ysize(),
+                           log(ans.sigma_x),
+                           log(ans.sigma_y),
+                           ans.angle,
+                           ans.intensity[0] / colorFactor[0],
+                           ans.intensity[1] / colorFactor[1],
+                           ans.intensity[2] / colorFactor[2],
+                           ans.bgColor[0] / colorFactor[0],
+                           ans.bgColor[1] / colorFactor[1],
+                           ans.bgColor[2] / colorFactor[2]};
+  auto p = optimize::RunSimplex(11, 0.01, 77, init, l2_loss);
+
+  ans.l2_loss = p[0];
+  ans.x = img.xsize() * p[1];
+  ans.y = img.ysize() * p[2];
+  ans.sigma_x = exp(p[3]);
+  ans.sigma_y = exp(p[4]);
+  ans.angle = p[5];
+  for (int c = 0; c < 3; c++) {
+    ans.intensity[c] = colorFactor[c] * p[6 + c];
+    ans.bgColor[c] = colorFactor[c] * p[9 + c];
+  }
+  return ans;
+}
+
+GaussianEllipse FitGaussian(const ConnectedComponent& cc, const ImageF& energy,
+                            const Image3F& img, const Image3F& background) {
+  auto ellipse = FitGaussianFast(cc, energy, img, background);
+  if (ellipse.sigma_x < ellipse.sigma_y) {
+    std::swap(ellipse.sigma_x, ellipse.sigma_y);
+    ellipse.angle += kPi / 2.0;
+  }
+  ellipse.angle -= kPi * std::floor(ellipse.angle / kPi);
+  if (fabs(ellipse.angle - kPi) < 1e-6 || fabs(ellipse.angle) < 1e-6) {
+    ellipse.angle = 0.0;
+  }
+  JXL_CHECK(ellipse.angle >= 0 && ellipse.angle <= kPi &&
+            ellipse.sigma_x >= ellipse.sigma_y);
+  JXL_DEBUG(JXL_DEBUG_DOT_DETECT,
+            "Ellipse mu=(%lf,%lf) sigma=(%lf,%lf) angle=%lf "
+            "intensity=(%lf,%lf,%lf) bg=(%lf,%lf,%lf) l2_loss=%lf "
+            "custom_loss=%lf, neg_pix=%zu, neg_v=(%lf,%lf,%lf)\n",
+            ellipse.x, ellipse.y, ellipse.sigma_x, ellipse.sigma_y,
+            ellipse.angle, ellipse.intensity[0], ellipse.intensity[1],
+            ellipse.intensity[2], ellipse.bgColor[0], ellipse.bgColor[1],
+            ellipse.bgColor[2], ellipse.l2_loss, ellipse.custom_loss,
+            ellipse.neg_pixels, ellipse.neg_value[0], ellipse.neg_value[1],
+            ellipse.neg_value[2]);
+  return ellipse;
+}
+
+}  // namespace
+
+std::vector<PatchInfo> DetectGaussianEllipses(
+    const Image3F& img, const GaussianDetectParams& params,
+    const EllipseQuantParams& qParams, ThreadPool* pool) {
+  PROFILER_FUNC;
+  std::vector<PatchInfo> dots;
+  Image3F smooth(img.xsize(), img.ysize());
+  ImageF energy = ComputeEnergyImage(img, &smooth, pool);
+#if JXL_DEBUG_DOT_DETECT
+  AuxOut aux;
+  aux.debug_prefix = "/tmp/sebastian/";
+  aux.DumpXybImage("smooth", smooth);
+  aux.DumpPlaneNormalized("energy", energy);
+#endif  // JXL_DEBUG_DOT_DETECT
+  std::vector<ConnectedComponent> components = FindCC(
+      energy, params.t_low, params.t_high, params.maxWinSize, params.minScore);
+  size_t numCC =
+      std::min(params.maxCC, (components.size() * params.percCC) / 100);
+  if (components.size() > numCC) {
+    std::sort(
+        components.begin(), components.end(),
+        [](const ConnectedComponent& a, const ConnectedComponent& b) -> bool {
+          return a.score > b.score;
+        });
+    components.erase(components.begin() + numCC, components.end());
+  }
+  for (const auto& cc : components) {
+    GaussianEllipse ellipse = FitGaussian(cc, energy, img, smooth);
+    if (ellipse.x < 0.0 ||
+        std::ceil(ellipse.x) >= static_cast<double>(img.xsize()) ||
+        ellipse.y < 0.0 ||
+        std::ceil(ellipse.y) >= static_cast<double>(img.ysize())) {
+      continue;
+    }
+    if (ellipse.neg_pixels > params.maxNegPixels) continue;
+    double intensity = 0.21 * ellipse.intensity[0] +
+                       0.72 * ellipse.intensity[1] +
+                       0.07 * ellipse.intensity[2];
+    double intensitySq = intensity * intensity;
+    // for (int c = 0; c < 3; c++) {
+    //  intensitySq += ellipse.intensity[c] * ellipse.intensity[c];
+    //}
+    double sqDistMeanMode = (ellipse.x - cc.mode.x) * (ellipse.x - cc.mode.x) +
+                            (ellipse.y - cc.mode.y) * (ellipse.y - cc.mode.y);
+    if (ellipse.l2_loss < params.maxL2Loss &&
+        ellipse.custom_loss < params.maxCustomLoss &&
+        intensitySq > (params.minIntensity * params.minIntensity) &&
+        sqDistMeanMode < params.maxDistMeanMode * params.maxDistMeanMode) {
+      QuantizedPatch patch;
+      patch.xsize = cc.bounds.xsize();
+      patch.ysize = cc.bounds.ysize();
+      size_t x0 = cc.bounds.x0();
+      size_t y0 = cc.bounds.y0();
+      for (size_t y = 0; y < patch.ysize; y++) {
+        for (size_t x = 0; x < patch.xsize; x++) {
+          for (size_t c = 0; c < 3; c++) {
+            patch.fpixels[c][y * patch.xsize + x] =
+                img.ConstPlaneRow(c, y0 + y)[x0 + x] -
+                smooth.ConstPlaneRow(c, y0 + y)[x0 + x];
+          }
+        }
+      }
+      dots.push_back(PatchInfo{patch, {{x0, y0}}});
+    }
+  }
+#if JXL_DEBUG_DOT_DETECT
+  JXL_DEBUG(JXL_DEBUG_DOT_DETECT, "Candidates: %zu, Dots: %zu\n",
+            components.size(), dots.size());
+  ApplyGaussianEllipses(&smooth, dots, 1.0);
+  aux.DumpXybImage("draw", smooth);
+  ApplyGaussianEllipses(&smooth, dots, -1.0);
+
+  auto qdots = QuantizeGaussianEllipses(dots, qParams);
+  auto deq = DequantizeGaussianEllipses(qdots, qParams);
+  ApplyGaussianEllipses(&smooth, deq, 1.0);
+  aux.DumpXybImage("qdraw", smooth);
+  ApplyGaussianEllipses(&smooth, deq, -1.0);
+#endif  // JXL_DEBUG_DOT_DETECT
+  return dots;
+}
+
+}  // namespace jxl
diff --git a/jxl/detect_dots.h b/jxl/detect_dots.h
new file mode 100644
index 000000000..86424c4e0
--- /dev/null
+++ b/jxl/detect_dots.h
@@ -0,0 +1,74 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// We attempt to remove dots, or speckle from images using Gaussian blur.
+#ifndef JXL_DETECT_DOTS_H_
+#define JXL_DETECT_DOTS_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/image.h"
+#include "jxl/patch_dictionary.h"
+
+namespace jxl {
+
+struct GaussianDetectParams {
+  double t_high;  // at least one pixel must have larger energy than t_high
+  double t_low;   // all pixels must have a larger energy than tLow
+  uint32_t maxWinSize;  // discard dots larger than this containing window
+  double maxL2Loss;
+  double maxCustomLoss;
+  double minIntensity;     // If the intensity is too low, discard it
+  double maxDistMeanMode;  // The mean and the mode must be close
+  size_t maxNegPixels;     // Maximum number of negative pixel
+  size_t minScore;
+  size_t maxCC = 50;   // Maximum number of CC to keep
+  size_t percCC = 15;  // Percentage in [0,100] of CC to keep
+};
+
+// Ellipse Quantization Params
+struct EllipseQuantParams {
+  size_t xsize;      // Image size in x
+  size_t ysize;      // Image size in y
+  size_t qPosition;  // Position quantization delta
+  // Quantization for the Gaussian sigma parameters
+  double minSigma;
+  double maxSigma;
+  size_t qSigma;  // number of quantization levels
+  // Quantization for the rotation angle (between -pi and pi)
+  size_t qAngle;
+  // Quantization for the intensity
+  std::array<double, 3> minIntensity;
+  std::array<double, 3> maxIntensity;
+  std::array<size_t, 3> qIntensity;  // number of quantization levels
+  // Extra parameters for the encoding
+  bool subtractQuantized;  // Should we subtract quantized or detected dots?
+  float ytox;
+  float ytob;
+
+  void QuantPositionSize(size_t* xsize, size_t* ysize) const;
+};
+
+// Detects dots in XYB image.
+std::vector<PatchInfo> DetectGaussianEllipses(
+    const Image3F& opsin, const GaussianDetectParams& params,
+    const EllipseQuantParams& qParams, ThreadPool* pool);
+
+}  // namespace jxl
+
+#endif  // JXL_DETECT_DOTS_H_
diff --git a/jxl/docs/color_management.md b/jxl/docs/color_management.md
new file mode 100644
index 000000000..23aec9626
--- /dev/null
+++ b/jxl/docs/color_management.md
@@ -0,0 +1,68 @@
+# Color Management
+
+[TOC]
+
+<!--*
+# Document freshness: For more information, see go/fresh-source.
+freshness: { owner: 'janwas' reviewed: '2019-02-01' }
+*-->
+
+## Why
+
+The vast majority of web images are still sRGB. However, wide-gamut material is
+increasingly being produced (photography, cinema, 4K). Screens covering most of
+the Adobe RGB gamut are readily available and some also cover most of DCI P3
+(iPhone, Pixel2) or even BT.2020.
+
+Currently, after a camera records a very saturated red pixel, most raw
+processors would clip it to the rather small sRGB gamut before saving as JPEG.
+In keeping with our high-quality goal, we prevent such loss by allowing wider
+input color spaces.
+
+## Which color space
+
+Even wide gamuts could be expressed relative to the sRGB primaries, but the
+resulting coordinates may be outside the valid 0..1 range. Surprisingly, such
+'unbounded' coordinates can be passed through color transforms provided the
+transfer functions are expressed as parametric functions (not lookup tables).
+However, most image file formats (including PNG and PNM) lack min/max metadata
+and thus do not support unbounded coordinates.
+
+Instead, we need a larger working gamut to ensure most pixel coordinates are
+within bounds and thus not clipped. However, larger gamuts result in lower
+precision/resolution when using <= 16 bit encodings (as opposed to 32-bit float
+in PFM). BT.2100 or P3 DCI appear to be good compromises.
+
+## CMS library
+
+Transforms with unbounded pixels are desirable because they reduce round-trip
+error in tests. This requires parametric curves, which are only supported for
+the common sRGB case in ICC v4 profiles. ArgyllCMS does not support v4. The
+other popular open-source CMS is LittleCMS. It is also used by color-managed
+editors (Krita/darktable), which increases the chances of interoperability.
+However, LCMS has race conditions and overflow issues that prevent fuzzing. We
+will later switch to the newer skcms. Note that this library does not intend to
+support multiProcessElements, so HDR transfer functions cannot be represented
+accurately. Thus in the long term, we will probably migrate away from ICC
+profiles entirely.
+
+## Which viewer
+
+On Linux, Krita and darktable support loading our PNG output images and their
+ICC profile.
+
+## How to compress/decompress
+
+### Embedded ICC profile
+
+-   Create an 8-bit or 16-bit PNG with an iCCP chunk, e.g. using darktable.
+-   Pass it to `cpik`, then `dpik` with no special arguments. The decoded output
+    will have the same bit depth (can override with `--output_bit_depth`) and
+    color space.
+
+### Images without metadata (e.g. HDR)
+
+-   Create a PGM/PPM/PFM file in a known color space.
+-   Invoke `cpik` with `-x color_space RGB_D65_202_Rel_Lin` (linear 2020). For
+    details/possible values, see color_encoding.cc `Description`.
+-   Invoke `dpik` as above.
diff --git a/jxl/docs/dc_predictor.md b/jxl/docs/dc_predictor.md
new file mode 100644
index 000000000..478b048f6
--- /dev/null
+++ b/jxl/docs/dc_predictor.md
@@ -0,0 +1,129 @@
+# DC prediction
+
+[TOC]
+
+<!--*
+# Document freshness: For more information, see go/fresh-source.
+freshness: { owner: 'janwas' reviewed: '2019-02-22' }
+*-->
+
+## Background
+
+After the DCT integral transform, we wish to reduce the encoded size of DC
+coefficients (or original pixels if the transform is omitted).
+pik/dc_predictor.h provides `Shrink` and `Expand` functions to reduce the
+magnitude of pixels by subtracting predicted values computed from their
+neighbors. In the ideal case of flat or slowly varying regions, the resulting
+"residuals" are zero and thus very efficiently encodable.
+
+## Design goals
+
+1.  Benefit from SIMD.
+1.  Use prior horizontal neighbor for prediction because it has the highest
+    correlation.
+1.  Use previously decoded planes (i.e. luminance) for improving prediction of
+    chrominance.
+1.  Lossless (no overflow/rounding errors) for signed integers in [-32768,
+    32768).
+
+## Prior approaches
+
+CALIC GAP and LOCO-I/JPEG-LS MED are somewhat adaptive: they choose from one of
+several simple predictors based on properties of the pixel neighborhood (let N
+denote the pixel above the current one, W = left, and NW above-left). History
+Based Blending introduced more freedom by adding adaptive weights and several
+more predictors including averaging, gradients and neighbors. We use a similar
+approach, but with selecting rather than blending predictors to avoid
+multiplications, and a more efficient cost estimator.
+
+## Basic algorithm
+
+```
+Compute all predictors for the already decoded N and W pixel.
+See which single predictor performs best (lowest total absolute residual).
+Residual := current pixel - best predictor's prediction for current pixel.
+To expand again, add this same prediction to the stored residual.
+```
+
+## Details
+
+### Dependencies between SIMD lanes
+
+The SIMD and prior horizontal neighbor requirements seem to be contradictory. If
+all steps are computed in units of SIMD vectors, then the prior neighbor may be
+part of the same vector and thus not computed beforehand. Instead, we compute
+multiple _predictors_ at a time using SIMD for the same pixel. This is feasible
+because several predictors use the same computation (e.g. Average) on
+independent inputs in the SIMD vector lanes.
+
+### Predictors
+
+We use several kinds of predictors.
+
+1.  **Average**: Noisy areas have outliers; computing the average is a simple
+    way to reduce their impact without expensive non-linear computations. We use
+    a (saturated) add and shift rather than the average+round SIMD instruction.
+1.  **Gradient**: Smooth areas benefit from a gradient; we find it helpful to
+    clamp the JPEG-LS-style N+W-NW to the min/max of all neighbor pixels.
+1.  **Edge**: Finally, unchanged N/W neighbor values are also helpful for
+    regions with edges because they violate the assumptions underlying the
+    average and gradient predictors (namely a shared central tendency or
+    smoothly varying neighborhood).
+
+We chose predictors subject to the constraint that 128-bit SIMD instruction sets
+can compute eight of them at a time, with a simple optimization algorithm that
+tried various permutations of these basic predictors. Note that order matters:
+the first predictor with min cost is chosen. The test data set was small, so it
+is possible that the choice of predictors could be improved, but that involves
+rewriting some intricate SIMD code.
+
+### Finding best predictor
+
+Given a SIMD vector with all prediction results, we compute each of their
+residuals by subtracting from the (broadcasted) current pixel. X86 SSE4 provides
+special support for finding the minimum element in u16x8 vectors. This lane can
+efficiently moved to the first lane via shuffling. On other architectures, we
+scan through the 8 lanes manually.
+
+### Threading
+
+No separate parallelization is needed because this is part of the JXL decoder,
+which ensures all pixel decoding steps are independent (in 512x512 groups) and
+computed in parallel.
+
+### Cross-channel correlations
+
+The adaptive predictor exploits correlations between neighboring pixels. To also
+reduce correlation between channels, we require the Y channel to be decoded
+first (green is in the middle of the frequency band and thus has higher
+correlation with red/blue). Then, rather than computing the cost (prediction
+residual) at N and W neighbors, we compute the cost at the same position in the
+previously decoded luminance image.
+Note: this assumes there is some remaining correlation (despite chroma from
+luma already being carried out), which is plausible because the current CfL
+model for DC is global, i.e. unable to remove all correlations. Once that
+changes, it may be better to no longer use Y to influence the chroma channels.
+
+### Bounds checking
+
+Predictors only use immediately adjacent pixels, and the cost estimator also
+applies predictors at immediately adjacent neighbors, so we need a border of two
+pixels on each side. For the first pixel, we have no predictor; subsequent
+pixels in the same row use their left neighbor. In the second row, we can apply
+the adaptive predictor while supplying an imaginary extra top row that is
+identical to the first row.
+
+### Side information
+
+Only causal (previously decoded in left to right, then top to bottom line scan
+order) neighbors are used, so the decoder-side `Expand` can run without any side
+information.
+
+### Related work
+
+lossless16 by Alex Rhatushnyak implements a more powerful predictor which uses
+the error history as feedback for the prediction weights. lossless16 is indeed
+better than dc_predictor for lossless photo compression. However, using
+lossless16 for encoding DCs has not yet proven to be advantageous - even after
+entropy-coding of the channel compact transforms, and encoding the entire DC
+image in a single call.
diff --git a/jxl/docs/entropy_coding_basic.md b/jxl/docs/entropy_coding_basic.md
new file mode 100644
index 000000000..973e62f59
--- /dev/null
+++ b/jxl/docs/entropy_coding_basic.md
@@ -0,0 +1,111 @@
+# Basic entropy encoders
+
+[TOC]
+
+<!--*
+# Document freshness: For more information, see go/fresh-source.
+freshness: { owner: 'user' reviewed: '2018-12-13' }
+*-->
+
+This document describes low level encodings used in JXL.
+
+## Uint32 field
+
+Field is a variable length encoding for values with predefined distribution.
+
+Distribution is described with \\(\text{distribution}\\) value.
+
+If \\(\text{distribution} > \texttt{0xFFFFFFDF} = 2^{32} - 32\\) then value is
+always encoded as \\(\text{distribution} - \texttt{0xFFFFFFDF}\\) bits.
+This mode is called "raw" encoding. Raw encoding is typically used for
+edge cases, e.g. when exactly 1 or 32 bit value needs to be encoded.
+
+In "regular" (non-raw) mode \\(\text{distribution}\\) is interpreted as array
+\\(\text{L}\\) containing 4 uint8 values.
+
+To decode regular field, first 2 bits are read;
+those bits represent "selector" value \\(\text{S}\\).
+
+Depending on mode \\(\text{M} = \text{L}[\text{S}]\\):
+
+-   if \\(\text{M} \ge \texttt{0x80}\\),
+    then \\(\text{M} - \texttt{0x80}\\) is the "direct" encoded value
+-   else if \\(\text{M} \ge \texttt{0x40}\\),
+    let \\(\text{V}\\) be the \\((\text{M} \& \texttt{0x7}) + 1\\)
+    following bits, "shifted" encoded value is
+    \\(\text{V} + ((\text{M} \gg 3) \& \texttt{0x7}) + 1\\)
+-   otherwise \\(\text{M}\\) following bits represent the encoded value
+
+Source code: cs/jxl/fields.h
+
+## Uint64 field
+
+This field supports bigger values than [Uint32](#uint32-field), but has single
+fixed distribution.
+
+Value is decoded as following:
+
+-   "selector" \\(\text{S}\\) 2 bits
+-   if \\(\text{S} = 0\\) then value is 0
+-   if \\(\text{S} = 1\\) then next 4 bits represent \\(\text{value} - 1\\)
+-   if \\(\text{S} = 2\\) then next 8 bits represent \\(\text{value} - 17\\)
+-   if \\(\text{S} = 3\\) then:
+    -   12 bits represent the lowest 12 bits of value
+    -   while next bit is 1:
+        -   if less than 60 value bits are already read,
+            then 8 bits represent higher 8 bits of value
+        -   otherwise 4 bits represent the highest 4 bits of value and 'while'
+            loop is finished
+
+Source code: cs/jxl/fields.h
+
+## Byte array field
+
+Byte array field holds (optionally compressed) array of bytes. Byte array is
+encoded as:
+
+-   \\(\text{type}\\) [field](#uint32-field) /
+    L = [direct 0, direct 1, direct 2, 3 bits + 3]
+-   if \\(\text{type}\\) is \\(\text{None} = 0\\), then byte array is empty
+-   if \\(\text{type}\\) is \\(\text{Raw} = 1\\) or \\(\text{Brotli} = 2\\),
+    then:
+    -   \\(\text{size}\\) [field](#uint32-field) /
+        L = [8 bits, 16 bits, 24 bits, 32 bits]
+    -   uint8 repeated \\(\text{size}\\) times;
+    -   payload is compressed if \\(\text{type}\\) is \\(\text{Brotli}\\)
+
+Source code: cs/jxl/fields.h
+
+## VarSignedMantissa
+
+VarMantissa is a mapping from \\(\text{L}\\) bits to signed values.
+The principle is that \\(\text{L} + 1\\) bits may only represent values whose
+absolute values are bigger than absolute values that could be represented with
+\\(\text{L}\\) bits.
+
+-   0-bit values represent \\(\{0\}\\)
+-   1-bit values represent \\(\{-1, 1\}\\)
+-   2-bit values represent \\(\{-3, -2, 2, 3\}\\)
+-   L-bit values represent \\(\{\pm 2^{L - 1} .. \pm(2^L - 1)\}\\)
+
+## VarUnsignedMantissa
+
+Analogous to [VarSignedMantissa](#varsignedmantissa), but for unsigned values.
+
+TODO: provide examples
+
+## VarLenUint8
+
+-   \\(\text{zero}\\) 1 bit
+-   if \\(\text{zero}\\) is \\(0\\), then value is \\(0\\), otherwise:
+    -   \\(\text{L}\\) 3 bits
+    -   \\(\text{value} - 1\\) encoded as
+        \\(\text{L}\\)-bit [VarUnsignedMantissa](#vaunrsignedmantissa)
+
+## VarLenUint16
+
+-   \\(\text{zero}\\) 1 bit
+-   if \\(\text{zero}\\) is \\(0\\), then value is \\(0\\), otherwise:
+    -   \\(\text{L}\\) 4 bits
+    -   \\(\text{value} - 1\\) encoded as
+        \\(\text{L}\\)-bit [VarUnsignedMantissa](#varunsignedmantissa)
diff --git a/jxl/docs/file_format.md b/jxl/docs/file_format.md
new file mode 100644
index 000000000..429ee669d
--- /dev/null
+++ b/jxl/docs/file_format.md
@@ -0,0 +1,129 @@
+# File format overview
+
+[TOC]
+
+<!--*
+# Document freshness: For more information, see go/fresh-source.
+freshness: { owner: 'user' reviewed: '2018-12-17' }
+
+!!!!!!WARNING: copied into working draft, update that instead!!!!!!
+*-->
+
+This document describes high level PIK format and metadata.
+
+## FileHeader
+
+Topmost structure that contains most general image related metadata and a file
+signature used to distinguish PIK files.
+
+Structure:
+
+-   "signature" 32 bits; must be equal to little-endian representation of
+    \\(\texttt{0x0A4CD74A}\\) value; \\(\texttt{0x0A}\\) causes files opened in
+    text mode to be rejected, and \\(\texttt{0xD7}\\) detects 7-bit transfers
+-   "xsize_minus_1" [field](entropy_coding_basic.md#uint32-field) /
+    L = [9 bits, 11 bits, 13 bits, 32 bits]; 13-bit values support most existing
+    cameras (up to 8K x 8K images), 32-bit values cover provide support for up
+    to 4G x 4G images; "xsize" is derived from this field by adding \\(1\\),
+    thus zero-width images are not supported
+-   "ysize_minus_1" [field](entropy_coding_basic.md#uint32-field) /
+    L = [9 bits, 11 bits, 13 bits, 32 bits]; similar to "xsize_minus_1"
+-   orientation indicates 8 possible orientations, as defined in EXIF.
+-   nested ["metadata"](#metadata) structure
+-   nested ["preview"](#preview) structure
+-   nested ["animation"](#animation) structure
+-   ["extensions"](#extensions) stub that allows future format extension
+
+## Metadata
+
+[Optional structure](#optional-structures) that contains meta-information about
+image and other supportive information.
+
+Structure:
+-   "all_default" 1 bit; see ["optional structures"](#optional-structures)
+-   nested ["transcoded"](#transcoded) structure
+-   "target_nits_div50" [field](entropy_coding_basic.md#uint32-field) /
+    L = [direct 2, direct 5, direct 80, 8 bits];
+    most common values (100, 250, 4000) are encoded directly; maximal
+    expressible intensity is 12750
+-   "exif" compressed [byte array](entropy_coding_basic.md#byte-array-field);
+    original image metadata
+-   "iptc" compressed [byte array](entropy_coding_basic.md#byte-array-field);
+    original image metadata
+-   "xmp" compressed [byte array](entropy_coding_basic.md#byte-array-field);
+    original image metadata
+
+## Transcoded
+
+[Optional structure](#optional-structures) that contains information original
+image.
+
+Structure:
+
+-   "all_default" 1 bit; see ["optional structures"](#optional-structures)
+-   "original_bit_depth" [field](entropy_coding_basic.md#uint32-field) /
+    L = [direct 8, direct 16, direct 32, 5 bits]
+-   "original_color_encoding" nested [ColorEncoding](#colorencoding) structure
+-   "original_bytes_per_alpha" [field](entropy_coding_basic.md#uint32-field) /
+    L = [direct 0, direct 1, direct 2, direct 4]
+
+
+## Preview
+
+[Optional structure](#optional-structures) that contains information about
+"discardable preview" image. Unlike early "progressive" passes, this image is
+completely independent and optimized for better "preview" experience, i.e.
+appropriately preprocessed and non-power-of-two scaled.
+
+Preview images have different size constraints than main image and currently
+limited to 8K x 8K; size limit is set to be 128MiB.
+
+Structure:
+
+-   "all_default" 1 bit; see ["optional structures"](#optional-structures)
+-   "size_bits" [field](entropy_coding_basic.md#uint32-field) /
+    L = [12 bits, 16 bits, 20 bits, 28 bits]
+-   "xsize" [field](entropy_coding_basic.md#uint32-field) /
+    L = [7 bits, 9 bits, 11 bits, 13 bits]
+-   "ysize" [field](entropy_coding_basic.md#uint32-field) /
+    L = [7 bits, 9 bits, 11 bits, 13 bits]
+
+## Animation
+
+[Optional structure](#optional-structures) that contains meta-information about
+animation (image sequence).
+
+Structure:
+
+-   "all_default" 1 bit; see ["optional structures"](#optional-structures)
+-   "num_loops" [field](entropy_coding_basic.md#uint32-field) /
+    L = [direct 0, 3 bits, 16 bits, 32 bits]; \\(0\\) means to repeat infinitely
+-   "ticks_numerator" [field](entropy_coding_basic.md#uint32-field) /
+    L = [direct 1, 9 bits, 20 bits, 32 bits]
+-   "ticks_denominator" [field](entropy_coding_basic.md#uint32-field) /
+    L = [direct 1, 9 bits, 20 bits, 32 bits]
+
+## Extensions
+
+This "structure" is usually put at the end of other structures which would be
+extended in future. It allows earlier versions of decoders to skip the newer
+fields.
+
+Structure:
+
+-   "extensions" [field](entropy_coding_basic.md#uint64-field)
+-   if "extensions" is \\(0\\), then no extra information follows
+-   otherwise:
+    -   "extension_bits" [field](entropy_coding_basic.md#uint64-field) - number
+        of bits to be skipped by decoder that does not expect extensions here
+
+## Optional structures
+
+Some structures are "optional". In case all the field values are equal to their
+defaults, encoder is eligible to represent the structure with a single bit.
+
+Structures that contain non-empty [extension](#extensions) tail are ineligible
+for 1-bit encoding.
+
+Technically, this bit is represented as "all_default" field that comes first; if
+the value of this field is \\(1\\), then the rest of structure is not decoded.
diff --git a/jxl/docs/upsample.md b/jxl/docs/upsample.md
new file mode 100644
index 000000000..bcbc54c58
--- /dev/null
+++ b/jxl/docs/upsample.md
@@ -0,0 +1,151 @@
+# Upsampling
+
+[TOC]
+
+<!--*
+# Document freshness: For more information, see go/fresh-source.
+freshness: { owner: 'janwas' reviewed: '2019-02-22' }
+*-->
+
+jxl/resample.h provides `Upsampler8` for fast and high-quality 8x8 upsampling by
+4x4 (separable/non-separable) or 6x6 (non-separable) floating-point kernels.
+This was previously used for the "smooth predictor", which has been removed. It
+would still be useful for a progressive mode that upsamples DC for a preview,
+though this code is not yet used for that purpose.
+
+See 'Separability' section below for the surprising result that non-separable
+can be faster than separable and possibly better.
+
+## Performance evaluation
+
+### 4x4 separable/non-separable
+
+__Single-core__: 5.1 GB/s (single-channel, floating-point, 160x96 input)
+
+*   40x speedup vs. an unoptimized Nehab/Hoppe bicubic upsampler
+*   25x or 6x speedup vs. [Pillow 4.3](http://python-pillow.org/pillow-perf/)
+    bicubic (AVX2: 66M RGB 8-bit per second) in terms of bytes or samples
+*   2.6x AVX2 speedup vs. our SSE4 version (similar algorithm)
+
+__12 independent instances on 12 cores__: same speed for each instance (linear
+scalability: not limited by memory bandwidth).
+
+__Multicore__: 15-18 GB/s (single-channel, floating-point, 320x192 input)
+
+*   9-11x or 2.3-2.8x speedup vs. a parallel Halide bicubic (1.6G single-channel
+    8-bit per second, 320x192 input) in terms of bytes or samples.
+
+### 6x6 non-separable
+
+Note that a separable (outer-product) kernel only requires 6+6 (12)
+multiplications per output pixel/vector. However, we do not assume anything
+about the kernel rank (separability), and thus require 6x6 (36) multiplications.
+
+__Single-core__: 2.8 GB/s (single-channel, floating-point, 320x192 input)
+
+*   5x speedup vs. optimized (outer-product of two 1D) Lanczos without SIMD
+
+__Multicore__: 9-10 GB/s (single-channel, floating-point, 320x192 input)
+
+*   7-8x or 2x speedup vs. a parallel Halide 6-tap tensor-product Lanczos (1.3G
+    single-channel 8-bit per second, 320x192 input) in terms of bytes or
+    samples.
+
+## Implementation details
+
+### Data type
+
+Our input pixels are 16-bit, so 8-bit integer multiplications are insufficient.
+16-bit fixed-point arithmetic is fast but risks overflow or loss of precision
+unless the value intervals are carefully managed. 32-bit integers reduce this
+concern, but 32-bit multiplies are slow on Haswell (one mullo32 per cycle with
+10+1 cycle latency). We instead use single-precision float, which enables 2 FMA
+per cycle with 5 cycle latency. The extra bandwidth vs. 16-bit types types may
+be a concern, but we can run 12 instances (on a 12-core socket) with zero
+slowdown, indicating that memory bandwidth is not the bottleneck at the moment.
+
+### Pixel layout
+
+We require planar inputs, e.g. all red channel samples in a 2D matrix. This
+allows better utilization of 8-lane SIMD compared to dedicating four SIMD lanes
+to R,G,B,A samples. If upsampling is the only operation, this requires an extra
+deinterleaving step, but our application involves a larger processing pipeline.
+
+### No prefilter
+
+Nehab and Hoppe (http://hhoppe.com/filtering.pdf) advocate generalized sampling
+with an additional digital filter step. They claim "significantly higher
+quality" for upsampling operations, which is contradicted by minimal MSSIM
+differences between cardinal and ordinary cubic interpolators in their
+experiment on page 65. We also see only minor Butteraugli differences between
+Catmull-Rom and B-spline3 or OMOMS3 when upsampling 8x. As they note on page 29,
+a prefilter is often omitted when upsampling because the reconstruction filter's
+frequency cutoff is lower than that of the prefilter. The prefilter is claimed
+to be efficient (relative to normal separable convolutions), but still involves
+separate horizontal and vertical passes. To avoid cache thrashing would require
+a separate ring buffer of rows, which may be less efficient than our single-pass
+algorithm which only writes final outputs.
+
+### 8x upsampling
+
+Our code is currently specific to 8x upsampling because that is what is required
+for our (DCT prediction) application. This happens to be a particularly good fit
+for both 4-lane and 8-lane (AVX2) SIMD AVX2. It would be relatively easy to
+adapt the code to 4x upsampling.
+
+### Kernel support
+
+Even (asymmetric) kernels are often used to reduce computation. Many upsampling
+applications use 4-tap cubic interpolation kernels but at such extreme
+magnifications (8x) we find 6x6 to be better.
+
+### Separability
+
+Separable kernels can be expressed as the (outer) product of two 1D kernels. For
+n x n kernels, this requires n + n multiplications per pixel rather than n x n.
+However, the Fourier transform of such kernels (except the Gaussian) has a
+square rather than disc shape.
+
+We instead allow arbitrary (possibly non-separable) kernels. This can avoid
+structural dependencies between output pixels within the same 8x8 block.
+
+Surprisingly, 4x4 non-separable is actually faster than the separable version
+due to better utilization of FMA units. For 6x6, we only implement the
+non-separable version because our application benefits from such kernels.
+
+### Dual grid
+
+A primal grid with `n` grid points at coordinates `k/n` would be convenient for
+index computations, but is asymmetric at the borders (0 and `(n-1)/n` != 1) and
+does not compensate for the asymmetric (even) kernel support. We instead use a
+dual grid with coordinates offset by half the sample spacing, which only adds a
+integer shift term to the index computations. Note that the offset still leads
+to four identical input pixels in 8x upsampling, which is convenient for 4 and
+even 8-lane SIMD.
+
+### Kernel
+
+We use Catmull-Rom splines out of convenience. Computational cost does not
+matter because the weights are precomputed. Also, Catmull-Rom splines pass
+through the control points, but that does not matter in this application.
+B-splines or other kernels (arbitrary coefficients, even non-separable) can also
+be used.
+
+### Single pixel loads
+
+For SIMD convolution, we have the choice whether to broadcast inputs or weights.
+To ensure we write a unit-stride vector of output pixels, we need to broadcast
+the input pixels and vary the weights. This has the additional benefit of
+avoiding complexity at the borders, where it is not safe to load an entire
+vector. Instead, we only load and broadcast single pixels, with bounds checking
+at the borders but not in the interior.
+
+### Single pass
+
+Separable 2D convolutions are often implemented as two separate 1D passes.
+However, this leads to cache thrashing in the vertical pass, assuming the image
+does not fit into L2 caches. We instead use a single-pass algorithm that
+generates four (see "Kernel support" above) horizontal convolution results and
+immediately convolves them in the vertical direction to produce a final output.
+This is simpler than sliding windows and has a smaller working set, thus
+enabling the major speedups reported above.
diff --git a/jxl/dot_dictionary.cc b/jxl/dot_dictionary.cc
new file mode 100644
index 000000000..920899d6d
--- /dev/null
+++ b/jxl/dot_dictionary.cc
@@ -0,0 +1,104 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/dot_dictionary.h"
+
+#include <stddef.h>
+#include <string.h>
+
+#include <array>
+#include <utility>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/override.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_xyb.h"
+#include "jxl/detect_dots.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_params.h"
+#include "jxl/enc_xyb.h"
+#include "jxl/image.h"
+
+namespace jxl {
+
+// Private implementation of Dictionary Encode/Decode
+namespace {
+
+/* Quantization constants for Ellipse dots */
+const size_t kEllipsePosQ = 2;        // Quantization level for the position
+const double kEllipseMinSigma = 0.1;  // Minimum sigma value
+const double kEllipseMaxSigma = 3.1;  // Maximum Sigma value
+const size_t kEllipseSigmaQ = 16;     // Number of quantization levels for sigma
+const size_t kEllipseAngleQ = 8;      // Quantization level for the angle
+// TODO: fix these values.
+const std::array<double, 3> kEllipseMinIntensity{-0.05, 0.0, -0.5};
+const std::array<double, 3> kEllipseMaxIntensity{0.05, 1.0, 0.4};
+const std::array<size_t, 3> kEllipseIntensityQ{10, 36, 10};
+}  // namespace
+
+namespace {
+
+GaussianDetectParams GetConstDetectParams() {
+  GaussianDetectParams ans;
+  memset(&ans, 0, sizeof(ans));
+  ans.t_high = 0.04;
+  ans.t_low = 0.02;
+  ans.maxWinSize = 5;
+  ans.maxL2Loss = 0.005;
+  ans.maxCustomLoss = 300;
+  ans.minIntensity = 0.12;
+  ans.maxDistMeanMode = 1.0;
+  ans.maxNegPixels = 0;
+  ans.minScore = 12.0;
+  ans.maxCC = 100;
+  ans.percCC = 100;
+  return ans;
+}
+
+const GaussianDetectParams kEllipseDetectParams = GetConstDetectParams();
+
+};  // namespace
+
+std::vector<PatchInfo> FindDotDictionary(const CompressParams& cparams,
+                                         const Image3F& opsin,
+                                         const ColorCorrelationMap& cmap,
+                                         ThreadPool* pool) {
+  if (ApplyOverride(cparams.dots,
+                    cparams.butteraugli_distance >= kMinButteraugliForDots)) {
+    EllipseQuantParams qParams{opsin.xsize(),
+                               opsin.ysize(),
+                               kEllipsePosQ,
+                               kEllipseMinSigma,
+                               kEllipseMaxSigma,
+                               kEllipseSigmaQ,
+                               kEllipseAngleQ,
+                               kEllipseMinIntensity,
+                               kEllipseMaxIntensity,
+                               kEllipseIntensityQ,
+                               kEllipsePosQ <= 5,
+                               cmap.YtoXRatio(kColorOffset),
+                               cmap.YtoBRatio(kColorOffset)};
+    // cparams.butteraugli_distance < 1.5};
+    GaussianDetectParams eDetectParams = kEllipseDetectParams;
+    /*if (cparams.butteraugli_distance > 2.5) {
+      eDetectParams.maxNegPixels = 30;
+    }*/
+
+    return DetectGaussianEllipses(opsin, eDetectParams, qParams, pool);
+  }
+  return {};
+}
+}  // namespace jxl
diff --git a/jxl/dot_dictionary.h b/jxl/dot_dictionary.h
new file mode 100644
index 000000000..34354865a
--- /dev/null
+++ b/jxl/dot_dictionary.h
@@ -0,0 +1,45 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_DOT_DICTIONARY_H_
+#define JXL_DOT_DICTIONARY_H_
+
+// Dots are stored in a dictionary to avoid storing similar dots multiple
+// times.
+
+#include <stddef.h>
+
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_params.h"
+#include "jxl/image.h"
+#include "jxl/patch_dictionary.h"
+
+namespace jxl {
+
+std::vector<PatchInfo> FindDotDictionary(const CompressParams& cparams,
+                                         const Image3F& opsin,
+                                         const ColorCorrelationMap& cmap,
+                                         ThreadPool* pool);
+
+}  // namespace jxl
+
+#endif  // JXL_DOT_DICTIONARY_H_
diff --git a/jxl/enc_ac_strategy.cc b/jxl/enc_ac_strategy.cc
new file mode 100644
index 000000000..e18aaccd4
--- /dev/null
+++ b/jxl/enc_ac_strategy.cc
@@ -0,0 +1,849 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_ac_strategy.h"
+
+#include <stdint.h>
+#include <string.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdio>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/ans_params.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/convolve.h"
+#include "jxl/enc_params.h"
+#include "jxl/entropy_coder.h"
+
+namespace jxl {
+
+// Parameters of the heuristic are marked with a OPTIMIZE comment.
+
+namespace {
+
+// Debugging utilities.
+
+// Returns a linear sRGB color (as bytes) for each AC strategy.
+static const uint8_t* TypeColor(const uint8_t& raw_strategy) {
+  JXL_ASSERT(AcStrategy::IsRawStrategyValid(raw_strategy));
+  static_assert(AcStrategy::kNumValidStrategies == 18, "Change colors");
+  static constexpr uint8_t kColors[][3] = {
+      {0x00, 0xBB, 0xBB},  // DCT8
+      {0x00, 0xFF, 0xFF},  // IDENTITY
+      {0x00, 0xF8, 0xF8},  // DCT2x2
+      {0x00, 0xF0, 0xF0},  // DCT4x4
+      {0x00, 0x77, 0x77},  // DCT16x16
+      {0x00, 0x33, 0x33},  // DCT32x32
+      {0x00, 0x99, 0x99},  // DCT16x8
+      {0x00, 0x99, 0x99},  // DCT8x16
+      {0x00, 0x55, 0x55},  // DCT32x8
+      {0x00, 0x55, 0x55},  // DCT8x32
+      {0x00, 0x44, 0x44},  // DCT32x16
+      {0x00, 0x44, 0x44},  // DCT16x32
+      {0x00, 0xE8, 0xE8},  // DCT4x8
+      {0x00, 0xE8, 0xE8},  // DCT8x4
+      {0x00, 0xFF, 0xFF},  // AFV0
+      {0x00, 0xFF, 0xFF},  // AFV1
+      {0x00, 0xFF, 0xFF},  // AFV2
+      {0x00, 0xFF, 0xFF},  // AFV3
+  };
+  return kColors[raw_strategy];
+}
+
+static const uint8_t* TypeMask(const uint8_t& raw_strategy) {
+  JXL_ASSERT(AcStrategy::IsRawStrategyValid(raw_strategy));
+  static_assert(AcStrategy::kNumValidStrategies == 18, "Add masks");
+  static constexpr uint8_t kMask[][64] = {
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // DCT8
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 1, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 1, 0, 0, 0,  //
+          0, 0, 0, 0, 1, 0, 0, 0,  //
+          0, 0, 0, 0, 1, 0, 0, 0,  //
+          0, 0, 0, 0, 1, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // IDENTITY
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 1, 0, 0, 1, 0, 0,  //
+          0, 0, 0, 0, 0, 1, 0, 0,  //
+          0, 0, 0, 0, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 0, 0, 0, 0,  //
+          0, 0, 1, 1, 1, 1, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // 2x2
+      {
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+      },                           // 4x4
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 1, 1, 1, 1, 0, 0,  //
+          0, 0, 1, 1, 1, 1, 0, 0,  //
+          0, 0, 1, 1, 1, 1, 0, 0,  //
+          0, 0, 1, 1, 1, 1, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // DCT16x16
+      {
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+      },                           // DCT32x32
+      {
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+      },                           // DCT16x8
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // DCT8x16
+      {
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+      },                           // DCT32x8
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // DCT8x32
+      {
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+      },                           // DCT32x16
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // DCT16x32
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          1, 1, 1, 1, 1, 1, 1, 1,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // DCT4x8
+      {
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+          0, 0, 0, 1, 1, 0, 0, 0,  //
+      },                           // DCT8x4
+      {
+          1, 1, 0, 0, 0, 0, 0, 0,  //
+          1, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // AFV0
+      {
+          0, 0, 0, 0, 0, 0, 1, 1,  //
+          0, 0, 0, 0, 0, 0, 0, 1,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+      },                           // AFV1
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          1, 0, 0, 0, 0, 0, 0, 0,  //
+          1, 1, 0, 0, 0, 0, 0, 0,  //
+      },                           // AFV2
+      {
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 0,  //
+          0, 0, 0, 0, 0, 0, 0, 1,  //
+          0, 0, 0, 0, 0, 0, 1, 1,  //
+      },                           // AFV3
+  };
+  return kMask[raw_strategy];
+}
+
+void DumpAcStrategy(const AcStrategyImage& ac_strategy, size_t xsize,
+                    size_t ysize, const char* tag, AuxOut* aux_out) {
+  Image3F color_acs(xsize, ysize);
+  for (size_t y = 0; y < ysize; y++) {
+    float* JXL_RESTRICT rows[3] = {
+        color_acs.PlaneRow(0, y),
+        color_acs.PlaneRow(1, y),
+        color_acs.PlaneRow(2, y),
+    };
+    const AcStrategyRow acs_row = ac_strategy.ConstRow(y / kBlockDim);
+    for (size_t x = 0; x < xsize; x++) {
+      AcStrategy acs = acs_row[x / kBlockDim];
+      const uint8_t* JXL_RESTRICT color = TypeColor(acs.RawStrategy());
+      for (size_t c = 0; c < 3; c++) {
+        rows[c][x] = color[c];
+      }
+    }
+  }
+  size_t stride = color_acs.PixelsPerRow();
+  const uint8_t highlight_color[3] = {0xFF, 0xFF, 0x00};
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t by = 0; by < DivCeil(ysize, kBlockDim); by++) {
+      float* JXL_RESTRICT row = color_acs.PlaneRow(c, by * kBlockDim);
+      const AcStrategyRow acs_row = ac_strategy.ConstRow(by);
+      for (size_t bx = 0; bx < DivCeil(xsize, kBlockDim); bx++) {
+        AcStrategy acs = acs_row[bx];
+        if (!acs.IsFirstBlock()) continue;
+        const uint8_t* JXL_RESTRICT mask = TypeMask(acs.RawStrategy());
+        size_t xstart = (acs.covered_blocks_x() - 1) * kBlockDim / 2;
+        size_t ystart = (acs.covered_blocks_y() - 1) * kBlockDim / 2;
+        for (size_t iy = 0;
+             iy < kBlockDim && by * kBlockDim + iy + ystart < ysize; iy++) {
+          for (size_t ix = 0;
+               ix < kBlockDim && bx * kBlockDim + ix + xstart < xsize; ix++) {
+            if (mask[iy * kBlockDim + ix]) {
+              row[(iy + ystart) * stride + bx * kBlockDim + ix + xstart] =
+                  highlight_color[c];
+            }
+          }
+        }
+      }
+    }
+  }
+  aux_out->DumpImage(tag, color_acs);
+}
+
+// AC strategy selection: utility struct and entropy estimation.
+
+// Highest observed token > 64.
+constexpr size_t kNumTokens = ANS_MAX_ALPHA_SIZE;
+
+struct ACSConfig {
+  const DequantMatrices* JXL_RESTRICT dequant;
+  float token_bits[kNumTokens];
+  float info_loss_multiplier;
+  float* JXL_RESTRICT quant_field_row;
+  size_t quant_field_stride;
+  const float* JXL_RESTRICT src_rows[3];
+  size_t src_stride;
+  const float& Pixel(size_t c, size_t x, size_t y) const {
+    return src_rows[c][y * src_stride + x];
+  }
+  float& Quant(size_t bx, size_t by) const {
+    return quant_field_row[by * quant_field_stride + bx];
+  }
+};
+
+void ComputeTokenBits(float butteraugli_target, float* token_bits) {
+  const double kSmallValueBase = 1;       // OPTIMIZE
+  const double kSmallValueMul = 3.8;      // OPTIMIZE
+  const double kLargeValueFactor = 0.02;  // OPTIMIZE
+
+  const double kMaxCost = ANS_LOG_TAB_SIZE;
+
+  const double kLargeParam = std::max(
+      0.05f, 0.01f * std::pow(butteraugli_target, 0.1f) - 0.015f);  // OPTIMIZE
+  const double kSmallParam = 8.25f * kLargeParam - 0.363f;          // OPTIMIZE
+
+  for (size_t i = 0; i < 16; i++) {
+    token_bits[i] =
+        kSmallValueBase + kSmallValueMul *
+                              (pow((i + 1) / 2, kSmallParam) +
+                               pow(i ? (i - 1) / 2 : 0, kSmallParam)) *
+                              0.5f;
+  }
+  for (size_t i = 16; i < kNumTokens; i++) {
+    token_bits[i] = std::min(
+        kMaxCost, std::exp(kLargeParam * i * (i * kLargeValueFactor + 1)));
+  }
+}
+
+float EstimateEntropy(const AcStrategy& acs, size_t x, size_t y,
+                      const ACSConfig& config,
+                      const float* JXL_RESTRICT cmap_factors) {
+  const size_t size = (1 << acs.log2_covered_blocks()) * kDCTBlockSize;
+
+  // Apply transform.
+  HWY_ALIGN float block[3 * AcStrategy::kMaxCoeffArea];
+  for (size_t c = 0; c < 3; c++) {
+    float* JXL_RESTRICT block_c = block + size * c;
+    std::fill(block_c, block_c + size, 0.0f);
+    acs.TransformFromPixels(&config.Pixel(c, x, y), config.src_stride, block_c);
+  }
+
+  float quant = 0;
+  // Load QF value
+  for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+    for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+      quant = std::max(quant, config.Quant(x / 8 + ix, y / 8 + iy));
+    }
+  }
+
+  // Compute entropy.
+  float entropy = 0.0;
+  float info_loss = 0.0;
+  const coeff_order_t* JXL_RESTRICT order = acs.NaturalCoeffOrder();
+  for (size_t c = 0; c < 3; c++) {
+    int extra_nbits = 0;
+    float extra_tbits = 0.0;
+    size_t num_nzeros = 0;
+    size_t cx = acs.covered_blocks_x();
+    size_t cy = acs.covered_blocks_y();
+    CoefficientLayout(&cy, &cx);
+    for (size_t y = 0; y < cy * kBlockDim; y++) {
+      for (size_t x = 0; x < cx * kBlockDim; x++) {
+        // Leave out the lowest frequencies.
+        size_t k = order[y * cx * kBlockDim + x];
+        if (x < cx && y < cy) {
+          continue;
+        }
+        float val = (block[c * size + k] - block[size + k] * cmap_factors[c]) *
+                    config.dequant->InvMatrix(acs.RawStrategy(), c)[k];
+        val *= quant;
+
+        uint32_t token, nbits, bits;
+        int v = std::lrint(val);
+        info_loss += std::fabs(v - val);
+        EncodeHybridVarLenUint(PackSigned(v), &token, &nbits, &bits);
+        // nbits + bits for token. Skip trailing zeros in natural coeff order.
+        extra_nbits += nbits;
+        extra_tbits += config.token_bits[token];
+        if (v != 0) {
+          num_nzeros++;
+          entropy += extra_nbits + extra_tbits;
+          extra_tbits = 0;
+          extra_nbits = 0;
+        }
+      }
+    }
+
+    // Add #bit of num_nonzeros, as an estimate of the cost for encoding the
+    // number of non-zeros of the block.
+    size_t nbits = CeilLog2Nonzero(num_nzeros + 1) + 1;
+    // Also add #bit of #bit of num_nonzeros, to estimate the ANS cost, with a
+    // bias.
+    entropy += CeilLog2Nonzero(nbits + 17) + nbits;
+  }
+  info_loss /= quant;  // Scale of information loss should not depend on quant.
+  float ret = entropy + config.info_loss_multiplier * info_loss;
+  return ret;
+}
+
+// AC strategy selection: recursive block splitting.
+
+template <size_t N>
+size_t ACSCandidates(const AcStrategy::Type (&in)[N],
+                     AcStrategy::Type* JXL_RESTRICT out) {
+  memcpy(out, in, N * sizeof(AcStrategy::Type));
+  return N;
+}
+
+// Order in which transforms are tested for max delta: the first
+// acceptable one is chosen as initial guess.
+constexpr AcStrategy::Type kACSOrder[] = {
+    AcStrategy::Type::DCT32X32,
+    AcStrategy::Type::DCT32X16,
+    AcStrategy::Type::DCT16X32,
+    AcStrategy::Type::DCT16X16,
+    AcStrategy::Type::DCT8X32,
+    AcStrategy::Type::DCT32X8,
+    AcStrategy::Type::DCT16X8,
+    AcStrategy::Type::DCT8X16,
+    // DCT8x8 is the "fallback" option if no bigger transform can be used.
+    AcStrategy::Type::DCT,
+};
+
+constexpr size_t kEnabledTransforms = sizeof(kACSOrder) / sizeof(*kACSOrder);
+
+size_t ACSPossibleReplacements(AcStrategy::Type current,
+                               AcStrategy::Type* JXL_RESTRICT out) {
+  // TODO(veluca): is this decision tree optimal?
+  if (current == AcStrategy::Type::DCT32X32) {
+    return ACSCandidates(
+        {AcStrategy::Type::DCT32X16, AcStrategy::Type::DCT16X32}, out);
+  }
+  if (current == AcStrategy::Type::DCT32X16) {
+    return ACSCandidates(
+        {/*AcStrategy::Type::DCT32X8,*/ AcStrategy::Type::DCT16X16}, out);
+  }
+  if (current == AcStrategy::Type::DCT16X32) {
+    return ACSCandidates(
+        {/*AcStrategy::Type::DCT8X32,*/ AcStrategy::Type::DCT16X16}, out);
+  }
+  if (current == AcStrategy::Type::DCT32X8) {
+    return ACSCandidates({AcStrategy::Type::DCT16X8, AcStrategy::Type::DCT},
+                         out);
+  }
+  if (current == AcStrategy::Type::DCT8X32) {
+    return ACSCandidates({AcStrategy::Type::DCT8X16, AcStrategy::Type::DCT},
+                         out);
+  }
+  if (current == AcStrategy::Type::DCT16X16) {
+    return ACSCandidates({AcStrategy::Type::DCT8X16, AcStrategy::Type::DCT16X8},
+                         out);
+  }
+  if (current == AcStrategy::Type::DCT16X8 ||
+      current == AcStrategy::Type::DCT8X16) {
+    return ACSCandidates({AcStrategy::Type::DCT}, out);
+  }
+  if (current == AcStrategy::Type::DCT) {
+    return ACSCandidates({AcStrategy::Type::DCT4X8, AcStrategy::Type::DCT8X4,
+                          AcStrategy::Type::DCT4X4, AcStrategy::Type::DCT2X2,
+                          AcStrategy::Type::IDENTITY, AcStrategy::Type::AFV0,
+                          AcStrategy::Type::AFV1, AcStrategy::Type::AFV2,
+                          AcStrategy::Type::AFV3},
+                         out);
+  }
+  // Other 8x8 have no replacements - they already were chosen as the best
+  // between all the 8x8s.
+  return 0;
+}
+
+void MaybeReplaceACS(size_t bx, size_t by, const ACSConfig& config,
+                     const float* JXL_RESTRICT cmap_factors,
+                     AcStrategyImage* JXL_RESTRICT ac_strategy,
+                     float* JXL_RESTRICT entropy_estimate) {
+  AcStrategy::Type current =
+      AcStrategy::Type(ac_strategy->ConstRow(by)[bx].RawStrategy());
+  AcStrategy::Type candidates[AcStrategy::kNumValidStrategies];
+  size_t num_candidates = ACSPossibleReplacements(current, candidates);
+  if (num_candidates == 0) return;
+  size_t best = num_candidates;
+  size_t best_ee = entropy_estimate[0];
+  if (current == AcStrategy::Type::DCT) {
+    // Discourage special transforms.
+    best_ee *= 0.9;  // OPTIMIZE
+  }
+  // For each candidate replacement strategy, keep track of its entropy
+  // estimate.
+  float ee_val[AcStrategy::kNumValidStrategies][AcStrategy::kMaxCoeffBlocks];
+  AcStrategy current_acs = AcStrategy::FromRawStrategy(current);
+  for (size_t cand = 0; cand < num_candidates; cand++) {
+    AcStrategy acs = AcStrategy::FromRawStrategy(candidates[cand]);
+    size_t idx = 0;
+    float total_entropy = 0;
+    for (size_t iy = 0; iy < current_acs.covered_blocks_y();
+         iy += acs.covered_blocks_y()) {
+      for (size_t ix = 0; ix < current_acs.covered_blocks_x();
+           ix += acs.covered_blocks_x()) {
+        float entropy = EstimateEntropy(acs, (bx + ix) * 8, (by + iy) * 8,
+                                        config, cmap_factors);
+        ee_val[cand][idx] = entropy;
+        total_entropy += entropy;
+        idx++;
+      }
+    }
+    if (acs.RawStrategy() == AcStrategy::Type::DCT4X4) {
+      total_entropy *= 0.91;
+    }
+    if (acs.RawStrategy() == AcStrategy::Type::AFV0 ||
+        acs.RawStrategy() == AcStrategy::Type::AFV1 ||
+        acs.RawStrategy() == AcStrategy::Type::AFV2 ||
+        acs.RawStrategy() == AcStrategy::Type::AFV3) {
+      total_entropy *= 0.97;
+    }
+    if (total_entropy < best_ee) {
+      best_ee = total_entropy;
+      best = cand;
+    }
+  }
+  // Nothing changed.
+  if (best == num_candidates) return;
+  AcStrategy acs = AcStrategy::FromRawStrategy(candidates[best]);
+  size_t idx = 0;
+  for (size_t y = 0; y < current_acs.covered_blocks_y();
+       y += acs.covered_blocks_y()) {
+    for (size_t x = 0; x < current_acs.covered_blocks_x();
+         x += acs.covered_blocks_x()) {
+      ac_strategy->Set(bx + x, by + y, candidates[best]);
+      for (size_t iy = y; iy < y + acs.covered_blocks_y(); iy++) {
+        for (size_t ix = x; ix < x + acs.covered_blocks_x(); ix++) {
+          entropy_estimate[iy * 4 + ix] = ee_val[best][idx];
+        }
+      }
+      idx++;
+    }
+  }
+}
+}  // namespace
+
+HWY_ATTR void FindBestAcStrategy(const Image3F& src,
+                                 PassesEncoderState* JXL_RESTRICT enc_state,
+                                 ThreadPool* pool, AuxOut* aux_out) {
+  PROFILER_FUNC;
+  const CompressParams& cparams = enc_state->cparams;
+  const float butteraugli_target = cparams.butteraugli_distance;
+  AcStrategyImage* ac_strategy = &enc_state->shared.ac_strategy;
+
+  const size_t xsize_blocks = enc_state->shared.frame_dim.xsize_blocks;
+  const size_t ysize_blocks = enc_state->shared.frame_dim.ysize_blocks;
+  // In Falcon mode, use DCT8 everywhere and uniform quantization.
+  if (cparams.speed_tier == SpeedTier::kFalcon) {
+    ac_strategy->FillDCT8();
+    return;
+  }
+
+  // Maximum delta that every strategy type is allowed to have in the area
+  // it covers. Ignored for 8x8 transforms.
+  const float kMaxDelta = 0.06f * butteraugli_target;  // OPTIMIZE
+
+  // Scale of channels when computing delta.
+  const float kDeltaScale[3] = {
+      17.0f,  // OPTIMIZE
+      1.0f,   // OPTIMIZE
+      0.9f,   // OPTIMIZE
+  };
+
+  ACSConfig config;
+  config.dequant = &enc_state->shared.matrices;
+
+  // Entropy estimate is composed of two factors:
+  //  - estimate of the number of bits that will be used by the block
+  //  - information loss due to quantization
+  // The following constant controls the relative weights of these components.
+  config.info_loss_multiplier = 42.0f;  // OPTIMIZE
+
+  ComputeTokenBits(butteraugli_target, config.token_bits);
+
+  // Image row pointers and strides.
+  config.quant_field_row = enc_state->initial_quant_field.Row(0);
+  config.quant_field_stride = enc_state->initial_quant_field.PixelsPerRow();
+
+  config.src_rows[0] = src.ConstPlaneRow(0, 0);
+  config.src_rows[1] = src.ConstPlaneRow(1, 0);
+  config.src_rows[2] = src.ConstPlaneRow(2, 0);
+  config.src_stride = src.PixelsPerRow();
+
+  size_t xsize32 = DivCeil(xsize_blocks, 4);
+  size_t ysize32 = DivCeil(ysize_blocks, 4);
+  const auto compute_initial_acs_guess = [&](int block32, int _) HWY_ATTR {
+    size_t bx = block32 % xsize32;
+    size_t by = block32 / xsize32;
+    size_t tx = bx * 4 / kColorTileDimInBlocks;
+    size_t ty = by * 4 / kColorTileDimInBlocks;
+    const float cmap_factors[3] = {
+        enc_state->shared.cmap.YtoXRatio(
+            enc_state->shared.cmap.ytox_map.ConstRow(ty)[tx]),
+        0.0f,
+        enc_state->shared.cmap.YtoBRatio(
+            enc_state->shared.cmap.ytob_map.ConstRow(ty)[tx]),
+    };
+    HWY_CAPPED(float, kBlockDim) d;
+    HWY_CAPPED(uint32_t, kBlockDim) du;
+    const auto abs_mask = BitCast(d, Set(du, 0x7FFFFFFFu));
+    // Pre-compute maximum delta in each 8x8 block.
+    // Find a minimum delta of three options:
+    // 1) all, 2) not accounting vertical, 3) not accounting horizontal
+    HWY_ALIGN float pixels[3][64];
+    float max_delta[3][16] = {};
+    float entropy_estimate[16] = {};
+    for (size_t c = 0; c < 3; c++) {
+      for (size_t iy = 0; iy < 4; iy++) {
+        size_t dy = by * 4 + iy;
+        if (dy >= ysize_blocks) continue;
+        for (size_t ix = 0; ix < 4; ix++) {
+          size_t dx = bx * 4 + ix;
+          if (dx >= xsize_blocks) continue;
+          for (size_t c = 0; c < 3; c++) {
+            for (size_t y = 0; y < 8; y++) {
+              for (size_t x = 0; x < 8; x += d.N) {
+                const auto v =
+                    Load(d, &config.Pixel(c, dx * 8 + x, dy * 8 + y));
+                Store(v, d, &pixels[c][y * 8 + x]);
+              }
+            }
+          }
+          for (size_t c = 0; c < 3; c++) {
+            float top[8] = {0};
+            float side[8] = {0};
+            for (size_t y = 0; y < 8; y++) {
+              for (size_t x = 0; x < 8; x++) {
+                side[y] += pixels[c][y * 8 + x];
+                top[x] += pixels[c][y * 8 + x];
+              }
+            }
+            for (size_t y = 0; y < 8; y++) {
+              for (size_t x = 0; x < 8; x++) {
+                pixels[c][y * 8 + x] -= (1.0 / 8.0) * (top[x] + side[y]);
+              }
+            }
+          }
+          auto delta = Zero(d);
+          for (size_t x = 0; x < 8; x += d.N) {
+            for (size_t y = 1; y < 7; y++) {
+              float* pix = &pixels[c][y * 8 + x];
+              const auto p = Load(d, pix);
+              const auto n = Load(d, pix + 8);
+              const auto s = Load(d, pix - 8);
+              const auto w = LoadU(d, pix - 1);
+              const auto e = LoadU(d, pix + 1);
+              // Compute amount of per-pixel variation.
+              const auto m1 = Max(abs_mask & (n - p), abs_mask & (s - p));
+              const auto m2 = Max(abs_mask & (w - p), abs_mask & (e - p));
+              const auto m3 = Max(abs_mask & (e - w), abs_mask & (s - n));
+              const auto m4 = Max(m1, m2);
+              const auto m5 = Max(m3, m4);
+              delta = Max(delta, m5);
+            }
+            HWY_ALIGN float lmax[d.N];
+            Store(delta, d, lmax);
+            float mdelta = 0;
+            for (size_t i = 0; i < d.N; i++) {
+              int ioff = (i + x) & 7;
+              if (ioff != 0 && ioff != 7) {
+                mdelta = std::max(mdelta, lmax[i]);
+              }
+            }
+            max_delta[c][iy * 4 + ix] =
+                std::max(max_delta[c][iy * 4 + ix], mdelta);
+          }
+        }
+      }
+    }
+    for (size_t i = 0; i < 3; ++i) {
+      for (size_t k = 0; k < 16; ++k) {
+        max_delta[i][k] *= kDeltaScale[i];
+      }
+    }
+    // Choose the first transform that can be used to cover each block.
+    uint8_t chosen_mask[16] = {};
+    for (size_t iy = 0; iy < 4 && by * 4 + iy < ysize_blocks; iy++) {
+      for (size_t ix = 0; ix < 4 && bx * 4 + ix < xsize_blocks; ix++) {
+        if (chosen_mask[iy * 4 + ix]) continue;
+        for (size_t i = 0; i < kEnabledTransforms; i++) {
+          AcStrategy acs = AcStrategy::FromRawStrategy(kACSOrder[i]);
+          size_t cx = acs.covered_blocks_x();
+          size_t cy = acs.covered_blocks_y();
+          float max_delta_v[3] = {max_delta[0][iy * 4 + ix],
+                                  max_delta[1][iy * 4 + ix],
+                                  max_delta[2][iy * 4 + ix]};
+          float max_delta_acs = std::max(
+              std::max(max_delta_v[0], max_delta_v[1]), max_delta_v[2]);
+          float min_delta_v[3] = {1e30, 1e30, 1e30};
+          float ave_delta_v[3] = {};
+          // Check if strategy is usable
+          if (cx != 1 || cy != 1) {
+            // Alignment
+            if ((iy & (cy - 1)) != 0) continue;
+            if ((ix & (cx - 1)) != 0) continue;
+            // Out of block32 bounds
+            if (iy + cy > 4) continue;
+            if (ix + cx > 4) continue;
+            // Out of image bounds
+            if (by * 4 + iy + cy > ysize_blocks) continue;
+            if (bx * 4 + ix + cx > xsize_blocks) continue;
+            // Block would overwrite an already-chosen block
+            bool overwrites_covered = false;
+            for (size_t y = 0; y < cy; y++) {
+              for (size_t x = 0; x < cx; x++) {
+                if (chosen_mask[(y + iy) * 4 + x + ix])
+                  overwrites_covered = true;
+              }
+            }
+            if (overwrites_covered) continue;
+            for (size_t c = 0; c < 3; ++c) {
+              max_delta_v[c] = 0;
+              min_delta_v[c] = 1e30f;
+              ave_delta_v[c] = 0;
+              // Max delta in covered area
+              for (size_t y = 0; y < cy; y++) {
+                for (size_t x = 0; x < cx; x++) {
+                  int pix = (iy + y) * 4 + ix + x;
+                  max_delta_v[c] = std::max(max_delta_v[c], max_delta[c][pix]);
+                  min_delta_v[c] = std::min(min_delta_v[c], max_delta[c][pix]);
+                  ave_delta_v[c] += max_delta[c][pix];
+                }
+              }
+              ave_delta_v[c] -= max_delta_v[c];
+              ave_delta_v[c] /= (cy * cx - 1);
+              max_delta_v[c] -= 0.25 * min_delta_v[c];
+              max_delta_v[c] -= 0.25 * ave_delta_v[c];
+            }
+            max_delta_acs = max_delta_v[0] + max_delta_v[1] + max_delta_v[2];
+            if (max_delta_acs > kMaxDelta) continue;
+          }
+          // Estimate entropy and qf value
+          float entropy = 0.0f;
+          // In modes faster than Wombat mode, AC strategy replacement is not
+          // attempted: no need to estimate entropy.
+          if (cparams.speed_tier <= SpeedTier::kWombat) {
+            entropy = EstimateEntropy(acs, bx * 32 + ix * 8, by * 32 + iy * 8,
+                                      config, cmap_factors);
+          }
+          // In modes faster than Hare mode, we don't use InitialQuantField -
+          // hence, we need to come up with quant field values.
+          if (cparams.speed_tier > SpeedTier::kHare) {
+            // OPTIMIZE
+            float quant = 1.1f / (1.0f + max_delta_acs) / butteraugli_target;
+            for (size_t y = 0; y < cy; y++) {
+              for (size_t x = 0; x < cx; x++) {
+                config.Quant(bx * 4 + ix + x, by * 4 + iy + y) = quant;
+              }
+            }
+          }
+          // Mark blocks as chosen and write to acs image.
+          ac_strategy->Set(bx * 4 + ix, by * 4 + iy, kACSOrder[i]);
+          for (size_t y = 0; y < cy; y++) {
+            for (size_t x = 0; x < cx; x++) {
+              chosen_mask[(y + iy) * 4 + x + ix] = 1;
+              entropy_estimate[(iy + y) * 4 + ix + x] = entropy;
+            }
+          }
+          break;
+        }
+      }
+    }
+    // Do not try to replace ACS in modes faster than wombat mode.
+    if (cparams.speed_tier > SpeedTier::kWombat) return;
+    // Iterate through the 32-block attempting to replace the current strategy.
+    // If replaced, repeat for the top-left new block and let the other ones be
+    // taken care of by future iterations.
+    uint8_t computed_mask[16] = {};
+    for (size_t iy = 0; iy < 4; iy++) {
+      if (by * 4 + iy >= ysize_blocks) continue;
+      for (size_t ix = 0; ix < 4; ix++) {
+        if (bx * 4 + ix >= xsize_blocks) continue;
+        if (computed_mask[iy * 4 + ix]) continue;
+        uint8_t prev = AcStrategy::kNumValidStrategies;
+        while (prev !=
+               ac_strategy->ConstRow(by * 4 + iy)[bx * 4 + ix].RawStrategy()) {
+          prev = ac_strategy->ConstRow(by * 4 + iy)[bx * 4 + ix].RawStrategy();
+          MaybeReplaceACS(bx * 4 + ix, by * 4 + iy, config, cmap_factors,
+                          ac_strategy, entropy_estimate + (iy * 4 + ix));
+        }
+        AcStrategy acs = ac_strategy->ConstRow(by * 4 + iy)[bx * 4 + ix];
+        for (size_t y = 0; y < acs.covered_blocks_y(); y++) {
+          for (size_t x = 0; x < acs.covered_blocks_x(); x++) {
+            computed_mask[(iy + y) * 4 + ix + x] = 1;
+          }
+        }
+      }
+    }
+  };
+  RunOnPool(pool, 0, xsize32 * ysize32, ThreadPool::SkipInit(),
+            compute_initial_acs_guess, "ChooseACS");
+
+  // Accounting and debug output.
+  if (aux_out != nullptr) {
+    aux_out->num_dct2_blocks =
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT2X2);
+    aux_out->num_dct4_blocks =
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT4X4);
+    aux_out->num_dct4x8_blocks =
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT4X8) +
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT8X4);
+    aux_out->num_afv_blocks = ac_strategy->CountBlocks(AcStrategy::Type::AFV0) +
+                              ac_strategy->CountBlocks(AcStrategy::Type::AFV1) +
+                              ac_strategy->CountBlocks(AcStrategy::Type::AFV2) +
+                              ac_strategy->CountBlocks(AcStrategy::Type::AFV3);
+    aux_out->num_dct8_blocks = ac_strategy->CountBlocks(AcStrategy::Type::DCT);
+    aux_out->num_dct8x16_blocks =
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT8X16) +
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT16X8);
+    aux_out->num_dct8x32_blocks =
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT8X32) +
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT32X8);
+    aux_out->num_dct16_blocks =
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT16X16);
+    aux_out->num_dct16x32_blocks =
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT16X32) +
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT32X16);
+    aux_out->num_dct32_blocks =
+        ac_strategy->CountBlocks(AcStrategy::Type::DCT32X32);
+  }
+
+  if (WantDebugOutput(aux_out)) {
+    DumpAcStrategy(*ac_strategy, enc_state->shared.frame_dim.xsize,
+                   enc_state->shared.frame_dim.ysize, "ac_strategy", aux_out);
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_ac_strategy.h b/jxl/enc_ac_strategy.h
new file mode 100644
index 000000000..4e7bb1a72
--- /dev/null
+++ b/jxl/enc_ac_strategy.h
@@ -0,0 +1,47 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_AC_STRATEGY_H_
+#define JXL_ENC_AC_STRATEGY_H_
+
+#include <hwy/static_targets.h>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/common.h"
+#include "jxl/dec_ans.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_params.h"
+#include "jxl/image.h"
+#include "jxl/quant_weights.h"
+
+// `FindBestAcStrategy` uses heuristics to choose which AC strategy should be
+// used in each block, as well as the initial quantization field.
+
+namespace jxl {
+
+// `quant_field` will be the initial quantization field for this image.  `src`
+// is the input image in the XYB color space. `ac_strategy` is the output
+// strategy.
+HWY_ATTR void FindBestAcStrategy(const Image3F& src,
+                                 PassesEncoderState* JXL_RESTRICT enc_state,
+                                 ThreadPool* pool, AuxOut* aux_out);
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_AC_STRATEGY_H_
diff --git a/jxl/enc_adaptive_quantization.cc b/jxl/enc_adaptive_quantization.cc
new file mode 100644
index 000000000..42def700c
--- /dev/null
+++ b/jxl/enc_adaptive_quantization.cc
@@ -0,0 +1,1176 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_adaptive_quantization.h"
+
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <cmath>
+#include <hwy/static_targets.h>
+#include <string>
+#include <vector>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/block.h"
+#include "jxl/butteraugli/butteraugli.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/convolve.h"
+#include "jxl/dct.h"
+#include "jxl/dec_cache.h"
+#include "jxl/dec_group.h"
+#include "jxl/dec_reconstruct.h"
+#include "jxl/enc_butteraugli_comparator.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_group.h"
+#include "jxl/enc_params.h"
+#include "jxl/gauss_blur.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/multiframe.h"
+#include "jxl/noise.h"
+#include "jxl/opsin_params.h"
+#include "jxl/quant_weights.h"
+
+bool FLAGS_log_search_state = false;
+// If true, prints the quantization maps at each iteration.
+bool FLAGS_dump_quant_state = false;
+
+namespace jxl {
+namespace {
+
+static const double kQuant64[64] = {
+    0.0,
+    1.3627035631195761,
+    3.75394603948046884,
+    3.7552562789889619,
+    3.738923115258169,
+    3.2658460689961202,
+    3.3644109735743761,
+    3.407927898760664,
+    3.3435304472886553,
+    0.912932235161943,
+    0.86013076305762415,
+    3.2181857425503768,
+    2.5344889514395561,
+    1.1030971121374131,
+    0.86654135268900234,
+    1.1790809890137122,
+    1.1325855042326589,
+    2.1036916840563111,
+    2.1662829197292539,
+    1.0855127071756325,
+    1.4837636416580766,
+    0.79528165577401788,
+    0.84472454415339238,
+    1.4156418573711551,
+    1.4477267104680289,
+    1.3169293134214741,
+    1.1435971405444745,
+    0.44922571728240968,
+    0.66437130514818066,
+    0.56811538880030854,
+    0.88663683723756648,
+    0.75797820670535709,
+    1.2141376049363837,
+    1.8082860500557918,
+    0.34785192011414331,
+    0.86658716645380518,
+    0.84457566277095242,
+    0.365886424294040194,
+    0.27942131498024103,
+    0.48621389791680419,
+    0.2848216363151751,
+    0.37182998793445576,
+    0.37191251587301016,
+    0.3434673885351725,
+    0.50423064557670205,
+    0.30742301163233954,
+    0.38866381308020725,
+    0.46912603786452162,
+    0.38233313062142565,
+    0.41786679581133063,
+    0.3695788988347995,
+    0.63197592721834761,
+    0.73351761078380862,
+    0.40450080085057888,
+    0.39291869306279764,
+    0.51847970794880838,
+    0.43559898503478928,
+    0.32837421314561244,
+    0.51318960017980164,
+    0.79565824960993681,
+    0.21456893657335676,
+    0.47552975726356284,
+    0.4411023380122629,
+    0.32950033621891689,
+};
+
+void ComputeMask(float* JXL_RESTRICT out_pos) {
+  constexpr float kBase = 1.4731057596473099f;
+  constexpr float kMul1 = 0.011521457315309454f;
+  constexpr float kOffset1 = 0.0079611186521877063f;
+  constexpr float kMul2 = -0.19590586155132378f;
+  constexpr float kOffset2 = 0.074575093726693686f;
+  const float val = *out_pos;
+  // Avoid division by zero.
+  const float div = std::max(val + kOffset1, 1e-3f);
+  *out_pos = kBase + kMul1 / div + kMul2 / (val * val + kOffset2);
+}
+
+// Increase precision in 8x8 blocks that are complicated in DCT space.
+HWY_ATTR void DctModulation(const size_t x, const size_t y, const ImageF& xyb,
+                            const coeff_order_t* natural_coeff_order,
+                            const float* JXL_RESTRICT dct_rescale,
+                            float* JXL_RESTRICT out_pos) {
+  HWY_ALIGN float dct[kDCTBlockSize] = {0};
+  for (size_t dy = 0; dy < 8; ++dy) {
+    const size_t yclamp = std::min(y + dy, xyb.ysize() - 1);
+    const float* const JXL_RESTRICT row_in = xyb.Row(yclamp);
+    for (size_t dx = 0; dx < 8; ++dx) {
+      const size_t xclamp = std::min(x + dx, xyb.xsize() - 1);
+      dct[dy * 8 + dx] = row_in[xclamp];
+    }
+  }
+  ComputeTransposedScaledDCT<8>()(FromBlock<8>(dct), ToBlock<8>(dct));
+  double entropyQL2 = 0;
+  double entropyQL4 = 0;
+  double entropyQL8 = 0;
+  for (size_t k = 1; k < kDCTBlockSize; ++k) {
+    const coeff_order_t i = natural_coeff_order[k];
+    const float scale = dct_rescale[i];
+    double v = static_cast<double>(dct[i] * scale);
+    v *= v;
+    static const double kPow = 1.7656070913325459;
+    double q = std::pow(kQuant64[k], kPow);
+    entropyQL2 += q * v;
+    v *= v;
+    entropyQL4 += q * v;
+    v *= v;
+    entropyQL8 += q * v;
+  }
+  entropyQL2 = std::sqrt(entropyQL2);
+  entropyQL4 = std::sqrt(std::sqrt(entropyQL4));
+  entropyQL8 = std::pow(entropyQL8, 0.125);
+  static const double mulQL2 = 0.00064095761586667813;
+  static const double mulQL4 = -0.93103691258798293;
+  static const double mulQL8 = 0.20682345500923968;
+  double v = mulQL2 * entropyQL2 + mulQL4 * entropyQL4 + mulQL8 * entropyQL8;
+  double kMul = 1.0833857206487167;
+  *out_pos += kMul * v;
+}
+
+// Increase precision in 8x8 blocks that have high dynamic range.
+HWY_ATTR void RangeModulation(const size_t x, const size_t y, const ImageF& xyb,
+                              float* JXL_RESTRICT out_pos) {
+  float minval = 1e30f;
+  float maxval = -1e30f;
+  for (size_t dy = 0; dy < 8 && y + dy < xyb.ysize(); ++dy) {
+    const float* const JXL_RESTRICT row_in = xyb.Row(y + dy);
+    for (size_t dx = 0; dx < 8 && x + dx < xyb.xsize(); ++dx) {
+      float v = row_in[x + dx];
+      if (minval > v) {
+        minval = v;
+      }
+      if (maxval < v) {
+        maxval = v;
+      }
+    }
+  }
+  float range = maxval - minval;
+  constexpr float mul = 0.66697599699046262f;
+  *out_pos += mul * range;
+}
+
+// Change precision in 8x8 blocks that have high frequency content.
+void HfModulation(const size_t x, const size_t y, const ImageF& xyb,
+                  float* JXL_RESTRICT out_pos) {
+  float sum = 0;
+  int n = 0;
+  for (size_t dy = 0; dy < 8 && y + dy < xyb.ysize(); ++dy) {
+    const float* JXL_RESTRICT row_in = xyb.Row(y + dy);
+    for (size_t dx = 0; dx < 7 && x + dx + 1 < xyb.xsize(); ++dx) {
+      float v = std::fabs(row_in[x + dx] - row_in[x + dx + 1]);
+      sum += v;
+      ++n;
+    }
+  }
+  for (size_t dy = 0; dy < 7 && y + dy + 1 < xyb.ysize(); ++dy) {
+    const float* JXL_RESTRICT row_in = xyb.Row(y + dy);
+    const float* JXL_RESTRICT row_in_next = xyb.Row(y + dy + 1);
+    for (size_t dx = 0; dx < 8 && x + dx < xyb.xsize(); ++dx) {
+      float v = std::abs(row_in[x + dx] - row_in_next[x + dx]);
+      sum += v;
+      ++n;
+    }
+  }
+  if (n != 0) {
+    sum /= n;
+  }
+  constexpr float kMul = 0.70810081505707823f;
+  sum *= kMul;
+  *out_pos += sum;
+}
+
+HWY_ATTR void PerBlockModulations(const ImageF& xyb, const float scale,
+                                  ThreadPool* pool, ImageF* out) {
+  JXL_ASSERT(DivCeil(xyb.xsize(), kBlockDim) == out->xsize());
+  JXL_ASSERT(DivCeil(xyb.ysize(), kBlockDim) == out->ysize());
+  const coeff_order_t* natural_coeff_order =
+      AcStrategy::FromRawStrategy(AcStrategy::Type::DCT).NaturalCoeffOrder();
+  float dct_rescale[kDCTBlockSize] = {0};
+  {
+    const float* dct_scale = DCTScales<8>();
+    for (size_t i = 0; i < kDCTBlockSize; ++i) {
+      dct_rescale[i] = dct_scale[i / 8] * dct_scale[i % 8];
+    }
+  }
+
+  RunOnPool(
+      pool, 0, static_cast<int>(DivCeil(xyb.ysize(), kBlockDim)),
+      ThreadPool::SkipInit(),
+      [&](const int task, const int /*thread*/) {
+        const size_t iy = static_cast<size_t>(task);
+        const size_t y = iy * 8;
+        float* const JXL_RESTRICT row_out = out->Row(iy);
+        for (size_t x = 0; x < xyb.xsize(); x += 8) {
+          float* JXL_RESTRICT out_pos = row_out + x / 8;
+          ComputeMask(out_pos);
+          DctModulation(x, y, xyb, natural_coeff_order, dct_rescale, out_pos);
+          RangeModulation(x, y, xyb, out_pos);
+          HfModulation(x, y, xyb, out_pos);
+
+          // We want multiplicative quantization field, so everything
+          // until this point has been modulating the exponent.
+          *out_pos = std::exp(*out_pos) * scale;
+        }
+      },
+      "AQ per block modulation");
+}
+
+static double SimpleGamma(double v) {
+  // A simple HDR compatible gamma function.
+  // mul and mul2 represent a scaling difference between jxl and butteraugli.
+  static const double mul = 103.72874071313939;
+  static const double mul2 = 1.0 / (67.781877502322729);
+
+  v *= mul;
+
+  static const double kRetMul = mul2 * 18.6580932135;
+  static const double kRetAdd = mul2 * -20.2789020414;
+  static const double kVOffset = 7.14672470003;
+
+  if (v < 0) {
+    // This should happen rarely, but may lead to a NaN, which is rather
+    // undesirable. Since negative photons don't exist we solve the NaNs by
+    // clamping here.
+    v = 0;
+  }
+  return kRetMul * log(v + kVOffset) + kRetAdd;
+}
+
+static double RatioOfCubicRootToSimpleGamma(double v) {
+  // The opsin space in jxl is the cubic root of photons, i.e., v * v * v
+  // is related to the number of photons.
+  //
+  // SimpleGamma(v * v * v) is the psychovisual space in butteraugli.
+  // This ratio allows quantization to move from jxl's opsin space to
+  // butteraugli's log-gamma space.
+  return v / SimpleGamma(v * v * v);
+}
+
+// Returns image (padded to multiple of 8x8) of local pixel differences.
+ImageF DiffPrecompute(const Image3F& xyb, const FrameDimensions& frame_dim,
+                      float cutoff, ThreadPool* pool) {
+  PROFILER_ZONE("aq DiffPrecompute");
+  const size_t xsize = frame_dim.xsize;
+  const size_t ysize = frame_dim.ysize;
+  const size_t padded_xsize = RoundUpToBlockDim(xsize);
+  const size_t padded_ysize = RoundUpToBlockDim(ysize);
+  JXL_ASSERT(xsize > 1);
+  JXL_ASSERT(ysize > 1);
+  ImageF padded_diff(padded_xsize, padded_ysize);
+  constexpr float mul0 = 0.046072108343079003f;
+
+  // The XYB gamma is 3.0 to be able to decode faster with two muls.
+  // Butteraugli's gamma is matching the gamma of human eye, around 2.6.
+  // We approximate the gamma difference by adding one cubic root into
+  // the adaptive quantization. This gives us a total gamma of 2.6666
+  // for quantization uses.
+  constexpr float match_gamma_offset = 0.25084239333070085f;
+
+  RunOnPool(
+      pool, 0, static_cast<int>(ysize), ThreadPool::SkipInit(),
+      [&](const int task, int /*thread*/) {
+        const size_t y = static_cast<size_t>(task);
+        size_t y2;
+        if (y + 1 < ysize) {
+          y2 = y + 1;
+        } else if (y > 0) {
+          y2 = y - 1;
+        } else {
+          y2 = y;
+        }
+        size_t y1;
+        if (y == 0 && ysize >= 2) {
+          y1 = y + 1;
+        } else if (y > 0) {
+          y1 = y - 1;
+        } else {
+          y1 = y;
+        }
+        const float* row_in = xyb.PlaneRow(1, y);
+        const float* row_in1 = xyb.PlaneRow(1, y1);
+        const float* row_in2 = xyb.PlaneRow(1, y2);
+        float* JXL_RESTRICT row_out = padded_diff.Row(y);
+
+        for (size_t x = 0; x + 1 < xsize; ++x) {
+          size_t x2;
+          if (x + 1 < xsize) {
+            x2 = x + 1;
+          } else if (x > 0) {
+            x2 = x - 1;
+          } else {
+            x2 = x;
+          }
+          size_t x1;
+          if (x == 0 && xsize >= 2) {
+            x1 = x + 1;
+          } else if (x > 0) {
+            x1 = x - 1;
+          } else {
+            x1 = x;
+          }
+          float diff = mul0 * (std::abs(row_in[x] - row_in[x2]) +
+                               std::abs(row_in[x] - row_in2[x]) +
+                               std::abs(row_in[x] - row_in[x1]) +
+                               std::abs(row_in[x] - row_in1[x]) +
+                               3 * (std::abs(row_in2[x] - row_in1[x]) +
+                                    std::abs(row_in[x1] - row_in[x2])));
+          diff *= RatioOfCubicRootToSimpleGamma(row_in[x] + match_gamma_offset);
+          row_out[x] = std::min(cutoff, diff);
+        }
+        // Last pixel of the row.
+        {
+          const size_t x = xsize - 1;
+          float diff = 7.0f * mul0 * (std::abs(row_in[x] - row_in2[x]));
+          diff *= RatioOfCubicRootToSimpleGamma(row_in[x] + match_gamma_offset);
+          row_out[x] = std::min(cutoff, diff);
+        }
+
+        // Extend to multiple of 8 columns
+        float lastval = row_out[xsize - 1];
+        if (xsize >= 3) {
+          lastval += row_out[xsize - 3];
+          lastval += row_out[xsize - 2];
+          lastval *= 1.0f / 3;
+        } else if (xsize >= 2) {
+          lastval += row_out[xsize - 2];
+          lastval *= 0.5f;
+        }
+        for (size_t x = xsize; x < padded_diff.xsize(); ++x) {
+          row_out[x] = lastval;
+        }
+      },
+      "AQ DiffPrecompute");
+
+  // Last row.
+  {
+    const size_t y = ysize - 1;
+    const float* const JXL_RESTRICT row_in = xyb.PlaneRow(1, y);
+    float* const JXL_RESTRICT row_out = padded_diff.Row(y);
+    for (size_t x = 0; x + 1 < xsize; ++x) {
+      const size_t x2 = x + 1;
+      float diff = 7.0f * mul0 * std::abs(row_in[x] - row_in[x2]);
+      diff *= RatioOfCubicRootToSimpleGamma(row_in[x] + match_gamma_offset);
+      row_out[x] = std::min(cutoff, diff);
+    }
+    // Last pixel of the last row.
+    {
+      const size_t x = xsize - 1;
+      row_out[x] = row_out[x - 1];
+    }
+  }
+  // Extend to multiple of 8 rows
+  if (ysize != padded_diff.ysize()) {
+    const float* JXL_RESTRICT last_row = padded_diff.Row(ysize - 1);
+    for (size_t x = 0; x < padded_diff.xsize(); ++x) {
+      float lastval = last_row[x];
+      if (ysize >= 3) {
+        lastval += padded_diff.Row(ysize - 2)[x];
+        lastval += padded_diff.Row(ysize - 3)[x];
+        lastval *= 1.0f / 3;
+      } else if (ysize >= 2) {
+        lastval += padded_diff.Row(ysize - 2)[x];
+        lastval *= 0.5f;
+      }
+      for (size_t y = ysize; y < padded_diff.ysize(); ++y) {
+        padded_diff.Row(y)[x] = lastval;
+      }
+    }
+  }
+
+  return padded_diff;
+}
+
+ImageF TileDistMap(const ImageF& distmap, int tile_size, int margin,
+                   const AcStrategyImage& ac_strategy) {
+  PROFILER_FUNC;
+  const int tile_xsize = (distmap.xsize() + tile_size - 1) / tile_size;
+  const int tile_ysize = (distmap.ysize() + tile_size - 1) / tile_size;
+  ImageF tile_distmap(tile_xsize, tile_ysize);
+  size_t distmap_stride = tile_distmap.PixelsPerRow();
+  for (int tile_y = 0; tile_y < tile_ysize; ++tile_y) {
+    AcStrategyRow ac_strategy_row = ac_strategy.ConstRow(tile_y);
+    float* JXL_RESTRICT dist_row = tile_distmap.Row(tile_y);
+    for (int tile_x = 0; tile_x < tile_xsize; ++tile_x) {
+      AcStrategy acs = ac_strategy_row[tile_x];
+      if (!acs.IsFirstBlock()) continue;
+      int this_tile_xsize = acs.covered_blocks_x() * tile_size;
+      int this_tile_ysize = acs.covered_blocks_y() * tile_size;
+      int y_begin = std::max<int>(0, tile_size * tile_y - margin);
+      int y_end = std::min<int>(distmap.ysize(),
+                                tile_size * tile_y + this_tile_ysize + margin);
+      int x_begin = std::max<int>(0, tile_size * tile_x - margin);
+      int x_end = std::min<int>(distmap.xsize(),
+                                tile_size * tile_x + this_tile_xsize + margin);
+      float dist_norm = 0.0;
+      double pixels = 0;
+      for (int y = y_begin; y < y_end; ++y) {
+        float ymul = 1.0;
+        static const float kBorderMul = 0.98f;
+        static const float kCornerMul = 0.7f;
+        if (margin != 0 && (y == y_begin || y == y_end - 1)) {
+          ymul = kBorderMul;
+        }
+        const float* const JXL_RESTRICT row = distmap.Row(y);
+        for (int x = x_begin; x < x_end; ++x) {
+          float xmul = ymul;
+          if (margin != 0 && (x == x_begin || x == x_end - 1)) {
+            if (xmul == 1.0) {
+              xmul = kBorderMul;
+            } else {
+              xmul = kCornerMul;
+            }
+          }
+          float v = row[x];
+          v *= v;
+          v *= v;
+          v *= v;
+          v *= v;
+          dist_norm += xmul * v;
+          pixels += xmul;
+        }
+      }
+      if (pixels == 0) pixels = 1;
+      // 16th norm is less than the max norm, we reduce the difference
+      // with this normalization factor.
+      static const double kTileNorm = 1.2;
+      const double tile_dist =
+          kTileNorm * std::pow(dist_norm / pixels, 1.0 / 16);
+      dist_row[tile_x] = tile_dist;
+      for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+        for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+          dist_row[tile_x + distmap_stride * iy + ix] = tile_dist;
+        }
+      }
+    }
+  }
+  return tile_distmap;
+}
+
+ImageF DistToPeakMap(const ImageF& field, float peak_min, int local_radius,
+                     float peak_weight) {
+  ImageF result(field.xsize(), field.ysize());
+  FillImage(-1.0f, &result);
+  for (size_t y0 = 0; y0 < field.ysize(); ++y0) {
+    for (size_t x0 = 0; x0 < field.xsize(); ++x0) {
+      int x_min = std::max<int>(0, static_cast<int>(x0) - local_radius);
+      int y_min = std::max<int>(0, static_cast<int>(y0) - local_radius);
+      int x_max = std::min<size_t>(field.xsize(), x0 + 1 + local_radius);
+      int y_max = std::min<size_t>(field.ysize(), y0 + 1 + local_radius);
+      float local_max = peak_min;
+      for (int y = y_min; y < y_max; ++y) {
+        for (int x = x_min; x < x_max; ++x) {
+          local_max = std::max(local_max, field.Row(y)[x]);
+        }
+      }
+      if (field.Row(y0)[x0] >
+          (1.0f - peak_weight) * peak_min + peak_weight * local_max) {
+        for (int y = y_min; y < y_max; ++y) {
+          for (int x = x_min; x < x_max; ++x) {
+            float dist = std::max(std::abs(y - static_cast<int>(y0)),
+                                  std::abs(x - static_cast<int>(x0)));
+            float cur_dist = result.Row(y)[x];
+            if (cur_dist < 0.0 || cur_dist > dist) {
+              result.Row(y)[x] = dist;
+            }
+          }
+        }
+      }
+    }
+  }
+  return result;
+}
+
+bool AdjustQuantVal(float* const JXL_RESTRICT q, const float d,
+                    const float factor, const float quant_max) {
+  if (*q >= 0.999f * quant_max) return false;
+  const float inv_q = 1.0f / *q;
+  const float adj_inv_q = inv_q - factor / (d + 1.0f);
+  *q = 1.0f / std::max(1.0f / quant_max, adj_inv_q);
+  return true;
+}
+
+void DumpHeatmap(const AuxOut* aux_out, const std::string& label,
+                 const ImageF& image, float good_threshold,
+                 float bad_threshold) {
+  Image3B heatmap =
+      butteraugli::CreateHeatMapImage(image, good_threshold, bad_threshold);
+  char filename[200];
+  snprintf(filename, sizeof(filename), "%s%05d", label.c_str(),
+           aux_out->num_butteraugli_iters);
+  aux_out->DumpImage(filename, heatmap);
+}
+
+void DumpHeatmaps(const AuxOut* aux_out, float ba_target,
+                  const ImageF& quant_field, const ImageF& tile_heatmap) {
+  if (!WantDebugOutput(aux_out)) return;
+  ImageF inv_qmap(quant_field.xsize(), quant_field.ysize());
+  for (size_t y = 0; y < quant_field.ysize(); ++y) {
+    const float* JXL_RESTRICT row_q = quant_field.ConstRow(y);
+    float* JXL_RESTRICT row_inv_q = inv_qmap.Row(y);
+    for (size_t x = 0; x < quant_field.xsize(); ++x) {
+      row_inv_q[x] = 1.0f / row_q[x];  // never zero
+    }
+  }
+  DumpHeatmap(aux_out, "quant_heatmap", inv_qmap, 4.0f * ba_target,
+              6.0f * ba_target);
+  DumpHeatmap(aux_out, "tile_heatmap", tile_heatmap, ba_target,
+              1.5f * ba_target);
+}
+
+void AdjustQuantField(const AcStrategyImage& ac_strategy, ImageF* quant_field) {
+  // Replace the whole quant_field in non-8x8 blocks with the maximum of each
+  // 8x8 block.
+  size_t stride = quant_field->PixelsPerRow();
+  for (size_t y = 0; y < quant_field->ysize(); ++y) {
+    AcStrategyRow ac_strategy_row = ac_strategy.ConstRow(y);
+    float* JXL_RESTRICT quant_row = quant_field->Row(y);
+    for (size_t x = 0; x < quant_field->xsize(); ++x) {
+      AcStrategy acs = ac_strategy_row[x];
+      if (!acs.IsFirstBlock()) continue;
+      JXL_ASSERT(x + acs.covered_blocks_x() <= quant_field->xsize());
+      JXL_ASSERT(y + acs.covered_blocks_y() <= quant_field->ysize());
+      float max = quant_row[x];
+      for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+        for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+          max = std::max(quant_row[x + ix + iy * stride], max);
+        }
+      }
+      for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+        for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+          quant_row[x + ix + iy * stride] = max;
+        }
+      }
+    }
+  }
+}
+
+static const float kDcQuantPow = 0.55;
+static const float kDcQuant = 1.13;
+static const float kAcQuant = 0.84;
+
+void FindBestQuantization(const ImageBundle& linear, const Image3F& opsin,
+                          PassesEncoderState* enc_state, ThreadPool* pool,
+                          AuxOut* aux_out) {
+  const CompressParams& cparams = enc_state->cparams;
+  Quantizer& quantizer = enc_state->shared.quantizer;
+  ImageI& raw_quant_field = enc_state->shared.raw_quant_field;
+  ImageF& quant_field = enc_state->initial_quant_field;
+
+  const float intensity_multiplier = cparams.GetIntensityMultiplier();
+  const float butteraugli_target = cparams.butteraugli_distance;
+  ButteraugliComparator comparator(cparams.hf_asymmetry);
+  ImageMetadata metadata;
+  JXL_CHECK(comparator.SetReferenceImage(linear));
+  bool lower_is_better =
+      (comparator.GoodQualityScore() < comparator.BadQualityScore());
+  const float initial_quant_dc =
+      InitialQuantDC(butteraugli_target, intensity_multiplier);
+  AdjustQuantField(enc_state->shared.ac_strategy, &quant_field);
+  ImageF tile_distmap;
+  ImageF tile_distmap_localopt;
+  ImageF initial_quant_field = CopyImage(quant_field);
+  ImageF last_quant_field = CopyImage(initial_quant_field);
+  ImageF last_tile_distmap_localopt;
+
+  float initial_qf_min, initial_qf_max;
+  ImageMinMax(initial_quant_field, &initial_qf_min, &initial_qf_max);
+  float initial_qf_ratio = initial_qf_max / initial_qf_min;
+  float qf_max_deviation_low = std::sqrt(250 / initial_qf_ratio);
+  float asymmetry = 2;
+  if (qf_max_deviation_low < asymmetry) asymmetry = qf_max_deviation_low;
+  float qf_lower = initial_qf_min / (asymmetry * qf_max_deviation_low);
+  float qf_higher = initial_qf_max * (qf_max_deviation_low / asymmetry);
+
+  JXL_ASSERT(qf_higher / qf_lower < 253);
+
+  constexpr int kOriginalComparisonRound = 1;
+  constexpr float kMaximumDistanceIncreaseFactor = 1.015;
+
+  for (int i = 0; i < cparams.max_butteraugli_iters + 1; ++i) {
+    if (FLAGS_dump_quant_state) {
+      printf("\nQuantization field:\n");
+      for (size_t y = 0; y < quant_field.ysize(); ++y) {
+        for (size_t x = 0; x < quant_field.xsize(); ++x) {
+          printf(" %.5f", quant_field.Row(y)[x]);
+        }
+        printf("\n");
+      }
+    }
+
+    quantizer.SetQuantField(initial_quant_dc, quant_field, &raw_quant_field);
+    ImageMetadata metadata;
+    metadata.bits_per_sample = 32;
+    metadata.color_encoding = ColorManagement::LinearSRGB();
+    ImageBundle linear(&metadata);
+    linear.SetFromImage(RoundtripImage(opsin, enc_state, pool),
+                        metadata.color_encoding);
+    PROFILER_ZONE("enc Butteraugli");
+    float score;
+    ImageF diffmap;
+    JXL_CHECK(comparator.CompareWith(linear, &diffmap, &score));
+    if (!lower_is_better) {
+      score = -score;
+      diffmap = ScaleImage(-1.0f, diffmap);
+    }
+    static const int kMargins[100] = {0, 0, 0, 0, 0, 0, 0, 0, 0};
+    tile_distmap =
+        TileDistMap(diffmap, 8, kMargins[i], enc_state->shared.ac_strategy);
+    tile_distmap_localopt =
+        TileDistMap(diffmap, 8, 2, enc_state->shared.ac_strategy);
+    if (WantDebugOutput(aux_out)) {
+      DumpHeatmaps(aux_out, butteraugli_target, quant_field, tile_distmap);
+    }
+    if (aux_out != nullptr) ++aux_out->num_butteraugli_iters;
+    if (FLAGS_log_search_state) {
+      float minval, maxval;
+      ImageMinMax(quant_field, &minval, &maxval);
+      printf("\nButteraugli iter: %d/%d\n", i, cparams.max_butteraugli_iters);
+      printf("Butteraugli distance: %f\n", score);
+      printf("quant range: %f ... %f  DC quant: %f\n", minval, maxval,
+             initial_quant_dc);
+      if (FLAGS_dump_quant_state) {
+        quantizer.DumpQuantizationMap(raw_quant_field);
+      }
+    }
+
+    if (i > kOriginalComparisonRound) {
+      // Undo last round if it made things worse (i.e. increased the quant value
+      // AND the distance in nearby pixels by at least some percentage).
+      for (size_t y = 0; y < quant_field.ysize(); ++y) {
+        float* const JXL_RESTRICT row_q = quant_field.Row(y);
+        const float* const JXL_RESTRICT row_dist = tile_distmap_localopt.Row(y);
+        const float* const JXL_RESTRICT row_last_dist =
+            last_tile_distmap_localopt.Row(y);
+        const float* const JXL_RESTRICT row_last_q = last_quant_field.Row(y);
+        for (size_t x = 0; x < quant_field.xsize(); ++x) {
+          if (row_q[x] > row_last_q[x] &&
+              row_dist[x] > kMaximumDistanceIncreaseFactor * row_last_dist[x]) {
+            row_q[x] = row_last_q[x];
+          }
+        }
+      }
+    }
+    last_quant_field = CopyImage(quant_field);
+    last_tile_distmap_localopt = CopyImage(tile_distmap_localopt);
+    if (i == cparams.max_butteraugli_iters) break;
+
+    double kPow[8] = {
+        0.2, 0.2, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+    };
+    double kPowMod[8] = {
+        0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0,
+    };
+    if (i == kOriginalComparisonRound) {
+      // Don't allow optimization to make the quant field a lot worse than
+      // what the initial guess was. This allows the AC field to have enough
+      // precision to reduce the oscillations due to the dc reconstruction.
+      double kInitMul = 0.6;
+      const double kOneMinusInitMul = 1.0 - kInitMul;
+      for (size_t y = 0; y < quant_field.ysize(); ++y) {
+        float* const JXL_RESTRICT row_q = quant_field.Row(y);
+        const float* const JXL_RESTRICT row_init = initial_quant_field.Row(y);
+        for (size_t x = 0; x < quant_field.xsize(); ++x) {
+          double clamp = kOneMinusInitMul * row_q[x] + kInitMul * row_init[x];
+          if (row_q[x] < clamp) {
+            row_q[x] = clamp;
+            if (row_q[x] > qf_higher) row_q[x] = qf_higher;
+            if (row_q[x] < qf_lower) row_q[x] = qf_lower;
+          }
+        }
+      }
+    }
+
+    double cur_pow = 0.0;
+    if (i < 7) {
+      cur_pow = kPow[i] + (butteraugli_target - 1.0) * kPowMod[i];
+      if (cur_pow < 0) {
+        cur_pow = 0;
+      }
+    }
+    // pow(x, 0) == 1, so skip pow.
+    if (cur_pow == 0.0) {
+      for (size_t y = 0; y < quant_field.ysize(); ++y) {
+        const float* const JXL_RESTRICT row_dist = tile_distmap.Row(y);
+        float* const JXL_RESTRICT row_q = quant_field.Row(y);
+        for (size_t x = 0; x < quant_field.xsize(); ++x) {
+          const float diff = row_dist[x] / butteraugli_target;
+          if (diff > 1.0f) {
+            float old = row_q[x];
+            row_q[x] *= diff;
+            int qf_old = old * quantizer.InvGlobalScale() + 0.5;
+            int qf_new = row_q[x] * quantizer.InvGlobalScale() + 0.5;
+            if (qf_old == qf_new) {
+              row_q[x] = old + quantizer.Scale();
+            }
+          }
+          if (row_q[x] > qf_higher) row_q[x] = qf_higher;
+          if (row_q[x] < qf_lower) row_q[x] = qf_lower;
+        }
+      }
+    } else {
+      for (size_t y = 0; y < quant_field.ysize(); ++y) {
+        const float* const JXL_RESTRICT row_dist = tile_distmap.Row(y);
+        float* const JXL_RESTRICT row_q = quant_field.Row(y);
+        for (size_t x = 0; x < quant_field.xsize(); ++x) {
+          const float diff = row_dist[x] / butteraugli_target;
+          if (diff <= 1.0f) {
+            row_q[x] *= std::pow(diff, cur_pow);
+          } else {
+            float old = row_q[x];
+            row_q[x] *= diff;
+            int qf_old = old * quantizer.InvGlobalScale() + 0.5;
+            int qf_new = row_q[x] * quantizer.InvGlobalScale() + 0.5;
+            if (qf_old == qf_new) {
+              row_q[x] = old + quantizer.Scale();
+            }
+          }
+          if (row_q[x] > qf_higher) row_q[x] = qf_higher;
+          if (row_q[x] < qf_lower) row_q[x] = qf_lower;
+        }
+      }
+    }
+  }
+  quantizer.SetQuantField(initial_quant_dc, quant_field, &raw_quant_field);
+}
+
+void FindBestQuantizationMaxError(const Image3F& opsin,
+                                  PassesEncoderState* enc_state,
+                                  ThreadPool* pool, AuxOut* aux_out) {
+  const CompressParams& cparams = enc_state->cparams;
+  Quantizer& quantizer = enc_state->shared.quantizer;
+  ImageI& raw_quant_field = enc_state->shared.raw_quant_field;
+  ImageF& quant_field = enc_state->initial_quant_field;
+
+  // TODO(veluca): better choice of this value.
+  const float initial_quant_dc =
+      16 * std::sqrt(0.1f / cparams.butteraugli_distance);
+  AdjustQuantField(enc_state->shared.ac_strategy, &quant_field);
+
+  const float inv_max_err[3] = {1.0f / enc_state->cparams.max_error[0],
+                                1.0f / enc_state->cparams.max_error[1],
+                                1.0f / enc_state->cparams.max_error[2]};
+
+  for (int i = 0; i < cparams.max_butteraugli_iters + 1; ++i) {
+    quantizer.SetQuantField(initial_quant_dc, quant_field, &raw_quant_field);
+    if (aux_out)
+      aux_out->DumpXybImage(("ops" + std::to_string(i)).c_str(), opsin);
+    Image3F decoded =
+        RoundtripImage(opsin, enc_state, pool, /*save_decompressed=*/false,
+                       /*apply_color_transform=*/false);
+    if (aux_out)
+      aux_out->DumpXybImage(("dec" + std::to_string(i)).c_str(), decoded);
+
+    for (size_t by = 0; by < enc_state->shared.frame_dim.ysize_blocks; by++) {
+      AcStrategyRow ac_strategy_row =
+          enc_state->shared.ac_strategy.ConstRow(by);
+      for (size_t bx = 0; bx < enc_state->shared.frame_dim.xsize_blocks; bx++) {
+        AcStrategy acs = ac_strategy_row[bx];
+        if (!acs.IsFirstBlock()) continue;
+        float max_error = 0;
+        for (size_t c = 0; c < 3; c++) {
+          for (size_t y = by * kBlockDim;
+               y < (by + acs.covered_blocks_y()) * kBlockDim; y++) {
+            if (y >= decoded.ysize()) continue;
+            const float* JXL_RESTRICT in_row = opsin.ConstPlaneRow(c, y);
+            const float* JXL_RESTRICT dec_row = decoded.ConstPlaneRow(c, y);
+            for (size_t x = bx * kBlockDim;
+                 x < (bx + acs.covered_blocks_x()) * kBlockDim; x++) {
+              if (x >= decoded.xsize()) continue;
+              max_error = std::max(
+                  std::abs(in_row[x] - dec_row[x]) * inv_max_err[c], max_error);
+            }
+          }
+        }
+        // Target an error between max_error/2 and max_error.
+        // If the error in the varblock is above the target, increase the qf to
+        // compensate. If the error is below the target, decrease the qf.
+        // However, to avoid an excessive increase of the qf, only do so if the
+        // error is less than half the maximum allowed error.
+        float qf_mul = max_error < 0.5f ? max_error * 2.0f
+                                        : max_error > 1.0f ? max_error : 1.0f;
+        for (size_t qy = by; qy < by + acs.covered_blocks_y(); qy++) {
+          float* JXL_RESTRICT quant_field_row = quant_field.Row(qy);
+          for (size_t qx = bx; qx < bx + acs.covered_blocks_x(); qx++) {
+            quant_field_row[qx] *= qf_mul;
+          }
+        }
+      }
+    }
+  }
+  quantizer.SetQuantField(initial_quant_dc, quant_field, &raw_quant_field);
+}
+
+void FindBestQuantizationHQ(const ImageBundle& linear, const Image3F& opsin,
+                            PassesEncoderState* enc_state, ThreadPool* pool,
+                            AuxOut* aux_out) {
+  const CompressParams& cparams = enc_state->cparams;
+  Quantizer& quantizer = enc_state->shared.quantizer;
+  ImageI& raw_quant_field = enc_state->shared.raw_quant_field;
+  ImageF& quant_field = enc_state->initial_quant_field;
+  const AcStrategyImage& ac_strategy = enc_state->shared.ac_strategy;
+
+  ButteraugliComparator comparator(cparams.hf_asymmetry);
+  ImageMetadata metadata;
+  JXL_CHECK(comparator.SetReferenceImage(linear));
+  AdjustQuantField(ac_strategy, &quant_field);
+  ImageF best_quant_field = CopyImage(quant_field);
+  bool lower_is_better =
+      (comparator.GoodQualityScore() < comparator.BadQualityScore());
+  float best_score = 1000000.0f;
+  ImageF tile_distmap;
+  static const int kMaxOuterIters = 2;
+  int outer_iter = 0;
+  int butteraugli_iter = 0;
+  int search_radius = 0;
+  float quant_ceil = 5.0f;
+  float quant_dc = 1.2f;
+  float best_quant_dc = quant_dc;
+  int num_stalling_iters = 0;
+  int max_iters = cparams.max_butteraugli_iters_guetzli_mode;
+  const float butteraugli_target = cparams.butteraugli_distance;
+
+  for (;;) {
+    if (FLAGS_dump_quant_state) {
+      printf("\nQuantization field:\n");
+      for (size_t y = 0; y < quant_field.ysize(); ++y) {
+        for (size_t x = 0; x < quant_field.xsize(); ++x) {
+          printf(" %.5f", quant_field.Row(y)[x]);
+        }
+        printf("\n");
+      }
+    }
+    float qmin, qmax;
+    ImageMinMax(quant_field, &qmin, &qmax);
+    ++butteraugli_iter;
+    float score = 0.0;
+    ImageF diffmap;
+    quantizer.SetQuantField(quant_dc, quant_field, &raw_quant_field);
+    ImageMetadata metadata;
+    metadata.bits_per_sample = 32;
+    metadata.color_encoding = ColorManagement::LinearSRGB();
+    ImageBundle linear(&metadata);
+    linear.SetFromImage(RoundtripImage(opsin, enc_state, pool),
+                        metadata.color_encoding);
+    JXL_CHECK(comparator.CompareWith(linear, &diffmap, &score));
+
+    if (!lower_is_better) {
+      score = -score;
+      ScaleImage(-1.0f, &diffmap);
+    }
+    bool best_quant_updated = false;
+    if (score <= best_score) {
+      best_quant_field = CopyImage(quant_field);
+      best_score = std::max<float>(score, butteraugli_target);
+      best_quant_updated = true;
+      best_quant_dc = quant_dc;
+      num_stalling_iters = 0;
+    } else if (outer_iter == 0) {
+      ++num_stalling_iters;
+    }
+    tile_distmap = TileDistMap(diffmap, 8, 0, ac_strategy);
+    if (WantDebugOutput(aux_out)) {
+      DumpHeatmaps(aux_out, butteraugli_target, quant_field, tile_distmap);
+    }
+    if (aux_out) {
+      ++aux_out->num_butteraugli_iters;
+    }
+    if (FLAGS_log_search_state) {
+      float minval, maxval;
+      ImageMinMax(quant_field, &minval, &maxval);
+      printf("\nButteraugli iter: %d/%d%s\n", butteraugli_iter, max_iters,
+             best_quant_updated ? " (*)" : "");
+      printf("Butteraugli distance: %f\n", score);
+      printf(
+          "quant range: %f ... %f  DC quant: "
+          "%f\n",
+          minval, maxval, quant_dc);
+      printf("search radius: %d\n", search_radius);
+      if (FLAGS_dump_quant_state) {
+        quantizer.DumpQuantizationMap(raw_quant_field);
+      }
+    }
+    if (butteraugli_iter >= max_iters) {
+      break;
+    }
+    bool changed = false;
+    while (!changed && score > butteraugli_target) {
+      for (int radius = 0; radius <= search_radius && !changed; ++radius) {
+        ImageF dist_to_peak_map =
+            DistToPeakMap(tile_distmap, butteraugli_target, radius, 0.0);
+        for (size_t y = 0; y < quant_field.ysize(); ++y) {
+          float* const JXL_RESTRICT row_q = quant_field.Row(y);
+          const float* const JXL_RESTRICT row_dist = dist_to_peak_map.Row(y);
+          for (size_t x = 0; x < quant_field.xsize(); ++x) {
+            if (row_dist[x] >= 0.0f) {
+              static const float kAdjSpeed[kMaxOuterIters] = {0.1f, 0.04f};
+              const float factor =
+                  kAdjSpeed[outer_iter] * tile_distmap.Row(y)[x];
+              if (AdjustQuantVal(&row_q[x], row_dist[x], factor, quant_ceil)) {
+                changed = true;
+              }
+            }
+          }
+        }
+      }
+      if (!changed || num_stalling_iters >= 3) {
+        // Try to extend the search parameters.
+        if ((search_radius < 4) &&
+            (qmax < 0.99f * quant_ceil || quant_ceil >= 3.0f + search_radius)) {
+          ++search_radius;
+          continue;
+        }
+        if (quant_dc < 0.4f * quant_ceil - 0.8f) {
+          quant_dc += 0.2f;
+          changed = true;
+          continue;
+        }
+        if (quant_ceil < 8.0f) {
+          quant_ceil += 0.5f;
+          continue;
+        }
+        break;
+      }
+    }
+    if (!changed) {
+      if (++outer_iter == kMaxOuterIters) break;
+      static const float kQuantScale = 0.75f;
+      for (size_t y = 0; y < quant_field.ysize(); ++y) {
+        for (size_t x = 0; x < quant_field.xsize(); ++x) {
+          quant_field.Row(y)[x] *= kQuantScale;
+        }
+      }
+      num_stalling_iters = 0;
+    }
+  }
+  quantizer.SetQuantField(best_quant_dc, best_quant_field, &raw_quant_field);
+}
+
+ImageF AdaptiveQuantizationMap(const Image3F& opsin, const ImageF& intensity_ac,
+                               const FrameDimensions& frame_dim, float scale,
+                               ThreadPool* pool) {
+  PROFILER_ZONE("aq AdaptiveQuantMap");
+  if (frame_dim.xsize <= 1 || frame_dim.ysize <= 1) {
+    ImageF out(1, 1);
+    FillImage(1.0f, &out);
+    return out;
+  }
+
+  const float kSigma = 8.2553856725566153f;
+  static const int kRadius = static_cast<int>(2 * kSigma + 0.5f);
+  std::vector<float> kernel = GaussianKernel(kRadius, kSigma);
+
+  constexpr float kDiffCutoff = 0.11883287948847132f;
+  ImageF out = DiffPrecompute(opsin, frame_dim, kDiffCutoff, pool);
+  JXL_ASSERT(out.xsize() % kBlockDim == 0 && out.ysize() % kBlockDim == 0);
+  out = ConvolveAndSample(out, kernel, kBlockDim);
+  PerBlockModulations(intensity_ac, scale, pool, &out);
+  return out;
+}
+
+namespace kernel {
+
+struct GaussianDC {
+  JXL_INLINE const Weights3x3& Weights() const {
+    constexpr float w0 = 0.320356f;
+    constexpr float w1 = 0.122822f;
+    constexpr float w2 = 0.047089f;
+    static constexpr Weights3x3 weights = {
+        {HWY_REP4(w2)}, {HWY_REP4(w1)}, {HWY_REP4(w2)},
+        {HWY_REP4(w1)}, {HWY_REP4(w0)}, {HWY_REP4(w1)},
+        {HWY_REP4(w2)}, {HWY_REP4(w1)}, {HWY_REP4(w2)}};
+    return weights;
+  }
+};
+
+}  // namespace kernel
+
+ImageF IntensityAcEstimate(const ImageF& opsin_y,
+                           const FrameDimensions& frame_dim, ThreadPool* pool) {
+  const Rect rect(0, 0, frame_dim.xsize, frame_dim.ysize);  // not padded
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+
+  const kernel::GaussianDC kernel;
+  ImageF smoothed(xsize, ysize);
+  if (xsize < kConvolveMinWidth) {
+    using Convolution = slow::General3x3Convolution<1, WrapMirror>;
+    Convolution::Run(opsin_y, rect, kernel, &smoothed);
+  } else {
+    using Conv3 = ConvolveT<strategy::Symmetric3>;
+    Conv3::Run(opsin_y, rect, kernel, pool, &smoothed);
+  }
+
+  RunOnPool(
+      pool, 0, static_cast<int>(ysize), ThreadPool::SkipInit(),
+      [xsize, &opsin_y, &smoothed](const int task, int /*thread*/) {
+        const size_t y = static_cast<size_t>(task);
+        const float* JXL_RESTRICT row_opsin = opsin_y.ConstRow(y);
+        float* JXL_RESTRICT row_smooth = smoothed.Row(y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row_smooth[x] = row_opsin[x] - row_smooth[x];
+        }
+      },
+      "AQ subtract");
+  return smoothed;
+}
+
+}  // namespace
+
+float InitialQuantDC(float butteraugli_target, float intensity_multiplier) {
+  const float intensity_multiplier3 = std::cbrt(intensity_multiplier);
+  const float butteraugli_target_dc =
+      std::min<float>(butteraugli_target,
+                      2.5 * std::pow(0.4 * butteraugli_target, kDcQuantPow));
+  // We want the maximum DC value to be at most 2**15 * kInvDCQuant / quant_dc.
+  // The maximum DC value might not be in the kXybRange because of inverse
+  // gaborish, so we add some slack to the maximum theoretical quant obtained
+  // this way (64), and some more if the intensity multiplier is more than the
+  // default of 1.
+  return std::min(kDcQuant / butteraugli_target_dc,
+                  50.f / intensity_multiplier3);
+}
+
+ImageF InitialQuantField(const float butteraugli_target, const Image3F& opsin,
+                         const FrameDimensions& frame_dim, ThreadPool* pool,
+                         float rescale) {
+  PROFILER_FUNC;
+  const float quant_ac = kAcQuant / butteraugli_target;
+  ImageF intensity_ac = IntensityAcEstimate(opsin.Plane(1), frame_dim, pool);
+  return AdaptiveQuantizationMap(opsin, intensity_ac, frame_dim,
+                                 quant_ac * rescale, pool);
+}
+
+void FindBestQuantizer(const ImageBundle& linear, const Image3F& opsin,
+                       PassesEncoderState* enc_state, ThreadPool* pool,
+                       AuxOut* aux_out, double rescale) {
+  const CompressParams& cparams = enc_state->cparams;
+  Quantizer& quantizer = enc_state->shared.quantizer;
+  ImageI& raw_quant_field = enc_state->shared.raw_quant_field;
+  const float intensity_multiplier = cparams.GetIntensityMultiplier();
+  if (cparams.max_error_mode) {
+    PROFILER_ZONE("enc find best maxerr");
+    FindBestQuantizationMaxError(opsin, enc_state, pool, aux_out);
+  } else if (cparams.speed_tier == SpeedTier::kFalcon) {
+    const float quant_dc =
+        InitialQuantDC(cparams.butteraugli_distance, intensity_multiplier);
+    // TODO(veluca): tune constant.
+    const float quant_ac = kAcQuant / cparams.butteraugli_distance;
+    quantizer.SetQuant(quant_dc, quant_ac, &raw_quant_field);
+  } else if (cparams.uniform_quant > 0.0) {
+    quantizer.SetQuant(cparams.uniform_quant * rescale,
+                       cparams.uniform_quant * rescale, &raw_quant_field);
+  } else if (cparams.speed_tier > SpeedTier::kKitten) {
+    PROFILER_ZONE("enc fast quant");
+    const float quant_dc =
+        InitialQuantDC(cparams.butteraugli_distance, intensity_multiplier);
+    AdjustQuantField(enc_state->shared.ac_strategy,
+                     &enc_state->initial_quant_field);
+    quantizer.SetQuantField(quant_dc, enc_state->initial_quant_field,
+                            &raw_quant_field);
+  } else {
+    // Normal encoding to a butteraugli score.
+    PROFILER_ZONE("enc find best2");
+    if (cparams.speed_tier == SpeedTier::kTortoise) {
+      FindBestQuantizationHQ(linear, opsin, enc_state, pool, aux_out);
+    } else {
+      FindBestQuantization(linear, opsin, enc_state, pool, aux_out);
+    }
+  }
+}
+
+Image3F RoundtripImage(const Image3F& opsin, PassesEncoderState* enc_state,
+                       ThreadPool* pool, bool save_decompressed,
+                       bool apply_color_transform) {
+  PROFILER_ZONE("enc roundtrip");
+  PassesDecoderState dec_state;
+  dec_state.shared = &enc_state->shared;
+  JXL_ASSERT(opsin.ysize() % kBlockDim == 0);
+
+  const size_t xsize_groups = DivCeil(opsin.xsize(), kGroupDim);
+  const size_t ysize_groups = DivCeil(opsin.ysize(), kGroupDim);
+  const size_t num_groups = xsize_groups * ysize_groups;
+
+  // Dummy metadata with grayscale = off.
+  ImageMetadata metadata;
+  metadata.color_encoding = ColorManagement::SRGB();
+
+  InitializePassesEncoder(opsin, pool, enc_state, nullptr);
+  dec_state.Init(pool);
+
+  Image3F idct(opsin.xsize(), opsin.ysize());
+
+  const auto allocate_storage = [&](size_t num_threads) {
+    dec_state.EnsureStorage(num_threads);
+    return true;
+  };
+  const auto process_group = [&](const int group_index, const int thread) {
+    ComputeCoefficients(group_index, enc_state, nullptr);
+    JXL_CHECK(DecodeGroupForRoundtrip(
+        enc_state->coeffs, group_index, &dec_state, thread, &idct, nullptr,
+        save_decompressed, apply_color_transform));
+  };
+  RunOnPool(pool, 0, num_groups, allocate_storage, process_group, "AQ loop");
+
+  // Fine to do a JXL_ASSERT instead of error handling, since this only happens
+  // on the encoder side where we can't be fed with invalid data.
+  JXL_CHECK(FinalizeFrameDecoding(&idct, &dec_state, pool, nullptr,
+                                  save_decompressed, apply_color_transform));
+  return idct;
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_adaptive_quantization.h b/jxl/enc_adaptive_quantization.h
new file mode 100644
index 000000000..34e278920
--- /dev/null
+++ b/jxl/enc_adaptive_quantization.h
@@ -0,0 +1,74 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_ADAPTIVE_QUANTIZATION_H_
+#define JXL_ENC_ADAPTIVE_QUANTIZATION_H_
+
+#include <stddef.h>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/common.h"
+#include "jxl/dot_dictionary.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_params.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/loop_filter.h"
+#include "jxl/multiframe.h"
+#include "jxl/patch_dictionary.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+#include "jxl/splines.h"
+
+// Heuristics to find a good quantizer for a given image. InitialQuantField
+// produces a quantization field (i.e. relative quantization amounts for each
+// block) out of an opsin-space image. `InitialQuantField` uses heuristics,
+// `FindBestQuantizer` (in non-fast mode) will run multiple encoding-decoding
+// steps and try to improve the given quant field.
+
+namespace jxl {
+
+// Computes the decoded image for a given set of compression parameters. Mainly
+// used in the FindBestQuantization loops and in some tests.
+// TODO(veluca): this doesn't seem the best possible file for this function.
+Image3F RoundtripImage(const Image3F& opsin, PassesEncoderState* enc_state,
+                       ThreadPool* pool, bool save_decompressed = false,
+                       bool apply_color_transform = true);
+
+// Returns an image subsampled by kBlockDim in each direction. If the value
+// at pixel (x,y) in the returned image is greater than 1.0, it means that
+// more fine-grained quantization should be used in the corresponding block
+// of the input image, while a value less than 1.0 indicates that less
+// fine-grained quantization should be enough.
+ImageF InitialQuantField(float butteraugli_target, const Image3F& opsin,
+                         const FrameDimensions& frame_dim, ThreadPool* pool,
+                         float rescale);
+
+float InitialQuantDC(float butteraugli_target, float intensity_multiplier);
+
+// Returns a quantizer that uses an adjusted version of the provided
+// quant_field. Also computes the dequant_map corresponding to the given
+// dequant_float_map and chosen quantization levels.
+// `linear` is only used in Kitten mode or slower.
+void FindBestQuantizer(const ImageBundle& linear, const Image3F& opsin,
+                       PassesEncoderState* enc_state, ThreadPool* pool,
+                       AuxOut* aux_out, double rescale = 1.0);
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_ADAPTIVE_QUANTIZATION_H_
diff --git a/jxl/enc_ans.cc b/jxl/enc_ans.cc
new file mode 100644
index 000000000..1d3cbb625
--- /dev/null
+++ b/jxl/enc_ans.cc
@@ -0,0 +1,750 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_ans.h"
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <array>
+#include <numeric>
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "c/enc/context_map_encode.h"
+#include "c/enc/write_bits.h"
+#include "jxl/ans_common.h"
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/fast_log.h"
+#include "jxl/enc_cluster.h"
+#include "jxl/enc_context_map.h"
+
+namespace jxl {
+
+namespace {
+
+static const int kMaxNumSymbolsForSmallCode = 4;
+
+HWY_ATTR void ANSBuildInfoTable(const ANSHistBin* counts,
+                                const AliasTable::Entry* table,
+                                size_t alphabet_size, ANSEncSymbolInfo* info) {
+  // create valid alias table for empty streams.
+  for (size_t s = 0; s < std::max<size_t>(1, alphabet_size); ++s) {
+    const ANSHistBin freq = s == alphabet_size ? ANS_TAB_SIZE : counts[s];
+    info[s].freq_ = static_cast<uint16_t>(freq);
+#ifdef USE_MULT_BY_RECIPROCAL
+    if (freq != 0) {
+      info[s].ifreq_ =
+          ((1ull << RECIPROCAL_PRECISION) + info[s].freq_ - 1) / info[s].freq_;
+    } else {
+      info[s].ifreq_ = 1;  // shouldn't matter (symbol shouldn't occur), but...
+    }
+#endif
+    info[s].reverse_map_.resize(freq);
+  }
+  for (int i = 0; i < ANS_TAB_SIZE; i++) {
+    AliasTable::Symbol s = AliasTable::Lookup(table, i);
+    info[s.value].reverse_map_[s.offset] = i;
+  }
+}
+
+float EstimateDataBits(const ANSHistBin* histogram, const ANSHistBin* counts,
+                       size_t len) {
+  float sum = 0.0f;
+  int total_histogram = 0;
+  int total_counts = 0;
+  for (size_t i = 0; i < len; ++i) {
+    total_histogram += histogram[i];
+    total_counts += counts[i];
+    if (histogram[i] > 0) {
+      JXL_ASSERT(counts[i] > 0);
+      // += histogram[i] * -log(counts[i]/total_counts)
+      sum += histogram[i] * (ANS_LOG_TAB_SIZE - FastLog2(counts[i]));
+    }
+  }
+  if (total_histogram > 0) {
+    JXL_ASSERT(total_counts == ANS_TAB_SIZE);
+  }
+  return sum;
+}
+
+float EstimateDataBitsFlat(const ANSHistBin* histogram, size_t len) {
+  const float flat_bits = FastLog2(len);
+  int total_histogram = 0;
+  for (size_t i = 0; i < len; ++i) {
+    total_histogram += histogram[i];
+  }
+  return total_histogram * flat_bits;
+}
+
+// Static Huffman code for encoding logcounts. The last symbol is used as RLE
+// sequence.
+static const uint8_t kLogCountBitLengths[ANS_LOG_TAB_SIZE + 2] = {
+    5, 4, 4, 4, 4, 4, 3, 3, 3, 3, 3, 6, 7, 7,
+};
+static const uint8_t kLogCountSymbols[ANS_LOG_TAB_SIZE + 2] = {
+    17, 11, 15, 3, 9, 7, 4, 2, 5, 6, 0, 33, 1, 65,
+};
+
+// Returns the difference between largest count that can be represented and is
+// smaller than "count" and smallest representable count larger than "count".
+static int SmallestIncrement(uint32_t count, uint32_t shift) {
+  int bits = count == 0 ? -1 : FloorLog2Nonzero(count);
+  int drop_bits = bits - GetPopulationCountPrecision(bits, shift);
+  return (1 << drop_bits);
+}
+
+template <bool minimize_error_of_sum>
+bool RebalanceHistogram(const float* targets, int max_symbol, int table_size,
+                        uint32_t shift, int* omit_pos, ANSHistBin* counts) {
+  int sum = 0;
+  float sum_nonrounded = 0.0;
+  int remainder_pos = 0;  // if all of them are handled in first loop
+  int remainder_log = -1;
+  for (int n = 0; n < max_symbol; ++n) {
+    if (targets[n] > 0 && targets[n] < 1.0f) {
+      counts[n] = 1;
+      sum_nonrounded += targets[n];
+      sum += counts[n];
+    }
+  }
+  const float discount_ratio =
+      (table_size - sum) / (table_size - sum_nonrounded);
+  JXL_ASSERT(discount_ratio > 0);
+  JXL_ASSERT(discount_ratio <= 1.0f);
+  // Invariant for minimize_error_of_sum == true:
+  // abs(sum - sum_nonrounded)
+  //   <= SmallestIncrement(max(targets[])) + max_symbol
+  for (int n = 0; n < max_symbol; ++n) {
+    if (targets[n] >= 1.0f) {
+      sum_nonrounded += targets[n];
+      counts[n] =
+          static_cast<ANSHistBin>(targets[n] * discount_ratio);  // truncate
+      if (counts[n] == 0) counts[n] = 1;
+      if (counts[n] == table_size) counts[n] = table_size - 1;
+      // Round the count to the closest nonzero multiple of SmallestIncrement
+      // (when minimize_error_of_sum is false) or one of two closest so as to
+      // keep the sum as close as possible to sum_nonrounded.
+      int inc = SmallestIncrement(counts[n], shift);
+      counts[n] -= counts[n] & (inc - 1);
+      // TODO(robryk): Should we rescale targets[n]?
+      const float target =
+          minimize_error_of_sum ? (sum_nonrounded - sum) : targets[n];
+      if (counts[n] == 0 ||
+          (target > counts[n] + inc / 2 && counts[n] + inc < table_size)) {
+        counts[n] += inc;
+      }
+      sum += counts[n];
+      const int count_log = FloorLog2Nonzero(static_cast<uint32_t>(counts[n]));
+      if (count_log > remainder_log) {
+        remainder_pos = n;
+        remainder_log = count_log;
+      }
+    }
+  }
+  JXL_ASSERT(remainder_pos != -1);
+  // NOTE: This is the only place where counts could go negative. We could
+  // detect that, return false and make ANSHistBin uint32_t.
+  counts[remainder_pos] -= sum - table_size;
+  *omit_pos = remainder_pos;
+  return counts[remainder_pos] > 0;
+}
+
+Status NormalizeCounts(ANSHistBin* counts, int* omit_pos, const int length,
+                       const int precision_bits, uint32_t shift,
+                       int* num_symbols, int* symbols) {
+  const int32_t table_size = 1 << precision_bits;  // target sum / table size
+  uint64_t total = 0;
+  int max_symbol = 0;
+  int symbol_count = 0;
+  for (int n = 0; n < length; ++n) {
+    total += counts[n];
+    if (counts[n] > 0) {
+      if (symbol_count < kMaxNumSymbolsForSmallCode) {
+        symbols[symbol_count] = n;
+      }
+      ++symbol_count;
+      max_symbol = n + 1;
+    }
+  }
+  *num_symbols = symbol_count;
+  if (symbol_count == 0) {
+    return true;
+  }
+  if (symbol_count == 1) {
+    counts[symbols[0]] = table_size;
+    return true;
+  }
+  if (symbol_count > table_size)
+    return JXL_FAILURE("Too many entries in an ANS histogram");
+
+  const float norm = 1.f * table_size / total;
+  std::vector<float> targets(max_symbol);
+  for (size_t n = 0; n < targets.size(); ++n) {
+    targets[n] = norm * counts[n];
+  }
+  if (!RebalanceHistogram<false>(&targets[0], max_symbol, table_size, shift,
+                                 omit_pos, counts)) {
+    // Use an alternative rebalancing mechanism if the one above failed
+    // to create a histogram that is positive wherever the original one was.
+    if (!RebalanceHistogram<true>(&targets[0], max_symbol, table_size, shift,
+                                  omit_pos, counts)) {
+      return JXL_FAILURE("Logic error: couldn't rebalance a histogram");
+    }
+  }
+  return true;
+}
+
+struct SizeWriter {
+  size_t size = 0;
+  void Write(size_t num, size_t bits) { size += num; }
+};
+
+template <typename Writer>
+void StoreVarLenUint8(size_t n, Writer* writer) {
+  if (n == 0) {
+    writer->Write(1, 0);
+  } else {
+    writer->Write(1, 1);
+    size_t nbits = FloorLog2Nonzero(n);
+    writer->Write(3, nbits);
+    writer->Write(nbits, n - (1ULL << nbits));
+  }
+}
+
+template <typename Writer>
+void EncodeCounts(const ANSHistBin* counts, const int alphabet_size,
+                  const int omit_pos, const int num_symbols, uint32_t shift,
+                  const int* symbols, Writer* writer) {
+  if (num_symbols <= 2) {
+    // Small tree marker to encode 1-2 symbols.
+    writer->Write(1, 1);
+    if (num_symbols == 0) {
+      writer->Write(1, 0);
+      StoreVarLenUint8(0, writer);
+    } else {
+      writer->Write(1, num_symbols - 1);
+      for (int i = 0; i < num_symbols; ++i) {
+        StoreVarLenUint8(symbols[i], writer);
+      }
+    }
+    if (num_symbols == 2) {
+      writer->Write(ANS_LOG_TAB_SIZE, counts[symbols[0]]);
+    }
+  } else {
+    // Mark non-small tree.
+    writer->Write(1, 0);
+    // Mark non-flat histogram.
+    writer->Write(1, 0);
+
+    // Precompute sequences for RLE encoding. Contains the number of identical
+    // values starting at a given index. Only contains the value at the first
+    // element of the series.
+    std::vector<uint32_t> same(alphabet_size, 0);
+    int last = 0;
+    for (int i = 1; i < alphabet_size; i++) {
+      // Store the sequence length once different symbol reached, or we're at
+      // the end, or the length is longer than we can encode, or we are at
+      // the omit_pos. We don't support including the omit_pos in an RLE
+      // sequence because this value may use a different amount of log2 bits
+      // than standard, it is too complex to handle in the decoder.
+      if (counts[i] != counts[last] || i + 1 == alphabet_size ||
+          (i - last) >= 255 || i == omit_pos || i == omit_pos + 1) {
+        same[last] = (i - last);
+        last = i + 1;
+      }
+    }
+
+    int length = 0;
+    std::vector<int> logcounts(alphabet_size);
+    int omit_log = 0;
+    for (int i = 0; i < alphabet_size; ++i) {
+      JXL_ASSERT(counts[i] <= ANS_TAB_SIZE);
+      JXL_ASSERT(counts[i] >= 0);
+      if (i == omit_pos) {
+        length = i + 1;
+      } else if (counts[i] > 0) {
+        logcounts[i] = FloorLog2Nonzero(static_cast<uint32_t>(counts[i])) + 1;
+        length = i + 1;
+        if (i < omit_pos) {
+          omit_log = std::max(omit_log, logcounts[i] + 1);
+        } else {
+          omit_log = std::max(omit_log, logcounts[i]);
+        }
+      }
+    }
+    logcounts[omit_pos] = omit_log;
+
+    // Elias gamma-like code for shift. Only difference is that if the number
+    // of bits to be encoded is equal to FloorLog2(ANS_LOG_TAB_SIZE+1), we skip
+    // the terminating 0 in unary coding.
+    int upper_bound_log = FloorLog2Nonzero(ANS_LOG_TAB_SIZE + 1);
+    int log = FloorLog2Nonzero(shift + 1);
+    writer->Write(log, (1 << log) - 1);
+    if (log != upper_bound_log) writer->Write(1, 0);
+    writer->Write(log, ((1 << log) - 1) & (shift + 1));
+
+    // Since num_symbols >= 3, we know that length >= 3, therefore we encode
+    // length - 3.
+    StoreVarLenUint8(length - 3, writer);
+
+    // The logcount values are encoded with a static Huffman code.
+    static const size_t kMinReps = 4;
+    size_t rep = ANS_LOG_TAB_SIZE + 1;
+    for (int i = 0; i < length; ++i) {
+      if (i > 0 && same[i - 1] > kMinReps) {
+        // Encode the RLE symbol and skip the repeated ones.
+        writer->Write(kLogCountBitLengths[rep], kLogCountSymbols[rep]);
+        StoreVarLenUint8(same[i - 1] - kMinReps - 1, writer);
+        i += same[i - 1] - 2;
+        continue;
+      }
+      writer->Write(kLogCountBitLengths[logcounts[i]],
+                    kLogCountSymbols[logcounts[i]]);
+    }
+    for (int i = 0; i < length; ++i) {
+      if (i > 0 && same[i - 1] > kMinReps) {
+        // Skip symbols encoded by RLE.
+        i += same[i - 1] - 2;
+        continue;
+      }
+      if (logcounts[i] > 1 && i != omit_pos) {
+        int bitcount = GetPopulationCountPrecision(logcounts[i] - 1, shift);
+        int drop_bits = logcounts[i] - 1 - bitcount;
+        JXL_CHECK((counts[i] & ((1 << drop_bits) - 1)) == 0);
+        writer->Write(bitcount, (counts[i] >> drop_bits) - (1 << bitcount));
+      }
+    }
+  }
+}
+
+void EncodeFlatHistogram(const int alphabet_size, BitWriter* writer) {
+  // Mark non-small tree.
+  writer->Write(1, 0);
+  // Mark uniform histogram.
+  writer->Write(1, 1);
+  JXL_ASSERT(alphabet_size > 0);
+  // Encode alphabet size.
+  StoreVarLenUint8(alphabet_size - 1, writer);
+}
+
+float ComputeHistoAndDataCost(const ANSHistBin* histogram, size_t alphabet_size,
+                              uint32_t method) {
+  if (method == 0) {  // Flat code
+    return ANS_LOG_TAB_SIZE + 2 +
+           EstimateDataBitsFlat(histogram, alphabet_size);
+  }
+  // Non-flat: shift = method-1.
+  uint32_t shift = method - 1;
+  std::vector<ANSHistBin> counts(histogram, histogram + alphabet_size);
+  int omit_pos = 0;
+  int num_symbols;
+  int symbols[kMaxNumSymbolsForSmallCode] = {};
+  JXL_CHECK(NormalizeCounts(counts.data(), &omit_pos, alphabet_size,
+                            ANS_LOG_TAB_SIZE, shift, &num_symbols, symbols));
+  SizeWriter writer;
+  EncodeCounts(counts.data(), alphabet_size, omit_pos, num_symbols, shift,
+               symbols, &writer);
+  return writer.size +
+         EstimateDataBits(histogram, counts.data(), alphabet_size);
+}
+
+uint32_t ComputeBestMethod(const ANSHistBin* histogram, size_t alphabet_size,
+                           size_t* cost) {
+  uint32_t method = 0;
+  float fcost = ComputeHistoAndDataCost(histogram, alphabet_size, 0);
+  for (uint32_t shift = 0; shift <= ANS_LOG_TAB_SIZE; shift++) {
+    float c = ComputeHistoAndDataCost(histogram, alphabet_size, shift + 1);
+    if (c < fcost) {
+      method = shift + 1;
+      fcost = c;
+    }
+  }
+  *cost = fcost;
+  return method;
+}
+
+}  // namespace
+
+// Returns an estimate of the cost of encoding this histogram and the
+// corresponding data.
+size_t BuildAndStoreANSEncodingData(const ANSHistBin* histogram,
+                                    size_t alphabet_size, bool use_prefix_code,
+                                    ANSEncSymbolInfo* info, BitWriter* writer) {
+  if (use_prefix_code) {
+    if (alphabet_size <= 1) return 0;
+    uint32_t histo[ANS_MAX_ALPHA_SIZE];
+    size_t total = 0;
+    for (size_t i = 0; i < alphabet_size; i++) {
+      histo[i] = histogram[i];
+      JXL_CHECK(histogram[i] >= 0);
+      total += histo[i];
+    }
+    uint8_t depths[ANS_MAX_ALPHA_SIZE] = {};
+    uint16_t bits[ANS_MAX_ALPHA_SIZE];
+    uint8_t storage[1024] = {};
+    brunsli::Storage st(storage, 1024);
+    brunsli::BuildAndStoreHuffmanTree(histo, alphabet_size, depths, bits, &st);
+    for (size_t i = 0; i < alphabet_size; i++) {
+      info[i].bits = depths[i] == 0 ? 0 : bits[i];
+      info[i].depth = depths[i];
+    }
+    if (writer) {
+      size_t pos = 0;
+      while ((pos + 1) * 8 < st.pos) {
+        writer->Write(8, storage[pos++]);
+      }
+      writer->Write(st.pos - pos * 8, storage[pos]);
+    }
+    // Estimate data cost.
+    size_t cost = st.pos;
+    for (size_t i = 0; i < alphabet_size; i++) {
+      cost += histogram[i] * depths[i];
+    }
+    return cost;
+  }
+  JXL_ASSERT(alphabet_size <= ANS_TAB_SIZE);
+  // Ensure we ignore trailing zeros in the histogram.
+  if (alphabet_size != 0) {
+    size_t largest_symbol = 0;
+    for (size_t i = 0; i < alphabet_size; i++) {
+      if (histogram[i] != 0) largest_symbol = i;
+    }
+    alphabet_size = largest_symbol + 1;
+  }
+  size_t cost;
+  uint32_t method = ComputeBestMethod(histogram, alphabet_size, &cost);
+  int num_symbols;
+  int symbols[kMaxNumSymbolsForSmallCode] = {};
+  std::vector<ANSHistBin> counts(histogram, histogram + alphabet_size);
+  if (!counts.empty()) {
+    size_t sum = 0;
+    for (size_t i = 0; i < counts.size(); i++) {
+      sum += counts[i];
+    }
+    if (sum == 0) {
+      counts[0] = ANS_TAB_SIZE;
+    }
+  }
+  if (method == 0) {
+    counts = CreateFlatHistogram(alphabet_size, ANS_TAB_SIZE);
+    AliasTable::Entry a[ANS_MAX_ALPHA_SIZE];
+    InitAliasTable(counts, ANS_TAB_SIZE, a);
+    ANSBuildInfoTable(counts.data(), a, alphabet_size, info);
+    if (writer != nullptr) {
+      EncodeFlatHistogram(alphabet_size, writer);
+    }
+    return cost;
+  }
+  int omit_pos = 0;
+  uint32_t shift = method - 1;
+  JXL_CHECK(NormalizeCounts(counts.data(), &omit_pos, alphabet_size,
+                            ANS_LOG_TAB_SIZE, shift, &num_symbols, symbols));
+  AliasTable::Entry a[ANS_MAX_ALPHA_SIZE];
+  InitAliasTable(counts, ANS_TAB_SIZE, a);
+  ANSBuildInfoTable(counts.data(), a, alphabet_size, info);
+  if (writer != nullptr) {
+    EncodeCounts(counts.data(), alphabet_size, omit_pos, num_symbols, shift,
+                 symbols, writer);
+  }
+  return cost;
+}
+
+float ANSPopulationCost(const ANSHistBin* data, size_t alphabet_size) {
+  // TODO(veluca): this function really ought to just run the next three
+  // lines, but this seems to make numbers worse.
+  //
+  // size_t c;
+  // uint32_t method = ComputeBestMethod(data, alphabet_size, &c);
+  // return ComputeHistoAndDataCost(data, alphabet_size, method);
+
+  size_t total_count = std::accumulate(data, data + alphabet_size, 0);
+  uint32_t shift = ANS_TAB_SIZE / 2;
+  if (total_count == 0) {
+    return 7;
+  }
+
+  float entropy_bits = total_count * ANS_LOG_TAB_SIZE;
+  int histogram_bits = 0;
+  size_t count = 0;
+  size_t length = 0;
+  if (total_count > ANS_TAB_SIZE) {
+    size_t total = total_count;
+    for (size_t i = 0; i < alphabet_size; ++i) {
+      if (data[i] > 0) {
+        ++count;
+        length = i;
+      }
+    }
+    if (count == 1) {
+      return 7;
+    }
+    ++length;
+    const uint64_t max0 = (total * length) >> ANS_LOG_TAB_SIZE;
+    const uint64_t max1 = (max0 * length) >> ANS_LOG_TAB_SIZE;
+    const uint32_t min_base = (total + max0 + max1) >> ANS_LOG_TAB_SIZE;
+    total += min_base * count;
+    const int64_t kFixBits = 32;
+    const int64_t kFixOne = 1LL << kFixBits;
+    const int64_t kDescaleBits = kFixBits - ANS_LOG_TAB_SIZE;
+    const int64_t kDescaleOne = 1LL << kDescaleBits;
+    const int64_t kDescaleMask = kDescaleOne - 1;
+    const uint32_t mult = kFixOne / total;
+    const uint32_t error = kFixOne % total;
+    uint32_t cumul = error;
+    if (error < kDescaleOne) {
+      cumul += (kDescaleOne - error) >> 1;
+    }
+    if (data[0] > 0) {
+      uint64_t c = (uint64_t)(data[0] + min_base) * mult + cumul;
+      float log2count = FastLog2(c >> kDescaleBits);
+      entropy_bits -= data[0] * log2count;
+      cumul = c & kDescaleMask;
+    }
+    for (size_t i = 1; i < length; ++i) {
+      if (data[i] > 0) {
+        uint64_t c = (uint64_t)(data[i] + min_base) * mult + cumul;
+        float log2count = FastLog2(c >> kDescaleBits);
+        int log2floor = static_cast<int>(log2count);
+        entropy_bits -= data[i] * log2count;
+        histogram_bits += log2floor;
+        histogram_bits += kLogCountBitLengths[log2floor + 1];
+        cumul = c & kDescaleMask;
+      } else {
+        histogram_bits += kLogCountBitLengths[0];
+      }
+    }
+  } else {
+    float log2norm = ANS_LOG_TAB_SIZE - FastLog2(total_count);
+    if (data[0] > 0) {
+      float log2count = FastLog2(data[0]) + log2norm;
+      entropy_bits -= data[0] * log2count;
+      length = 0;
+      ++count;
+    }
+    for (size_t i = 1; i < alphabet_size; ++i) {
+      if (data[i] > 0) {
+        float log2count = FastLog2(data[i]) + log2norm;
+        int log2floor = static_cast<int>(log2count);
+        entropy_bits -= data[i] * log2count;
+        if (log2floor >= static_cast<int>(ANS_LOG_TAB_SIZE)) {
+          log2floor = ANS_LOG_TAB_SIZE - 1;
+        }
+        histogram_bits += GetPopulationCountPrecision(log2floor, shift);
+        histogram_bits += kLogCountBitLengths[log2floor + 1];
+        length = i;
+        ++count;
+      } else {
+        histogram_bits += kLogCountBitLengths[0];
+      }
+    }
+    ++length;
+  }
+
+  if (count == 1) {
+    return 7;
+  }
+
+  if (count == 2) {
+    return static_cast<int>(entropy_bits) + 1 + 12 + ANS_LOG_TAB_SIZE;
+  }
+
+  int max_bits =
+      alphabet_size == 1 ? 0 : 1 + FloorLog2Nonzero(alphabet_size - 1);
+  histogram_bits += max_bits;
+
+  return histogram_bits + static_cast<int>(entropy_bits) + 1;
+}
+
+namespace {
+
+class HistogramBuilder {
+ public:
+  explicit HistogramBuilder(const size_t num_contexts)
+      : histograms_(num_contexts) {}
+
+  void VisitSymbol(int symbol, size_t histo_idx) {
+    JXL_ASSERT(histo_idx < histograms_.size());
+    histograms_[histo_idx].Add(symbol);
+  }
+
+  // NOTE: `layer` is only for clustered_entropy; caller does ReclaimAndCharge.
+  size_t BuildAndStoreEntropyCodes(const HistogramParams params,
+                                   EntropyEncodingData* codes,
+                                   std::vector<uint8_t>* context_map,
+                                   bool use_prefix_code,
+                                   const BitWriter::Allotment& allotment,
+                                   BitWriter* writer, size_t layer,
+                                   AuxOut* aux_out) const {
+    size_t cost = 0;
+    codes->encoding_info.clear();
+    std::vector<Histogram> clustered_histograms(histograms_);
+    context_map->resize(histograms_.size());
+    if (histograms_.size() > 1) {
+      std::vector<uint32_t> histogram_symbols;
+      ClusterHistograms(params, histograms_, histograms_.size(), kClustersLimit,
+                        &clustered_histograms, &histogram_symbols);
+      for (size_t c = 0; c < histograms_.size(); ++c) {
+        (*context_map)[c] = static_cast<uint8_t>(histogram_symbols[c]);
+      }
+      if (writer != nullptr) {
+        EncodeContextMap(*context_map, clustered_histograms.size(), allotment,
+                         writer);
+      }
+    }
+    if (aux_out != nullptr) {
+      for (size_t i = 0; i < clustered_histograms.size(); ++i) {
+        aux_out->layers[layer].clustered_entropy +=
+            clustered_histograms[i].ShannonEntropy();
+      }
+    }
+    cost += 1;
+    if (writer) writer->Write(1, use_prefix_code);
+    codes->use_prefix_code = use_prefix_code;
+    if (use_prefix_code) {
+      SizeWriter size_writer;  // Used if writer == nullptr to estimate costs.
+      for (size_t c = 0; c < clustered_histograms.size(); ++c) {
+        size_t num_symbol = 1;
+        for (size_t i = 0; i < ANS_MAX_ALPHA_SIZE; i++) {
+          if (clustered_histograms[c].data_[i]) num_symbol = i + 1;
+        }
+        if (writer) {
+          StoreVarLenUint8(num_symbol - 1, writer);
+        } else {
+          StoreVarLenUint8(num_symbol - 1, &size_writer);
+        }
+      }
+      cost += size_writer.size;
+    }
+    for (size_t c = 0; c < clustered_histograms.size(); ++c) {
+      size_t num_symbol = 1;
+      for (size_t i = 0; i < ANS_MAX_ALPHA_SIZE; i++) {
+        if (clustered_histograms[c].data_[i]) num_symbol = i + 1;
+      }
+      codes->encoding_info.emplace_back();
+      codes->encoding_info.back().resize(std::max<size_t>(1, num_symbol));
+
+      cost += BuildAndStoreANSEncodingData(
+          clustered_histograms[c].data_, num_symbol, use_prefix_code,
+          codes->encoding_info.back().data(), writer);
+    }
+    return cost;
+  }
+
+ private:
+  std::vector<Histogram> histograms_;
+};
+}  // namespace
+
+size_t BuildAndEncodeHistograms(const HistogramParams& params,
+                                size_t num_contexts,
+                                const std::vector<std::vector<Token>>& tokens,
+                                EntropyEncodingData* codes,
+                                std::vector<uint8_t>* context_map,
+                                BitWriter* writer, size_t layer,
+                                AuxOut* aux_out) {
+  size_t total_bits = 0;
+  size_t total_tokens = 0;
+  // Build histograms.
+  HistogramBuilder builder(num_contexts);
+  for (size_t i = 0; i < tokens.size(); ++i) {
+    for (size_t j = 0; j < tokens[i].size(); ++j) {
+      const Token token = tokens[i][j];
+      total_tokens++;
+      builder.VisitSymbol(token.symbol, token.context);
+    }
+  }
+
+  // TODO(veluca): better heuristics.
+  bool use_prefix_code = total_tokens < 100;
+
+  // Encode histograms.
+  const size_t max_contexts = std::min(num_contexts, kClustersLimit);
+  BitWriter::Allotment allotment(writer, 8192 * (max_contexts + 4));
+  total_bits = builder.BuildAndStoreEntropyCodes(params, codes, context_map,
+                                                 use_prefix_code, allotment,
+                                                 writer, layer, aux_out);
+  allotment.FinishedHistogram(writer);
+  ReclaimAndCharge(writer, &allotment, layer, aux_out);
+
+  if (aux_out != nullptr) {
+    aux_out->layers[layer].num_clustered_histograms +=
+        codes->encoding_info.size();
+  }
+  return total_bits;
+}
+
+size_t WriteTokens(const std::vector<Token>& tokens,
+                   const EntropyEncodingData& codes,
+                   const std::vector<uint8_t>& context_map,
+                   const BitWriter::Allotment& allotment, BitWriter* writer) {
+  size_t num_extra_bits = 0;
+  if (codes.use_prefix_code) {
+    for (size_t i = 0; i < tokens.size(); i++) {
+      const uint8_t histo_idx = context_map[tokens[i].context];
+      writer->Write(codes.encoding_info[histo_idx][tokens[i].symbol].depth,
+                    codes.encoding_info[histo_idx][tokens[i].symbol].bits);
+      writer->Write(tokens[i].nbits, tokens[i].bits);
+      num_extra_bits += tokens[i].nbits;
+    }
+    return num_extra_bits;
+  }
+  std::vector<std::pair<uint32_t, uint32_t>> out;
+  out.reserve(tokens.size());
+  const int end = tokens.size();
+  ANSCoder ans;
+  for (int i = end - 1; i >= 0; --i) {
+    const Token token = tokens[i];
+    const uint8_t histo_idx = context_map[token.context];
+    const ANSEncSymbolInfo& info = codes.encoding_info[histo_idx][token.symbol];
+    uint8_t nbits = 0;
+    uint32_t bits = ans.PutSymbol(info, &nbits);
+    if (nbits == 16) {
+      out.push_back(std::make_pair(i, bits));
+    }
+  }
+  const uint32_t state = ans.GetState();
+  writer->Write(32, state);
+  int tokenidx = 0;
+  for (int i = out.size(); i >= 0; --i) {
+    int nextidx = i > 0 ? out[i - 1].first : end;
+    for (; tokenidx < nextidx; ++tokenidx) {
+      const Token token = tokens[tokenidx];
+      writer->Write(token.nbits, token.bits);
+      num_extra_bits += token.nbits;
+    }
+    if (i > 0) {
+      writer->Write(16, out[i - 1].second);
+    }
+  }
+  return num_extra_bits;
+}
+
+void WriteTokens(const std::vector<Token>& tokens,
+                 const EntropyEncodingData& codes,
+                 const std::vector<uint8_t>& context_map, BitWriter* writer,
+                 size_t layer, AuxOut* aux_out) {
+  BitWriter::Allotment allotment(writer, 32 * tokens.size() + 32 * 1024 * 4);
+  size_t num_extra_bits =
+      WriteTokens(tokens, codes, context_map, allotment, writer);
+  ReclaimAndCharge(writer, &allotment, layer, aux_out);
+  if (aux_out != nullptr) {
+    aux_out->layers[layer].extra_bits += num_extra_bits;
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_ans.h b/jxl/enc_ans.h
new file mode 100644
index 000000000..3a20ddcf2
--- /dev/null
+++ b/jxl/enc_ans.h
@@ -0,0 +1,144 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_ANS_H_
+#define JXL_ENC_ANS_H_
+
+// Library to encode the ANS population counts to the bit-stream and encode
+// symbols based on the respective distributions.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <vector>
+
+#include "c/dec/huffman_table.h"
+#include "jxl/ans_common.h"
+#include "jxl/ans_params.h"
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/enc_bit_writer.h"
+
+namespace jxl {
+
+#define USE_MULT_BY_RECIPROCAL
+
+// precision must be equal to:  #bits(state_) + #bits(freq)
+#define RECIPROCAL_PRECISION (32 + ANS_LOG_TAB_SIZE)
+
+// Data structure representing one element of the encoding table built
+// from a distribution.
+// TODO(veluca): split this up, or use an union.
+struct ANSEncSymbolInfo {
+  // ANS
+  uint16_t freq_;
+  std::vector<uint16_t> reverse_map_;
+#ifdef USE_MULT_BY_RECIPROCAL
+  uint64_t ifreq_;
+#endif
+  // Prefix coding.
+  uint8_t depth;
+  uint16_t bits;
+};
+
+class ANSCoder {
+ public:
+  ANSCoder() : state_(ANS_SIGNATURE << 16) {}
+
+  uint32_t PutSymbol(const ANSEncSymbolInfo& t, uint8_t* nbits) {
+    uint32_t bits = 0;
+    *nbits = 0;
+    if ((state_ >> (32 - ANS_LOG_TAB_SIZE)) >= t.freq_) {
+      bits = state_ & 0xffff;
+      state_ >>= 16;
+      *nbits = 16;
+    }
+#ifdef USE_MULT_BY_RECIPROCAL
+    // We use mult-by-reciprocal trick, but that requires 64b calc.
+    const uint32_t v = (state_ * t.ifreq_) >> RECIPROCAL_PRECISION;
+    const uint32_t offset = t.reverse_map_[state_ - v * t.freq_];
+    state_ = (v << ANS_LOG_TAB_SIZE) + offset;
+#else
+    state_ = ((state_ / t.freq_) << ANS_LOG_TAB_SIZE) +
+             t.reverse_map_[state_ % t.freq_];
+#endif
+    return bits;
+  }
+
+  uint32_t GetState() const { return state_; }
+
+ private:
+  uint32_t state_;
+};
+
+// RebalanceHistogram requires a signed type.
+using ANSHistBin = int32_t;
+
+struct EntropyEncodingData {
+  std::vector<std::vector<ANSEncSymbolInfo>> encoding_info;
+  bool use_prefix_code;
+};
+
+// Token to be encoded by the ANS. Uses context c (16 bits), writing symbol s
+// (8 bits), and adds up to 32 (nb) extra bits (b) that are interleaved in the
+// ANS stream.
+struct Token {
+  Token(uint32_t c, uint32_t s, uint32_t nb, uint32_t b)
+      : bits(b), context(c), nbits(nb), symbol(s) {
+    JXL_DASSERT(c < (1UL << 16));
+    static_assert(sizeof(Token) == 8, "Token must be a 8 byte struct!");
+  }
+  uint32_t bits;
+  uint16_t context;
+  uint8_t nbits;
+  uint8_t symbol;
+};
+
+// Returns an estimate of the number of bits required to encode the given
+// histogram (header bits plus data bits).
+float ANSPopulationCost(const ANSHistBin* data, size_t alphabet_size);
+
+// Apply context clustering, compute histograms and encode them. Returns an
+// estimate of the total bits used for encoding the stream. If `writer` ==
+// nullptr, the bit estimate will not take into account the context map (which
+// does not get written if `num_contexts` == 1).
+size_t BuildAndEncodeHistograms(const HistogramParams& params,
+                                size_t num_contexts,
+                                const std::vector<std::vector<Token>>& tokens,
+                                EntropyEncodingData* codes,
+                                std::vector<uint8_t>* context_map,
+                                BitWriter* writer, size_t layer,
+                                AuxOut* aux_out);
+
+// Write the tokens to a string.
+void WriteTokens(const std::vector<Token>& tokens,
+                 const EntropyEncodingData& codes,
+                 const std::vector<uint8_t>& context_map, BitWriter* writer,
+                 size_t layer, AuxOut* aux_out);
+
+// Same as above, but assumes allotment created by caller.
+size_t WriteTokens(const std::vector<Token>& tokens,
+                   const EntropyEncodingData& codes,
+                   const std::vector<uint8_t>& context_map,
+                   const BitWriter::Allotment& allotment, BitWriter* writer);
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_ANS_H_
diff --git a/jxl/enc_bit_writer.cc b/jxl/enc_bit_writer.cc
new file mode 100644
index 000000000..a14c5aae8
--- /dev/null
+++ b/jxl/enc_bit_writer.cc
@@ -0,0 +1,223 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_bit_writer.h"
+
+#include <string.h>  // memcpy
+
+#include "jxl/base/byte_order.h"
+#include "jxl/dec_bit_reader.h"
+
+namespace jxl {
+
+BitWriter::Allotment::Allotment(BitWriter* JXL_RESTRICT writer, size_t max_bits)
+    : max_bits_(max_bits) {
+  if (writer == nullptr) return;
+  prev_bits_written_ = writer->BitsWritten();
+  const size_t prev_bytes = writer->storage_.size();
+  const size_t next_bytes = DivCeil(max_bits, kBitsPerByte);
+  writer->storage_.resize(prev_bytes + next_bytes);
+  parent_ = writer->current_allotment_;
+  writer->current_allotment_ = this;
+}
+
+BitWriter::Allotment::~Allotment() {
+  if (!called_) {
+    // Not calling is a bug - unused storage will not be reclaimed.
+    JXL_ABORT("Did not call Allotment::ReclaimUnused");
+  }
+}
+
+void BitWriter::Allotment::FinishedHistogram(BitWriter* JXL_RESTRICT writer) {
+  if (writer == nullptr) return;
+  JXL_ASSERT(!called_);              // Call before ReclaimUnused
+  JXL_ASSERT(histogram_bits_ == 0);  // Do not call twice
+  JXL_ASSERT(writer->BitsWritten() >= prev_bits_written_);
+  histogram_bits_ = writer->BitsWritten() - prev_bits_written_;
+}
+
+void BitWriter::Allotment::PrivateReclaim(BitWriter* JXL_RESTRICT writer,
+                                          size_t* JXL_RESTRICT used_bits,
+                                          size_t* JXL_RESTRICT unused_bits) {
+  JXL_ASSERT(!called_);  // Do not call twice
+  called_ = true;
+  if (writer == nullptr) return;
+
+  JXL_ASSERT(writer->BitsWritten() >= prev_bits_written_);
+  *used_bits = writer->BitsWritten() - prev_bits_written_;
+  JXL_ASSERT(*used_bits <= max_bits_);
+  *unused_bits = max_bits_ - *used_bits;
+
+  // Reclaim unused bytes whole bytes from writer's allotment.
+  const size_t unused_bytes = *unused_bits / kBitsPerByte;  // truncate
+  JXL_ASSERT(writer->storage_.size() >= unused_bytes);
+  writer->storage_.resize(writer->storage_.size() - unused_bytes);
+  writer->current_allotment_ = parent_;
+  // Ensure we don't also charge the parent for these bits.
+  if (parent_ != nullptr) {
+    parent_->prev_bits_written_ += *used_bits;
+  }
+}
+
+void BitWriter::AppendByteAligned(const BitWriter& other) {
+  JXL_ASSERT(other.BitsWritten() % kBitsPerByte == 0);
+  const size_t other_bytes = other.BitsWritten() / kBitsPerByte;
+  JXL_ASSERT(other_bytes != 0);
+  storage_.resize(storage_.size() + other_bytes + 1);  // extra zero padding
+
+  // Concatenate by copying bytes because both source and destination are bytes.
+  JXL_ASSERT(BitsWritten() % kBitsPerByte == 0);
+  size_t pos = BitsWritten() / kBitsPerByte;
+  const Span<const uint8_t> span = other.GetSpan();
+  memcpy(storage_.data() + pos, span.data(), span.size());
+  pos += span.size();
+  storage_[pos++] = 0;  // for next Write
+  JXL_ASSERT(pos <= storage_.size());
+  bits_written_ += other_bytes * kBitsPerByte;
+}
+
+void BitWriter::AppendByteAligned(const std::vector<BitWriter>& others) {
+  // Total size to add so we can preallocate
+  size_t other_bytes = 0;
+  for (const BitWriter& writer : others) {
+    JXL_ASSERT(writer.BitsWritten() % kBitsPerByte == 0);
+    other_bytes += writer.BitsWritten() / kBitsPerByte;
+  }
+  if (other_bytes == 0) {
+    // No bytes to append: this happens for example when creating per-group
+    // storage for groups, but not writing anything in them for e.g. lossless
+    // images with no alpha. Do nothing.
+    return;
+  }
+  storage_.resize(storage_.size() + other_bytes + 1);  // extra zero padding
+
+  // Concatenate by copying bytes because both source and destination are bytes.
+  JXL_ASSERT(BitsWritten() % kBitsPerByte == 0);
+  size_t pos = BitsWritten() / kBitsPerByte;
+  for (const BitWriter& writer : others) {
+    const Span<const uint8_t> span = writer.GetSpan();
+    memcpy(storage_.data() + pos, span.data(), span.size());
+    pos += span.size();
+  }
+  storage_[pos++] = 0;  // for next Write
+  JXL_ASSERT(pos <= storage_.size());
+  bits_written_ += other_bytes * kBitsPerByte;
+}
+
+HWY_ATTR BitWriter& BitWriter::operator+=(const BitWriter& other) {
+  // Required for correctness, otherwise owned[bits_written_] is out of bounds.
+  if (other.bits_written_ == 0) return *this;
+  const size_t other_bytes = DivCeil(other.bits_written_, kBitsPerByte);
+  const size_t prev_bytes = storage_.size();
+  storage_.resize(prev_bytes + other_bytes + 1);  // extra zero padding
+
+  if (bits_written_ % kBitsPerByte == 0) {
+    // Only copy fully-initialized bytes.
+    const size_t full_bytes = other.bits_written_ / kBitsPerByte;  // truncated
+    memcpy(&storage_[bits_written_ / kBitsPerByte], other.storage_.data(),
+           full_bytes);
+    storage_[bits_written_ / kBitsPerByte + full_bytes] = 0;  // for next Write
+    bits_written_ += full_bytes * kBitsPerByte;
+
+    const size_t leftovers = other.bits_written_ % kBitsPerByte;
+    if (leftovers != 0) {
+      BitReader reader(Span<const uint8_t>(other.storage_.data() + full_bytes,
+                                           other_bytes - full_bytes));
+      Write(leftovers, reader.ReadBits(leftovers));
+      JXL_CHECK(reader.Close());
+    }
+    return *this;
+  }
+
+  constexpr size_t N = kMaxBitsPerCall < BitReader::kMaxBitsPerCall
+                           ? kMaxBitsPerCall
+                           : BitReader::kMaxBitsPerCall;
+
+  // Do not use GetSpan because other may not be byte-aligned.
+  BitReader reader(other.storage_);
+  size_t i = 0;
+  for (; i + N <= other.bits_written_; i += N) {
+    Write(N, reader.ReadFixedBits<N>());
+  }
+  const size_t leftovers = other.bits_written_ - i;
+  if (leftovers != 0) {
+    Write(leftovers, reader.ReadBits(leftovers));
+  }
+  JXL_CHECK(reader.Close());
+  return *this;
+}
+
+// Example: let's assume that 3 bits (Rs below) have been written already:
+// BYTE+0       BYTE+1       BYTE+2
+// 0000 0RRR    ???? ????    ???? ????
+//
+// Now, we could write up to 5 bits by just shifting them left by 3 bits and
+// OR'ing to BYTE-0.
+//
+// For n > 5 bits, we write the lowest 5 bits as above, then write the next
+// lowest bits into BYTE+1 starting from its lower bits and so on.
+void BitWriter::Write(size_t n_bits, uint64_t bits) {
+  JXL_DASSERT((bits >> n_bits) == 0);
+  JXL_DASSERT(n_bits <= kMaxBitsPerCall);
+  uint8_t* p = &storage_[bits_written_ / kBitsPerByte];
+  const size_t bits_in_first_byte = bits_written_ % kBitsPerByte;
+  bits <<= bits_in_first_byte;
+#if JXL_BYTE_ORDER_LITTLE
+  uint64_t v = *p;
+  // Last (partial) or next byte to write must be zero-initialized!
+  // PaddedBytes initializes the first, and Write/Append maintain this.
+  JXL_DASSERT(v >> bits_in_first_byte == 0);
+  v |= bits;
+  memcpy(p, &v, sizeof(v));  // Write bytes: possibly more than n_bits/8
+#else
+  *p++ |= static_cast<uint8_t>(bits & 0xFF);
+  for (size_t bits_left_to_write = n_bits + bits_in_first_byte;
+       bits_left_to_write >= 9; bits_left_to_write -= 8) {
+    bits >>= 8;
+    *p++ = static_cast<uint8_t>(bits & 0xFF);
+  }
+  *p = 0;
+#endif
+  bits_written_ += n_bits;
+}
+
+HWY_ATTR BitWriter& BitWriter::operator+=(const PaddedBytes& other) {
+  const size_t other_bytes = other.size();
+  // Required for correctness, otherwise owned[bits_written_] is out of bounds.
+  if (other_bytes == 0) return *this;
+  const size_t other_bits = other_bytes * kBitsPerByte;
+
+  storage_.resize(storage_.size() + other_bytes + 1);
+  if (bits_written_ % kBitsPerByte == 0) {
+    memcpy(&storage_[bits_written_ / kBitsPerByte], other.data(), other_bytes);
+    storage_[bits_written_ / kBitsPerByte + other_bytes] = 0;  // for next Write
+    bits_written_ += other_bits;
+    return *this;
+  }
+  constexpr size_t N = kMaxBitsPerCall < BitReader::kMaxBitsPerCall
+                           ? kMaxBitsPerCall
+                           : BitReader::kMaxBitsPerCall;
+
+  BitReader reader(other);
+  size_t i = 0;
+  for (; i + N <= other_bits; i += N) {
+    Write(N, reader.ReadFixedBits<N>());
+  }
+  const size_t leftovers = other_bits - i;
+  Write(leftovers, reader.ReadBits(leftovers));
+  JXL_CHECK(reader.Close());
+  return *this;
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_bit_writer.h b/jxl/enc_bit_writer.h
new file mode 100644
index 000000000..7887c5425
--- /dev/null
+++ b/jxl/enc_bit_writer.h
@@ -0,0 +1,151 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_BIT_WRITER_H_
+#define JXL_ENC_BIT_WRITER_H_
+
+// BitWriter class: unbuffered writes using unaligned 64-bit stores.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <utility>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+
+namespace jxl {
+
+struct BitWriter {
+  // Upper bound on `n_bits` in each call to Write. We shift a 64-bit word by
+  // 7 bits (max already valid bits in the last byte) and at least 1 bit is
+  // needed to zero-initialize the bit-stream ahead (i.e. if 7 bits are valid
+  // and we write 57 bits, then the next write will access a byte that was not
+  // yet zero-initialized).
+  static constexpr size_t kMaxBitsPerCall = 56;
+
+  BitWriter() : bits_written_(0) {}
+
+  // Disallow copying - may lead to bugs.
+  BitWriter(const BitWriter&) = delete;
+  BitWriter& operator=(const BitWriter&) = delete;
+  BitWriter(BitWriter&&) = default;
+  BitWriter& operator=(BitWriter&&) = default;
+
+  explicit BitWriter(PaddedBytes&& donor)
+      : bits_written_(donor.size() * kBitsPerByte),
+        storage_(std::move(donor)) {}
+
+  size_t BitsWritten() const { return bits_written_; }
+
+  Span<const uint8_t> GetSpan() const {
+    // Callers must ensure byte alignment to avoid uninitialized bits.
+    JXL_ASSERT(bits_written_ % kBitsPerByte == 0);
+    return Span<const uint8_t>(storage_.data(), bits_written_ / kBitsPerByte);
+  }
+
+  // Example usage: bytes = std::move(writer).TakeBytes(); Useful for the
+  // top-level encoder which returns PaddedBytes, not a BitWriter.
+  // *this must be an rvalue reference and is invalid afterwards.
+  PaddedBytes&& TakeBytes() && {
+    // Callers must ensure byte alignment to avoid uninitialized bits.
+    JXL_ASSERT(bits_written_ % kBitsPerByte == 0);
+    storage_.resize(bits_written_ / kBitsPerByte);
+    return std::move(storage_);
+  }
+
+  // Must be byte-aligned before calling.
+  // NOTE: no allotment needed, the other BitWriters have already been charged.
+  void AppendByteAligned(const BitWriter& other);
+  void AppendByteAligned(const std::vector<BitWriter>& others);
+
+  class Allotment {
+   public:
+    // Expands a BitWriter's storage. Must happen before calling Write or
+    // ZeroPadToByte. Must call ReclaimUnused after writing to reclaim the
+    // unused storage so that BitWriter memory use remains tightly bounded.
+    Allotment(BitWriter* JXL_RESTRICT writer, size_t max_bits);
+    ~Allotment();
+
+    size_t MaxBits() const { return max_bits_; }
+
+    // Call after writing a histogram, but before ReclaimUnused.
+    void FinishedHistogram(BitWriter* JXL_RESTRICT writer);
+
+    size_t HistogramBits() const {
+      JXL_ASSERT(called_);
+      return histogram_bits_;
+    }
+
+    // Do not call directly - use ::ReclaimAndCharge instead, which ensures
+    // the bits are charged to a layer.
+    void PrivateReclaim(BitWriter* JXL_RESTRICT writer,
+                        size_t* JXL_RESTRICT used_bits,
+                        size_t* JXL_RESTRICT unused_bits);
+
+   private:
+    size_t prev_bits_written_;
+    const size_t max_bits_;
+    size_t histogram_bits_ = 0;
+    bool called_ = false;
+    Allotment* parent_;
+  };
+
+  // WARNING: think twice before using this. Concatenating two BitWriters that
+  // pad to bytes is NOT the same as one contiguous BitWriter.
+  BitWriter& operator+=(const BitWriter& other);
+
+  // TODO(janwas): remove once all callers use BitWriter
+  BitWriter& operator+=(const PaddedBytes& other);
+
+  // Writes bits into bytes in increasing addresses, and within a byte
+  // least-significant-bit first.
+  //
+  // The function can write up to 56 bits in one go.
+  void Write(size_t n_bits, uint64_t bits);
+
+  // This should only rarely be used - e.g. when the current location will be
+  // referenced via byte offset (TOCs point to groups), or byte-aligned reading
+  // is required for speed. WARNING: this interacts badly with operator+=,
+  // see above.
+  void ZeroPadToByte() {
+    const size_t remainder_bits =
+        RoundUpBitsToByteMultiple(bits_written_) - bits_written_;
+    if (remainder_bits == 0) return;
+    Write(remainder_bits, 0);
+    JXL_ASSERT(bits_written_ % kBitsPerByte == 0);
+  }
+
+  // TODO(janwas): remove? only called from ANS
+  void RewindStorage(const size_t pos0) {
+    JXL_ASSERT(pos0 <= bits_written_);
+    bits_written_ = pos0;
+    static const uint8_t kRewindMasks[8] = {0x0, 0x1,  0x3,  0x7,
+                                            0xf, 0x1f, 0x3f, 0x7f};
+    storage_[pos0 >> 3] &= kRewindMasks[pos0 & 7];
+  }
+
+ private:
+  size_t bits_written_;
+  PaddedBytes storage_;
+  Allotment* current_allotment_ = nullptr;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_BIT_WRITER_H_
diff --git a/jxl/enc_butteraugli_comparator.cc b/jxl/enc_butteraugli_comparator.cc
new file mode 100644
index 000000000..d8b1d19ee
--- /dev/null
+++ b/jxl/enc_butteraugli_comparator.cc
@@ -0,0 +1,101 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_butteraugli_comparator.h"
+
+#include <algorithm>
+#include <vector>
+
+#include "jxl/color_management.h"
+
+namespace jxl {
+
+ButteraugliComparator::ButteraugliComparator(float hf_asymmetry)
+    : hf_asymmetry_(hf_asymmetry) {}
+
+Status ButteraugliComparator::SetReferenceImage(const ImageBundle& ref) {
+  const ImageBundle* ref_linear_srgb;
+  ImageMetadata metadata = *ref.metadata();
+  ImageBundle store(&metadata);
+  if (!TransformIfNeeded(ref, ColorManagement::LinearSRGB(ref.IsGray()),
+                         /*pool=*/nullptr, &store, &ref_linear_srgb)) {
+    return false;
+  }
+
+  comparator_.reset(new butteraugli::ButteraugliComparator(
+      ref_linear_srgb->color(), hf_asymmetry_));
+  xsize_ = ref.xsize();
+  ysize_ = ref.ysize();
+  return true;
+}
+
+Status ButteraugliComparator::CompareWith(const ImageBundle& actual,
+                                          ImageF* diffmap, float* score) {
+  if (!comparator_) {
+    return JXL_FAILURE("Must set reference image first");
+  }
+  if (xsize_ != actual.xsize() || ysize_ != actual.ysize()) {
+    return JXL_FAILURE("Images must have same size");
+  }
+
+  const ImageBundle* actual_linear_srgb;
+  ImageMetadata metadata = *actual.metadata();
+  ImageBundle store(&metadata);
+  if (!TransformIfNeeded(actual, ColorManagement::LinearSRGB(actual.IsGray()),
+                         /*pool=*/nullptr, &store, &actual_linear_srgb)) {
+    return false;
+  }
+
+  ImageF temp_diffmap(xsize_, ysize_);
+  comparator_->Diffmap(actual_linear_srgb->color(), temp_diffmap);
+
+  if (score != nullptr) {
+    *score = butteraugli::ButteraugliScoreFromDiffmap(temp_diffmap);
+  }
+  if (diffmap != nullptr) {
+    diffmap->Swap(temp_diffmap);
+  }
+
+  return true;
+}
+
+float ButteraugliComparator::GoodQualityScore() const {
+  return butteraugli::ButteraugliFuzzyInverse(1.5);
+}
+
+float ButteraugliComparator::BadQualityScore() const {
+  return butteraugli::ButteraugliFuzzyInverse(0.5);
+}
+
+float ButteraugliDistance(const ImageBundle& rgb0, const ImageBundle& rgb1,
+                          float hf_asymmetry, ImageF* distmap,
+                          ThreadPool* pool) {
+  ButteraugliComparator comparator(hf_asymmetry);
+  return ComputeScore(rgb0, rgb1, &comparator, distmap, pool);
+}
+
+float ButteraugliDistance(const CodecInOut& rgb0, const CodecInOut& rgb1,
+                          float hf_asymmetry, ImageF* distmap,
+                          ThreadPool* pool) {
+  ButteraugliComparator comparator(hf_asymmetry);
+  JXL_ASSERT(rgb0.frames.size() == rgb1.frames.size());
+  float max_dist = 0.0f;
+  for (size_t i = 0; i < rgb0.frames.size(); ++i) {
+    max_dist = std::max(max_dist, ComputeScore(rgb0.frames[i], rgb1.frames[i],
+                                               &comparator, distmap, pool));
+  }
+  return max_dist;
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_butteraugli_comparator.h b/jxl/enc_butteraugli_comparator.h
new file mode 100644
index 000000000..142f76ab3
--- /dev/null
+++ b/jxl/enc_butteraugli_comparator.h
@@ -0,0 +1,63 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_BUTTERAUGLI_COMPARATOR_H_
+#define JXL_ENC_BUTTERAUGLI_COMPARATOR_H_
+
+#include <stddef.h>
+
+#include <memory>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/butteraugli/butteraugli.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/enc_comparator.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+
+class ButteraugliComparator : public Comparator {
+ public:
+  explicit ButteraugliComparator(float hf_asymmetry);
+
+  Status SetReferenceImage(const ImageBundle& ref) override;
+
+  Status CompareWith(const ImageBundle& actual, ImageF* diffmap,
+                     float* score) override;
+
+  float GoodQualityScore() const override;
+  float BadQualityScore() const override;
+
+ private:
+  float hf_asymmetry_;
+  std::unique_ptr<butteraugli::ButteraugliComparator> comparator_;
+  size_t xsize_ = 0;
+  size_t ysize_ = 0;
+};
+
+// Returns the butteraugli distance between rgb0 and rgb1.
+// If distmap is not null, it must be the same size as rgb0 and rgb1.
+float ButteraugliDistance(const ImageBundle& rgb0, const ImageBundle& rgb1,
+                          float hf_asymmetry, ImageF* distmap = nullptr,
+                          ThreadPool* pool = nullptr);
+
+float ButteraugliDistance(const CodecInOut& rgb0, const CodecInOut& rgb1,
+                          float hf_asymmetry, ImageF* distmap = nullptr,
+                          ThreadPool* pool = nullptr);
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_BUTTERAUGLI_COMPARATOR_H_
diff --git a/jxl/enc_cache.cc b/jxl/enc_cache.cc
new file mode 100644
index 000000000..384abe5cb
--- /dev/null
+++ b/jxl/enc_cache.cc
@@ -0,0 +1,201 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_cache.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <type_traits>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/span.h"
+#include "jxl/color_encoding.h"
+#include "jxl/common.h"
+#include "jxl/compressed_dc.h"
+#include "jxl/dct_util.h"
+#include "jxl/dec_frame.h"
+#include "jxl/enc_frame.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/passes_state.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+void InitializePassesEncoder(const Image3F& opsin, ThreadPool* pool,
+                             PassesEncoderState* enc_state, AuxOut* aux_out) {
+  PROFILER_FUNC;
+  constexpr size_t N = kBlockDim;
+
+  PassesSharedState& JXL_RESTRICT shared = enc_state->shared;
+
+  const size_t xsize_blocks = shared.frame_dim.xsize_blocks;
+  const size_t ysize_blocks = shared.frame_dim.ysize_blocks;
+
+  if (shared.frame_header.color_transform == ColorTransform::kXYB) {
+    enc_state->x_qm_multiplier =
+        std::pow(2.0f, 0.5f * shared.frame_header.x_qm_scale - 0.5f);
+  } else {
+    enc_state->x_qm_multiplier = 1.0f;  // don't scale X quantization in YCbCr
+  }
+
+  if (enc_state->coeffs.size() != shared.frame_header.passes.num_passes) {
+    enc_state->coeffs.resize(shared.frame_header.passes.num_passes);
+    for (size_t i = 0; i < shared.frame_header.passes.num_passes; i++) {
+      static_assert(std::is_same<float, ac_qcoeff_t>::value,
+                    "float != ac_coeff_t");
+      // Allocate enough coefficients for each group on every row.
+      enc_state->coeffs[i] =
+          ACImage3(kGroupDim * kGroupDim, shared.frame_dim.num_groups);
+    }
+  }
+  Image3F dc = Image3F(xsize_blocks, ysize_blocks);
+
+  size_t dc_stride = dc.PixelsPerRow();
+  size_t opsin_stride = opsin.PixelsPerRow();
+
+  auto compute_coeffs = [&](int group_index, int /* thread */) {
+    const size_t gx = group_index % shared.frame_dim.xsize_groups;
+    const size_t gy = group_index / shared.frame_dim.xsize_groups;
+    size_t offset = 0;
+    float* JXL_RESTRICT rows[3] = {
+        enc_state->coeffs[0].PlaneRow(0, group_index),
+        enc_state->coeffs[0].PlaneRow(1, group_index),
+        enc_state->coeffs[0].PlaneRow(2, group_index),
+    };
+    for (size_t by = gy * kGroupDimInBlocks;
+         by < ysize_blocks && by < (gy + 1) * kGroupDimInBlocks; ++by) {
+      const float* JXL_RESTRICT opsin_rows[3] = {
+          opsin.ConstPlaneRow(0, by * N),
+          opsin.ConstPlaneRow(1, by * N),
+          opsin.ConstPlaneRow(2, by * N),
+      };
+      float* JXL_RESTRICT dc_rows[3] = {
+          dc.PlaneRow(0, by),
+          dc.PlaneRow(1, by),
+          dc.PlaneRow(2, by),
+      };
+      for (size_t bx = gx * kGroupDimInBlocks;
+           bx < xsize_blocks && bx < (gx + 1) * kGroupDimInBlocks; ++bx) {
+        AcStrategy acs = shared.ac_strategy.ConstRow(by)[bx];
+        if (!acs.IsFirstBlock()) continue;
+        for (size_t c = 0; c < 3; ++c) {
+          acs.TransformFromPixels(opsin_rows[c] + bx * N, opsin_stride,
+                                  rows[c] + offset);
+          acs.DCFromLowestFrequencies(rows[c] + offset, dc_rows[c] + bx,
+                                      dc_stride);
+        }
+        offset += kDCTBlockSize << acs.log2_covered_blocks();
+      }
+    }
+  };
+
+  RunOnPool(pool, 0, shared.frame_dim.num_groups, ThreadPool::SkipInit(),
+            compute_coeffs, "Compute coeffs");
+  if (aux_out != nullptr) {
+    aux_out->InspectImage3F("compressed_image:InitializeFrameEncCache:dc", dc);
+  }
+
+  if (shared.frame_header.flags & FrameHeader::kUseDcFrame) {
+    CompressParams cparams = enc_state->cparams;
+    // Guess a distance that produces good initial results.
+    cparams.butteraugli_distance =
+        std::max(kMinButteraugliDistance,
+                 enc_state->cparams.butteraugli_distance * 0.1f);
+    cparams.dots = Override::kOff;
+    cparams.patches = Override::kOff;
+    cparams.gaborish = Override::kOff;
+    cparams.adaptive_reconstruction = Override::kOff;
+    cparams.max_error_mode = true;
+    for (size_t c = 0; c < 3; c++) {
+      cparams.max_error[c] = shared.quantizer.MulDC()[c];
+    }
+    FrameDimensions frame_dim;
+    frame_dim.Set(enc_state->shared.frame_dim.xsize << (3 * cparams.dc_level),
+                  enc_state->shared.frame_dim.ysize << (3 * cparams.dc_level));
+    cparams.dc_level++;
+    cparams.progressive_dc--;
+    // Use kPasses in max_error_mode for intermediate progressive DC,
+    // and kModularGroup for the smallest DC (first in the bitstream)
+    if (cparams.progressive_dc == 0) {
+      cparams.modular_group_mode = true;
+      cparams.quality_pair.first = cparams.quality_pair.second =
+          99.f - enc_state->cparams.butteraugli_distance * 2.f;
+    }
+    ImageMetadata metadata;
+    metadata.color_encoding = ColorManagement::LinearSRGB();
+    ImageBundle ib(&metadata);
+    // This is a lie - dc is in XYB
+    // (but EncodeFrame will skip RGB->XYB conversion anyway)
+    ib.SetFromImage(std::move(dc), ColorManagement::LinearSRGB());
+    PassesEncoderState state;
+    enc_state->special_frames.emplace_back();
+    JXL_CHECK(EncodeFrame(cparams, nullptr, ib, &state, pool,
+                          &enc_state->special_frames.back(), nullptr,
+                          enc_state->shared.multiframe));
+    const Span<const uint8_t> encoded =
+        enc_state->special_frames.back().GetSpan();
+    BitReader br(encoded);
+    ImageBundle decoded(&metadata);
+    JXL_CHECK(DecodeFrame({}, encoded, nullptr, &frame_dim, shared.multiframe,
+                          pool, &br, nullptr, &decoded));
+    shared.dc_storage =
+        CopyImage(*shared.multiframe->SavedDc(cparams.dc_level));
+    JXL_CHECK(br.Close());
+  } else {
+    const size_t xsize_dc_groups = DivCeil(xsize_blocks, kDcGroupDimInBlocks);
+    const size_t ysize_dc_groups = DivCeil(ysize_blocks, kDcGroupDimInBlocks);
+
+    enc_state->dc_tokens =
+        std::vector<std::vector<Token>>(xsize_dc_groups * ysize_dc_groups);
+    // Disable extra levels when compressing very high quality images: this
+    // allows us to use a finer quantization step.
+    enc_state->extra_dc_levels.resize(
+        xsize_dc_groups * ysize_dc_groups,
+        (enc_state->cparams.butteraugli_distance < 0.5 ? 0 : 1));
+
+    auto compute_dc_coeffs = [&](int group_index, int /* thread */) {
+      TokenizeDC(group_index, dc, enc_state, aux_out);
+    };
+    RunOnPool(pool, 0, shared.frame_dim.num_dc_groups, ThreadPool::SkipInit(),
+              compute_dc_coeffs, "Compute DC coeffs");
+    // TODO(veluca): this is only useful in tests and if inspection is enabled.
+    if ((shared.frame_header.flags & FrameHeader::kSkipAdaptiveDCSmoothing) ==
+        0) {
+      AdaptiveDCSmoothing(shared.dc_quant_field, &shared.dc_storage, pool);
+    }
+  }
+
+  if (aux_out != nullptr) {
+    aux_out->InspectImage3F("compressed_image:InitializeFrameEncCache:dc_dec",
+                            shared.dc_storage);
+  }
+}
+
+void EncCache::InitOnce() {
+  PROFILER_FUNC;
+
+  if (num_nzeroes.xsize() == 0) {
+    num_nzeroes = Image3I(kGroupDimInBlocks, kGroupDimInBlocks);
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_cache.h b/jxl/enc_cache.h
new file mode 100644
index 000000000..d851d8491
--- /dev/null
+++ b/jxl/enc_cache.h
@@ -0,0 +1,105 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_CACHE_H_
+#define JXL_ENC_CACHE_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/coeff_order.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/common.h"
+#include "jxl/dct_util.h"
+#include "jxl/dot_dictionary.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_params.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/passes_state.h"
+#include "jxl/patch_dictionary.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+// Contains encoder state.
+struct PassesEncoderState {
+  PassesSharedState shared;
+
+  ImageF initial_quant_field;  // Invalid in Falcon mode.
+
+  // Per-pass DCT coefficients for the image. One row per group.
+  // Used for both quantized and non-quantized coefficients (only coeffs[0]).
+  // WARNING: assumes ac_qcoeff_t == float!
+  std::vector<ACImage3> coeffs;
+
+  // Vector of tokens for each DC group.
+  std::vector<std::vector<Token>> dc_tokens;
+
+  // Raw data for special (reference+DC) frames.
+  std::vector<BitWriter> special_frames;
+
+  // Quantized DC.
+  Image3S dc;
+
+  // Number of extra DC levels, per group.
+  std::vector<int> extra_dc_levels;
+
+  // Storage for reference frames. More than one to allow for photographic and
+  // non-photographic patches, as well as mixing previous frames and special
+  // frames as sources.
+  Image3F reference_frames[kMaxNumReferenceFrames];
+
+  // Per-dc16-group-tokens.
+  std::vector<std::vector<Token>> downsampled_dc_tokens;
+
+  CompressParams cparams;
+
+  struct PassData {
+    std::vector<std::vector<Token>> ac_tokens;
+    std::vector<uint8_t> context_map;
+    EntropyEncodingData codes;
+  };
+
+  std::vector<PassData> passes;
+
+  // Multiplier to be applied to the quant matrices of the x channel.
+  float x_qm_multiplier = 1.0f;
+};
+
+// Initialize per-frame information.
+void InitializePassesEncoder(const Image3F& opsin, ThreadPool* pool,
+                             PassesEncoderState* passes_enc_state,
+                             AuxOut* aux_out);
+
+// Working area for ComputeCoefficients (per-group!)
+struct EncCache {
+  // Allocates memory when first called, shrinks images to current group size.
+  void InitOnce();
+
+  // TokenizeCoefficients
+  Image3I num_nzeroes;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_CACHE_H_
diff --git a/jxl/enc_cluster.cc b/jxl/enc_cluster.cc
new file mode 100644
index 000000000..533ec512b
--- /dev/null
+++ b/jxl/enc_cluster.cc
@@ -0,0 +1,473 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_cluster.h"
+
+#include <stdint.h>
+
+#include <limits>
+#include <map>
+#include <memory>
+
+#include "jxl/base/fast_log.h"
+#include "jxl/base/profiler.h"
+
+namespace jxl {
+
+namespace {
+inline double CrossEntropy(const ANSHistBin* counts, const size_t counts_len,
+                           const ANSHistBin* codes, const size_t codes_len) {
+  // TODO(veluca): use FastLog2f?
+  double sum = 0.0f;
+  uint32_t total_count = 0;
+  uint32_t total_codes = 0;
+  for (size_t i = 0; i < codes_len; ++i) {
+    if (codes[i] > 0) {
+      if (i < counts_len && counts[i] > 0) {
+        sum -= counts[i] * std::log2(codes[i]);
+        total_count += counts[i];
+      }
+      total_codes += codes[i];
+    }
+  }
+  if (total_codes > 0) {
+    sum += total_count * std::log2(total_codes);
+  }
+  return sum;
+}
+
+inline double ShannonEntropy(const ANSHistBin* data, const size_t data_size) {
+  return CrossEntropy(data, data_size, data, data_size);
+}
+}  // namespace
+
+double Histogram::ShannonEntropy() const {
+  return jxl::ShannonEntropy(data_, ANS_MAX_ALPHA_SIZE);
+}
+
+namespace {
+
+struct HistogramPair {
+  uint32_t idx1;
+  uint32_t idx2;
+  float cost_combo;
+  float cost_diff;
+};
+
+inline bool operator<(const HistogramPair& p1, const HistogramPair& p2) {
+  if (p1.cost_diff != p2.cost_diff) {
+    return p1.cost_diff > p2.cost_diff;
+  }
+  return std::abs(static_cast<int32_t>(p1.idx1) -
+                  static_cast<int32_t>(p1.idx2)) >
+         std::abs(static_cast<int32_t>(p2.idx1) -
+                  static_cast<int32_t>(p2.idx2));
+}
+
+// Returns entropy reduction of the context map when we combine two clusters.
+inline float ClusterCostDiff(int size_a, int size_b) {
+  int size_c = size_a + size_b;
+  return size_a * FastLog2(size_a) + size_b * FastLog2(size_b) -
+         size_c * FastLog2(size_c);
+}
+
+// Computes the bit cost reduction by combining out[idx1] and out[idx2] and if
+// it is below a threshold, stores the pair (idx1, idx2) in the *pairs queue.
+void CompareAndPushToQueue(const Histogram* out, const int* cluster_size,
+                           const float* bit_cost, int idx1, int idx2,
+                           std::vector<HistogramPair>* pairs) {
+  if (idx1 == idx2) {
+    return;
+  }
+  if (idx2 < idx1) {
+    uint32_t t = idx2;
+    idx2 = idx1;
+    idx1 = t;
+  }
+  bool store_pair = false;
+  HistogramPair p;
+  p.idx1 = idx1;
+  p.idx2 = idx2;
+  p.cost_diff = 0.5f * ClusterCostDiff(cluster_size[idx1], cluster_size[idx2]);
+  p.cost_diff -= bit_cost[idx1];
+  p.cost_diff -= bit_cost[idx2];
+
+  if (out[idx1].total_count_ == 0) {
+    p.cost_combo = bit_cost[idx2];
+    store_pair = true;
+  } else if (out[idx2].total_count_ == 0) {
+    p.cost_combo = bit_cost[idx1];
+    store_pair = true;
+  } else {
+    const float threshold = pairs->empty()
+                                ? std::numeric_limits<float>::max()
+                                : std::max(0.0f, (*pairs)[0].cost_diff);
+    Histogram combo = out[idx1];
+    combo.AddHistogram(out[idx2]);
+    float cost_combo = combo.PopulationCost();
+    if (cost_combo + p.cost_diff < threshold) {
+      p.cost_combo = cost_combo;
+      store_pair = true;
+    }
+  }
+  if (store_pair) {
+    p.cost_diff += p.cost_combo;
+    if (!pairs->empty() && (pairs->front() < p)) {
+      // Replace the top of the queue if needed.
+      pairs->push_back(pairs->front());
+      pairs->front() = p;
+    } else {
+      pairs->push_back(p);
+    }
+  }
+}
+
+int HistogramCombine(Histogram* out, int* cluster_size, float* bit_cost,
+                     uint32_t* symbols, size_t symbols_size,
+                     size_t max_clusters) {
+  float cost_diff_threshold = 0.0f;
+  size_t min_cluster_size = 1;
+
+  // Uniquify the list of symbols after merging empty clusters.
+  std::vector<uint32_t> clusters;
+  clusters.reserve(symbols_size);
+  int64_t sum_of_totals = 0;
+  int first_zero_pop_count_symbol = -1;
+  for (size_t i = 0; i < symbols_size; ++i) {
+    if (out[symbols[i]].total_count_ == 0) {
+      // Merge the zero pop count histograms into one.
+      if (first_zero_pop_count_symbol == -1) {
+        first_zero_pop_count_symbol = symbols[i];
+        clusters.push_back(symbols[i]);
+      } else {
+        symbols[i] = first_zero_pop_count_symbol;
+      }
+    } else {
+      // Insert all histograms with non-zero pop counts.
+      clusters.push_back(symbols[i]);
+      sum_of_totals += out[symbols[i]].total_count_;
+    }
+  }
+  if (sum_of_totals < 160) {
+    // Use a single histogram if there are only a few samples.
+    // This helps with small images (like 64x64 size) where the
+    // context map is more expensive than the related savings.
+    // TODO: Estimate the the actual difference in bitcost to
+    // make the final decision of this strategy and clustering.
+    *cluster_size = 1;
+    Histogram combo = out[symbols[0]];
+    for (size_t i = 1; i < symbols_size; ++i) {
+      combo.AddHistogram(out[symbols[i]]);
+    }
+    out[symbols[0]] = combo;
+    for (size_t i = 1; i < symbols_size; ++i) {
+      symbols[i] = symbols[0];
+    }
+    return 1;
+  }
+  std::sort(clusters.begin(), clusters.end());
+  clusters.resize(std::unique(clusters.begin(), clusters.end()) -
+                  clusters.begin());
+
+  // We maintain a priority queue of histogram pairs, ordered by the bit cost
+  // reduction. For efficiency, only the front of the queue matters, the rest of
+  // it is unordered.
+  std::vector<HistogramPair> pairs;
+  for (size_t idx1 = 0; idx1 < clusters.size(); ++idx1) {
+    for (size_t idx2 = idx1 + 1; idx2 < clusters.size(); ++idx2) {
+      CompareAndPushToQueue(out, cluster_size, bit_cost, clusters[idx1],
+                            clusters[idx2], &pairs);
+    }
+  }
+
+  while (clusters.size() > min_cluster_size) {
+    if (pairs[0].cost_diff >= cost_diff_threshold) {
+      cost_diff_threshold = std::numeric_limits<float>::max();
+      min_cluster_size = max_clusters;
+      continue;
+    }
+
+    // Take the best pair from the top of queue.
+    uint32_t best_idx1 = pairs[0].idx1;
+    uint32_t best_idx2 = pairs[0].idx2;
+    out[best_idx1].AddHistogram(out[best_idx2]);
+    bit_cost[best_idx1] = pairs[0].cost_combo;
+    cluster_size[best_idx1] += cluster_size[best_idx2];
+    for (size_t i = 0; i < symbols_size; ++i) {
+      if (symbols[i] == best_idx2) {
+        symbols[i] = best_idx1;
+      }
+    }
+    for (auto cluster = clusters.begin(); cluster != clusters.end();
+         ++cluster) {
+      if (*cluster >= best_idx2) {
+        clusters.erase(cluster);
+        break;
+      }
+    }
+
+    // Remove pairs intersecting the just combined best pair.
+    auto copy_to = pairs.begin();
+    for (size_t i = 0; i < pairs.size(); ++i) {
+      HistogramPair& p = pairs[i];
+      if (p.idx1 == best_idx1 || p.idx2 == best_idx1 || p.idx1 == best_idx2 ||
+          p.idx2 == best_idx2) {
+        // Remove invalid pair from the queue.
+        continue;
+      }
+      if (pairs.front() < p) {
+        // Replace the top of the queue if needed.
+        auto front = pairs.front();
+        pairs.front() = p;
+        *copy_to = front;
+      } else {
+        *copy_to = p;
+      }
+      ++copy_to;
+    }
+    pairs.resize(copy_to - pairs.begin());
+
+    // Push new pairs formed with the combined histogram to the queue.
+    for (size_t i = 0; i < clusters.size(); ++i) {
+      CompareAndPushToQueue(out, cluster_size, bit_cost, best_idx1, clusters[i],
+                            &pairs);
+    }
+  }
+  return clusters.size();
+}
+
+// -----------------------------------------------------------------------------
+// Histogram refinement
+
+// What is the bit cost of moving histogram from cur_symbol to candidate.
+float HistogramBitCostDistance(const Histogram& histogram,
+                               const Histogram& candidate,
+                               const float candidate_bit_cost) {
+  if (histogram.total_count_ == 0) {
+    return 0.0;
+  }
+  Histogram tmp = histogram;
+  tmp.AddHistogram(candidate);
+  return tmp.PopulationCost() - candidate_bit_cost;
+}
+
+// Find the best 'out' histogram for each of the 'in' histograms.
+// Note: we assume that out[]->bit_cost_ is already up-to-date.
+void HistogramRemap(const Histogram* in, int in_size, Histogram* out,
+                    float* bit_cost, uint32_t* symbols) {
+  // Uniquify the list of symbols.
+  std::vector<int> all_symbols(symbols, symbols + in_size);
+  std::sort(all_symbols.begin(), all_symbols.end());
+  all_symbols.resize(std::unique(all_symbols.begin(), all_symbols.end()) -
+                     all_symbols.begin());
+
+  for (int i = 0; i < in_size; ++i) {
+    int best_out = i == 0 ? symbols[0] : symbols[i - 1];
+    float best_bits =
+        HistogramBitCostDistance(in[i], out[best_out], bit_cost[best_out]);
+    for (auto k : all_symbols) {
+      const float cur_bits =
+          HistogramBitCostDistance(in[i], out[k], bit_cost[k]);
+      if (cur_bits < best_bits) {
+        best_bits = cur_bits;
+        best_out = k;
+      }
+    }
+    symbols[i] = best_out;
+  }
+
+  // Recompute each out based on raw and symbols.
+  for (auto k : all_symbols) {
+    out[k].Clear();
+  }
+  for (int i = 0; i < in_size; ++i) {
+    out[symbols[i]].AddHistogram(in[i]);
+  }
+}
+
+// Reorder histograms in *out so that the new symbols in *symbols come in
+// increasing order.
+void HistogramReindex(std::vector<Histogram>* out,
+                      std::vector<uint32_t>* symbols) {
+  std::vector<Histogram> tmp(*out);
+  std::map<int, int> new_index;
+  int next_index = 0;
+  for (size_t i = 0; i < symbols->size(); ++i) {
+    if (new_index.find((*symbols)[i]) == new_index.end()) {
+      new_index[(*symbols)[i]] = next_index;
+      (*out)[next_index] = tmp[(*symbols)[i]];
+      ++next_index;
+    }
+  }
+  out->resize(next_index);
+  for (size_t i = 0; i < symbols->size(); ++i) {
+    (*symbols)[i] = new_index[(*symbols)[i]];
+  }
+}
+
+float Entropy(size_t count, float total) {
+  if (count == 0) return 0;
+  // TODO(veluca): FastLog2f?
+  return count * std::log2(total / count);
+}
+
+void HistogramEntropy(const Histogram& a) {
+  a.entropy_ = 0;
+  if (a.total_count_ == 0) return;
+  float tot = a.total_count_;
+  for (size_t i = 0; i < ANS_MAX_ALPHA_SIZE; i++) {
+    a.entropy_ += Entropy(a.data_[i], tot);
+  }
+}
+
+float HistogramDistance(const Histogram& a, const Histogram& b) {
+  if (a.total_count_ == 0 || b.total_count_ == 0) return 0;
+  float tot = a.total_count_ + b.total_count_;
+  float total_distance = -a.entropy_ - b.entropy_;
+  for (size_t i = 0; i < ANS_MAX_ALPHA_SIZE; i++) {
+    size_t va = a.data_[i];
+    size_t vb = b.data_[i];
+    total_distance += Entropy(va + vb, tot);
+  }
+  return total_distance;
+}
+
+// First step of a k-means clustering with a fancy distance metric.
+void FastClusterHistograms(const std::vector<Histogram>& in,
+                           const size_t num_contexts, size_t max_histograms,
+                           std::vector<Histogram>* out,
+                           std::vector<uint32_t>* histogram_symbols) {
+  PROFILER_FUNC;
+  size_t largest_idx = 0;
+  for (size_t i = 0; i < num_contexts; i++) {
+    HistogramEntropy(in[i]);
+    if (in[i].total_count_ > in[largest_idx].total_count_) {
+      largest_idx = i;
+    }
+  }
+  out->clear();
+  out->reserve(max_histograms);
+  std::vector<float> dists(num_contexts, std::numeric_limits<float>::max());
+  histogram_symbols->clear();
+  histogram_symbols->resize(num_contexts, max_histograms);
+
+  constexpr float kMinDistanceForDistinct = 64.0f;
+  while (out->size() < max_histograms && out->size() < num_contexts) {
+    (*histogram_symbols)[largest_idx] = out->size();
+    out->push_back(in[largest_idx]);
+    largest_idx = 0;
+    for (size_t i = 0; i < num_contexts; i++) {
+      dists[i] = std::min(HistogramDistance(in[i], out->back()), dists[i]);
+      // Avoid repeating histograms
+      if ((*histogram_symbols)[i] != max_histograms) continue;
+      if (dists[i] > dists[largest_idx]) largest_idx = i;
+    }
+    if (dists[largest_idx] < kMinDistanceForDistinct) break;
+  }
+
+  for (size_t i = 0; i < num_contexts; i++) {
+    if ((*histogram_symbols)[i] != max_histograms) continue;
+    size_t best = 0;
+    float best_dist = HistogramDistance(in[i], (*out)[best]);
+    for (size_t j = 1; j < out->size(); j++) {
+      float dist = HistogramDistance(in[i], (*out)[j]);
+      if (dist < best_dist) {
+        best = j;
+        best_dist = dist;
+      }
+    }
+    (*out)[best].AddHistogram(in[i]);
+    HistogramEntropy((*out)[best]);
+    (*histogram_symbols)[i] = best;
+  }
+  // Convert the context map to a canonical form.
+  HistogramReindex(out, histogram_symbols);
+}
+
+// Clusters together all the histograms.
+// Note that most of the speedup from FastestClusterHistograms actually comes
+// from having a single context during ANS encoding.
+void FastestClusterHistograms(const std::vector<Histogram>& in,
+                              const size_t num_contexts, size_t max_histograms,
+                              std::vector<Histogram>* out,
+                              std::vector<uint32_t>* histogram_symbols) {
+  PROFILER_FUNC;
+  histogram_symbols->resize(num_contexts, 0);
+  out->resize(1);
+  (*out)[0] = in[0];
+  for (size_t i = 1; i < num_contexts; i++) {
+    (*out)[0].AddHistogram(in[i]);
+  }
+  return;
+}
+
+}  // namespace
+
+// Clusters similar histograms in 'in' together, the selected histograms are
+// placed in 'out', and for each index in 'in', *histogram_symbols will
+// indicate which of the 'out' histograms is the best approximation.
+void ClusterHistograms(const HistogramParams params,
+                       const std::vector<Histogram>& in,
+                       const size_t num_contexts, size_t max_histograms,
+                       std::vector<Histogram>* out,
+                       std::vector<uint32_t>* histogram_symbols) {
+  if (params.clustering == HistogramParams::ClusteringType::kFastest) {
+    return FastestClusterHistograms(in, num_contexts, max_histograms, out,
+                                    histogram_symbols);
+  }
+  if (params.clustering == HistogramParams::ClusteringType::kFast) {
+    return FastClusterHistograms(in, num_contexts, max_histograms, out,
+                                 histogram_symbols);
+  }
+  PROFILER_FUNC;
+
+  const int in_size = num_contexts;
+  std::vector<int> cluster_size(in_size, 1);
+  std::vector<float> bit_cost(in_size);
+  out->resize(in_size);
+  histogram_symbols->resize(in_size);
+
+  for (int i = 0; i < in_size; ++i) {
+    (*out)[i] = in[i];
+    bit_cost[i] = in[i].PopulationCost();
+    (*histogram_symbols)[i] = i;
+  }
+
+  // Collapse similar histograms within a block type.
+  if (num_contexts > 1) {
+    HistogramCombine(&(*out)[0], &cluster_size[0], &bit_cost[0],
+                     &(*histogram_symbols)[0], num_contexts, max_histograms);
+  }
+
+  static const int kMaxClustersForHistogramRemap = 255;
+
+  int num_clusters = 0;
+  {
+    PROFILER_ZONE("CLUSTER final");
+    // There are no longer "block groups", so need a final round of clustering.
+    num_clusters =
+        HistogramCombine(&(*out)[0], &cluster_size[0], &bit_cost[0],
+                         &(*histogram_symbols)[0], in_size, max_histograms);
+    // Find the optimal map from original histograms to the final ones.
+    if (num_clusters >= 2 && num_clusters <= kMaxClustersForHistogramRemap) {
+      HistogramRemap(&in[0], in_size, &(*out)[0], &bit_cost[0],
+                     &(*histogram_symbols)[0]);
+    }
+  }
+
+  // Convert the context map to a canonical form.
+  HistogramReindex(out, histogram_symbols);
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_cluster.h b/jxl/enc_cluster.h
new file mode 100644
index 000000000..bd83414d3
--- /dev/null
+++ b/jxl/enc_cluster.h
@@ -0,0 +1,71 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Functions for clustering similar histograms together.
+
+#ifndef JXL_ENC_CLUSTER_H_
+#define JXL_ENC_CLUSTER_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <cmath>
+#include <complex>
+#include <limits>
+#include <map>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "jxl/ans_params.h"
+#include "jxl/enc_ans.h"
+
+namespace jxl {
+
+struct Histogram {
+  Histogram() { total_count_ = 0; }
+  void Clear() {
+    memset(data_, 0, ANS_MAX_ALPHA_SIZE * sizeof(data_[0]));
+    total_count_ = 0;
+  }
+  void Add(size_t symbol) {
+    JXL_DASSERT(symbol < ANS_MAX_ALPHA_SIZE);
+    ++data_[symbol];
+    ++total_count_;
+  }
+  void AddHistogram(const Histogram& other) {
+    for (size_t i = 0; i < ANS_MAX_ALPHA_SIZE; ++i) {
+      data_[i] += other.data_[i];
+    }
+    total_count_ += other.total_count_;
+  }
+  float PopulationCost() const {
+    return ANSPopulationCost(data_, ANS_MAX_ALPHA_SIZE);
+  }
+  double ShannonEntropy() const;
+
+  ANSHistBin data_[ANS_MAX_ALPHA_SIZE] = {};
+  size_t total_count_;
+  mutable float entropy_;  // WARNING: not kept up-to-date.
+};
+
+void ClusterHistograms(HistogramParams params, const std::vector<Histogram>& in,
+                       size_t num_contexts, size_t max_histograms,
+                       std::vector<Histogram>* out,
+                       std::vector<uint32_t>* histogram_symbols);
+}  // namespace jxl
+
+#endif  // JXL_ENC_CLUSTER_H_
diff --git a/jxl/enc_comparator.cc b/jxl/enc_comparator.cc
new file mode 100644
index 000000000..9bdf61ce8
--- /dev/null
+++ b/jxl/enc_comparator.cc
@@ -0,0 +1,161 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_comparator.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <algorithm>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/profiler.h"
+#include "jxl/color_management.h"
+#include "jxl/enc_gamma_correct.h"
+
+namespace jxl {
+namespace {
+
+// color is linear, but blending happens in gamma-compressed space using
+// (gamma-compressed) grayscale background color, alpha image represents
+// weights of the sRGB colors in the [0 .. (1 << bit_depth) - 1] interval,
+// output image is in linear space.
+void AlphaBlend(const Image3F& in, const size_t c, float background_linear255,
+                const ImageU& alpha, const uint16_t opaque, Image3F* out) {
+  const float background = LinearToSrgb8Direct(background_linear255);
+
+  for (size_t y = 0; y < out->ysize(); ++y) {
+    const uint16_t* JXL_RESTRICT row_a = alpha.ConstRow(y);
+    const float* JXL_RESTRICT row_i = in.ConstPlaneRow(c, y);
+    float* JXL_RESTRICT row_o = out->PlaneRow(c, y);
+    for (size_t x = 0; x < out->xsize(); ++x) {
+      const uint16_t a = row_a[x];
+      if (a == 0) {
+        row_o[x] = background_linear255;
+      } else if (a == opaque) {
+        row_o[x] = row_i[x];
+      } else {
+        const float w_fg = a * 1.0f / opaque;
+        const float w_bg = 1.0f - w_fg;
+        const float fg = w_fg * LinearToSrgb8Direct(row_i[x]);
+        const float bg = w_bg * background;
+        row_o[x] = Srgb8ToLinearDirect(fg + bg);
+      }
+    }
+  }
+}
+
+const Image3F* AlphaBlend(const ImageBundle& ib, const Image3F& linear,
+                          float background_linear255, Image3F* copy) {
+  // No alpha => all opaque.
+  if (!ib.HasAlpha()) return &linear;
+
+  *copy = Image3F(linear.xsize(), linear.ysize());
+  const uint16_t opaque = (1U << ib.metadata()->alpha_bits) - 1;
+  for (size_t c = 0; c < 3; ++c) {
+    AlphaBlend(linear, c, background_linear255, ib.alpha(), opaque, copy);
+  }
+  return copy;
+}
+
+void AlphaBlend(float background_linear255, ImageBundle* io_linear_srgb) {
+  // No alpha => all opaque.
+  if (!io_linear_srgb->HasAlpha()) return;
+
+  const uint16_t opaque = (1U << io_linear_srgb->metadata()->alpha_bits) - 1;
+  for (size_t c = 0; c < 3; ++c) {
+    AlphaBlend(io_linear_srgb->color(), c, background_linear255,
+               io_linear_srgb->alpha(), opaque, io_linear_srgb->MutableColor());
+  }
+}
+
+float ComputeScoreImpl(const ImageBundle& rgb0, const ImageBundle& rgb1,
+                       Comparator* comparator, ImageF* distmap) {
+  JXL_CHECK(comparator->SetReferenceImage(rgb0));
+  float score;
+  JXL_CHECK(comparator->CompareWith(rgb1, distmap, &score));
+  return score;
+}
+
+}  // namespace
+
+float ComputeScore(const ImageBundle& rgb0, const ImageBundle& rgb1,
+                   Comparator* comparator, ImageF* diffmap, ThreadPool* pool) {
+  PROFILER_FUNC;
+  // Convert to linear sRGB (unless already in that space)
+  ImageMetadata metadata0 = *rgb0.metadata();
+  ImageBundle store0(&metadata0);
+  const ImageBundle* linear_srgb0;
+  JXL_CHECK(TransformIfNeeded(rgb0, ColorManagement::LinearSRGB(rgb0.IsGray()),
+                              pool, &store0, &linear_srgb0));
+  ImageMetadata metadata1 = *rgb1.metadata();
+  ImageBundle store1(&metadata1);
+  const ImageBundle* linear_srgb1;
+  JXL_CHECK(TransformIfNeeded(rgb1, ColorManagement::LinearSRGB(rgb1.IsGray()),
+                              pool, &store1, &linear_srgb1));
+
+  return ComputeScoreImpl(*linear_srgb0, *linear_srgb1, comparator, diffmap);
+  // No alpha: skip blending, only need a single call to Butteraugli.
+  if (!rgb0.HasAlpha() && !rgb1.HasAlpha()) {
+    return ComputeScoreImpl(*linear_srgb0, *linear_srgb1, comparator, diffmap);
+  }
+
+  // Blend on black and white backgrounds
+
+  const float black = 0.0f;
+  ImageBundle blended_black0(&metadata0);
+  blended_black0.SetFromImage(CopyImage(linear_srgb0->color()),
+                              linear_srgb0->c_current());
+  ImageBundle blended_black1(&metadata1);
+  blended_black1.SetFromImage(CopyImage(linear_srgb1->color()),
+                              linear_srgb1->c_current());
+  AlphaBlend(black, &blended_black0);
+  AlphaBlend(black, &blended_black1);
+
+  // TODO(lode): this is incorrect in case of intensity multiplier, consider
+  // making intensity multiplier part of comparator
+  const float white = 255.0f;
+  ImageBundle blended_white0(&metadata0);
+  blended_white0.SetFromImage(CopyImage(linear_srgb0->color()),
+                              linear_srgb0->c_current());
+  ImageBundle blended_white1(&metadata1);
+  blended_white1.SetFromImage(CopyImage(linear_srgb1->color()),
+                              linear_srgb1->c_current());
+  AlphaBlend(white, &blended_white0);
+  AlphaBlend(white, &blended_white1);
+
+  ImageF diffmap_black, diffmap_white;
+  const float dist_black = ComputeScoreImpl(blended_black0, blended_black1,
+                                            comparator, &diffmap_black);
+  const float dist_white = ComputeScoreImpl(blended_white0, blended_white1,
+                                            comparator, &diffmap_white);
+
+  // diffmap and return values are the max of diffmap_black/white.
+  if (diffmap != nullptr) {
+    const size_t xsize = rgb0.xsize();
+    const size_t ysize = rgb0.ysize();
+    *diffmap = ImageF(xsize, ysize);
+    for (size_t y = 0; y < ysize; ++y) {
+      const float* JXL_RESTRICT row_black = diffmap_black.ConstRow(y);
+      const float* JXL_RESTRICT row_white = diffmap_white.ConstRow(y);
+      float* JXL_RESTRICT row_out = diffmap->Row(y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row_out[x] = std::max(row_black[x], row_white[x]);
+      }
+    }
+  }
+  return std::max(dist_black, dist_white);
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_comparator.h b/jxl/enc_comparator.h
new file mode 100644
index 000000000..fec84ea67
--- /dev/null
+++ b/jxl/enc_comparator.h
@@ -0,0 +1,61 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_COMPARATOR_H_
+#define JXL_ENC_COMPARATOR_H_
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+
+class Comparator {
+ public:
+  virtual ~Comparator() = default;
+
+  // Sets the reference image, the first to compare
+  // Image must be in linear sRGB (gamma expanded) in range 0.0f-255.0f as
+  // the range from standard black point to standard white point, but values
+  // outside permitted.
+  virtual Status SetReferenceImage(const ImageBundle& ref) = 0;
+
+  // Sets the actual image (with loss), the second to compare
+  // Image must be in linear sRGB (gamma expanded) in range 0.0f-255.0f as
+  // the range from standard black point to standard white point, but values
+  // outside permitted.
+  // In diffmap it outputs the local score per pixel, while in score it outputs
+  // a single score. Any one may be set to nullptr to not compute it.
+  virtual Status CompareWith(const ImageBundle& actual, ImageF* diffmap,
+                             float* score) = 0;
+
+  // Quality thresholds for diffmap and score values.
+  // The good score must represent a value where the images are considered to
+  // be perceptually indistinguishable (but not identical)
+  // The bad value must be larger than good to indicate "lower means better"
+  // and smaller than good to indicate "higher means better"
+  virtual float GoodQualityScore() const = 0;
+  virtual float BadQualityScore() const = 0;
+};
+
+// Computes the score given images in any RGB color model, optionally with
+// alpha channel.
+float ComputeScore(const ImageBundle& rgb0, const ImageBundle& rgb1,
+                   Comparator* comparator, ImageF* diffmap = nullptr,
+                   ThreadPool* pool = nullptr);
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_COMPARATOR_H_
diff --git a/jxl/enc_context_map.cc b/jxl/enc_context_map.cc
new file mode 100644
index 000000000..bc0245eba
--- /dev/null
+++ b/jxl/enc_context_map.cc
@@ -0,0 +1,128 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Library to encode the context map.
+
+#include "jxl/enc_context_map.h"
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <cstddef>
+#include <vector>
+
+#include "jxl/base/bits.h"
+#include "jxl/base/status.h"
+#include "jxl/enc_ans.h"
+#include "jxl/entropy_coder.h"
+
+namespace jxl {
+
+namespace {
+
+size_t IndexOf(const std::vector<uint8_t>& v, uint8_t value) {
+  size_t i = 0;
+  for (; i < v.size(); ++i) {
+    if (v[i] == value) return i;
+  }
+  return i;
+}
+
+void MoveToFront(std::vector<uint8_t>* v, size_t index) {
+  uint8_t value = (*v)[index];
+  for (size_t i = index; i != 0; --i) {
+    (*v)[i] = (*v)[i - 1];
+  }
+  (*v)[0] = value;
+}
+
+std::vector<uint8_t> MoveToFrontTransform(const std::vector<uint8_t>& v) {
+  if (v.empty()) return v;
+  uint8_t max_value = *std::max_element(v.begin(), v.end());
+  std::vector<uint8_t> mtf(max_value + 1);
+  for (size_t i = 0; i <= max_value; ++i) mtf[i] = i;
+  std::vector<uint8_t> result(v.size());
+  for (size_t i = 0; i < v.size(); ++i) {
+    size_t index = IndexOf(mtf, v[i]);
+    JXL_ASSERT(index < mtf.size());
+    result[i] = static_cast<uint8_t>(index);
+    MoveToFront(&mtf, index);
+  }
+  return result;
+}
+
+}  // namespace
+
+void EncodeContextMap(const std::vector<uint8_t>& context_map,
+                      size_t num_histograms,
+                      const BitWriter::Allotment& allotment,
+                      BitWriter* writer) {
+  if (num_histograms == 1) {
+    // Simple code
+    writer->Write(1, 1);
+    // 0 bits per entry.
+    writer->Write(2, 0);
+    return;
+  }
+
+  std::vector<uint8_t> transformed_symbols = MoveToFrontTransform(context_map);
+  std::vector<Token> tokens;
+  auto encode_rl = [&](size_t len) {
+    uint32_t nbits, bits;
+    EncodeVarLenUint(len, &nbits, &bits);
+    tokens.emplace_back(0, nbits + 8, nbits, bits);
+  };
+  int zero_start = -1;
+  for (size_t i = 0; i < transformed_symbols.size(); i++) {
+    if (zero_start != -1 && transformed_symbols[i] != 0) {
+      encode_rl(i - zero_start - 2);
+      zero_start = -1;
+    }
+    if (transformed_symbols[i] == 0) {
+      if (zero_start == -1 && i + 1 != transformed_symbols.size() &&
+          transformed_symbols[i + 1] == 0) {
+        zero_start = i;
+      }
+      if (zero_start != -1) continue;
+    }
+    uint32_t nbits, bits;
+    EncodeVarLenUint(transformed_symbols[i], &nbits, &bits);
+    JXL_ASSERT(nbits < 8);
+    tokens.emplace_back(0, nbits, nbits, bits);
+  }
+  if (zero_start != -1) {
+    encode_rl(transformed_symbols.size() - zero_start - 2);
+  }
+  EntropyEncodingData codes;
+  std::vector<uint8_t> dummy_context_map;
+  size_t ans_cost =
+      BuildAndEncodeHistograms(HistogramParams{}, 1, {tokens}, &codes,
+                               &dummy_context_map, nullptr, 0, nullptr);
+  size_t entry_bits = CeilLog2Nonzero(num_histograms);
+  size_t simple_cost = entry_bits * context_map.size();
+  if (entry_bits < 4 && simple_cost < ans_cost) {
+    writer->Write(1, 1);
+    writer->Write(2, entry_bits);
+    for (size_t i = 0; i < context_map.size(); i++) {
+      writer->Write(entry_bits, context_map[i]);
+    }
+  } else {
+    writer->Write(1, 0);
+    BuildAndEncodeHistograms(HistogramParams{}, 1, {tokens}, &codes,
+                             &dummy_context_map, writer, 0, nullptr);
+    WriteTokens(tokens, codes, dummy_context_map, allotment, writer);
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_context_map.h b/jxl/enc_context_map.h
new file mode 100644
index 000000000..118627924
--- /dev/null
+++ b/jxl/enc_context_map.h
@@ -0,0 +1,41 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_CONTEXT_MAP_H_
+#define JXL_ENC_CONTEXT_MAP_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/enc_bit_writer.h"
+
+namespace jxl {
+
+// Context map uses uint8_t.
+constexpr size_t kMaxClusters = 256;
+// Max limit is 255 because encoding assumes numbers < 255
+// More clusters can help compression, but makes encode/decode somewhat slower
+static const size_t kClustersLimit = 128;
+
+// Encodes the given context map to the bit stream. The number of different
+// histogram ids is given by num_histograms.
+void EncodeContextMap(const std::vector<uint8_t>& context_map,
+                      size_t num_histograms,
+                      const BitWriter::Allotment& allotment, BitWriter* writer);
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_CONTEXT_MAP_H_
diff --git a/jxl/enc_file.cc b/jxl/enc_file.cc
new file mode 100644
index 000000000..796243fe1
--- /dev/null
+++ b/jxl/enc_file.cc
@@ -0,0 +1,245 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_file.h"
+
+#include <stddef.h>
+
+#include <type_traits>
+#include <utility>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_frame.h"
+#include "jxl/frame_header.h"
+#include "jxl/headers.h"
+#include "jxl/icc_codec.h"
+#include "jxl/image_bundle.h"
+#include "jxl/multiframe.h"
+
+namespace jxl {
+
+namespace {
+
+// DC + 'Very Low Frequency'
+PassDefinition progressive_passes_dc_vlf[] = {
+    {/*num_coefficients=*/2, /*shift=*/0, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/4}};
+
+PassDefinition progressive_passes_dc_lf[] = {
+    {/*num_coefficients=*/2, /*shift=*/0, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/4},
+    {/*num_coefficients=*/3, /*shift=*/0, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/2}};
+
+PassDefinition progressive_passes_dc_lf_salient_ac[] = {
+    {/*num_coefficients=*/2, /*shift=*/0, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/4},
+    {/*num_coefficients=*/3, /*shift=*/0, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/2},
+    {/*num_coefficients=*/8, /*shift=*/0, /*salient_only=*/true,
+     /*suitable_for_downsampling_of_at_least=*/0}};
+
+PassDefinition progressive_passes_dc_lf_salient_ac_other_ac[] = {
+    {/*num_coefficients=*/2, /*shift=*/0, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/4},
+    {/*num_coefficients=*/3, /*shift=*/0, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/2},
+    {/*num_coefficients=*/8, /*shift=*/0, /*salient_only=*/true,
+     /*suitable_for_downsampling_of_at_least=*/0},
+    {/*num_coefficients=*/8, /*shift=*/0, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/0}};
+
+PassDefinition progressive_passes_dc_quant_ac_full_ac[] = {
+    {/*num_coefficients=*/8, /*shift=*/2, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/4},
+    {/*num_coefficients=*/8, /*shift=*/1, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/2},
+    {/*num_coefficients=*/8, /*shift=*/0, /*salient_only=*/false,
+     /*suitable_for_downsampling_of_at_least=*/0},
+};
+
+Status EncodePreview(const CompressParams& cparams, const ImageBundle& ib,
+                     ThreadPool* pool, BitWriter* JXL_RESTRICT writer) {
+  BitWriter preview_writer;
+  // TODO(janwas): also support generating preview by downsampling
+  if (ib.HasColor()) {
+    AuxOut aux_out;
+    Multiframe multiframe;
+    const AnimationFrame* animation_frame = nullptr;
+    PassesEncoderState passes_enc_state;
+    JXL_RETURN_IF_ERROR(EncodeFrame(cparams, animation_frame, ib,
+                                    &passes_enc_state, pool, &preview_writer,
+                                    &aux_out, &multiframe));
+    preview_writer.ZeroPadToByte();
+  }
+
+  if (preview_writer.BitsWritten() != 0) {
+    writer->ZeroPadToByte();
+    writer->AppendByteAligned(preview_writer);
+  }
+
+  return true;
+}
+
+// Returns whether an encoder/decoder must send/receive an ICC profile.
+bool KeepICC(bool lossless, const ColorEncoding& color_encoding) {
+  // color_encoding fields fully describe the profile => skip
+  if (!color_encoding.received_icc) return false;
+
+  // Reconstructed profile may be slightly different (quantization), which is
+  // unacceptable in fully lossless mode, so keep the original profile.
+  if (lossless) return true;
+
+  // Some fields are unknown, so keep ICC.
+  if (color_encoding.opaque_icc) return true;
+
+  return false;
+}
+
+Status MakeImageMetadata(const CompressParams& cparams, const CodecInOut* io,
+                         ImageMetadata* metadata) {
+  *metadata = io->metadata;
+  metadata->have_icc = KeepICC(
+      cparams.brunsli_group_mode ||
+          (cparams.modular_group_mode && cparams.quality_pair.first == 100.0f),
+      io->metadata.color_encoding);
+  metadata->SetIntensityTarget(cparams.intensity_target);
+  return true;
+}
+
+Status WriteHeaders(const CompressParams& cparams, const CodecInOut* io,
+                    ImageMetadata* metadata, BitWriter* writer,
+                    AuxOut* aux_out) {
+  // Marker/signature
+  BitWriter::Allotment allotment(writer, 16);
+  writer->Write(8, 0xFF);
+  writer->Write(8, kMarkerShort);
+  ReclaimAndCharge(writer, &allotment, kLayerHeader, aux_out);
+
+  SizeHeader size;
+  JXL_RETURN_IF_ERROR(size.Set(io->xsize(), io->ysize()));
+  JXL_RETURN_IF_ERROR(WriteSizeHeader(size, writer, kLayerHeader, aux_out));
+
+  JXL_RETURN_IF_ERROR(MakeImageMetadata(cparams, io, metadata));
+  JXL_RETURN_IF_ERROR(
+      WriteImageMetadata(*metadata, writer, kLayerHeader, aux_out));
+
+  if (metadata->m2.have_preview) {
+    JXL_RETURN_IF_ERROR(
+        WritePreviewHeader(io->preview, writer, kLayerHeader, aux_out));
+  }
+
+  if (metadata->m2.have_animation) {
+    JXL_RETURN_IF_ERROR(
+        WriteAnimationHeader(io->animation, writer, kLayerHeader, aux_out));
+  }
+
+  return true;
+}
+
+}  // namespace
+
+Status EncodeFile(const CompressParams& cparams, const CodecInOut* io,
+                  PassesEncoderState* passes_enc_state, PaddedBytes* compressed,
+                  AuxOut* aux_out, ThreadPool* pool) {
+  io->CheckMetadata();
+  BitWriter writer;
+
+  ImageMetadata metadata;
+  JXL_RETURN_IF_ERROR(WriteHeaders(cparams, io, &metadata, &writer, aux_out));
+
+  // Only send ICC (at least several hundred bytes) if fields aren't enough.
+  if (metadata.have_icc) {
+    JXL_RETURN_IF_ERROR(
+        WriteICC(metadata.color_encoding.icc, &writer, kLayerHeader, aux_out));
+  }
+
+  if (metadata.m2.have_preview) {
+    JXL_RETURN_IF_ERROR(
+        EncodePreview(cparams, io->preview_frame, pool, &writer));
+  }
+
+  // Each frame should start on byte boundaries.
+  writer.ZeroPadToByte();
+
+  Multiframe multiframe;
+  if (cparams.progressive_mode || cparams.qprogressive_mode) {
+    if (cparams.saliency_map != nullptr) {
+      multiframe.SetSaliencyMap(cparams.saliency_map);
+    }
+    multiframe.SetSaliencyThreshold(cparams.saliency_threshold);
+    if (cparams.qprogressive_mode) {
+      multiframe.SetProgressiveMode(
+          ProgressiveMode{progressive_passes_dc_quant_ac_full_ac});
+    } else {
+      switch (cparams.saliency_num_progressive_steps) {
+        case 1:
+          multiframe.SetProgressiveMode(
+              ProgressiveMode{progressive_passes_dc_vlf});
+          break;
+        case 2:
+          multiframe.SetProgressiveMode(
+              ProgressiveMode{progressive_passes_dc_lf});
+          break;
+        case 3:
+          multiframe.SetProgressiveMode(
+              ProgressiveMode{progressive_passes_dc_lf_salient_ac});
+          break;
+        case 4:
+          if (cparams.saliency_threshold == 0.0f) {
+            // No need for a 4th pass if saliency-threshold regards everything
+            // as salient.
+            multiframe.SetProgressiveMode(
+                ProgressiveMode{progressive_passes_dc_lf_salient_ac});
+          } else {
+            multiframe.SetProgressiveMode(
+                ProgressiveMode{progressive_passes_dc_lf_salient_ac_other_ac});
+          }
+          break;
+        default:
+          return JXL_FAILURE("Invalid saliency_num_progressive_steps.");
+      }
+    }
+  }
+
+  if (metadata.m2.have_animation) {
+    JXL_CHECK(io->animation_frames.size() == io->frames.size());
+    for (size_t i = 0; i < io->frames.size(); i++) {
+      AnimationFrame animation_frame = io->animation_frames[i];
+      animation_frame.nonserialized_have_timecode =
+          io->animation.have_timecodes;
+      animation_frame.is_last = i == io->frames.size() - 1;
+      JXL_RETURN_IF_ERROR(EncodeFrame(cparams, &animation_frame, io->frames[i],
+                                      passes_enc_state, pool, &writer, aux_out,
+                                      &multiframe));
+    }
+  } else {
+    const AnimationFrame* animation_frame = nullptr;
+    JXL_RETURN_IF_ERROR(EncodeFrame(cparams, animation_frame, io->frames[0],
+                                    passes_enc_state, pool, &writer, aux_out,
+                                    &multiframe));
+  }
+
+  *compressed = std::move(writer).TakeBytes();
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_file.h b/jxl/enc_file.h
new file mode 100644
index 000000000..af3592eac
--- /dev/null
+++ b/jxl/enc_file.h
@@ -0,0 +1,50 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_FILE_H_
+#define JXL_ENC_FILE_H_
+
+// Facade for JXL encoding.
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_params.h"
+
+namespace jxl {
+
+// Compresses pixels from `io` (given in any ColorEncoding).
+// `io->metadata.original` must be set.
+Status EncodeFile(const CompressParams& params, const CodecInOut* io,
+                  PassesEncoderState* passes_enc_state, PaddedBytes* compressed,
+                  AuxOut* aux_out = nullptr, ThreadPool* pool = nullptr);
+
+// Backwards-compatible interface. Don't use in new code.
+// TODO(deymo): Remove this function once we migrate users to C encoder API.
+struct FrameEncCache {};
+JXL_INLINE Status EncodeFile(const CompressParams& params, const CodecInOut* io,
+                             FrameEncCache* /* unused */,
+                             PaddedBytes* compressed, AuxOut* aux_out = nullptr,
+                             ThreadPool* pool = nullptr) {
+  PassesEncoderState passes_enc_state;
+  return EncodeFile(params, io, &passes_enc_state, compressed, aux_out, pool);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_FILE_H_
diff --git a/jxl/enc_frame.cc b/jxl/enc_frame.cc
new file mode 100644
index 000000000..4fa9c34f2
--- /dev/null
+++ b/jxl/enc_frame.cc
@@ -0,0 +1,1016 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_frame.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <array>
+#include <atomic>
+#include <vector>
+
+#include "jxl/ac_context.h"
+#include "jxl/ac_strategy.h"
+#include "jxl/ans_params.h"
+#include "jxl/ar_control_field.h"
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/override.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/brunsli.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/coeff_order.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/compressed_dc.h"
+#include "jxl/dct_util.h"
+#include "jxl/dot_dictionary.h"
+#include "jxl/enc_ac_strategy.h"
+#include "jxl/enc_adaptive_quantization.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_group.h"
+#include "jxl/enc_modular.h"
+#include "jxl/enc_params.h"
+#include "jxl/enc_xyb.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/fields.h"
+#include "jxl/frame_header.h"
+#include "jxl/gaborish.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/loop_filter.h"
+#include "jxl/multiframe.h"
+#include "jxl/noise.h"
+#include "jxl/patch_dictionary.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+#include "jxl/splines.h"
+#include "jxl/toc.h"
+
+namespace jxl {
+namespace {
+
+uint64_t FrameFlagsFromParams(const CompressParams& cparams) {
+  uint64_t flags = 0;
+
+  const float dist = cparams.butteraugli_distance;
+
+  // We don't add noise at low butteraugli distances because the original
+  // noise is stored within the compressed image and adding noise makes things
+  // worse.
+  if (ApplyOverride(cparams.noise, dist >= kMinButteraugliForNoise)) {
+    flags |= FrameHeader::kNoise;
+  }
+
+  if (cparams.progressive_dc) {
+    flags |= FrameHeader::kUseDcFrame;
+  }
+
+  return flags;
+}
+
+Status LoopFilterFromParams(const CompressParams& cparams,
+                            LoopFilter* JXL_RESTRICT loop_filter) {
+  // Gaborish defaults to enabled in Hare or slower.
+  loop_filter->gab =
+      ApplyOverride(cparams.gaborish, cparams.speed_tier <= SpeedTier::kHare);
+
+  loop_filter->epf = ApplyOverride(
+      cparams.adaptive_reconstruction,
+      cparams.butteraugli_distance >= kMinButteraugliForAdaptiveReconstruction);
+
+  return true;
+}
+
+// Returns the target size based on whether bitrate or direct targetsize is
+// given.
+size_t TargetSize(const CompressParams& cparams,
+                  const FrameDimensions& frame_dim) {
+  if (cparams.target_size > 0) {
+    return cparams.target_size;
+  }
+  if (cparams.target_bitrate > 0.0) {
+    return 0.5 + cparams.target_bitrate * frame_dim.xsize * frame_dim.ysize /
+                     kBitsPerByte;
+  }
+  return 0;
+}
+
+// Also modifies the block, e.g. by removing patches.
+Status JxlLossyFrameHeuristics(PassesEncoderState* enc_state,
+                               const ImageBundle& linear, Image3F* opsin,
+                               ThreadPool* pool, AuxOut* aux_out) {
+  CompressParams& cparams = enc_state->cparams;
+  const FrameDimensions& frame_dim = enc_state->shared.frame_dim;
+  size_t target_size = TargetSize(cparams, frame_dim);
+  size_t opsin_target_size = target_size;
+  if (cparams.target_size > 0 || cparams.target_bitrate > 0.0) {
+    cparams.target_size = opsin_target_size;
+  } else if (cparams.butteraugli_distance < 0) {
+    return JXL_FAILURE("Expected non-negative distance");
+  }
+
+  PROFILER_ZONE("JxlLossyFrameHeuristics uninstrumented");
+
+  FindBestDequantMatrices(cparams, *opsin, &enc_state->shared.matrices);
+
+  // Non-default cmap is on only for Hare or slower.
+  if (cparams.speed_tier <= SpeedTier::kHare) {
+    FindBestColorCorrelationMap(*opsin, enc_state->shared.matrices, pool,
+                                &enc_state->shared.cmap);
+  }
+
+  FindBestAcStrategy(*opsin, enc_state, pool, aux_out);
+
+  FindBestArControlField(*opsin, enc_state, pool);
+
+  FindBestQuantizer(linear, *opsin, enc_state, pool, aux_out);
+  return true;
+}
+
+Status MakeFrameHeader(const CompressParams& cparams,
+                       const FrameDimensions& frame_dim,
+                       const AnimationFrame* animation_frame_or_null,
+                       const ImageBundle& ib, Multiframe* multiframe,
+                       FrameHeader* JXL_RESTRICT frame_header,
+                       LoopFilter* JXL_RESTRICT loop_filter) {
+  frame_header->has_animation = animation_frame_or_null != nullptr;
+  if (frame_header->has_animation) {
+    frame_header->animation_frame = *animation_frame_or_null;
+  }
+
+  multiframe->InitPasses(&frame_header->passes);
+
+  if (cparams.modular_group_mode) {
+    frame_header->encoding = FrameEncoding::kModularGroup;
+  }
+
+  if (!ib.jpeg_quant_table.empty()) {
+    // we are transcoding a JPEG, so we don't get to choose
+    frame_header->encoding = FrameEncoding::kPasses;
+    frame_header->color_transform = ib.color_transform;
+    frame_header->chroma_subsampling = ib.chroma_subsampling;
+  } else {
+    frame_header->color_transform = cparams.color_transform;
+    frame_header->chroma_subsampling = cparams.chroma_subsampling;
+  }
+
+  if (cparams.brunsli_group_mode) {
+    frame_header->encoding = FrameEncoding::kJpegGroup;
+  }
+
+  if (frame_header->IsLossy()) {
+    frame_header->flags = FrameFlagsFromParams(cparams);
+
+    JXL_RETURN_IF_ERROR(LoopFilterFromParams(cparams, loop_filter));
+  }
+
+  if (cparams.dc_level) {
+    frame_header->dc_level = cparams.dc_level;
+  }
+  frame_header->save_as_reference = cparams.save_as_reference;
+  if (cparams.save_as_reference == 0 && cparams.dc_level == 0) {
+    frame_header->is_displayed = true;
+  }
+
+  frame_header->has_alpha = ib.HasAlpha();
+  if (!frame_header->IsDisplayed()) frame_header->has_alpha = false;
+
+  return true;
+}
+
+}  // namespace
+
+class LossyFrameEncoder {
+ public:
+  LossyFrameEncoder(const CompressParams& cparams,
+                    const FrameHeader& frame_header,
+                    const LoopFilter& loop_filter,
+                    const ImageMetadata& image_metadata,
+                    const FrameDimensions& frame_dim,
+                    PassesEncoderState* JXL_RESTRICT enc_state,
+                    Multiframe* multiframe, ThreadPool* pool,
+                    const std::function<Status(size_t)>& pool_init,
+                    AuxOut* aux_out, std::vector<AuxOut>* aux_outs)
+      : enc_state_(enc_state),
+        pool_(pool),
+        pool_init_(pool_init),
+        aux_out_(aux_out),
+        aux_outs_(aux_outs) {
+    JXL_CHECK(InitializePassesSharedState(
+        frame_header, loop_filter, image_metadata, frame_dim, multiframe,
+        &enc_state_->shared, /*encoder=*/true));
+    enc_state_->cparams = cparams;
+    enc_state_->passes.clear();
+  }
+
+  Status ComputeEncodingData(const ImageBundle& linear,
+                             Image3F* JXL_RESTRICT opsin, ThreadPool* pool,
+                             BitWriter* JXL_RESTRICT writer,
+                             FrameHeader* frame_header) {
+    PROFILER_ZONE("ComputeEncodingData uninstrumented");
+    JXL_ASSERT((opsin->xsize() % kBlockDim) == 0 &&
+               (opsin->ysize() % kBlockDim) == 0);
+    PassesSharedState& shared = enc_state_->shared;
+
+    if (!enc_state_->cparams.max_error_mode) {
+      float x_qm_scale_steps[3] = {1.2f, 2.4f, 4.8f};
+      shared.frame_header.x_qm_scale = 0;
+      for (size_t i = 0; i < 3; i++) {
+        if (enc_state_->cparams.butteraugli_distance > x_qm_scale_steps[i]) {
+          shared.frame_header.x_qm_scale++;
+        }
+      }
+    }
+
+    if (enc_state_->cparams.speed_tier != SpeedTier::kFalcon) {
+      // Call InitialQuantField only in Hare mode or slower. Otherwise, rely
+      // on simple heuristics in FindBestAcStrategy.
+      if (enc_state_->cparams.speed_tier > SpeedTier::kHare) {
+        enc_state_->initial_quant_field = ImageF(shared.frame_dim.xsize_blocks,
+                                                 shared.frame_dim.ysize_blocks);
+      } else {
+        // Call this here, as it relies on pre-gaborish values.
+        // TODO(veluca): adjust to post-gaborish values.
+        // TODO(veluca): call after image features.
+        enc_state_->initial_quant_field =
+            InitialQuantField(enc_state_->cparams.butteraugli_distance, *opsin,
+                              shared.frame_dim, pool, 1.0f);
+      }
+    }
+
+    if (shared.frame_header.flags & FrameHeader::kNoise) {
+      PROFILER_ZONE("enc GetNoiseParam");
+      // Don't start at zero amplitude since adding noise is expensive -- it
+      // significantly slows down decoding, and this is unlikely to
+      // completely go away even with advanced optimizations. After the
+      // kNoiseModelingRampUpDistanceRange we have reached the full level,
+      // i.e. noise is no longer represented by the compressed image, so we
+      // can add full noise by the noise modeling itself.
+      static const float kNoiseModelingRampUpDistanceRange = 0.6;
+      static const float kNoiseLevelAtStartOfRampUp = 0.25;
+      static const float kNoiseRampupStart = 1.0;
+      // TODO(user) test and properly select quality_coef with smooth
+      // filter
+      float quality_coef = 1.0f;
+      const float rampup =
+          (enc_state_->cparams.butteraugli_distance - kNoiseRampupStart) /
+          kNoiseModelingRampUpDistanceRange;
+      if (rampup < 1.0f) {
+        quality_coef = kNoiseLevelAtStartOfRampUp +
+                       (1.0f - kNoiseLevelAtStartOfRampUp) * rampup;
+      }
+      if (rampup < 0.0f) {
+        quality_coef = kNoiseRampupStart;
+      }
+      if (!GetNoiseParameter(*opsin, &shared.image_features.noise_params,
+                             quality_coef)) {
+        shared.frame_header.flags &= ~FrameHeader::kNoise;
+      }
+    }
+
+    shared.multiframe->DecorrelateOpsin(opsin);
+
+    if (enc_state_->cparams.speed_tier <= SpeedTier::kSquirrel) {
+      shared.image_features.splines = FindSplines(*opsin);
+      shared.image_features.splines.SubtractFrom(opsin,
+                                                 enc_state_->shared.cmap);
+    }
+
+    if (ApplyOverride(enc_state_->cparams.patches,
+                      enc_state_->cparams.speed_tier <= SpeedTier::kSquirrel)) {
+      FindBestPatchDictionary(*opsin, enc_state_, pool, aux_out_);
+      enc_state_->shared.image_features.patches.SubtractFrom(opsin);
+    }
+
+    if (shared.image_features.loop_filter.gab) {
+      *opsin = GaborishInverse(*opsin, 0.9908511000000001, pool);
+    }
+
+    JXL_RETURN_IF_ERROR(
+        JxlLossyFrameHeuristics(enc_state_, linear, opsin, pool_, aux_out_));
+
+    InitializePassesEncoder(*opsin, pool_, enc_state_, aux_out_);
+
+    enc_state_->passes.resize(shared.multiframe->GetNumPasses());
+    for (PassesEncoderState::PassData& pass : enc_state_->passes) {
+      pass.ac_tokens.resize(shared.frame_dim.num_groups);
+    }
+
+    {
+      PROFILER_ZONE("PixelsToGroupCoefficients");
+      const auto compute_group_cache_init = [&](const size_t num_threads) {
+        group_caches_.resize(num_threads);
+        return pool_init_(num_threads);
+      };
+      const auto compute_group_cache = [&](const int group_index,
+                                           const int thread) {
+        // Compute coefficients and coefficient split.
+        AuxOut* my_aux_out = aux_out_ ? &(*aux_outs_)[thread] : nullptr;
+        ComputeCoefficients(group_index, enc_state_, my_aux_out);
+      };
+      RunOnPool(pool_, 0, shared.frame_dim.num_groups, compute_group_cache_init,
+                compute_group_cache, "PixelsToGroupCoefficients");
+    }
+
+    shared.num_histograms = 1;
+
+    ComputeAllCoeffOrders(shared.frame_dim);
+
+    const auto tokenize_group_init = [&](const size_t num_threads) {
+      group_caches_.resize(num_threads);
+      return true;
+    };
+    const auto tokenize_group = [&](const int group_index, const int thread) {
+      // Tokenize coefficients.
+      const Rect rect = shared.BlockGroupRect(group_index);
+      for (size_t idx_pass = 0; idx_pass < enc_state_->passes.size();
+           idx_pass++) {
+        const ac_qcoeff_t* JXL_RESTRICT ac_rows[3] = {
+            enc_state_->coeffs[idx_pass].ConstPlaneRow(0, group_index),
+            enc_state_->coeffs[idx_pass].ConstPlaneRow(1, group_index),
+            enc_state_->coeffs[idx_pass].ConstPlaneRow(2, group_index),
+        };
+        // Ensure group cache is initialized.
+        group_caches_[thread].InitOnce();
+        TokenizeCoefficients(
+            &shared.coeff_orders[idx_pass * kCoeffOrderSize], rect, ac_rows,
+            shared.ac_strategy, &group_caches_[thread].num_nzeroes,
+            &enc_state_->passes[idx_pass].ac_tokens[group_index]);
+      }
+    };
+    RunOnPool(pool_, 0, shared.frame_dim.num_groups, tokenize_group_init,
+              tokenize_group, "TokenizeGroup");
+
+    if (!shared.frame_header.animation_frame.is_last &&
+        !shared.frame_header.animation_frame.have_crop) {
+      RoundtripImage(*opsin, enc_state_, pool_,
+                     /*save_decompressed=*/true);
+    }
+
+    if (aux_out_ && aux_out_->testing_aux.dc) {
+      *aux_out_->testing_aux.dc = CopyImage(shared.dc_storage);
+    }
+    if (aux_out_ && aux_out_->testing_aux.decoded) {
+      *aux_out_->testing_aux.decoded =
+          RoundtripImage(*opsin, enc_state_, pool_,
+                         /*save_decompressed=*/false);
+    }
+
+    *frame_header = shared.frame_header;
+    return true;
+  }
+
+  Status ComputeJPEGTranscodingData(const Image3F& opsin_orig,
+                                    const std::vector<int>& quant_table,
+                                    FrameHeader* frame_header) {
+    constexpr size_t N = kBlockDim;
+    PROFILER_ZONE("ComputeJPEGTranscodingData uninstrumented");
+    PassesSharedState& shared = enc_state_->shared;
+
+    FrameDimensions frame_dim;
+    frame_dim.Set(opsin_orig.xsize(), opsin_orig.ysize());
+
+    const size_t xsize = opsin_orig.xsize();
+    const size_t ysize = opsin_orig.ysize();
+    const size_t xsize_blocks = xsize / N;
+    const size_t ysize_blocks = ysize / N;
+
+    shared.frame_header.x_qm_scale = 0;
+
+    // no-op chroma from luma
+    shared.cmap = ColorCorrelationMap(xsize, ysize, false);
+    shared.ac_strategy.FillDCT8();
+    FillImage(uint8_t(0), &shared.epf_sharpness);
+
+    enc_state_->coeffs.clear();
+    enc_state_->coeffs.emplace_back(
+        ACImage3(kGroupDim * kGroupDim, frame_dim.num_groups));
+
+    // convert JPEG quantization table to a Quantizer object
+    float dcquantization[3] = {8.0f / quant_table[0],
+                               8.0f / quant_table[1 * 64],
+                               8.0f / quant_table[2 * 64]};
+    shared.matrices.SetCustomDC(dcquantization);
+    std::vector<QuantEncoding> qe(DequantMatrices::kNum,
+                                  QuantEncoding::Library(0));
+    std::vector<int> qt(192);
+    for (size_t c = 0; c < 3; c++) {
+      for (size_t y = 0; y < 8; y++) {
+        for (size_t x = 0; x < 8; x++) {
+          // JPEG XL transposes the DCT, JPEG doesn't.
+          qt[c * 64 + 8 * x + y] = quant_table[c * 64 + 8 * y + x];
+        }
+      }
+    }
+    qe[AcStrategy::Type::DCT] = QuantEncoding::RAW(qt);
+    shared.matrices.SetCustom(qe);
+    // Recompute MulDC() and InvMulDC().
+    shared.quantizer.RecomputeFromGlobalScale();
+
+    // Per-block dequant scaling should be 1.
+    FillImage(static_cast<int>(shared.quantizer.InvGlobalScale()),
+              &shared.raw_quant_field);
+
+    Image3F dc = Image3F(xsize_blocks, ysize_blocks);
+    enc_state_->dc = Image3S(xsize_blocks, ysize_blocks);
+    intptr_t onerow = opsin_orig.Plane(0).PixelsPerRow();
+    bool DCzero =
+        (shared.frame_header.color_transform == ColorTransform::kYCbCr);
+    // Compute chroma-from-luma for AC (doesn't seem to be useful for DC)
+    if (frame_header->chroma_subsampling == YCbCrChromaSubsampling::k444) {
+      for (size_t c : {0, 2}) {
+        ImageB* map = (c == 0 ? &shared.cmap.ytox_map : &shared.cmap.ytob_map);
+        const float kScale = kDefaultColorFactor;
+        const int kOffset = kColorOffset;
+        const float kBase = c == 0 ? shared.cmap.YtoXRatio(kOffset)
+                                   : shared.cmap.YtoBRatio(kOffset);
+        const float kZeroThresh =
+            kScale * kZeroBiasDefault[c] *
+            0.9999f;  // just epsilon less for better rounding
+
+        auto process_row = [&](int task, int thread) {
+          size_t ty = task;
+          uint8_t* JXL_RESTRICT row_out = map->Row(ty);
+          for (size_t tx = 0; tx < map->xsize(); ++tx) {
+            const size_t y0 = ty * kColorTileDimInBlocks * kBlockDim;
+            const size_t x0 = tx * kColorTileDimInBlocks * kBlockDim;
+            const size_t y1 =
+                std::min(opsin_orig.ysize(),
+                         (ty + 1) * kColorTileDimInBlocks * kBlockDim);
+            const size_t x1 =
+                std::min(opsin_orig.xsize(),
+                         (tx + 1) * kColorTileDimInBlocks * kBlockDim);
+            int32_t d_num_zeros[257] = {0};
+            for (size_t y = y0; y < y1; ++y) {
+              const float* JXL_RESTRICT row_m = opsin_orig.PlaneRow(1, y);
+              const float* JXL_RESTRICT row_s = opsin_orig.PlaneRow(c, y);
+              for (size_t x = x0; x < x1; ++x) {
+                int coeffpos = (x % kBlockDim) + kBlockDim * (y % kBlockDim);
+                if (coeffpos == 0) continue;
+                const float scaled_m = row_m[x] * quant_table[64 + coeffpos] /
+                                       quant_table[64 * c + coeffpos];
+                const float scaled_s =
+                    kScale * row_s[x] + (kOffset - kBase * kScale) * scaled_m;
+                if (std::abs(scaled_m) > 1e-8f) {
+                  float from, to;
+                  if (scaled_m > 0) {
+                    from = (scaled_s - kZeroThresh) / scaled_m;
+                    to = (scaled_s + kZeroThresh) / scaled_m;
+                  } else {
+                    from = (scaled_s + kZeroThresh) / scaled_m;
+                    to = (scaled_s - kZeroThresh) / scaled_m;
+                  }
+                  if (from < 0.0f) {
+                    from = 0.0f;
+                  }
+                  if (to > 255.0f) {
+                    to = 255.0f;
+                  }
+                  // Instead of clamping the both values
+                  // we just check that range is sane.
+                  if (from <= to) {
+                    d_num_zeros[static_cast<int>(std::ceil(from))]++;
+                    d_num_zeros[static_cast<int>(std::floor(to + 1))]--;
+                  }
+                }
+              }
+            }
+            int best = 0;
+            int32_t best_sum = 0;
+            FindIndexOfSumMaximum(d_num_zeros, 256, &best, &best_sum);
+            int32_t offset_sum = 0;
+            for (size_t i = 0; i < 256; ++i) {
+              if (i <= kOffset) {
+                offset_sum += d_num_zeros[i];
+              }
+            }
+            if (best_sum > offset_sum + 1)
+              row_out[tx] = best;
+            else
+              row_out[tx] = kOffset;
+          }
+        };
+
+        RunOnPool(pool_, 0, map->ysize(), ThreadPool::SkipInit(), process_row,
+                  "FindCorrelation");
+      }
+    }
+
+    for (size_t c : {1, 0, 2}) {
+      ImageB& map = (c == 0 ? shared.cmap.ytox_map : shared.cmap.ytob_map);
+      for (size_t group_index = 0; group_index < frame_dim.num_groups;
+           group_index++) {
+        const size_t gx = group_index % frame_dim.xsize_groups;
+        const size_t gy = group_index / frame_dim.xsize_groups;
+        size_t offset = 0;
+        float* JXL_RESTRICT ac = enc_state_->coeffs[0].PlaneRow(c, group_index);
+        for (size_t by = gy * kGroupDimInBlocks;
+             by < ysize_blocks && by < (gy + 1) * kGroupDimInBlocks; ++by) {
+          const float* JXL_RESTRICT inputjpeg = opsin_orig.PlaneRow(c, by * 8);
+          int16_t* JXL_RESTRICT idc = enc_state_->dc.PlaneRow(c, by);
+          const float* JXL_RESTRICT inputjpegY = opsin_orig.PlaneRow(1, by * 8);
+          float* JXL_RESTRICT fdc = dc.PlaneRow(c, by);
+          const uint8_t* JXL_RESTRICT cm =
+              map.ConstRow(by / kColorTileDimInBlocks);
+          for (size_t bx = gx * kGroupDimInBlocks;
+               bx < xsize_blocks && bx < (gx + 1) * kGroupDimInBlocks; ++bx) {
+            if (DCzero)
+              idc[bx] = inputjpeg[bx * 8];
+            else
+              idc[bx] = (inputjpeg[bx * 8] * quant_table[c * 64] + 1024.f) /
+                        quant_table[c * 64];
+            fdc[bx] = idc[bx] / dcquantization[c];
+            if (c == 1) {
+              for (int i = 0; i < 64; i++) {
+                ac[offset + i] = inputjpeg[8 * bx + (i % 8) * onerow + (i / 8)];
+              }
+            } else {
+              const float scale =
+                  (c == 0
+                       ? shared.cmap.YtoXRatio(cm[bx / kColorTileDimInBlocks])
+                       : shared.cmap.YtoBRatio(cm[bx / kColorTileDimInBlocks]));
+
+              for (int i = 0; i < 64; i++) {
+                float Y = inputjpegY[8 * bx + (i % 8) * onerow + (i / 8)] *
+                          quant_table[64 + i];
+                float QChroma = inputjpeg[8 * bx + (i % 8) * onerow + (i / 8)];
+                // Apply it like this to keep it reversible
+                int QCR = QChroma - (int)(Y * scale / quant_table[64 * c + i]);
+                ac[offset + i] = QCR;
+              }
+            }
+            offset += 64;
+          }
+        }
+      }
+    }
+
+    // disable DC frame for now
+    shared.frame_header.UpdateFlag(false, FrameHeader::kUseDcFrame);
+    const size_t xsize_dc_groups = DivCeil(xsize_blocks, kDcGroupDimInBlocks);
+    const size_t ysize_dc_groups = DivCeil(ysize_blocks, kDcGroupDimInBlocks);
+    enc_state_->dc_tokens =
+        std::vector<std::vector<Token>>(xsize_dc_groups * ysize_dc_groups);
+    enc_state_->extra_dc_levels.resize(xsize_dc_groups * ysize_dc_groups, 0);
+    auto compute_dc_coeffs = [&](int group_index, int /* thread */) {
+      TokenizeDC(group_index, dc, enc_state_, aux_out_);
+    };
+    RunOnPool(pool_, 0, shared.frame_dim.num_dc_groups, ThreadPool::SkipInit(),
+              compute_dc_coeffs, "Compute DC coeffs");
+
+    // Must happen before WriteFrameHeader!
+    shared.frame_header.UpdateFlag(true, FrameHeader::kSkipAdaptiveDCSmoothing);
+
+    enc_state_->passes.resize(shared.multiframe->GetNumPasses());
+    for (PassesEncoderState::PassData& pass : enc_state_->passes) {
+      pass.ac_tokens.resize(shared.frame_dim.num_groups);
+    }
+
+    JXL_CHECK(enc_state_->passes.size() ==
+              1);  // skipping coeff splitting so need to have only one pass
+
+    // Number of histograms and coefficient orders, per pass (always 1 for
+    // now). Encoded as shared.num_histograms - 1.
+    shared.num_histograms = 1;
+    JXL_ASSERT(shared.num_histograms <= shared.frame_dim.num_groups);
+
+    ComputeAllCoeffOrders(frame_dim);
+
+    const auto tokenize_group_init = [&](const size_t num_threads) {
+      group_caches_.resize(num_threads);
+      return true;
+    };
+    const auto tokenize_group = [&](const int group_index, const int thread) {
+      // Tokenize coefficients.
+      const Rect rect = shared.BlockGroupRect(group_index);
+      for (size_t idx_pass = 0; idx_pass < enc_state_->passes.size();
+           idx_pass++) {
+        const ac_qcoeff_t* JXL_RESTRICT ac_rows[3] = {
+            enc_state_->coeffs[idx_pass].ConstPlaneRow(0, group_index),
+            enc_state_->coeffs[idx_pass].ConstPlaneRow(1, group_index),
+            enc_state_->coeffs[idx_pass].ConstPlaneRow(2, group_index),
+        };
+        // Ensure group cache is initialized.
+        group_caches_[thread].InitOnce();
+        TokenizeCoefficients(
+            &shared.coeff_orders[idx_pass * kCoeffOrderSize], rect, ac_rows,
+            shared.ac_strategy, &group_caches_[thread].num_nzeroes,
+            &enc_state_->passes[idx_pass].ac_tokens[group_index]);
+      }
+    };
+    RunOnPool(pool_, 0, shared.frame_dim.num_groups, tokenize_group_init,
+              tokenize_group, "TokenizeGroup");
+    *frame_header = shared.frame_header;
+    return true;
+  }
+
+  Status EncodeGlobalDCInfo(const FrameHeader& frame_header,
+                            BitWriter* writer) const {
+    JXL_RETURN_IF_ERROR(enc_state_->shared.matrices.EncodeDC(
+        writer, kLayerDequantTables, aux_out_));
+
+    // Encode quantizer DC and global scale.
+    JXL_RETURN_IF_ERROR(
+        enc_state_->shared.quantizer.Encode(writer, kLayerQuant, aux_out_));
+    enc_state_->shared.cmap.EncodeDC(writer, kLayerDC, aux_out_);
+    return true;
+  }
+
+  Status EncodeDCGroup(size_t group_index, BitWriter* group_code,
+                       AuxOut* local_aux_out) {
+    return jxl::EncodeDCGroup(*enc_state_, group_index, group_code,
+                              local_aux_out);
+  }
+
+  Status EncodeGlobalACInfo(BitWriter* writer) {
+    if (enc_state_->shared.experiments.use_new_cmap) {
+      EncodeFullColorMap(enc_state_->shared.cmap,
+                         Rect(enc_state_->shared.cmap.ytob_map), writer,
+                         kLayerDC, aux_out_,
+                         enc_state_->shared.experiments.use_new_cmap);
+    }
+    JXL_RETURN_IF_ERROR(enc_state_->shared.matrices.Encode(
+        writer, kLayerDequantTables, aux_out_));
+    size_t num_histo_bits =
+        enc_state_->shared.frame_dim.num_groups == 1
+            ? 0
+            : CeilLog2Nonzero(enc_state_->shared.frame_dim.num_groups - 1);
+    if (num_histo_bits != 0) {
+      BitWriter::Allotment allotment(writer, num_histo_bits);
+      writer->Write(num_histo_bits, enc_state_->shared.num_histograms - 1);
+      ReclaimAndCharge(writer, &allotment, kLayerAC, aux_out_);
+    }
+    for (size_t i = 0; i < enc_state_->shared.multiframe->GetNumPasses(); i++) {
+      // Encode coefficient orders.
+      for (size_t histo = 0; histo < enc_state_->shared.num_histograms;
+           histo++) {
+        uint32_t used_orders = ComputeUsedOrders(
+            enc_state_->cparams.speed_tier, enc_state_->shared.ac_strategy,
+            Rect(enc_state_->shared.raw_quant_field));
+        size_t order_bits = 0;
+        JXL_RETURN_IF_ERROR(
+            U32Coder::CanEncode(kOrderEnc, used_orders, &order_bits));
+        BitWriter::Allotment allotment(writer, order_bits);
+        JXL_CHECK(U32Coder::Write(kOrderEnc, used_orders, writer));
+        ReclaimAndCharge(writer, &allotment, kLayerOrder, aux_out_);
+        EncodeCoeffOrders(used_orders,
+                          &enc_state_->shared.coeff_orders[i * kCoeffOrderSize],
+                          writer, kLayerOrder, aux_out_);
+      }
+
+      // Encode histograms.
+      HistogramParams hist_params(enc_state_->cparams.speed_tier, kNumContexts);
+      for (size_t histo = 0; histo < enc_state_->shared.num_histograms;
+           histo++) {
+        // Same histogram encoding even in fast mode - the cost of
+        // clustering is now lower (one per frame, not group), and it avoids
+        // huge size penalties for small images.
+        BuildAndEncodeHistograms(
+            hist_params, kNumContexts, enc_state_->passes[i].ac_tokens,
+            &enc_state_->passes[i].codes, &enc_state_->passes[i].context_map,
+            writer, kLayerAC, aux_out_);
+      }
+    }
+
+    return true;
+  }
+
+  Status EncodeACGroup(size_t pass, size_t group_index, BitWriter* group_code,
+                       AuxOut* local_aux_out) {
+    return EncodeGroupTokenizedCoefficients(group_index, pass, *enc_state_,
+                                            group_code, local_aux_out);
+  }
+
+  PassesEncoderState* State() { return enc_state_; }
+
+ private:
+  void ComputeAllCoeffOrders(const FrameDimensions& frame_dim) {
+    PROFILER_FUNC;
+    for (size_t histo = 0; histo < enc_state_->shared.num_histograms; histo++) {
+      for (size_t i = 0; i < enc_state_->shared.multiframe->GetNumPasses();
+           i++) {
+        uint32_t used_orders = 0;
+        // No coefficient reordering in Falcon mode.
+        if (enc_state_->cparams.speed_tier != SpeedTier::kFalcon) {
+          used_orders = ComputeUsedOrders(
+              enc_state_->cparams.speed_tier, enc_state_->shared.ac_strategy,
+              Rect(enc_state_->shared.raw_quant_field));
+        }
+        ComputeCoeffOrder(
+            enc_state_->coeffs[i], enc_state_->shared.ac_strategy, frame_dim,
+            used_orders, &enc_state_->shared.coeff_orders[i * kCoeffOrderSize]);
+      }
+    }
+  }
+
+  PassesEncoderState* JXL_RESTRICT enc_state_;
+  ThreadPool* pool_;
+  const std::function<Status(size_t)>& pool_init_;
+  AuxOut* aux_out_;
+  std::vector<AuxOut>* aux_outs_;
+  std::vector<EncCache> group_caches_;
+};
+
+Status EncodeFrame(const CompressParams& cparams_orig,
+                   const AnimationFrame* animation_frame_or_null,
+                   const ImageBundle& ib, PassesEncoderState* passes_enc_state,
+                   ThreadPool* pool, BitWriter* writer, AuxOut* aux_out,
+                   Multiframe* multiframe) {
+  ib.VerifyMetadata();
+  CompressParams cparams = cparams_orig;
+  if (cparams.dc_level + cparams.progressive_dc > 3) {
+    return JXL_FAILURE("Too many levels of progressive DC");
+  }
+
+  if (cparams.butteraugli_distance != 0 &&
+      cparams.butteraugli_distance < kMinButteraugliDistance) {
+    return JXL_FAILURE("Butteraugli distance is too low");
+  }
+
+  FrameDimensions frame_dim;
+  frame_dim.Set(ib.xsize(), ib.ysize());
+
+  if (!ib.jpeg_quant_table.empty()) {
+    cparams.gaborish = Override::kOff;
+    cparams.adaptive_reconstruction = Override::kOff;
+  }
+
+  const size_t xsize = ib.xsize();
+  const size_t ysize = ib.ysize();
+  const size_t num_groups = frame_dim.num_groups;
+  if (xsize == 0 || ysize == 0) return JXL_FAILURE("Empty image");
+
+  FrameHeader frame_header;
+  LoopFilter loop_filter;
+  JXL_RETURN_IF_ERROR(MakeFrameHeader(cparams, frame_dim,
+                                      animation_frame_or_null, ib, multiframe,
+                                      &frame_header, &loop_filter));
+
+  multiframe->StartFrame(frame_header);
+
+  Image3F opsin;
+  const ColorEncoding& c = ColorManagement::LinearSRGB(ib.IsGray());
+  ImageMetadata metadata;
+  metadata.color_encoding = c;
+  ImageBundle linear_storage(&metadata);
+  const ImageBundle* JXL_RESTRICT linear = &ib;
+
+  std::vector<AuxOut> aux_outs;
+  // LossyFrameEncoder stores a reference to a std::function<Status(size_t)>
+  // so we need to keep the std::function<Status(size_t)> being referenced
+  // alive while lossy_frame_encoder is used. We could make resize_aux_outs a
+  // lambda type by making LossyFrameEncoder a template instead, but this is
+  // simpler.
+  const std::function<Status(size_t)> resize_aux_outs =
+      [&aux_outs, aux_out](size_t num_threads) -> Status {
+    if (aux_out != nullptr) {
+      size_t old_size = aux_outs.size();
+      for (size_t i = num_threads; i < old_size; i++) {
+        aux_out->Assimilate(aux_outs[i]);
+      }
+      aux_outs.resize(num_threads);
+      // Each thread needs these INPUTS. Don't copy the entire AuxOut
+      // because it may contain stats which would be Assimilated multiple
+      // times below.
+      for (size_t i = old_size; i < aux_outs.size(); i++) {
+        aux_outs[i].testing_aux = aux_out->testing_aux;
+      }
+    }
+    return true;
+  };
+
+  LossyFrameEncoder lossy_frame_encoder(
+      cparams, frame_header, loop_filter, metadata, frame_dim, passes_enc_state,
+      multiframe, pool, resize_aux_outs, aux_out, &aux_outs);
+  BrunsliFrameEncoder jpeg_frame_encoder(frame_dim, pool);
+  ModularFrameEncoder modular_frame_encoder;
+
+  if (cparams.brunsli_group_mode) {
+    if (ib.jpeg_quant_table.empty()) {
+      return JXL_FAILURE("JpegGroup mode requires JPEG quant table");
+    }
+    JXL_RETURN_IF_ERROR(jpeg_frame_encoder.ReadSourceImage(
+        &ib, ib.jpeg_quant_table, frame_header.chroma_subsampling));
+  } else if (!ib.jpeg_quant_table.empty()) {
+    JXL_RETURN_IF_ERROR(lossy_frame_encoder.ComputeJPEGTranscodingData(
+        ib.color(), ib.jpeg_quant_table, &frame_header));
+  } else {
+    // The Butteraugli feedback loop is only run in Kitten mode or slower: no
+    // need to keep track of the original pixel values in faster modes.
+    ImageBundle* JXL_RESTRICT linear_storage_ptr =
+        cparams.speed_tier <= SpeedTier::kKitten &&
+                frame_header.encoding != FrameEncoding::kModularGroup
+            ? &linear_storage
+            : nullptr;
+    // Avoid a copy in PadImageToMultiple by allocating a large enough image
+    // to begin with.
+    opsin =
+        Image3F(RoundUpToBlockDim(ib.xsize()), RoundUpToBlockDim(ib.ysize()));
+    opsin.ShrinkTo(ib.xsize(), ib.ysize());
+
+    if (frame_header.color_transform == ColorTransform::kXYB &&
+        cparams.dc_level == 0 && cparams.save_as_reference == 0) {
+      ToXYB(ib, cparams.GetIntensityMultiplier(), pool, &opsin,
+            linear_storage_ptr);
+    } else {  // RGB or YCbCr: don't do anything (forward YCbCr is not
+              // implemented, this is only used when the input is already in
+              // YCbCr)
+              // If encoding a special DC or reference frame, don't do anything:
+              // input is already in XYB.
+      CopyImageTo(ib.color(), &opsin);
+    }
+    if (aux_out != nullptr) {
+      JXL_RETURN_IF_ERROR(
+          aux_out->InspectImage3F("enc_frame:OpsinDynamicsImage", opsin));
+    }
+    if (frame_header.encoding == FrameEncoding::kPasses) {
+      PadImageToBlockMultipleInPlace(&opsin);
+      JXL_RETURN_IF_ERROR(lossy_frame_encoder.ComputeEncodingData(
+          *linear, &opsin, pool, writer, &frame_header));
+    }
+  }
+  JXL_RETURN_IF_ERROR(modular_frame_encoder.ComputeEncodingData(
+      cparams, frame_header, ib, &opsin, lossy_frame_encoder.State(),
+      /* encode_color= */ frame_header.encoding ==
+          FrameEncoding::kModularGroup));
+
+  writer->AppendByteAligned(lossy_frame_encoder.State()->special_frames);
+  frame_header.UpdateFlag(
+      lossy_frame_encoder.State()->shared.image_features.patches.HasAny(),
+      FrameHeader::kPatches);
+  frame_header.UpdateFlag(
+      lossy_frame_encoder.State()->shared.image_features.splines.HasAny(),
+      FrameHeader::kSplines);
+  JXL_RETURN_IF_ERROR(WriteFrameHeader(frame_header, writer, aux_out));
+  if (frame_header.IsLossy()) {
+    JXL_RETURN_IF_ERROR(WriteLoopFilter(loop_filter, writer, aux_out));
+  }
+
+  const size_t num_passes = multiframe->GetNumPasses();
+
+  // DC global info + DC groups + AC global info + AC groups *
+  // num_passes.
+  const bool has_ac_global = !frame_header.IsJpeg();
+  std::vector<BitWriter> group_codes(NumTocEntries(frame_dim.num_groups,
+                                                   frame_dim.num_dc_groups,
+                                                   num_passes, has_ac_global));
+  const size_t global_ac_index = frame_dim.num_dc_groups + 1;
+  const size_t is_small_image = (frame_dim.num_groups == 1 && num_passes == 1);
+  const auto get_output = [&](const size_t index) {
+    return &group_codes[is_small_image ? 0 : index];
+  };
+  auto ac_group_code = [&](size_t pass, size_t group) {
+    return get_output(AcGroupIndex(pass, group, frame_dim.num_groups,
+                                   frame_dim.num_dc_groups, has_ac_global));
+  };
+
+  if (frame_header.flags & FrameHeader::kPatches) {
+    lossy_frame_encoder.State()->shared.image_features.patches.Encode(
+        get_output(0), kLayerDictionary, aux_out);
+  }
+
+  if (frame_header.flags & FrameHeader::kSplines) {
+    lossy_frame_encoder.State()->shared.image_features.splines.Encode(
+        get_output(0), kLayerSplines, aux_out);
+  }
+
+  if (frame_header.flags & FrameHeader::kNoise) {
+    EncodeNoise(lossy_frame_encoder.State()->shared.image_features.noise_params,
+                get_output(0), kLayerNoise, aux_out);
+  }
+
+  if (frame_header.IsJpeg()) {
+    // Heavy-lifting happens here.
+    JXL_RETURN_IF_ERROR(jpeg_frame_encoder.DoEncode());
+    JXL_RETURN_IF_ERROR(jpeg_frame_encoder.SerializeHeader(get_output(0)));
+  }
+  if (frame_header.IsLossy()) {
+    // encoding == kPasses
+    JXL_RETURN_IF_ERROR(
+        lossy_frame_encoder.EncodeGlobalDCInfo(frame_header, get_output(0)));
+    // Lossless has no DC info.
+  }
+  JXL_RETURN_IF_ERROR(
+      modular_frame_encoder.EncodeGlobalInfo(get_output(0), aux_out));
+
+  const auto process_dc_group = [&](const int group_index, const int thread) {
+    AuxOut* my_aux_out = aux_out ? &aux_outs[thread] : nullptr;
+    BitWriter* output = get_output(group_index + 1);
+    const size_t gx = group_index % frame_dim.xsize_dc_groups;
+    const size_t gy = group_index / frame_dim.xsize_dc_groups;
+    const Rect rect(gx * kDcGroupDim, gy * kDcGroupDim, kDcGroupDim,
+                    kDcGroupDim);
+    // minShift==3 because kDcGroupDim>>3 == kGroupDim
+    // maxShift==1000 is infinity
+    JXL_CHECK(modular_frame_encoder.EncodeGroup(rect, output, my_aux_out, 3,
+                                                1000, kLayerModularDcGroup));
+    if (frame_header.IsJpeg()) {
+      JXL_CHECK(
+          jpeg_frame_encoder.SerializeDcGroup(group_index, output, my_aux_out));
+    } else if (frame_header.IsLossy()) {
+      JXL_CHECK(
+          lossy_frame_encoder.EncodeDCGroup(group_index, output, my_aux_out));
+    }
+  };
+  RunOnPool(pool, 0, frame_dim.num_dc_groups, resize_aux_outs, process_dc_group,
+            "EncodeDCGroup");
+
+  const bool use_lossy_encoder =
+      (frame_header.encoding != FrameEncoding::kModularGroup &&
+       frame_header.encoding != FrameEncoding::kJpegGroup);
+
+  if (use_lossy_encoder) {
+    JXL_RETURN_IF_ERROR(
+        lossy_frame_encoder.EncodeGlobalACInfo(get_output(global_ac_index)));
+  }
+
+  std::atomic<int> num_errors{0};
+  const auto process_group = [&](const int group_index, const int thread) {
+    AuxOut* my_aux_out = aux_out ? &aux_outs[thread] : nullptr;
+
+    const size_t gx = group_index % frame_dim.xsize_groups;
+    const size_t gy = group_index / frame_dim.xsize_groups;
+    // For modular, don't limit to image dimensions here (is done in
+    // EncodeGroup)
+    const Rect mrect(gx * kGroupDim, gy * kGroupDim, kGroupDim, kGroupDim);
+    int maxShift = 2;
+    int minShift = 0;
+    for (size_t i = 0; i < multiframe->GetNumPasses(); i++) {
+      // Write all modular encoded data (color?, alpha, depth, extra channels)
+      for (uint32_t j = 0; j < frame_header.passes.num_downsample; ++j) {
+        if (i <= frame_header.passes.last_pass[j]) {
+          if (frame_header.passes.downsample[j] == 8) minShift = 3;
+          if (frame_header.passes.downsample[j] == 4) minShift = 2;
+          if (frame_header.passes.downsample[j] == 2) minShift = 1;
+          if (frame_header.passes.downsample[j] == 1) minShift = 0;
+        }
+      }
+      //      printf("Encoding shifts %i..%i\n",minShift,maxShift);
+      if (!modular_frame_encoder.EncodeGroup(
+              mrect, ac_group_code(i, group_index), my_aux_out, minShift,
+              maxShift, kLayerModularAcGroup)) {
+        num_errors.fetch_add(1, std::memory_order_relaxed);
+        return;
+      }
+      maxShift = minShift - 1;
+      minShift = 0;
+      if (frame_header.encoding == FrameEncoding::kJpegGroup) {
+        if (multiframe->GetNumPasses() != 1) {
+          num_errors.fetch_add(1, std::memory_order_relaxed);
+          return;
+        }
+        if (!jpeg_frame_encoder.SerializeAcGroup(
+                group_index, ac_group_code(i, group_index), my_aux_out)) {
+          num_errors.fetch_add(1, std::memory_order_relaxed);
+          return;
+        }
+      } else if (frame_header.encoding == FrameEncoding::kPasses) {
+        if (!lossy_frame_encoder.EncodeACGroup(
+                i, group_index, ac_group_code(i, group_index), my_aux_out)) {
+          num_errors.fetch_add(1, std::memory_order_relaxed);
+          return;
+        }
+      }
+    }
+  };
+  RunOnPool(pool, 0, num_groups, resize_aux_outs, process_group,
+            "EncodeGroupCoefficients");
+
+  // Resizing aux_outs to 0 also Assimilates the array.
+  static_cast<void>(resize_aux_outs(0));
+  JXL_RETURN_IF_ERROR(num_errors.load(std::memory_order_relaxed) == 0);
+
+  for (BitWriter& bw : group_codes) {
+    bw.ZeroPadToByte();  // end of group.
+  }
+
+  JXL_RETURN_IF_ERROR(WriteGroupOffsets(group_codes, writer, aux_out));
+  writer->AppendByteAligned(group_codes);
+  writer->ZeroPadToByte();  // end of frame.
+
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_frame.h b/jxl/enc_frame.h
new file mode 100644
index 000000000..315c03433
--- /dev/null
+++ b/jxl/enc_frame.h
@@ -0,0 +1,43 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_FRAME_H_
+#define JXL_ENC_FRAME_H_
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_params.h"
+#include "jxl/frame_header.h"
+#include "jxl/image_bundle.h"
+#include "jxl/multiframe.h"
+
+namespace jxl {
+
+// Encodes a single frame (including its header) into a byte stream. A frame is
+// either a single image, or animation frame (depending on multiframe),
+// and consists of one or more passes. Groups may be processed in parallel by
+// `pool`.
+Status EncodeFrame(const CompressParams& params,
+                   const AnimationFrame* animation_frame_or_null,
+                   const ImageBundle& ib, PassesEncoderState* passes_enc_state,
+                   ThreadPool* pool, BitWriter* writer, AuxOut* aux_out,
+                   Multiframe* multiframe);
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_FRAME_H_
diff --git a/jxl/enc_gamma_correct.h b/jxl/enc_gamma_correct.h
new file mode 100644
index 000000000..f3ea12d66
--- /dev/null
+++ b/jxl/enc_gamma_correct.h
@@ -0,0 +1,48 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_GAMMA_CORRECT_H_
+#define JXL_ENC_GAMMA_CORRECT_H_
+
+// Deprecated: sRGB transfer function. Use color_management.h instead.
+
+#include <cmath>
+
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+
+// Values are in [0, 255].
+static JXL_INLINE double Srgb8ToLinearDirect(double srgb8) {
+  if (srgb8 <= 0.0) return 0.0;
+  if (srgb8 <= 10.31475) return srgb8 / 12.92;
+  if (srgb8 >= 255.0) return 255.0;
+  const double srgb01 = srgb8 / 255.0;
+  const double linear01 = std::pow((srgb01 + 0.055) / 1.055, 2.4);
+  return linear01 * 255.0;
+}
+
+// Values are in [0, 255].
+static JXL_INLINE double LinearToSrgb8Direct(double linear) {
+  if (linear <= 0.0) return 0.0;
+  if (linear >= 255.0) return 255.0;
+  if (linear <= 10.31475 / 12.92) return linear * 12.92;
+  const double linear01 = linear / 255.0;
+  const double srgb01 = std::pow(linear01, 1.0 / 2.4) * 1.055 - 0.055;
+  return srgb01 * 255.0;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_GAMMA_CORRECT_H_
diff --git a/jxl/enc_group.cc b/jxl/enc_group.cc
new file mode 100644
index 000000000..add222e71
--- /dev/null
+++ b/jxl/enc_group.cc
@@ -0,0 +1,156 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_group.h"
+
+#include <utility>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/profiler.h"
+#include "jxl/common.h"
+#include "jxl/dct_util.h"
+#include "jxl/enc_params.h"
+#include "jxl/image.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+HWY_ATTR void ComputeCoefficients(size_t group_idx,
+                                  PassesEncoderState* enc_state,
+                                  AuxOut* aux_out) {
+  PROFILER_FUNC;
+  const Rect block_group_rect = enc_state->shared.BlockGroupRect(group_idx);
+  const Rect cmap_rect(
+      block_group_rect.x0() / kColorTileDimInBlocks,
+      block_group_rect.y0() / kColorTileDimInBlocks,
+      DivCeil(block_group_rect.xsize(), kColorTileDimInBlocks),
+      DivCeil(block_group_rect.ysize(), kColorTileDimInBlocks));
+
+  const size_t xsize_blocks = block_group_rect.xsize();
+  const size_t ysize_blocks = block_group_rect.ysize();
+
+  const ImageI& full_quant_field = enc_state->shared.raw_quant_field;
+  const CompressParams& cparams = enc_state->cparams;
+
+  // TODO(user): it would be better to find & apply correlation here, when
+  // quantization is chosen.
+
+  {
+    // Only use error diffusion in Wombat mode or slower.
+    const bool error_diffusion = cparams.speed_tier <= SpeedTier::kWombat;
+    constexpr HWY_CAPPED(float, kDCTBlockSize) d;
+
+    ac_qcoeff_t* JXL_RESTRICT coeffs[kMaxNumPasses][3];
+    size_t num_passes = enc_state->shared.multiframe->GetNumPasses();
+    JXL_DASSERT(num_passes > 0);
+    for (size_t i = 0; i < num_passes; i++) {
+      for (size_t c = 0; c < 3; c++) {
+        coeffs[i][c] = enc_state->coeffs[i].PlaneRow(c, group_idx);
+      }
+    }
+
+    HWY_ALIGN float roundtrip_y[AcStrategy::kMaxCoeffArea];
+    HWY_ALIGN ac_qcoeff_t quantized[3 * AcStrategy::kMaxCoeffArea];
+
+    size_t offset = 0;
+
+    for (size_t by = 0; by < ysize_blocks; ++by) {
+      const int32_t* JXL_RESTRICT row_quant_ac =
+          block_group_rect.ConstRow(full_quant_field, by);
+      size_t ty = by / kColorTileDimInBlocks;
+      const uint8_t* JXL_RESTRICT row_cmap[3] = {
+          cmap_rect.ConstRow(enc_state->shared.cmap.ytox_map, ty),
+          nullptr,
+          cmap_rect.ConstRow(enc_state->shared.cmap.ytob_map, ty),
+      };
+      AcStrategyRow ac_strategy_row =
+          enc_state->shared.ac_strategy.ConstRow(block_group_rect, by);
+      for (size_t tx = 0; tx < DivCeil(xsize_blocks, kColorTileDimInBlocks);
+           tx++) {
+        const auto x_factor =
+            Set(d, enc_state->shared.cmap.YtoXRatio(row_cmap[0][tx]));
+        const auto b_factor =
+            Set(d, enc_state->shared.cmap.YtoBRatio(row_cmap[2][tx]));
+        for (size_t bx = tx * kColorTileDimInBlocks;
+             bx < xsize_blocks && bx < (tx + 1) * kColorTileDimInBlocks; ++bx) {
+          const AcStrategy acs = ac_strategy_row[bx];
+          if (!acs.IsFirstBlock()) continue;
+
+          size_t xblocks = acs.covered_blocks_x();
+          size_t yblocks = acs.covered_blocks_y();
+          CoefficientLayout(&yblocks, &xblocks);
+          size_t size = kDCTBlockSize * xblocks * yblocks;
+
+          const int32_t quant_ac = row_quant_ac[bx];
+          enc_state->shared.quantizer.QuantizeRoundtripYBlockAC(
+              error_diffusion, quant_ac, acs.RawStrategy(), xblocks, yblocks,
+              kDefaultQuantBias, coeffs[0][1] + offset, quantized + size,
+              roundtrip_y);
+
+          // Unapply color correlation
+          for (size_t k = 0; k < size; k += d.N) {
+            const auto in_x = Load(d, coeffs[0][0] + offset + k);
+            const auto in_y = Load(d, roundtrip_y + k);
+            const auto in_b = Load(d, coeffs[0][2] + offset + k);
+            const auto out_x = in_x - x_factor * in_y;
+            const auto out_b = in_b - b_factor * in_y;
+            Store(out_x, d, coeffs[0][0] + offset + k);
+            Store(out_b, d, coeffs[0][2] + offset + k);
+          }
+
+          for (size_t c : {0, 2}) {
+            // Quantize
+            enc_state->shared.quantizer.QuantizeBlockAC(
+                error_diffusion, c, quant_ac,
+                c == 0 ? enc_state->x_qm_multiplier : 1.0f, acs.RawStrategy(),
+                xblocks, yblocks, coeffs[0][c] + offset, quantized + c * size);
+          }
+          enc_state->shared.multiframe->SplitACCoefficients(
+              quantized, size, acs, bx, by, offset, coeffs);
+          offset += size;
+        }
+      }
+    }
+  }
+}
+
+Status EncodeGroupTokenizedCoefficients(size_t group_idx, size_t pass_idx,
+                                        const PassesEncoderState& enc_state,
+                                        BitWriter* writer, AuxOut* aux_out) {
+  // Select which histogram to use among those of the current pass.
+  const size_t cur_histogram = 0;
+  const size_t num_histograms = enc_state.shared.num_histograms;
+  // num_histograms is 0 only for lossless.
+  JXL_ASSERT(num_histograms == 0 || cur_histogram < num_histograms);
+  size_t histo_selector_bits =
+      num_histograms == 1 ? 0 : CeilLog2Nonzero(num_histograms - 1);
+
+  if (histo_selector_bits != 0) {
+    BitWriter::Allotment allotment(writer, histo_selector_bits);
+    writer->Write(histo_selector_bits, cur_histogram);
+    ReclaimAndCharge(writer, &allotment, kLayerAC, aux_out);
+  }
+  WriteTokens(enc_state.passes[pass_idx].ac_tokens[group_idx],
+              enc_state.passes[pass_idx].codes,
+              enc_state.passes[pass_idx].context_map, writer, kLayerACTokens,
+              aux_out);
+
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_group.h b/jxl/enc_group.h
new file mode 100644
index 000000000..0e4ae6b99
--- /dev/null
+++ b/jxl/enc_group.h
@@ -0,0 +1,53 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_GROUP_H_
+#define JXL_ENC_GROUP_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/static_targets.h>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_params.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/frame_header.h"
+#include "jxl/multiframe.h"
+#include "jxl/noise.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+HWY_ATTR void ComputeCoefficients(size_t group_idx,
+                                  PassesEncoderState* enc_state,
+                                  AuxOut* aux_out);
+
+// `group_codes` contains one vector (of group writers) per pass.
+Status EncodeGroupTokenizedCoefficients(size_t group_idx, size_t pass_idx,
+                                        const PassesEncoderState& enc_state,
+                                        BitWriter* writer, AuxOut* aux_out);
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_GROUP_H_
diff --git a/jxl/enc_modular.cc b/jxl/enc_modular.cc
new file mode 100644
index 000000000..9aa222db3
--- /dev/null
+++ b/jxl/enc_modular.cc
@@ -0,0 +1,552 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_modular.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <array>
+#include <utility>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_params.h"
+#include "jxl/modular/encoding/encoding.h"
+#include "jxl/modular/image/image.h"
+#include "jxl/modular/transform/transform.h"
+
+namespace jxl {
+
+namespace {
+// Squeeze default quantization factors
+// these quantization factors are for -Q 50  (other qualities simply scale the
+// factors; things are rounded down and obviously cannot get below 1)
+static const float squeeze_quality_factor =
+    0.3;  // for easy tweaking of the quality range (decrease this number for
+          // higher quality)
+static const float squeeze_luma_factor =
+    1.2;  // for easy tweaking of the balance between luma (or anything
+          // non-chroma) and chroma (decrease this number for higher quality
+          // luma)
+
+static const float squeeze_luma_qtable[16] = {
+    163.84, 81.92, 40.96, 20.48, 10.24, 5.12, 2.56, 1.28,
+    0.64,   0.32,  0.16,  0.08,  0.04,  0.02, 0.01, 0.005};
+// for 8-bit input, the range of YCoCg chroma is -255..255 so basically this
+// does 4:2:0 subsampling (two most fine grained layers get quantized away)
+static const float squeeze_chroma_qtable[16] = {
+    1024, 512, 256, 128, 64, 32, 16, 8, 4, 2, 1, 0.5, 0.5, 0.5, 0.5, 0.5};
+
+// XYB multipliers. TODO: signal them.
+// Also may want to use chroma_from_luma instead of doing B-Y here...
+static const float kEncoderMul2[3] = {32768.0f, 2048.0f, 2048.0f};
+}  // namespace
+
+Status ModularFrameEncoder::ComputeEncodingData(
+    const CompressParams& orig_cparams, const FrameHeader& frame_header,
+    const ImageBundle& ib, Image3F* JXL_RESTRICT color,
+    PassesEncoderState* JXL_RESTRICT enc_state, bool encode_color) {
+  cparams = orig_cparams;
+  do_color = encode_color;
+
+  if (do_color && cparams.speed_tier < SpeedTier::kCheetah) {
+    FindBestPatchDictionary(*color, enc_state, nullptr, nullptr,
+                            cparams.color_transform == ColorTransform::kXYB);
+    enc_state->shared.image_features.patches.SubtractFrom(color);
+  }
+
+  // Convert ImageBundle to modular Image object
+  const size_t xsize = ib.xsize();
+  const size_t ysize = ib.ysize();
+
+  int nb_chans = 3;
+  if (ib.IsGray()) nb_chans = 1;
+  if (!do_color) nb_chans = 0;
+
+  if (frame_header.HasAlpha()) nb_chans++;
+  if (ib.HasDepth()) nb_chans++;
+  if (ib.HasExtraChannels() && frame_header.IsDisplayed()) {
+    nb_chans += ib.extra_channels().size();
+  }
+
+  int maxval = (1 << ib.metadata()->bits_per_sample) - 1;
+  if (cparams.color_transform == ColorTransform::kXYB)
+    maxval = 255;  // not true, but bits_per_sample doesn't matter either
+  Image gi(xsize, ysize, maxval, nb_chans);
+  int c = 0;
+  if (do_color) {
+    for (; c < 3; c++) {
+      if (ib.IsGray() &&
+          c != (cparams.color_transform == ColorTransform::kXYB ? 1 : 0))
+        continue;
+      int c_out = c;
+      // XYB is encoded as YX(B-Y)
+      if (cparams.color_transform == ColorTransform::kXYB && c < 2)
+        c_out = 1 - c_out;
+      float factor = maxval / 255.f;
+      if (cparams.color_transform == ColorTransform::kXYB)
+        factor *= kEncoderMul2[c];
+      if (c == 2 && cparams.color_transform == ColorTransform::kXYB) {
+        for (size_t y = 0; y < ysize; ++y) {
+          const float* const JXL_RESTRICT row_in = color->PlaneRow(c, y);
+          pixel_type* const JXL_RESTRICT row_out = gi.channel[c_out].Row(y);
+          pixel_type* const JXL_RESTRICT row_Y = gi.channel[0].Row(y);
+          for (size_t x = 0; x < xsize; ++x) {
+            row_out[x] = row_in[x] * factor + 0.5f;
+            row_out[x] -= row_Y[x];
+          }
+        }
+      } else {
+        for (size_t y = 0; y < ysize; ++y) {
+          const float* const JXL_RESTRICT row_in = color->PlaneRow(c, y);
+          pixel_type* const JXL_RESTRICT row_out = gi.channel[c_out].Row(y);
+          for (size_t x = 0; x < xsize; ++x) {
+            row_out[x] = row_in[x] * factor + 0.5f;
+          }
+        }
+      }
+    }
+    if (ib.IsGray()) c = 1;
+  }
+  if (frame_header.HasAlpha()) {
+    for (size_t y = 0; y < ysize; ++y) {
+      const uint16_t* const JXL_RESTRICT row_in = ib.alpha().Row(y);
+      pixel_type* const JXL_RESTRICT row_out = gi.channel[c].Row(y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row_out[x] = row_in[x];
+      }
+    }
+    c++;
+  }
+  if (ib.HasDepth()) {
+    gi.channel[c].resize(ib.depth().xsize(), ib.depth().ysize());
+    gi.channel[c].hshift = ib.metadata()->m2.depth_shift;
+    gi.channel[c].vshift = ib.metadata()->m2.depth_shift;
+    for (size_t y = 0; y < ib.depth().ysize(); ++y) {
+      const uint16_t* const JXL_RESTRICT row_in = ib.depth().Row(y);
+      pixel_type* const JXL_RESTRICT row_out = gi.channel[c].Row(y);
+      for (size_t x = 0; x < ib.depth().xsize(); ++x) {
+        row_out[x] = row_in[x];
+      }
+    }
+    c++;
+  }
+  if (ib.HasExtraChannels() && frame_header.IsDisplayed()) {
+    for (size_t ec = 0; ec < ib.extra_channels().size(); ec++, c++) {
+      for (size_t y = 0; y < ysize; ++y) {
+        const uint16_t* const JXL_RESTRICT row_in =
+            ib.extra_channels()[ec].Row(y);
+        pixel_type* const JXL_RESTRICT row_out = gi.channel[c].Row(y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row_out[x] = row_in[x];
+        }
+      }
+    }
+  }
+
+  // Set options and apply transformations
+
+  float quality = cparams.quality_pair.first;
+  float cquality = cparams.quality_pair.second;
+  if (cquality > 100) cquality = quality;
+
+  if (quality < 100 || cparams.near_lossless) {
+    if (cparams.palette_colors != 0)
+      JXL_DEBUG_V(3, "Lossy encode, not doing palette transforms");
+    cparams.channel_colors_pre_transform_percent = 0;
+    cparams.channel_colors_percent = 0;
+    cparams.palette_colors = 0;
+  }
+
+  // Global channel palette
+  if (cparams.channel_colors_pre_transform_percent > 0 && quality == 100) {
+    // single channel palette (like FLIF's ChannelCompact)
+    gi.recompute_minmax();
+    for (size_t i = 0; i < gi.nb_channels; i++) {
+      int colors = (gi.channel[gi.nb_meta_channels + i].maxval -
+                    gi.channel[gi.nb_meta_channels + i].minval + 1);
+      JXL_DEBUG_V(10, "Channel %zu: range=%i..%i", i,
+                  gi.channel[gi.nb_meta_channels + i].minval,
+                  gi.channel[gi.nb_meta_channels + i].maxval);
+      Transform maybe_palette_1(TRANSFORM_PALETTE);
+      maybe_palette_1.parameters.push_back(i + gi.nb_meta_channels);
+      maybe_palette_1.parameters.push_back(i + gi.nb_meta_channels);
+      // simple heuristic: if less than X percent of the values in the range
+      // actually occur, it is probably worth it to do a compaction
+      // (but only if the channel palette is less than 80% the size of the
+      // image itself)
+      maybe_palette_1.parameters.push_back(std::min(
+          (int)(xsize * ysize * 0.8),
+          (int)(cparams.channel_colors_pre_transform_percent / 100. * colors)));
+      gi.do_transform(maybe_palette_1);
+    }
+  }
+
+  gi.recompute_minmax();
+
+  // Global palette
+  if (cparams.palette_colors != 0 && cparams.speed_tier < SpeedTier::kFalcon) {
+    // all-channel palette (e.g. RGBA)
+    if (gi.nb_channels > 1) {
+      Transform maybe_palette(TRANSFORM_PALETTE);
+      maybe_palette.parameters.push_back(gi.nb_meta_channels);
+      maybe_palette.parameters.push_back(gi.nb_meta_channels + gi.nb_channels -
+                                         1);
+      maybe_palette.parameters.push_back(cparams.palette_colors);
+      gi.do_transform(maybe_palette);
+    }
+    // all-minus-one-channel palette (RGB with separate alpha, or CMY with
+    // separate K)
+    if (gi.nb_channels > 3) {
+      Transform maybe_palette_3(TRANSFORM_PALETTE);
+      maybe_palette_3.parameters.push_back(gi.nb_meta_channels);
+      maybe_palette_3.parameters.push_back(gi.nb_meta_channels +
+                                           gi.nb_channels - 2);
+      maybe_palette_3.parameters.push_back(cparams.palette_colors);
+      gi.do_transform(maybe_palette_3);
+    }
+  }
+
+  if (orig_cparams.color_transform == ColorTransform::kNone && do_color) {
+    if (cparams.colorspace == 1 ||
+        (cparams.colorspace < 0 && (quality < 100 || cparams.near_lossless ||
+                                    cparams.speed_tier > SpeedTier::kWombat))) {
+      gi.do_transform(Transform(TRANSFORM_YCoCg));
+    } else if (cparams.colorspace >= 2) {
+      Transform sg(TRANSFORM_RCT);
+      sg.parameters.push_back(cparams.colorspace - 2);
+      gi.do_transform(sg);
+    }
+  }
+  // use a sensible default if nothing explicit is specified:
+  // Squeeze for lossy, no squeeze for lossless
+  if (cparams.responsive < 0) {
+    if (quality == 100)
+      cparams.responsive = 0;
+    else
+      cparams.responsive = 1;
+  }
+  if (cparams.responsive) {
+    gi.do_transform(Transform(TRANSFORM_SQUEEZE));  // use default squeezing
+  }
+  if (quality < 100 || cquality < 100) {
+    JXL_DEBUG_V(
+        2,
+        "Adding quantization constants corresponding to luma quality %.2f "
+        "and chroma quality %.2f",
+        quality, cquality);
+    if (!cparams.responsive) {
+      JXL_DEBUG_V(1,
+                  "Warning: lossy compression without Squeeze "
+                  "transform is just color quantization.");
+      quality = (400 + quality) / 5;
+      cquality = (400 + cquality) / 5;
+    }
+    Transform quantize(TRANSFORM_QUANTIZE);
+    for (size_t i = 0; i < gi.nb_meta_channels; i++)
+      quantize.parameters.push_back(1);  // don't quantize metachannels
+
+    // convert 'quality' to quantization scaling factor
+    if (quality > 50)
+      quality = 200.0 - quality * 2.0;
+    else
+      quality = 900.0 - quality * 16.0;
+    if (cquality > 50)
+      cquality = 200.0 - cquality * 2.0;
+    else
+      cquality = 900.0 - cquality * 16.0;
+    quality *= 0.01f * maxval / 255.f;
+    cquality *= 0.01f * maxval / 255.f;
+
+    for (size_t i = gi.nb_meta_channels; i < gi.channel.size(); i++) {
+      Channel& ch = gi.channel[i];
+      int shift = ch.hcshift + ch.vcshift;  // number of pixel halvings
+      if (shift > 15) shift = 15;
+      int q;
+      // assuming default Squeeze here
+      int component = ((i - gi.nb_meta_channels) % gi.real_nb_channels);
+      // last 4 channels are final chroma residuals
+      if (gi.real_nb_channels > 2 && i >= gi.channel.size() - 4) component = 1;
+
+      if (cparams.colorspace != 0 && component > 0 && component < 3)
+        q = cquality * squeeze_quality_factor * squeeze_chroma_qtable[shift];
+      else
+        q = quality * squeeze_quality_factor * squeeze_luma_factor *
+            squeeze_luma_qtable[shift];
+      if (q < 1) q = 1;
+      quantize.parameters.push_back(q);
+    }
+    gi.do_transform(quantize);
+  }
+  if (cparams.options.predictor.size() == 0) {
+    // no explicit predictor(s) given, set a good default
+    if (cparams.near_lossless) {
+      // avg(top,left) predictor for near_lossless
+      cparams.options.predictor.push_back(3);
+    } else if (cparams.responsive) {
+      // zero predictor for Squeeze residues
+      cparams.options.predictor.push_back(0);
+    } else if (cparams.speed_tier < SpeedTier::kFalcon) {
+      // try median and weighted predictor for anything else
+      cparams.options.predictor.push_back(8);
+    } else {
+      // just weighted predictor in fastest mode
+      cparams.options.predictor.push_back(7);
+    }
+  }
+  switch (cparams.speed_tier) {
+    case SpeedTier::kFalcon:
+      cparams.options.nb_wp_modes = 1;
+      break;
+    case SpeedTier::kCheetah:
+      cparams.options.nb_wp_modes = 1;
+      break;
+    case SpeedTier::kHare:
+      cparams.options.nb_wp_modes = 1;
+      break;
+    case SpeedTier::kWombat:
+      cparams.options.nb_wp_modes = 1;
+      break;
+    case SpeedTier::kSquirrel:
+      cparams.options.nb_wp_modes = 1;
+      break;
+    case SpeedTier::kKitten:
+      cparams.options.nb_wp_modes = 2;
+      break;
+    case SpeedTier::kTortoise:
+      cparams.options.nb_wp_modes = 5;
+      break;
+  }
+
+  full_image = std::move(gi);
+  return true;
+}
+
+Status ModularFrameEncoder::EncodeGlobalInfo(BitWriter* writer,
+                                             AuxOut* aux_out) {
+  PaddedBytes compressed;
+  cparams.options.max_chan_size = kGroupDim;
+  modular_generic_compress(full_image, &compressed, &cparams.options, 0, false);
+  if (aux_out != nullptr)
+    aux_out->layers[kLayerModularGlobal].total_bits += compressed.size() * 8;
+  writer->ZeroPadToByte();
+  *writer += compressed;
+  return true;
+}
+Status ModularFrameEncoder::EncodeGroup(const Rect& rect, BitWriter* writer,
+                                        AuxOut* aux_out, size_t minShift,
+                                        size_t maxShift, size_t layer) {
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+  int maxval = full_image.maxval;
+  Image gi(xsize, ysize, maxval, 0);
+  // start at the first bigger-than-kGroupDim non-metachannel
+  int c = full_image.nb_meta_channels;
+  for (; c < full_image.channel.size(); c++) {
+    Channel& fc = full_image.channel[c];
+    if (fc.w > kGroupDim || fc.h > kGroupDim) break;
+  }
+  for (; c < full_image.channel.size(); c++) {
+    Channel& fc = full_image.channel[c];
+    int shift = std::min(fc.hshift, fc.vshift);
+    if (shift > maxShift) continue;
+    if (shift < minShift) continue;
+    Rect r(rect.x0() >> fc.hshift, rect.y0() >> fc.vshift,
+           rect.xsize() >> fc.hshift, rect.ysize() >> fc.vshift, fc.w, fc.h);
+    if (r.xsize() == 0 || r.ysize() == 0) continue;
+    Channel gc(r.xsize(), r.ysize(), 0, maxval);
+    gc.hshift = fc.hshift;
+    gc.vshift = fc.vshift;
+    for (size_t y = 0; y < r.ysize(); ++y) {
+      const pixel_type* const JXL_RESTRICT row_in = r.ConstRow(fc.plane, y);
+      pixel_type* const JXL_RESTRICT row_out = gc.Row(y);
+      for (size_t x = 0; x < r.xsize(); ++x) {
+        row_out[x] = row_in[x];
+      }
+    }
+    gi.channel.emplace_back(std::move(gc));
+  }
+  gi.nb_channels = gi.channel.size();
+  gi.real_nb_channels = gi.nb_channels;
+
+  // Do some per-group transforms
+
+  float quality = cparams.quality_pair.first;
+
+  // Local palette
+  if (cparams.palette_colors != 0 && cparams.speed_tier < SpeedTier::kCheetah) {
+    // all-channel palette (e.g. RGBA)
+    if (gi.nb_channels > 1) {
+      Transform maybe_palette(TRANSFORM_PALETTE);
+      maybe_palette.parameters.push_back(gi.nb_meta_channels);
+      maybe_palette.parameters.push_back(gi.nb_meta_channels + gi.nb_channels -
+                                         1);
+      maybe_palette.parameters.push_back(cparams.palette_colors);
+      gi.do_transform(maybe_palette);
+    }
+    // all-minus-one-channel palette (RGB with separate alpha, or CMY with
+    // separate K)
+    if (gi.nb_channels > 3) {
+      Transform maybe_palette_3(TRANSFORM_PALETTE);
+      maybe_palette_3.parameters.push_back(gi.nb_meta_channels);
+      maybe_palette_3.parameters.push_back(gi.nb_meta_channels +
+                                           gi.nb_channels - 2);
+      maybe_palette_3.parameters.push_back(cparams.palette_colors);
+      gi.do_transform(maybe_palette_3);
+    }
+  }
+
+  // Local channel palette
+  if (cparams.channel_colors_percent > 0 && quality == 100 &&
+      cparams.speed_tier < SpeedTier::kCheetah) {
+    // single channel palette (like FLIF's ChannelCompact)
+    gi.recompute_minmax();
+    for (size_t i = 0; i < gi.nb_channels; i++) {
+      int colors = (gi.channel[gi.nb_meta_channels + i].maxval -
+                    gi.channel[gi.nb_meta_channels + i].minval + 1);
+      JXL_DEBUG_V(10, "Channel %zu: range=%i..%i", i,
+                  gi.channel[gi.nb_meta_channels + i].minval,
+                  gi.channel[gi.nb_meta_channels + i].maxval);
+      Transform maybe_palette_1(TRANSFORM_PALETTE);
+      maybe_palette_1.parameters.push_back(i + gi.nb_meta_channels);
+      maybe_palette_1.parameters.push_back(i + gi.nb_meta_channels);
+      // simple heuristic: if less than X percent of the values in the range
+      // actually occur, it is probably worth it to do a compaction
+      // (but only if the channel palette is less than 80% the size of the
+      // image itself)
+      maybe_palette_1.parameters.push_back(
+          std::min((int)(xsize * ysize * 0.8),
+                   (int)(cparams.channel_colors_percent / 100. * colors)));
+      gi.do_transform(maybe_palette_1);
+    }
+  }
+
+  gi.recompute_minmax();
+  // lossless and no specific color transform specified: try Nothing, YCoCg,
+  // and 17 RCTs
+  if (cparams.color_transform == ColorTransform::kNone && quality == 100 &&
+      cparams.colorspace < 0 && gi.nb_channels > 2 && !cparams.near_lossless &&
+      cparams.responsive == false && do_color &&
+      cparams.speed_tier <= SpeedTier::kWombat) {
+#if JXL_DEBUG_V_LEVEL >= 5
+    int best = 0;
+#endif
+
+    PaddedBytes compressed, compressed2;
+    if (cparams.speed_tier <= SpeedTier::kKitten) {
+      modular_generic_compress(gi, &compressed, &cparams.options, 0, false);
+    }
+    gi.do_transform(Transform(TRANSFORM_YCoCg));
+    modular_generic_compress(gi, &compressed2, &cparams.options, 0, false);
+    if (compressed2.size() < compressed.size() ||
+        cparams.speed_tier > SpeedTier::kKitten) {
+      compressed = std::move(compressed2);
+#if JXL_DEBUG_V_LEVEL >= 5
+      best = 1;
+#endif
+    }
+    compressed2.clear();
+
+    Transform sg(TRANSFORM_RCT);
+    sg.parameters.push_back(0);
+
+    size_t nb_rcts_to_try = 0;
+    switch (cparams.speed_tier) {
+      case SpeedTier::kFalcon:
+        nb_rcts_to_try = 0;
+        break;
+      case SpeedTier::kCheetah:
+        nb_rcts_to_try = 1;
+        break;
+      case SpeedTier::kHare:
+        nb_rcts_to_try = 2;
+        break;
+      case SpeedTier::kWombat:
+        nb_rcts_to_try = 3;
+        break;
+      case SpeedTier::kSquirrel:
+        nb_rcts_to_try = 5;
+        break;
+      case SpeedTier::kKitten:
+        nb_rcts_to_try = 7;
+        break;
+      case SpeedTier::kTortoise:
+        nb_rcts_to_try = 17;
+        break;
+    }
+    // These should be 17 actually different transforms; the remaining ones
+    // are equivalent to one of these (or to do-nothing)
+    // modulo channel reordering (which only matters in the case
+    // of MA-with-prev-channels-properties) and/or sign (e.g. RmG vs GmR)
+    for (int i : {5, 9, 23, 35, 11, 17, 7, 4, 8, 13, 14, 15, 28, 29, 2, 1, 3}) {
+      if (nb_rcts_to_try == 0) break;
+      gi.undo_transforms(gi.transform.size() - 1);
+      sg.parameters[0] = i;
+      gi.do_transform(sg);
+      modular_generic_compress(gi, &compressed2, &cparams.options, 0, false);
+      if (compressed2.size() < compressed.size()) {
+        compressed = std::move(compressed2);
+#if JXL_DEBUG_V_LEVEL >= 5
+        best = 2 + i;
+#endif
+      }
+      compressed2.clear();
+      nb_rcts_to_try--;
+    }
+#if JXL_DEBUG_V_LEVEL >= 5
+    JXL_DEBUG_V(5, "Best color transform: %i", best);
+#endif
+    if (aux_out != nullptr)
+      aux_out->layers[layer].total_bits += compressed.size() * 8;
+    writer->ZeroPadToByte();
+    *writer += compressed;
+    return true;
+  }
+  if (cparams.near_lossless > 0) {
+    if (cparams.colorspace == 0) {
+      Transform nl(TRANSFORM_NEAR_LOSSLESS);
+      nl.parameters.push_back(0);
+      nl.parameters.push_back(gi.nb_channels - 1);
+      nl.parameters.push_back(cparams.near_lossless);
+      gi.do_transform(nl);
+    } else {
+      Transform nl(TRANSFORM_NEAR_LOSSLESS);
+      nl.parameters.push_back(0);
+      nl.parameters.push_back(0);
+      nl.parameters.push_back(cparams.near_lossless);
+      gi.do_transform(nl);
+      nl.parameters.clear();
+      nl.parameters.push_back(1);
+      nl.parameters.push_back(gi.nb_channels - 1);
+      nl.parameters.push_back(2 *
+                              cparams.near_lossless);  // more loss for chroma
+      gi.do_transform(nl);
+    }
+  }
+
+  PaddedBytes compressed;
+  modular_generic_compress(gi, &compressed, &cparams.options, 0, false);
+  if (aux_out != nullptr)
+    aux_out->layers[layer].total_bits += compressed.size() * 8;
+  writer->ZeroPadToByte();
+  *writer += compressed;
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_modular.h b/jxl/enc_modular.h
new file mode 100644
index 000000000..5207e53ff
--- /dev/null
+++ b/jxl/enc_modular.h
@@ -0,0 +1,57 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_MODULAR_H_
+#define JXL_ENC_MODULAR_H_
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/status.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_params.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+// Encodes a single frame into a byte stream using the modular image
+// sub-bitstream.
+Status EncodeModularRect(const CompressParams& params, const ImageBundle& ib,
+                         const Image3F& color, const Rect& rect,
+                         BitWriter* writer, AuxOut* aux_out);
+
+class ModularFrameEncoder {
+ public:
+  ModularFrameEncoder() {}
+  Status ComputeEncodingData(const CompressParams& orig_cparams,
+                             const FrameHeader& frame_header,
+                             const ImageBundle& ib, Image3F* JXL_RESTRICT color,
+                             PassesEncoderState* JXL_RESTRICT enc_state,
+                             bool encode_color);
+  Status EncodeGlobalInfo(BitWriter* writer, AuxOut* aux_out);
+  Status EncodeGroup(const Rect& rect, BitWriter* writer, AuxOut* aux_out,
+                     size_t minShift, size_t maxShift, size_t layer);
+
+ private:
+  Image full_image;
+  CompressParams cparams;
+  bool do_color;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_MODULAR_H_
diff --git a/jxl/enc_params.h b/jxl/enc_params.h
new file mode 100644
index 000000000..eb37cb94a
--- /dev/null
+++ b/jxl/enc_params.h
@@ -0,0 +1,233 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_PARAMS_H_
+#define JXL_ENC_PARAMS_H_
+
+// Parameters and flags that govern JXL compression.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <string>
+
+#include "jxl/base/override.h"
+#include "jxl/frame_header.h"
+#include "jxl/modular/encoding/encoding.h"
+
+namespace jxl {
+
+enum class SpeedTier {
+  // Turns on FindBestQuantizationHQ loop. Equivalent to "guetzli" mode.
+  kTortoise,
+  // Turns on FindBestQuantization butteraugli loop. Default.
+  kKitten,
+  // Turns on dots, patches, and spline detection by default, as well as full
+  // context clustering. Equivalent to "fast" mode.
+  kSquirrel,
+  // Turns on error diffusion and full AC strategy heuristics.
+  kWombat,
+  // Turns on gaborish by default, non-default cmap, initial quant field.
+  kHare,
+  // Turns on simple heuristics for AC strategy, quant field, and clustering;
+  // also enables coefficient reordering.
+  kCheetah,
+  // Turns off most encoder features, for the fastest possible encoding time.
+  kFalcon,
+};
+
+inline bool ParseSpeedTier(const std::string& s, SpeedTier* out) {
+  if (s == "falcon") {
+    *out = SpeedTier::kFalcon;
+    return true;
+  } else if (s == "cheetah") {
+    *out = SpeedTier::kCheetah;
+    return true;
+  } else if (s == "hare") {
+    *out = SpeedTier::kHare;
+    return true;
+  } else if (s == "wombat") {
+    *out = SpeedTier::kWombat;
+    return true;
+  } else if (s == "fast" || s == "squirrel") {
+    *out = SpeedTier::kSquirrel;
+    return true;
+  } else if (s == "kitten") {
+    *out = SpeedTier::kKitten;
+    return true;
+  } else if (s == "guetzli" || s == "tortoise") {
+    *out = SpeedTier::kTortoise;
+    return true;
+  }
+  return false;
+}
+
+inline const char* SpeedTierName(SpeedTier speed_tier) {
+  switch (speed_tier) {
+    case SpeedTier::kFalcon:
+      return "falcon";
+    case SpeedTier::kCheetah:
+      return "cheetah";
+    case SpeedTier::kHare:
+      return "hare";
+    case SpeedTier::kWombat:
+      return "wombat";
+    case SpeedTier::kSquirrel:
+      return "squirrel";
+    case SpeedTier::kKitten:
+      return "kitten";
+    case SpeedTier::kTortoise:
+      return "tortoise";
+  }
+}
+
+// NOLINTNEXTLINE(clang-analyzer-optin.performance.Padding)
+struct CompressParams {
+  // Only used for benchmarking (comparing vs libjpeg)
+  int jpeg_quality = 100;
+  bool jpeg_chroma_subsampling = false;
+  bool clear_metadata = false;
+
+  float butteraugli_distance = 1.0f;
+  size_t target_size = 0;
+  float target_bitrate = 0.0f;
+
+  // 0.0 means search for the adaptive quantization map that matches the
+  // butteraugli distance, positive values mean quantize everywhere with that
+  // value.
+  float uniform_quant = 0.0f;
+  float quant_border_bias = 0.0f;
+
+  // What reference frame number this should be saved as.
+  size_t save_as_reference = 0;
+
+  // Try to achieve a maximum pixel-by-pixel error on each channel.
+  bool max_error_mode = false;
+  float max_error[3] = {0.0, 0.0, 0.0};
+
+  // Encode a special DC frame (recursively, if > 1).
+  // Up to 3 pyramid levels - for up to 4096x downsampling.
+  size_t dc_level = 0;
+
+  SpeedTier speed_tier = SpeedTier::kKitten;
+
+  int max_butteraugli_iters = 4;
+
+  int max_butteraugli_iters_guetzli_mode = 100;
+
+  ColorTransform color_transform = ColorTransform::kXYB;
+  YCbCrChromaSubsampling chroma_subsampling = YCbCrChromaSubsampling::k444;
+
+  // If true, the "modular mode options" members below are used.
+  bool modular_group_mode = false;
+
+  // Use "lossless JPEG groups".
+  bool brunsli_group_mode = false;
+
+  Override preview = Override::kDefault;
+  Override noise = Override::kDefault;
+  Override dots = Override::kDefault;
+  Override patches = Override::kDefault;
+  Override adaptive_reconstruction = Override::kDefault;
+  Override gaborish = Override::kDefault;
+
+  // TODO(deymo): Remove "gradient" once all clients stop setting this value.
+  // This flag is already deprecated and is unused in the encoder.
+  Override gradient = Override::kOff;
+
+  // Progressive mode.
+  bool progressive_mode = false;
+
+  // Quantized-progressive mode.
+  bool qprogressive_mode = false;
+
+  size_t progressive_dc = 0;
+
+  // Progressive-mode saliency.
+  //
+  // How many progressive saliency-encoding steps to perform.
+  // - 1: Encode only DC and lowest-frequency AC. Does not need a saliency-map.
+  // - 2: Encode only DC+LF, dropping all HF AC data.
+  //      Does not need a saliency-map.
+  // - 3: Encode DC+LF+{salient HF}, dropping all non-salient HF data.
+  // - 4: Encode DC+LF+{salient HF}+{other HF}.
+  // - 5: Encode DC+LF+{quantized HF}+{low HF bits}.
+  size_t saliency_num_progressive_steps = 3;
+  // Every saliency-heatmap cell with saliency >= threshold will be considered
+  // as 'salient'. The default value of 0.0 will consider every AC-block
+  // as salient, hence not require a saliency-map, and not actually generate
+  // a 4th progressive step.
+  float saliency_threshold = 0.0f;
+  // Filename for the saliency-map (must be generated separately).
+  std::string saliency_map_filename;
+  // Saliency-map (owned by caller).
+  ImageF* saliency_map = nullptr;
+
+  // Input and output file name. Will be used to provide pluggable saliency
+  // extractor with paths.
+  const char* file_in = nullptr;
+  const char* file_out = nullptr;
+
+  // Prints extra information during/after encoding.
+  bool verbose = false;
+
+  // Multiplier for penalizing new HF artifacts more than blurring away
+  // features. 1.0=neutral.
+  float hf_asymmetry = 1.0f;
+
+  // Intended intensity target of the viewer after decoding, in nits (cd/m^2).
+  // There is no other way of knowing the target brightness - depends on source
+  // material. 709 typically targets 100 nits, BT.2100 PQ up to 10K, but HDR
+  // content is more typically mastered to 4K nits. The default requires no
+  // scaling for Butteraugli.
+  float intensity_target = kDefaultIntensityTarget;
+
+  float GetIntensityMultiplier() const {
+    return intensity_target * kIntensityMultiplier;
+  }
+
+  // modular mode options below
+  modular_options options;
+  int responsive = -1;
+  std::pair<float, float> quality_pair;
+  int colorspace = -1;
+  // Use Global channel palette if #colors < this percentage of range
+  float channel_colors_pre_transform_percent = 95.f;
+  // Use Local channel palette if #colors < this percentage of range
+  float channel_colors_percent = 80.f;
+  int near_lossless = 0;
+  int palette_colors = 1 << 10;  // up to 10-bit palette is probably worthwhile
+  bool ans = false;
+
+  CompressParams() {
+    quality_pair.first = 100.f;   // quality
+    quality_pair.second = 100.f;  // cquality
+    set_default_modular_options(options);
+  }
+};
+
+// Always on so we notice any changes.
+static constexpr float kMinButteraugliForAdaptiveReconstruction = 0.0f;
+static constexpr float kMinButteraugliForDots = 3.0f;
+static constexpr float kMinButteraugliToSubtractOriginalPatches = 3.0f;
+
+// Always off
+static constexpr float kMinButteraugliForNoise = 99.0f;
+
+// Minimum butteraugli distance the encoder accepts.
+static constexpr float kMinButteraugliDistance = 0.01f;
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_PARAMS_H_
diff --git a/jxl/enc_xyb.cc b/jxl/enc_xyb.cc
new file mode 100644
index 000000000..c65e31e39
--- /dev/null
+++ b/jxl/enc_xyb.cc
@@ -0,0 +1,266 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/enc_xyb.h"
+
+#ifndef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "jxl/enc_xyb.cc"
+
+#include <algorithm>
+#include <cstdlib>
+#include <hwy/runtime_dispatch.h>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/fields.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/opsin_params.h"
+
+namespace jxl {
+
+struct TestCubeRoot_T {
+  HWY_DECLARE(void, ())
+};
+
+// See enc_xyb.h.
+struct ToXYB_T {
+  HWY_DECLARE(void, (const ImageBundle& in, float linear_multiplier,
+                     ThreadPool* pool, Image3F* JXL_RESTRICT xyb,
+                     ImageBundle* JXL_RESTRICT linear = nullptr))
+};
+
+void TestCubeRoot() { hwy::TargetBitfield().Foreach(TestCubeRoot_T()); }
+
+void ToXYB(const ImageBundle& in, const float linear_multiplier,
+           ThreadPool* pool, Image3F* JXL_RESTRICT xyb,
+           ImageBundle* JXL_RESTRICT linear) {
+  Dispatch(hwy::TargetBitfield().Best(), ToXYB_T(), in, linear_multiplier, pool,
+           xyb, linear);
+}
+
+// DEPRECATED
+Image3F OpsinDynamicsImage(const Image3B& srgb8) {
+  ImageMetadata metadata;
+  metadata.bits_per_sample = 8;
+  metadata.color_encoding = ColorManagement::SRGB();
+  ImageBundle ib(&metadata);
+  ib.SetFromImage(StaticCastImage3<float>(srgb8), metadata.color_encoding);
+  JXL_CHECK(ib.TransformTo(ColorManagement::LinearSRGB(ib.IsGray())));
+  ThreadPool* null_pool = nullptr;
+  Image3F xyb(srgb8.xsize(), srgb8.ysize());
+  ToXYB(ib, 1.0f, null_pool, &xyb);
+  return xyb;
+}
+
+}  // namespace jxl
+
+#endif  // end of non-SIMD code
+
+#include <hwy/foreach_target.h>
+
+namespace jxl {
+namespace HWY_NAMESPACE {
+namespace {
+
+// Returns cbrt(x) + add with 6 ulp max error.
+// Modified from vectormath_exp.h, Apache 2 license.
+// https://www.agner.org/optimize/vectorclass.zip
+template <class V>
+HWY_ATTR V CubeRootAndAdd(const V x, const V add) {
+  const HWY_FULL(float) df;
+  const HWY_FULL(int32_t) di;
+
+  const auto kExpBias = Set(di, 0x54800000);  // cast(1.) + cast(1.) / 3
+  const auto kExpMul = Set(di, 0x002AAAAA);   // shifted 1/3
+  const auto k1_3 = Set(df, 1.0f / 3);
+  const auto k4_3 = Set(df, 4.0f / 3);
+
+  const auto xa = x;  // assume inputs never negative
+  const auto xa_3 = k1_3 * xa;
+
+  // Multiply exponent by -1/3
+  const auto m1 = BitCast(di, xa);
+  // Special case for 0. 0 is represented with an exponent of 0, so the
+  // "kExpBias - 1/3 * exp" below gives the wrong result. The IfThenZeroElse()
+  // sets those values as 0, which prevents having NaNs in the computations
+  // below.
+  const auto m2 = IfThenZeroElse(
+      m1 == Zero(di), kExpBias - (hwy::ShiftRight<23>(m1)) * kExpMul);
+  auto r = BitCast(df, m2);
+
+  // Newton-Raphson iterations
+  for (int i = 0; i < 3; i++) {
+    const auto r2 = r * r;
+    r = NegMulAdd(xa_3, r2 * r2, k4_3 * r);
+  }
+  // Final iteration
+  auto r2 = r * r;
+  r = MulAdd(k1_3, NegMulAdd(xa, r2 * r2, r), r);
+  r2 = r * r;
+  r = MulAdd(r2, x, add);
+
+  return r;
+}
+
+template <class V>
+HWY_ATTR void LinearXybTransform(const V r, V g, const V b,
+                                 float* JXL_RESTRICT valx,
+                                 float* JXL_RESTRICT valy,
+                                 float* JXL_RESTRICT valz) {
+  const HWY_FULL(float) d;
+  const V half = Set(d, 0.5f);
+  Store(half * (r - g), d, valx);
+  Store(half * (r + g), d, valy);
+  Store(b, d, valz);
+}
+
+// 4x3 matrix * 3x1 SIMD vectors
+template <class V>
+HWY_ATTR JXL_INLINE void OpsinAbsorbance(
+    const V r, const V g, const V b, const float* JXL_RESTRICT premul_absorb,
+    V* JXL_RESTRICT mixed0, V* JXL_RESTRICT mixed1, V* JXL_RESTRICT mixed2) {
+  const float* bias = &kOpsinAbsorbanceBias[0];
+  const HWY_FULL(float) d;
+  const auto m0 = Load(d, premul_absorb + 0 * d.N);
+  const auto m1 = Load(d, premul_absorb + 1 * d.N);
+  const auto m2 = Load(d, premul_absorb + 2 * d.N);
+  const auto m3 = Load(d, premul_absorb + 3 * d.N);
+  const auto m4 = Load(d, premul_absorb + 4 * d.N);
+  const auto m5 = Load(d, premul_absorb + 5 * d.N);
+  const auto m6 = Load(d, premul_absorb + 6 * d.N);
+  const auto m7 = Load(d, premul_absorb + 7 * d.N);
+  const auto m8 = Load(d, premul_absorb + 8 * d.N);
+  *mixed0 = MulAdd(m0, r, MulAdd(m1, g, MulAdd(m2, b, Set(d, bias[0]))));
+  *mixed1 = MulAdd(m3, r, MulAdd(m4, g, MulAdd(m5, b, Set(d, bias[1]))));
+  *mixed2 = MulAdd(m6, r, MulAdd(m7, g, MulAdd(m8, b, Set(d, bias[2]))));
+}
+
+// Converts one RGB vector to XYB.
+template <class V>
+HWY_ATTR void LinearToXyb(const V r, const V g, const V b,
+                          const float* JXL_RESTRICT premul_absorb,
+                          float* JXL_RESTRICT valx, float* JXL_RESTRICT valy,
+                          float* JXL_RESTRICT valz) {
+  V mixed0, mixed1, mixed2;
+  OpsinAbsorbance(r, g, b, premul_absorb, &mixed0, &mixed1, &mixed2);
+
+  // mixed* should be non-negative even for wide-gamut, so clamp to zero.
+  mixed0 = ZeroIfNegative(mixed0);
+  mixed1 = ZeroIfNegative(mixed1);
+  mixed2 = ZeroIfNegative(mixed2);
+
+  const HWY_FULL(float) d;
+  mixed0 = CubeRootAndAdd(mixed0, Load(d, premul_absorb + 9 * d.N));
+  mixed1 = CubeRootAndAdd(mixed1, Load(d, premul_absorb + 10 * d.N));
+  mixed2 = CubeRootAndAdd(mixed2, Load(d, premul_absorb + 11 * d.N));
+  LinearXybTransform(mixed0, mixed1, mixed2, valx, valy, valz);
+
+  // For wide-gamut inputs, r/g/b and valx (but not y/z) are often negative.
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+
+// Ensures infinity norm is bounded.
+HWY_ATTR void TestCubeRoot_T::HWY_FUNC() {
+  const HWY_FULL(float) d;
+  float max_err = 0.0f;
+  for (uint64_t x5 = 0; x5 < 2000000; x5++) {
+    const float x = x5 * 1E-5f;
+    const float expected = std::cbrt(x);
+    HWY_ALIGN float approx[d.N];
+    Store(HWY_NAMESPACE::CubeRootAndAdd(Set(d, x), Zero(d)), d, approx);
+
+    // All lanes are same
+    for (size_t i = 1; i < d.N; ++i) {
+      JXL_ASSERT(std::abs(approx[0] - approx[i]) <= 1.2E-7f);
+    }
+
+    const float err = std::abs(approx[0] - expected);
+    max_err = std::max(max_err, err);
+  }
+  // printf("max err %e\n", max_err);
+  JXL_ASSERT(max_err < 8E-7f);
+}
+
+// This is different from butteraugli::OpsinDynamicsImage() in the sense that
+// it does not contain a sensitivity multiplier based on the blurred image.
+HWY_ATTR void ToXYB_T::HWY_FUNC(const ImageBundle& in,
+                                const float linear_multiplier, ThreadPool* pool,
+                                Image3F* JXL_RESTRICT xyb,
+                                ImageBundle* JXL_RESTRICT linear) {
+  PROFILER_FUNC;
+
+  const size_t xsize = in.xsize();
+  const size_t ysize = in.ysize();
+  JXL_ASSERT(xyb->xsize() == xsize);
+  JXL_ASSERT(xyb->ysize() == ysize);
+
+  // Convert to linear sRGB (unless already in that space)
+  const ImageBundle* linear_srgb = &in;
+  const ColorEncoding& c = ColorManagement::LinearSRGB(in.IsGray());
+  ImageMetadata metadata;
+  metadata.color_encoding = c;
+  ImageBundle copy(&metadata);
+  JXL_CHECK(TransformIfNeeded(in, c, pool, linear == nullptr ? &copy : linear,
+                              &linear_srgb));
+  // Copy output to `linear` if TransformIfNeeded did not already do this.
+  if (linear != nullptr && linear != linear_srgb) {
+    *linear = linear_srgb->Copy();
+  }
+
+  const HWY_FULL(float) d;
+  HWY_ALIGN float premul_absorb[d.N * 12];
+  for (size_t i = 0; i < 9; ++i) {
+    const auto absorb = Set(d, kOpsinAbsorbanceMatrix[i]);
+    Store(absorb * Set(d, linear_multiplier), d, premul_absorb + i * d.N);
+  }
+  for (size_t i = 0; i < 3; ++i) {
+    const auto neg_bias_cbrt = Set(d, -std::cbrt(kOpsinAbsorbanceBias[i]));
+    Store(neg_bias_cbrt, d, premul_absorb + (9 + i) * d.N);
+  }
+
+  RunOnPool(
+      pool, 0, static_cast<int>(ysize), ThreadPool::SkipInit(),
+      [&](const int task, const int /*thread*/) HWY_ATTR {
+        const size_t y = static_cast<size_t>(task);
+        const float* JXL_RESTRICT row_in0 =
+            linear_srgb->color().ConstPlaneRow(0, y);
+        const float* JXL_RESTRICT row_in1 =
+            linear_srgb->color().ConstPlaneRow(1, y);
+        const float* JXL_RESTRICT row_in2 =
+            linear_srgb->color().ConstPlaneRow(2, y);
+        float* JXL_RESTRICT row_xyb0 = xyb->PlaneRow(0, y);
+        float* JXL_RESTRICT row_xyb1 = xyb->PlaneRow(1, y);
+        float* JXL_RESTRICT row_xyb2 = xyb->PlaneRow(2, y);
+        for (size_t x = 0; x < xsize; x += d.N) {
+          const auto in_x = Load(d, row_in0 + x);
+          const auto in_y = Load(d, row_in1 + x);
+          const auto in_b = Load(d, row_in2 + x);
+          HWY_NAMESPACE::LinearToXyb(in_x, in_y, in_b, premul_absorb,
+                                     row_xyb0 + x, row_xyb1 + x, row_xyb2 + x);
+        }
+      },
+      "ToXYB");
+}
+
+}  // namespace jxl
diff --git a/jxl/enc_xyb.h b/jxl/enc_xyb.h
new file mode 100644
index 000000000..9daf48309
--- /dev/null
+++ b/jxl/enc_xyb.h
@@ -0,0 +1,46 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENC_XYB_H_
+#define JXL_ENC_XYB_H_
+
+// Converts to XYB color space.
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+
+// Converts any color space to XYB. Copies internally to linear sRGB and
+// multiplies that by `linear_multiplier`. If `linear` is not a null pointer,
+// the pointee will contain a copy of the linear sRGB image bundle.
+// Runtime dispatch.
+void ToXYB(const ImageBundle& in, float linear_multiplier, ThreadPool* pool,
+           Image3F* JXL_RESTRICT xyb,
+           ImageBundle* JXL_RESTRICT linear = nullptr);
+
+// DEPRECATED, used by opsin_image_wrapper.
+Image3F OpsinDynamicsImage(const Image3B& srgb8);
+
+// For opsin_image_test.
+void TestCubeRoot();
+
+}  // namespace jxl
+
+#endif  // JXL_ENC_XYB_H_
diff --git a/jxl/entropy_coder.cc b/jxl/entropy_coder.cc
new file mode 100644
index 000000000..5b685a5cd
--- /dev/null
+++ b/jxl/entropy_coder.cc
@@ -0,0 +1,592 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/entropy_coder.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <algorithm>
+#include <hwy/static_targets.h>
+#include <utility>
+#include <vector>
+
+#include "jxl/ac_context.h"
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/coeff_order.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/common.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/epf.h"
+#include "jxl/image.h"
+#include "jxl/image_ops.h"
+#include "jxl/predictor.h"
+
+namespace jxl {
+namespace {
+
+constexpr size_t kNumResidualContexts = 8;
+constexpr size_t kControlFieldContexts = 4 + kNumResidualContexts;
+
+static_assert(kQuantFieldContexts == kControlFieldContexts,
+              "Invalid QF context number");
+
+// `correct` = 0..8 (number of predictors).
+static int Context(size_t correct, size_t badness) {
+  if (correct == 0) {
+    JXL_ASSERT(badness != 0);
+    badness = (badness + 1) >> 1;
+    uint32_t badness_offset =
+        std::min<size_t>(badness, kNumResidualContexts) - 1;
+    return badness_offset;
+  }
+  return kNumResidualContexts + CeilLog2Nonzero(9 - correct);
+}
+
+static int PredictionContext(size_t pred, size_t correct) {
+  return pred * kPerPredictionContexts + (correct > 5);
+}
+
+class AcStrategyCoder {
+ public:
+  using ErrorMetric = predictor::PackSigned;
+  // TODO(veluca): change predictors?
+  template <typename T>
+  using Predictor = predictor::ComputeResiduals<
+      T, ErrorMetric, predictor::Predictors1<predictor::XBPredictor>>;
+
+  struct Decoder : public Predictor<Decoder> {
+    Decoder(BitReader* JXL_RESTRICT br, ANSSymbolReader* decoder,
+            const std::vector<uint8_t>* context_map,
+            AcStrategyImage* JXL_RESTRICT ac_strategy, const Rect& rect,
+            size_t base_context, AuxOut* JXL_RESTRICT aux_out)
+        : br_(br),
+          decoder_(decoder),
+          context_map_(context_map),
+          ac_strategy_(ac_strategy),
+          rect_(rect),
+          base_context_(base_context),
+          aux_out_(aux_out) {}
+
+    Status Decode() {
+      Predictor<Decoder>::Run(rect_.xsize(), rect_.ysize(), aux_out_);
+      if (invalid_) {
+        return JXL_FAILURE("Invalid AC strategy");
+      }
+      return true;
+    }
+
+    HWY_ATTR JXL_INLINE void Prediction(
+        size_t x, size_t y, const int32_t* JXL_RESTRICT predictions,
+        const uint32_t* JXL_RESTRICT num_correct,
+        const uint32_t* JXL_RESTRICT min_error, int32_t* JXL_RESTRICT decoded) {
+      size_t ctx =
+          base_context_ + PredictionContext(predictions[0], num_correct[0]);
+
+      if (!ac_strategy_->IsValid(rect_.x0() + x, rect_.y0() + y)) {
+        uint32_t raw_strategy = decoder_->ReadSymbol((*context_map_)[ctx], br_);
+        if (!AcStrategy::IsRawStrategyValid(raw_strategy)) {
+          invalid_ = true;
+        }
+        AcStrategy acs = AcStrategy::FromRawStrategy(raw_strategy);
+        if (y + acs.covered_blocks_y() > rect_.ysize()) {
+          invalid_ = true;
+        }
+        if (x + acs.covered_blocks_x() > rect_.xsize()) {
+          invalid_ = true;
+        }
+        if (!invalid_) {
+          ac_strategy_->SetNoBoundsCheck(
+              rect_.x0() + x, rect_.y0() + y,
+              static_cast<AcStrategy::Type>(raw_strategy));
+        }
+      }
+      if (!invalid_) {
+        decoded[0] = ac_strategy_->ConstRow(rect_, y)[x].RawStrategy();
+      } else {
+        decoded[0] = 0;
+      }
+    }
+
+   private:
+    BitReader* JXL_RESTRICT br_;
+    ANSSymbolReader* decoder_;
+    const std::vector<uint8_t>* context_map_;
+    AcStrategyImage* JXL_RESTRICT ac_strategy_;
+    Rect rect_;
+    size_t base_context_;
+    AuxOut* JXL_RESTRICT aux_out_;
+    bool invalid_ = false;
+  };
+
+  struct Encoder : public Predictor<Encoder> {
+    Encoder(std::vector<Token>* JXL_RESTRICT tokens, const AcStrategyRow row,
+            size_t stride, size_t base_context, AuxOut* JXL_RESTRICT aux_out)
+        : tokens_(tokens),
+          row_(row),
+          stride_(stride),
+          base_context_(base_context),
+          aux_out_(aux_out) {}
+
+    void Encode(size_t xsize, size_t ysize) {
+      Predictor<Encoder>::Run(xsize, ysize, aux_out_);
+    }
+
+    HWY_ATTR JXL_INLINE void Prediction(
+        size_t x, size_t y, const int32_t* JXL_RESTRICT predictions,
+        const uint32_t* JXL_RESTRICT num_correct,
+        const uint32_t* JXL_RESTRICT min_error, int32_t* JXL_RESTRICT decoded) {
+      size_t ctx =
+          base_context_ + PredictionContext(predictions[0], num_correct[0]);
+      decoded[0] = row_[y * stride_ + x].RawStrategy();
+      JXL_ASSERT(AcStrategy::IsRawStrategyValid(decoded[0]));
+
+      if (row_[y * stride_ + x].IsFirstBlock()) {
+        tokens_->emplace_back(ctx, decoded[0], 0, 0);
+      }
+    }
+
+   private:
+    std::vector<Token>* JXL_RESTRICT tokens_;
+    const AcStrategyRow row_;
+    size_t stride_;
+    size_t base_context_;
+    AuxOut* JXL_RESTRICT aux_out_;
+  };
+};
+
+class QuantFieldCoder {
+ public:
+  using ErrorMetric = predictor::PackSignedRange<0, 255>;
+  // TODO(veluca): change predictors?
+  template <typename T>
+  using Predictor = predictor::ComputeResiduals<
+      T, ErrorMetric, predictor::Predictors1<predictor::XBPredictor>>;
+
+  struct Decoder : public Predictor<Decoder> {
+    Decoder(BitReader* JXL_RESTRICT br, ANSSymbolReader* decoder,
+            const std::vector<uint8_t>* context_map, int32_t* JXL_RESTRICT row,
+            size_t stride, AcStrategyRow acs_row, size_t acs_stride,
+            size_t base_context, AuxOut* JXL_RESTRICT aux_out)
+        : br_(br),
+          decoder_(decoder),
+          context_map_(context_map),
+          row_(row),
+          stride_(stride),
+          acs_row_(acs_row),
+          acs_stride_(acs_stride),
+          base_context_(base_context),
+          aux_out_(aux_out) {}
+
+    void Decode(size_t xsize, size_t ysize) {
+      Predictor<Decoder>::Run(xsize, ysize, aux_out_);
+    }
+
+    HWY_ATTR JXL_INLINE void Prediction(
+        size_t x, size_t y, const int32_t* JXL_RESTRICT predictions,
+        const uint32_t* JXL_RESTRICT num_correct,
+        const uint32_t* JXL_RESTRICT min_error, int32_t* JXL_RESTRICT decoded) {
+      size_t ctx = base_context_ + Context(num_correct[0], min_error[0]);
+
+      AcStrategy acs = acs_row_[y * acs_stride_ + x];
+      if (acs.IsFirstBlock()) {
+        uint32_t residual = ReadHybridUint(ctx, br_, decoder_, *context_map_);
+        int qf = ErrorMetric::Original(residual, predictions[0]);
+        if (qf < 0) qf = 0;
+        if (qf > 255) qf = 255;
+        for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+          for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+            row_[x + ix + (y + iy) * stride_] = qf + 1;
+          }
+        }
+      }
+      decoded[0] = row_[y * stride_ + x] - 1;
+    }
+
+   private:
+    BitReader* JXL_RESTRICT br_;
+    ANSSymbolReader* decoder_;
+    const std::vector<uint8_t>* context_map_;
+    int32_t* JXL_RESTRICT row_;
+    size_t stride_;
+    const AcStrategyRow acs_row_;
+    size_t acs_stride_;
+    size_t base_context_;
+    AuxOut* JXL_RESTRICT aux_out_;
+  };
+
+  struct Encoder : public Predictor<Encoder> {
+    Encoder(std::vector<Token>* JXL_RESTRICT tokens,
+            const int32_t* JXL_RESTRICT row, size_t stride,
+            AcStrategyRow acs_row, size_t acs_stride, size_t base_context,
+            AuxOut* JXL_RESTRICT aux_out)
+        : tokens_(tokens),
+          row_(row),
+          stride_(stride),
+          acs_row_(acs_row),
+          acs_stride_(acs_stride),
+          base_context_(base_context),
+          aux_out_(aux_out) {}
+
+    void Encode(size_t xsize, size_t ysize) {
+      Predictor<Encoder>::Run(xsize, ysize, aux_out_);
+    }
+
+    HWY_ATTR JXL_INLINE void Prediction(
+        size_t x, size_t y, const int32_t* JXL_RESTRICT predictions,
+        const uint32_t* JXL_RESTRICT num_correct,
+        const uint32_t* JXL_RESTRICT min_error, int32_t* JXL_RESTRICT decoded) {
+      size_t ctx = base_context_ + Context(num_correct[0], min_error[0]);
+
+      decoded[0] = row_[y * stride_ + x] - 1;
+
+      AcStrategy acs = acs_row_[y * acs_stride_ + x];
+      if (acs.IsFirstBlock()) {
+        uint32_t residual = ErrorMetric::Residual(decoded[0], predictions[0]);
+        TokenizeHybridUint(ctx, residual, tokens_);
+      }
+    }
+
+   private:
+    std::vector<Token>* JXL_RESTRICT tokens_;
+    const int32_t* JXL_RESTRICT row_;
+    size_t stride_;
+    const AcStrategyRow acs_row_;
+    size_t acs_stride_;
+    size_t base_context_;
+    AuxOut* JXL_RESTRICT aux_out_;
+  };
+};
+
+class ARParamsCoder {
+ public:
+  using ErrorMetric = predictor::PackSigned;
+  // TODO(veluca): change predictors?
+  template <typename T>
+  using Predictor = predictor::ComputeResiduals<
+      T, ErrorMetric, predictor::Predictors1<predictor::YPredictor>>;
+
+  struct Decoder : public Predictor<Decoder> {
+    Decoder(BitReader* JXL_RESTRICT br, ANSSymbolReader* decoder,
+            const std::vector<uint8_t>* context_map, uint8_t* JXL_RESTRICT row,
+            size_t stride, AcStrategyRow acs_row, size_t acs_stride,
+            size_t base_context, AuxOut* JXL_RESTRICT aux_out)
+        : br_(br),
+          decoder_(decoder),
+          context_map_(context_map),
+          row_(row),
+          stride_(stride),
+          acs_row_(acs_row),
+          acs_stride_(acs_stride),
+          base_context_(base_context),
+          aux_out_(aux_out) {}
+
+    void Decode(size_t xsize, size_t ysize) {
+      Predictor<Decoder>::Run(xsize, ysize, aux_out_);
+    }
+
+    HWY_ATTR JXL_INLINE void Prediction(
+        size_t x, size_t y, const int32_t* JXL_RESTRICT predictions,
+        const uint32_t* JXL_RESTRICT num_correct,
+        const uint32_t* JXL_RESTRICT min_error, int32_t* JXL_RESTRICT decoded) {
+      size_t ctx =
+          base_context_ + PredictionContext(predictions[0], num_correct[0]);
+      AcStrategy acs = acs_row_[y * acs_stride_ + x];
+      if (acs.IsFirstBlock()) {
+        uint32_t lut = decoder_->ReadSymbol((*context_map_)[ctx], br_);
+        if (lut > kNumEpfSharpness - 1) lut = kNumEpfSharpness - 1;
+        row_[y * stride_ + x] = lut;
+        ctx = base_context_ + kPerPredictionContexts * kNumEpfSharpness + lut;
+        for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+          for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+            if (ix == 0 && iy == 0) continue;
+            uint32_t new_lut = decoder_->ReadSymbol((*context_map_)[ctx], br_);
+            row_[(y + iy) * stride_ + x + ix] =
+                new_lut > kNumEpfSharpness - 1 ? kNumEpfSharpness - 1 : new_lut;
+          }
+        }
+      }
+
+      decoded[0] = row_[y * stride_ + x];
+    }
+
+   private:
+    BitReader* JXL_RESTRICT br_;
+    ANSSymbolReader* decoder_;
+    const std::vector<uint8_t>* context_map_;
+    uint8_t* JXL_RESTRICT row_;
+    size_t stride_;
+    const AcStrategyRow acs_row_;
+    size_t acs_stride_;
+    size_t base_context_;
+    AuxOut* JXL_RESTRICT aux_out_;
+  };
+
+  struct Encoder : public Predictor<Encoder> {
+    Encoder(std::vector<Token>* JXL_RESTRICT tokens,
+            const uint8_t* JXL_RESTRICT row, size_t stride,
+            AcStrategyRow acs_row, size_t acs_stride, size_t base_context,
+            AuxOut* JXL_RESTRICT aux_out)
+        : tokens_(tokens),
+          row_(row),
+          stride_(stride),
+          acs_row_(acs_row),
+          acs_stride_(acs_stride),
+          base_context_(base_context),
+          aux_out_(aux_out) {}
+
+    void Encode(size_t xsize, size_t ysize) {
+      Predictor<Encoder>::Run(xsize, ysize, aux_out_);
+    }
+
+    HWY_ATTR JXL_INLINE void Prediction(
+        size_t x, size_t y, const int32_t* JXL_RESTRICT predictions,
+        const uint32_t* JXL_RESTRICT num_correct,
+        const uint32_t* JXL_RESTRICT min_error, int32_t* JXL_RESTRICT decoded) {
+      size_t ctx =
+          base_context_ + PredictionContext(predictions[0], num_correct[0]);
+      AcStrategy acs = acs_row_[y * acs_stride_ + x];
+      if (acs.IsFirstBlock()) {
+        uint8_t lut = row_[y * stride_ + x];
+        tokens_->emplace_back(ctx, lut, 0, 0);
+        ctx = base_context_ + kPerPredictionContexts * kNumEpfSharpness + lut;
+        for (size_t iy = 0; iy < acs.covered_blocks_y(); iy++) {
+          for (size_t ix = 0; ix < acs.covered_blocks_x(); ix++) {
+            if (ix == 0 && iy == 0) continue;
+            uint8_t new_lut = row_[(y + iy) * stride_ + x + ix];
+            tokens_->emplace_back(ctx, new_lut, 0, 0);
+          }
+        }
+      }
+
+      decoded[0] = row_[y * stride_ + x];
+    }
+
+   private:
+    std::vector<Token>* JXL_RESTRICT tokens_;
+    const uint8_t* JXL_RESTRICT row_;
+    size_t stride_;
+    const AcStrategyRow acs_row_;
+    size_t acs_stride_;
+    size_t base_context_;
+    AuxOut* JXL_RESTRICT aux_out_;
+  };
+};
+
+}  // namespace
+
+void TokenizeQuantField(const Rect& rect, const ImageI& quant_field,
+                        const AcStrategyImage& ac_strategy,
+                        std::vector<Token>* JXL_RESTRICT output,
+                        size_t base_context) {
+  PROFILER_FUNC;
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+  const size_t stride = quant_field.PixelsPerRow();
+  const size_t acs_stride = ac_strategy.PixelsPerRow();
+
+  AuxOut aux_out;
+  QuantFieldCoder::Encoder enc(output, rect.ConstRow(quant_field, 0), stride,
+                               ac_strategy.ConstRow(rect, 0), acs_stride,
+                               base_context, &aux_out);
+  enc.Encode(xsize, ysize);
+}
+
+// The `rect_qf` argument specifies, in block units, the location we should
+// decode to inside the `quant_field` image, and the location we should read the
+// AC strategy from inside `ac_strategy`. It does *not* apply to the `hint`
+// argument.
+bool DecodeQuantField(BitReader* JXL_RESTRICT br,
+                      ANSSymbolReader* JXL_RESTRICT decoder,
+                      const std::vector<uint8_t>& context_map,
+                      const Rect& rect_qf,
+                      const AcStrategyImage& JXL_RESTRICT ac_strategy,
+                      ImageI* JXL_RESTRICT quant_field, size_t base_context) {
+  PROFILER_FUNC;
+  const size_t xsize = rect_qf.xsize();
+  const size_t ysize = rect_qf.ysize();
+  const size_t stride = quant_field->PixelsPerRow();
+  const size_t acs_stride = ac_strategy.PixelsPerRow();
+
+  AuxOut aux_out;
+  QuantFieldCoder::Decoder dec(
+      br, decoder, &context_map, rect_qf.Row(quant_field, 0), stride,
+      ac_strategy.ConstRow(rect_qf, 0), acs_stride, base_context, &aux_out);
+  dec.Decode(xsize, ysize);
+  return true;
+}
+
+void TokenizeAcStrategy(const Rect& rect, const AcStrategyImage& ac_strategy,
+                        std::vector<Token>* JXL_RESTRICT output,
+                        size_t base_context) {
+  PROFILER_FUNC;
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+  const size_t stride = ac_strategy.PixelsPerRow();
+
+  AuxOut aux_out;
+  AcStrategyCoder::Encoder enc(output, ac_strategy.ConstRow(rect, 0), stride,
+                               base_context, &aux_out);
+  enc.Encode(xsize, ysize);
+}
+
+bool DecodeAcStrategy(BitReader* JXL_RESTRICT br,
+                      ANSSymbolReader* JXL_RESTRICT decoder,
+                      const std::vector<uint8_t>& context_map, const Rect& rect,
+                      AcStrategyImage* JXL_RESTRICT ac_strategy,
+                      size_t base_context) {
+  PROFILER_FUNC;
+  AuxOut aux_out;
+  AcStrategyCoder::Decoder dec(br, decoder, &context_map, ac_strategy, rect,
+                               base_context, &aux_out);
+  JXL_RETURN_IF_ERROR(dec.Decode());
+  return true;
+}
+
+void TokenizeARParameters(const Rect& rect, const ImageB& epf_sharpness,
+                          const AcStrategyImage& ac_strategy,
+                          std::vector<Token>* JXL_RESTRICT output,
+                          size_t base_context) {
+  PROFILER_FUNC;
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+  const size_t stride = epf_sharpness.PixelsPerRow();
+  const size_t acs_stride = ac_strategy.PixelsPerRow();
+
+  AuxOut aux_out;
+  ARParamsCoder::Encoder enc(output, rect.ConstRow(epf_sharpness, 0), stride,
+                             ac_strategy.ConstRow(rect, 0), acs_stride,
+                             base_context, &aux_out);
+  enc.Encode(xsize, ysize);
+}
+
+bool DecodeARParameters(BitReader* br, ANSSymbolReader* decoder,
+                        const std::vector<uint8_t>& context_map,
+                        const Rect& rect, const AcStrategyImage& ac_strategy,
+                        ImageB* epf_sharpness, size_t base_context) {
+  PROFILER_FUNC;
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+  const size_t stride = epf_sharpness->PixelsPerRow();
+  const size_t acs_stride = ac_strategy.PixelsPerRow();
+
+  AuxOut aux_out;
+  ARParamsCoder::Decoder dec(
+      br, decoder, &context_map, rect.Row(epf_sharpness, 0), stride,
+      ac_strategy.ConstRow(rect, 0), acs_stride, base_context, &aux_out);
+  dec.Decode(xsize, ysize);
+  return true;
+}
+
+// The number of nonzeros of each block is predicted from the top and the left
+// blocks, with opportune scaling to take into account the number of blocks of
+// each strategy.  The predicted number of nonzeros divided by two is used as a
+// context; if this number is above 63, a specific context is used.  If the
+// number of nonzeros of a strategy is above 63, it is written directly using a
+// fixed number of bits (that depends on the size of the strategy).
+// TODO(veluca): consider predicting #zeros with predictor.h.
+void TokenizeCoefficients(const coeff_order_t* JXL_RESTRICT orders,
+                          const Rect& rect,
+                          const ac_qcoeff_t* JXL_RESTRICT* JXL_RESTRICT ac_rows,
+                          const AcStrategyImage& ac_strategy,
+                          Image3I* JXL_RESTRICT tmp_num_nzeroes,
+                          std::vector<Token>* JXL_RESTRICT output) {
+  const size_t xsize_blocks = rect.xsize();
+  const size_t ysize_blocks = rect.ysize();
+
+  // TODO(user): update the estimate: usually less coefficients are used.
+  output->reserve(output->size() +
+                  3 * xsize_blocks * ysize_blocks * kDCTBlockSize);
+
+  size_t offset = 0;
+  const size_t nzeros_stride = tmp_num_nzeroes->PixelsPerRow();
+  for (size_t by = 0; by < ysize_blocks; ++by) {
+    int32_t* JXL_RESTRICT row_nzeros[3] = {
+        tmp_num_nzeroes->PlaneRow(0, by),
+        tmp_num_nzeroes->PlaneRow(1, by),
+        tmp_num_nzeroes->PlaneRow(2, by),
+    };
+    const int32_t* JXL_RESTRICT row_nzeros_top[3] = {
+        by == 0 ? nullptr : tmp_num_nzeroes->ConstPlaneRow(0, by - 1),
+        by == 0 ? nullptr : tmp_num_nzeroes->ConstPlaneRow(1, by - 1),
+        by == 0 ? nullptr : tmp_num_nzeroes->ConstPlaneRow(2, by - 1),
+    };
+    AcStrategyRow acs_row = ac_strategy.ConstRow(rect, by);
+    for (size_t bx = 0; bx < xsize_blocks; ++bx) {
+      AcStrategy acs = acs_row[bx];
+      if (!acs.IsFirstBlock()) continue;
+      // Equal to number of LLF coefficients.
+      const size_t covered_blocks =
+          acs.covered_blocks_x() * acs.covered_blocks_y();
+      const size_t log2_covered_blocks =
+          NumZeroBitsBelowLSBNonzero(covered_blocks);
+      const size_t size = covered_blocks * kDCTBlockSize;
+
+      for (size_t c : {1, 0, 2}) {
+        size_t c_ctx = c == 1 ? 0 : 1;
+        const ac_qcoeff_t* JXL_RESTRICT block = ac_rows[c] + offset;
+        size_t nzeros = 0;
+        size_t cx = acs.covered_blocks_x();
+        size_t cy = acs.covered_blocks_y();
+        CoefficientLayout(&cy, &cx);
+        for (size_t y = 0; y < cy * kBlockDim; y++) {
+          for (size_t x = 0; x < cx * kBlockDim; x++) {
+            // Skip LLF.
+            if (y < cy && x < cx) continue;
+            nzeros += static_cast<int32_t>(block[y * cx * kBlockDim + x]) != 0;
+          }
+        }
+        for (size_t y = 0; y < acs.covered_blocks_y(); y++) {
+          for (size_t x = 0; x < acs.covered_blocks_x(); x++) {
+            row_nzeros[c][bx + x + y * nzeros_stride] =
+                (nzeros + covered_blocks - 1) >> log2_covered_blocks;
+          }
+        }
+
+        size_t ord = kStrategyOrder[acs.RawStrategy()];
+        const coeff_order_t* JXL_RESTRICT order =
+            &orders[(ord * 3 + c) * AcStrategy::kMaxCoeffArea];
+        ord = ord > 2 ? ord / 2 + 1 : ord;
+
+        int32_t predicted_nzeros =
+            PredictFromTopAndLeft(row_nzeros_top[c], row_nzeros[c], bx, 32);
+        const int32_t nzero_ctx =
+            NonZeroContext(predicted_nzeros, c_ctx * 5 + ord);
+        TokenizeHybridUint(nzero_ctx, nzeros, output);
+        const size_t histo_offset = ZeroDensityContextsOffset(c_ctx * 5 + ord);
+        // Skip LLF.
+        size_t prev = (nzeros > size / 16 ? 0 : 1);
+        for (size_t k = covered_blocks; k < size && nzeros != 0; ++k) {
+          int32_t coeff = static_cast<int32_t>(block[order[k]]);
+          size_t ctx =
+              histo_offset + ZeroDensityContext(nzeros, k, covered_blocks,
+                                                log2_covered_blocks, prev);
+          uint32_t u_coeff = PackSigned(coeff);
+          TokenizeHybridUint(ctx, u_coeff, output);
+          prev = coeff != 0;
+          nzeros -= prev;
+        }
+        JXL_ASSERT(nzeros == 0);
+      }
+      offset += size;
+    }
+  }
+}
+}  // namespace jxl
diff --git a/jxl/entropy_coder.h b/jxl/entropy_coder.h
new file mode 100644
index 000000000..a560acf2a
--- /dev/null
+++ b/jxl/entropy_coder.h
@@ -0,0 +1,267 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ENTROPY_CODER_H_
+#define JXL_ENTROPY_CODER_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+
+#include <hwy/static_targets.h>
+#include <memory>
+#include <utility>
+#include <vector>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/fast_log.h"
+#include "jxl/base/status.h"
+#include "jxl/coeff_order.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/common.h"
+#include "jxl/dct_util.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_cluster.h"
+#include "jxl/enc_context_map.h"
+#include "jxl/image.h"
+#include "jxl/quantizer.h"
+
+// Entropy coding and context modeling of DC and AC coefficients, as well as AC
+// strategy and quantization field.
+
+namespace jxl {
+
+constexpr uint32_t kNumEpfSharpness = 8;
+
+constexpr uint32_t kQuantFieldContexts = 12;
+constexpr uint32_t kPerPredictionContexts = 2;
+constexpr uint32_t kAcStrategyContexts =
+    kPerPredictionContexts * AcStrategy::kNumValidStrategies;
+constexpr uint32_t kARParamsContexts =
+    (1 + kPerPredictionContexts) * kNumEpfSharpness;
+
+constexpr uint32_t kNumControlFieldContexts =
+    kQuantFieldContexts + kAcStrategyContexts + kARParamsContexts;
+
+// Generate AC strategy tokens.
+// Only the subset "rect" [in units of blocks] within all images.
+// Appends one token per pixel to output.
+// See also DecodeAcStrategy.
+void TokenizeAcStrategy(const Rect& rect, const AcStrategyImage& ac_strategy,
+                        std::vector<Token>* JXL_RESTRICT output,
+                        size_t base_context = 0);
+
+// Generate quantization field tokens.
+// Only the subset "rect" [in units of blocks] within all images.
+// Appends one token per pixel to output.
+// TODO(user): quant field seems to be useful for all the AC strategies.
+// perhaps, we could just have different quant_ctx based on the block type.
+// See also DecodeQuantField.
+void TokenizeQuantField(const Rect& rect, const ImageI& quant_field,
+                        const AcStrategyImage& ac_strategy,
+                        std::vector<Token>* JXL_RESTRICT output,
+                        size_t base_context = 0);
+
+// Generate DCT NxN quantized AC values tokens.
+// Only the subset "rect" [in units of blocks] within all images.
+// See also DecodeACVarBlock.
+void TokenizeCoefficients(const coeff_order_t* JXL_RESTRICT orders,
+                          const Rect& rect,
+                          const ac_qcoeff_t* JXL_RESTRICT* JXL_RESTRICT ac_rows,
+                          const AcStrategyImage& ac_strategy,
+                          Image3I* JXL_RESTRICT tmp_num_nzeroes,
+                          std::vector<Token>* JXL_RESTRICT output);
+
+// Decode AC strategy. The `rect` argument does *not* apply to the hint!
+// See also TokenizeAcStrategy.
+bool DecodeAcStrategy(BitReader* JXL_RESTRICT br,
+                      ANSSymbolReader* JXL_RESTRICT decoder,
+                      const std::vector<uint8_t>& context_map, const Rect& rect,
+                      AcStrategyImage* JXL_RESTRICT ac_strategy,
+                      size_t base_context);
+
+void TokenizeARParameters(const Rect& rect, const ImageB& epf_sharpness,
+                          const AcStrategyImage& ac_strategy,
+                          std::vector<Token>* JXL_RESTRICT output,
+                          size_t base_context = 0);
+bool DecodeARParameters(BitReader* br, ANSSymbolReader* decoder,
+                        const std::vector<uint8_t>& context_map,
+                        const Rect& rect, const AcStrategyImage& ac_strategy,
+                        ImageB* epf_sharpness, size_t base_context = 0);
+
+// See TokenizeQuantField.
+bool DecodeQuantField(BitReader* JXL_RESTRICT br,
+                      ANSSymbolReader* JXL_RESTRICT decoder,
+                      const std::vector<uint8_t>& context_map,
+                      const Rect& rect_qf,
+                      const AcStrategyImage& JXL_RESTRICT ac_strategy,
+                      ImageI* JXL_RESTRICT quant_field, size_t base_context);
+
+// Encodes non-negative (X) into (2 * X), negative (-X) into (2 * X - 1)
+constexpr uint32_t PackSigned(int32_t value) {
+  return (static_cast<uint32_t>(value) << 1) ^
+         ((static_cast<uint32_t>(~value) >> 31) - 1);
+}
+
+// Reverse to PackSigned, i.e. UnpackSigned(PackSigned(X)) == X.
+constexpr intptr_t UnpackSigned(size_t value) {
+  return static_cast<intptr_t>((value >> 1) ^ (((~value) & 1) - 1));
+}
+
+// Encode non-negative integer as a pair (N, bits), where len(bits) == N.
+// 0 is encoded as (0, ''); X from range [2**N - 1, 2 * (2**N - 1)]
+// is encoded as (N, X + 1 - 2**N). In detail:
+// 0 -> (0, '')
+// 1 -> (1, '0')
+// 2 -> (1, '1')
+// 3 -> (2, '00')
+// 4 -> (2, '01')
+// 5 -> (2, '10')
+// 6 -> (2, '11')
+// 7 -> (3, '000')
+// ...
+// 65535 -> (16, '0000000000000000')
+static JXL_INLINE void EncodeVarLenUint(uint32_t value,
+                                        uint32_t* JXL_RESTRICT nbits,
+                                        uint32_t* JXL_RESTRICT bits) {
+  if (value == 0) {
+    *nbits = 0;
+    *bits = 0;
+  } else {
+    uint32_t len = FloorLog2Nonzero(value + 1);
+    *nbits = len;
+    *bits = (value + 1) & ((1 << len) - 1);
+  }
+}
+
+// Decode variable length non-negative value. Reverse to EncodeVarLenUint.
+constexpr uint32_t DecodeVarLenUint(size_t nbits, uint32_t bits) {
+  return (1u << nbits) + bits - 1;
+}
+
+// Pack signed integer and encode value.
+static JXL_INLINE void EncodeVarLenInt(int32_t value,
+                                       uint32_t* JXL_RESTRICT nbits,
+                                       uint32_t* JXL_RESTRICT bits) {
+  EncodeVarLenUint(PackSigned(value), nbits, bits);
+}
+
+// Decode value and unpack signed integer.
+constexpr int32_t DecodeVarLenInt(uint32_t nbits, uint32_t bits) {
+  return UnpackSigned(DecodeVarLenUint(nbits, bits));
+}
+
+// Experiments show that best performance is typically achieved for a
+// split-exponent of 3 or 4. Trend seems to be that '4' is better
+// for large-ish pictures, and '3' better for rather small-ish pictures.
+// This is plausible - the more special symbols we have, the better
+// statistics we need to get a benefit out of them.
+constexpr uint32_t kHybridEncodingDirectSplitExponent = 4;
+// constexpr uint32_t kHybridEncodingDirectSplitExponent = 3;
+constexpr int kHybridEncodingSplitToken = 1u
+                                          << kHybridEncodingDirectSplitExponent;
+
+// Our hybrid-encoding scheme has dedicated tokens for the smallest
+// (1 << kHybridEncodingDirectSplitExponent) numbers, and for the rest
+// encodes (number of bits) + (2 sub-leading binary digits) in the token,
+// with the remaining up to `bits` - 3 bits then being encoded as data.
+//
+// Numbers N in [0 .. kHybridEncodingSplitToken-1]:
+//   These get represented as (token=N, bits='').
+// Numbers N >= kHybridEncodingSplitToken:
+//   If n is such that 2**n <= N < 2**(n+1),
+//   and m = N - 2**n is the 'mantissa',
+//   these get represented as:
+// (token=kHybridEncodingSplitToken +
+//        ((n - kHybridEncodingDirectSplitExponent) * 4) +
+//        (m >> (n - 2)),
+//  bits=m & (1 << (n - 2)) - 1)
+// Specifically, for kHybridEncodingDirectSplitExponent = 4, i.e.
+// kHybridEncodingSplitToken=16, we would get:
+// N = 0 - 15:          (token=N, nbits=0, bits='')
+// N = 16 (10000):      (token=16, nbits=2, bits='00')
+// N = 17 (10001):      (token=16, nbits=2, bits='01')
+// N = 20 (10100):      (token=17, nbits=2, bits='00')
+// N = 24 (11000):      (token=18, nbits=2, bits='00')
+// N = 28 (11100):      (token=19, nbits=2, bits='00')
+// N = 32 (100000):     (token=20, nbits=3, bits='000')
+// N = 65535:           (token=63, nbits=13, bits='1111111111111')
+static JXL_INLINE void EncodeHybridVarLenUint(uint32_t value,
+                                              uint32_t* JXL_RESTRICT token,
+                                              uint32_t* JXL_RESTRICT nbits,
+                                              uint32_t* JXL_RESTRICT bits) {
+  if (value < kHybridEncodingSplitToken) {
+    *token = value;
+    *nbits = 0;
+    *bits = 0;
+  } else {
+    uint32_t n = FloorLog2Nonzero(value);
+    uint32_t m = value - (1 << n);
+    *token = kHybridEncodingSplitToken +
+             ((n - kHybridEncodingDirectSplitExponent) << 2) + (m >> (n - 2));
+    *nbits = n - 2;
+    *bits = value & ((1 << (n - 2)) - 1);
+  }
+}
+
+static JXL_INLINE void TokenizeHybridUint(
+    uint32_t ctx, uint32_t val, std::vector<Token>* JXL_RESTRICT tokens) {
+  uint32_t token, nbits, bits;
+  EncodeHybridVarLenUint(val, &token, &nbits, &bits);
+  JXL_ASSERT(static_cast<size_t>(nbits) <= sizeof(Token::bits) * kBitsPerByte);
+  tokens->emplace_back(ctx, token, nbits, bits);
+}
+
+static HWY_ATTR JXL_INLINE size_t
+ReadHybridUint(size_t ctx, BitReader* JXL_RESTRICT br,
+               ANSSymbolReader* JXL_RESTRICT decoder,
+               const std::vector<uint8_t>& context_map) {
+  br->Refill();  // covers ReadSymbolWithoutRefill + PeekBits
+  size_t token = decoder->ReadSymbolWithoutRefill(context_map[ctx], br);
+  // Fast-track version of hybrid integer decoding.
+  if (token < kHybridEncodingSplitToken) return token;
+  uint32_t nbits = kHybridEncodingDirectSplitExponent - 2 +
+                   ((token - kHybridEncodingSplitToken) >> 2);
+  // Max amount of bits for ReadBits is 32 and max valid left shift is 29 bits.
+  // However, for speed no error is propagated here, instead limit the nbits
+  // size. If nbits > 29, the code stream is invalid, but no error is returned.
+  nbits &= 31u;
+  const size_t bits = br->PeekBits(nbits);
+  br->Consume(nbits);
+  return ((4 | (token & 3)) << nbits) | bits;
+}
+
+static JXL_INLINE int32_t PredictFromTopAndLeft(
+    const int32_t* const JXL_RESTRICT row_top,
+    const int32_t* const JXL_RESTRICT row, size_t x, int32_t default_val) {
+  if (x == 0) {
+    return row_top == nullptr ? default_val : row_top[x];
+  }
+  if (row_top == nullptr) {
+    return row[x - 1];
+  }
+  return (row_top[x] + row[x - 1] + 1) / 2;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_ENTROPY_CODER_H_
diff --git a/jxl/entropy_coder_test.cc b/jxl/entropy_coder_test.cc
new file mode 100644
index 000000000..a96cfa07c
--- /dev/null
+++ b/jxl/entropy_coder_test.cc
@@ -0,0 +1,50 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/entropy_coder.h"
+
+#include <stdint.h>
+#include "gtest/gtest.h"
+
+namespace jxl {
+namespace {
+
+TEST(EntropyCoderTest, PackUnpack) {
+  for (int32_t i = -31; i < 32; ++i) {
+    uint32_t packed = PackSigned(i);
+    EXPECT_LT(packed, 63);
+    int32_t unpacked = UnpackSigned(packed);
+    EXPECT_EQ(i, unpacked);
+  }
+}
+
+TEST(EntropyCoderTest, EncodeDecodeVarUint) {
+  // When n == 0 there is only one, but most important case 0 <-> (0, 0).
+  for (int n = 0; n < 6; ++n) {
+    uint32_t count = 1 << n;
+    uint32_t base = count - 1;
+    for (uint32_t i = 0; i < count; ++i) {
+      uint32_t nbits = 0xFFFFFFFF;
+      uint32_t bits = 0xFFFFFFFF;
+      uint32_t value = base + i;
+      EncodeVarLenUint(value, &nbits, &bits);
+      EXPECT_EQ(n, nbits);
+      EXPECT_EQ(i, bits);
+      EXPECT_EQ(value, DecodeVarLenUint(nbits, bits));
+    }
+  }
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/epf.cc b/jxl/epf.cc
new file mode 100644
index 000000000..2e1f0e99e
--- /dev/null
+++ b/jxl/epf.cc
@@ -0,0 +1,532 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Edge-preserving smoothing: 7x8 weighted average based on L1 patch similarity.
+
+#include "jxl/epf.h"
+
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <atomic>
+#include <hwy/static_targets.h>
+#include <mutex>
+#include <numeric>  // std::accumulate
+#include <vector>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/convolve.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/loop_filter.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+using DF = HWY_CAPPED(float, 8);
+using DU = HWY_CAPPED(uint32_t, 8);
+
+using hwy::ext::AbsDiff;
+
+// kInvSigmaNum / 0.3
+constexpr float kMinSigma = -3.90524291751269967465540850526868f;
+
+DF df;
+
+HWY_ATTR JXL_INLINE hwy::VT<DF> Weight(hwy::VT<DF> sad, hwy::VT<DF> inv_sigma,
+                                       hwy::VT<DF> thres) {
+  auto v = MulAdd(sad, inv_sigma, Set(DF(), 1.0f));
+  auto v2 = v * v;
+  return hwy::IfThenZeroElse(v <= thres, v2);
+}
+
+template <bool aligned>
+HWY_ATTR JXL_INLINE void AddPixelStep1(
+    const float* JXL_RESTRICT* JXL_RESTRICT rows, size_t offset,
+    hwy::VT<DF> sad, size_t sad_id, hwy::VT<DF> sad_mul, hwy::VT<DF> inv_sigma,
+    const LoopFilter& lf, hwy::VT<DF>* JXL_RESTRICT X,
+    hwy::VT<DF>* JXL_RESTRICT Y, hwy::VT<DF>* JXL_RESTRICT B,
+    hwy::VT<DF>* JXL_RESTRICT w) {
+  auto cx =
+      aligned ? Load(DF(), rows[0] + offset) : LoadU(DF(), rows[0] + offset);
+  auto cy =
+      aligned ? Load(DF(), rows[1] + offset) : LoadU(DF(), rows[1] + offset);
+  auto cb =
+      aligned ? Load(DF(), rows[2] + offset) : LoadU(DF(), rows[2] + offset);
+
+  auto weight =
+      Weight(sad * sad_mul, inv_sigma, Set(df, lf.epf_pass1_zeroflush));
+  *w += weight;
+  *X = MulAdd(weight, cx, *X);
+  *Y = MulAdd(weight, cy, *Y);
+  *B = MulAdd(weight, cb, *B);
+}
+
+template <bool aligned>
+HWY_ATTR JXL_INLINE void AddPixelStep2(
+    float* JXL_RESTRICT* JXL_RESTRICT rows, size_t offset, hwy::VT<DF> rx,
+    hwy::VT<DF> ry, hwy::VT<DF> rb, hwy::VT<DF> sad_mul, hwy::VT<DF> inv_sigma,
+    const LoopFilter& lf, hwy::VT<DF>* JXL_RESTRICT X,
+    hwy::VT<DF>* JXL_RESTRICT Y, hwy::VT<DF>* JXL_RESTRICT B,
+    hwy::VT<DF>* JXL_RESTRICT w) {
+  auto cx =
+      aligned ? Load(DF(), rows[0] + offset) : LoadU(DF(), rows[0] + offset);
+  auto cy =
+      aligned ? Load(DF(), rows[1] + offset) : LoadU(DF(), rows[1] + offset);
+  auto cb =
+      aligned ? Load(DF(), rows[2] + offset) : LoadU(DF(), rows[2] + offset);
+
+  auto sad = AbsDiff(cx, rx) * Set(df, lf.epf_channel_scale[0]);
+  sad = MulAdd(AbsDiff(cy, ry), Set(df, lf.epf_channel_scale[1]), sad);
+  sad = MulAdd(AbsDiff(cb, rb), Set(df, lf.epf_channel_scale[2]), sad);
+
+  auto weight =
+      Weight(sad * sad_mul, inv_sigma, Set(df, lf.epf_pass2_zeroflush));
+
+  *w += weight;
+  *X = MulAdd(weight, cx, *X);
+  *Y = MulAdd(weight, cy, *Y);
+  *B = MulAdd(weight, cb, *B);
+}
+
+HWY_ATTR void GaborishRow(const float* JXL_RESTRICT* JXL_RESTRICT rows_in,
+                          size_t in_stride, size_t y_in, size_t x0_in,
+                          float* JXL_RESTRICT* JXL_RESTRICT rows_out,
+                          size_t out_stride, size_t y_out, size_t x0_out,
+                          size_t xsize, const float* JXL_RESTRICT gab_weights) {
+  for (size_t c = 0; c < 3; c++) {
+    const float* JXL_RESTRICT row_in = rows_in[c] + in_stride * y_in + x0_in;
+    float* JXL_RESTRICT row_out = rows_out[c] + out_stride * y_out + x0_out;
+    const auto w0 = Set(df, gab_weights[3 * c]);
+    const auto w1 = Set(df, gab_weights[3 * c + 1]);
+    const auto w2 = Set(df, gab_weights[3 * c + 2]);
+    for (size_t ix = 0; ix < xsize; ix += df.N) {
+      const auto t = Load(df, row_in - in_stride + ix);
+      const auto tl = LoadU(df, row_in - in_stride + ix - 1);
+      const auto tr = LoadU(df, row_in - in_stride + ix + 1);
+      const auto c = Load(df, row_in + ix);
+      const auto l = LoadU(df, row_in + ix - 1);
+      const auto r = LoadU(df, row_in + ix + 1);
+      const auto b = Load(df, row_in + in_stride + ix);
+      const auto bl = LoadU(df, row_in + in_stride + ix - 1);
+      const auto br = LoadU(df, row_in + in_stride + ix + 1);
+      const auto sum0 = c;
+      const auto sum1 = l + r + t + b;
+      const auto sum2 = tl + tr + bl + br;
+      auto pixels = MulAdd(sum2, w2, MulAdd(sum1, w1, sum0 * w0));
+      Store(pixels, df, row_out + ix);
+    }
+  }
+}
+
+// Step 1: 3x3 plus-shaped kernel with 5 SADs per pixel (also 3x3
+// plus-shaped).
+HWY_ATTR void Epf1Row(const LoopFilter& lf,
+                      const float* JXL_RESTRICT* JXL_RESTRICT rows_in,
+                      size_t in_stride, size_t y_in, size_t iy,
+                      const float* JXL_RESTRICT rows_sigma, size_t sigma_stride,
+                      size_t by_sigma,
+                      float* JXL_RESTRICT* JXL_RESTRICT rows_out,
+                      size_t out_stride, size_t y_out, size_t xsize) {
+  size_t ym2 = y_in < 2 ? y_in + kEpf1InputRows - 2 : y_in - 2;
+  size_t ym1 = y_in < 1 ? y_in + kEpf1InputRows - 1 : y_in - 1;
+  size_t yc = y_in;
+  size_t yp1 = y_in > kEpf1InputRows - 2 ? y_in + 1 - kEpf1InputRows : y_in + 1;
+  size_t yp2 = y_in > kEpf1InputRows - 3 ? y_in + 2 - kEpf1InputRows : y_in + 2;
+  y_out %= kEpf2InputRows;
+
+  HWY_ALIGN float sad_mul[kBlockDim] = {
+      lf.epf_border_sad_mul, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
+      lf.epf_border_sad_mul};
+
+  for (size_t x = kBlockDim - df.N; x < xsize + kBlockDim + df.N; x += df.N) {
+    size_t bx = x / kBlockDim;
+    size_t ix = x - kBlockDim * bx;
+    size_t sbx = bx + 1;
+    if (rows_sigma[by_sigma * sigma_stride + sbx] < kMinSigma) {
+      size_t dest_offset = y_out * out_stride + bx * kBlockDim + ix;
+      size_t source_offsetc = yc * in_stride + sbx * kBlockDim + ix;
+      for (size_t c = 0; c < 3; c++) {
+        auto px = Load(df, rows_in[c] + source_offsetc);
+        Store(px, df, rows_out[c] + dest_offset);
+      }
+      continue;
+    }
+    const auto inv_sigma = Set(DF(), rows_sigma[by_sigma * sigma_stride + sbx]);
+
+    const auto sm = iy == 0 || iy == kBlockDim - 1
+                        ? Set(df, lf.epf_border_sad_mul)
+                        : Load(df, sad_mul + ix);
+    size_t source_offsetm2 = ym2 * in_stride + sbx * kBlockDim + ix;
+    size_t source_offsetm1 = ym1 * in_stride + sbx * kBlockDim + ix;
+    size_t source_offsetc = yc * in_stride + sbx * kBlockDim + ix;
+    size_t source_offsetp1 = yp1 * in_stride + sbx * kBlockDim + ix;
+    size_t source_offsetp2 = yp2 * in_stride + sbx * kBlockDim + ix;
+    size_t dest_offset = y_out * out_stride + bx * kBlockDim + ix;
+
+    auto sad0 = Zero(df);
+    auto sad1 = Zero(df);
+    auto sad2 = Zero(df);
+    auto sad3 = Zero(df);
+
+    // compute sads
+    for (size_t c = 0; c < 3; c++) {
+      // center px = 22, px above = 21
+      auto t = Undefined(df);
+
+      const auto p20 = Load(df, rows_in[c] + source_offsetm2);
+      const auto p21 = Load(df, rows_in[c] + source_offsetm1);
+      auto sad0c = AbsDiff(p20, p21);  // SAD 2, 1
+
+      const auto p11 = LoadU(df, rows_in[c] + source_offsetm1 - 1);
+      auto sad1c = AbsDiff(p11, p21);  // SAD 1, 2
+
+      const auto p31 = LoadU(df, rows_in[c] + source_offsetm1 + 1);
+      auto sad2c = AbsDiff(p31, p21);  // SAD 3, 2
+
+      const auto p02 = LoadU(df, rows_in[c] + source_offsetc - 2);
+      const auto p12 = LoadU(df, rows_in[c] + source_offsetc - 1);
+      sad1c += AbsDiff(p02, p12);  // SAD 1, 2
+      sad0c += AbsDiff(p11, p12);  // SAD 2, 1
+
+      const auto p22 = LoadU(df, rows_in[c] + source_offsetc);
+      t = AbsDiff(p12, p22);
+      sad1c += t;  // SAD 1, 2
+      sad2c += t;  // SAD 3, 2
+      t = AbsDiff(p22, p21);
+      auto sad3c = t;  // SAD 2, 3
+      sad0c += t;      // SAD 2, 1
+
+      const auto p32 = LoadU(df, rows_in[c] + source_offsetc + 1);
+      sad0c += AbsDiff(p31, p32);  // SAD 2, 1
+      t = AbsDiff(p22, p32);
+      sad1c += t;  // SAD 1, 2
+      sad2c += t;  // SAD 3, 2
+
+      const auto p42 = LoadU(df, rows_in[c] + source_offsetc + 2);
+      sad2c += AbsDiff(p42, p32);  // SAD 3, 2
+
+      const auto p13 = LoadU(df, rows_in[c] + source_offsetp1 - 1);
+      sad3c += AbsDiff(p13, p12);  // SAD 2, 3
+
+      const auto p23 = Load(df, rows_in[c] + source_offsetp1);
+      t = AbsDiff(p22, p23);
+      sad0c += t;                  // SAD 2, 1
+      sad3c += t;                  // SAD 2, 3
+      sad1c += AbsDiff(p13, p23);  // SAD 1, 2
+
+      const auto p33 = LoadU(df, rows_in[c] + source_offsetp1 + 1);
+      sad2c += AbsDiff(p33, p23);  // SAD 3, 2
+      sad3c += AbsDiff(p33, p32);  // SAD 2, 3
+
+      const auto p24 = Load(df, rows_in[c] + source_offsetp2);
+      sad3c += AbsDiff(p24, p23);  // SAD 2, 3
+
+      auto scale = Set(df, lf.epf_channel_scale[c]);
+      sad0 = MulAdd(sad0c, scale, sad0);
+      sad1 = MulAdd(sad1c, scale, sad1);
+      sad2 = MulAdd(sad2c, scale, sad2);
+      sad3 = MulAdd(sad3c, scale, sad3);
+    }
+    const auto x_cc = Load(df, rows_in[0] + source_offsetc);
+    const auto y_cc = Load(df, rows_in[1] + source_offsetc);
+    const auto b_cc = Load(df, rows_in[2] + source_offsetc);
+
+    auto w = Set(df, 1);
+    auto X = x_cc;
+    auto Y = y_cc;
+    auto B = b_cc;
+
+    // Top row
+    AddPixelStep1</*aligned=*/true>(rows_in, source_offsetm1, sad0, 0, sm,
+                                    inv_sigma, lf, &X, &Y, &B, &w);
+    // Center
+    AddPixelStep1</*aligned=*/false>(rows_in, source_offsetc - 1, sad1, 1, sm,
+                                     inv_sigma, lf, &X, &Y, &B, &w);
+    AddPixelStep1</*aligned=*/false>(rows_in, source_offsetc + 1, sad2, 2, sm,
+                                     inv_sigma, lf, &X, &Y, &B, &w);
+    // Bottom
+    AddPixelStep1</*aligned=*/true>(rows_in, source_offsetp1, sad3, 3, sm,
+                                    inv_sigma, lf, &X, &Y, &B, &w);
+    auto inv_w = Set(df, 1.0f) / w;
+    Store(X * inv_w, df, rows_out[0] + dest_offset);
+    Store(Y * inv_w, df, rows_out[1] + dest_offset);
+    Store(B * inv_w, df, rows_out[2] + dest_offset);
+  }
+}
+
+// Step 2: 3x3 plus-shaped kernel with a single reference pixel, ran on
+// the output of the previous step.
+HWY_ATTR void Epf2Row(const LoopFilter& lf,
+                      float* JXL_RESTRICT* JXL_RESTRICT rows_in,
+                      size_t in_stride, size_t y_in,
+                      const float* JXL_RESTRICT rows_sigma, size_t sigma_stride,
+                      size_t by_sigma,
+                      float* JXL_RESTRICT* JXL_RESTRICT rows_out,
+                      size_t out_stride, size_t y_out, size_t xsize) {
+  HWY_ALIGN float sad_mul[kBlockDim] = {
+      lf.epf_border_sad_mul, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f,
+      lf.epf_border_sad_mul};
+
+  size_t iy = y_in % kBlockDim;
+  size_t ym1 = (y_in - 1) % kEpf2InputRows;
+  size_t yc = y_in % kEpf2InputRows;
+  size_t yp1 = (y_in + 1) % kEpf2InputRows;
+
+  for (size_t x = 0; x < xsize; x += df.N) {
+    size_t bx = x / kBlockDim;
+    size_t ix = x - kBlockDim * bx;
+    size_t dbx = bx;
+    size_t obx = bx + 2;
+
+    if (rows_sigma[by_sigma * sigma_stride + obx] < kMinSigma) {
+      size_t dest_offset = y_out * out_stride + dbx * kBlockDim + ix;
+      size_t source_offsetc = yc * in_stride + (bx + 1) * kBlockDim + ix;
+      for (size_t c = 0; c < 3; c++) {
+        auto px = Load(df, rows_in[c] + source_offsetc);
+        Store(px, df, rows_out[c] + dest_offset);
+      }
+      continue;
+    }
+
+    const auto inv_sigma =
+        Set(DF(), lf.epf_pass2_sigma_scale *
+                      rows_sigma[by_sigma * sigma_stride + obx]);
+
+    const auto sm = iy == 0 || iy == kBlockDim - 1
+                        ? Set(df, lf.epf_border_sad_mul)
+                        : Load(df, sad_mul + ix);
+    size_t source_offsetm1 = ym1 * in_stride + (bx + 1) * kBlockDim + ix;
+    size_t source_offsetc = yc * in_stride + (bx + 1) * kBlockDim + ix;
+    size_t source_offsetp1 = yp1 * in_stride + (bx + 1) * kBlockDim + ix;
+    size_t dest_offset = y_out * out_stride + dbx * kBlockDim + ix;
+
+    const auto x_cc = Load(df, rows_in[0] + source_offsetc);
+    const auto y_cc = Load(df, rows_in[1] + source_offsetc);
+    const auto b_cc = Load(df, rows_in[2] + source_offsetc);
+
+    auto w = Set(df, 1);
+    auto X = x_cc;
+    auto Y = y_cc;
+    auto B = b_cc;
+
+    // Top row
+    AddPixelStep2</*aligned=*/true>(rows_in, source_offsetm1, x_cc, y_cc, b_cc,
+                                    sm, inv_sigma, lf, &X, &Y, &B, &w);
+    // Center
+    AddPixelStep2</*aligned=*/false>(rows_in, source_offsetc - 1, x_cc, y_cc,
+                                     b_cc, sm, inv_sigma, lf, &X, &Y, &B, &w);
+    AddPixelStep2</*aligned=*/false>(rows_in, source_offsetc + 1, x_cc, y_cc,
+                                     b_cc, sm, inv_sigma, lf, &X, &Y, &B, &w);
+    // Bottom
+    AddPixelStep2</*aligned=*/true>(rows_in, source_offsetp1, x_cc, y_cc, b_cc,
+                                    sm, inv_sigma, lf, &X, &Y, &B, &w);
+
+    auto inv_w = Set(df, 1.0f) / w;
+    Store(X * inv_w, df, rows_out[0] + dest_offset);
+    Store(Y * inv_w, df, rows_out[1] + dest_offset);
+    Store(B * inv_w, df, rows_out[2] + dest_offset);
+  }
+}
+
+HWY_ATTR void EdgePreservingFilter(const LoopFilter& lf, const Rect& in_rect,
+                                   const Image3F& in, const Rect& sigma_rect,
+                                   const ImageF& sigma, const Rect& out_rect,
+                                   Image3F* JXL_RESTRICT out,
+                                   Image3F* JXL_RESTRICT storage1,
+                                   Image3F* JXL_RESTRICT storage2) {
+  JXL_CHECK(SameSize(in_rect, out_rect));
+  JXL_CHECK(in_rect.xsize() == sigma_rect.xsize() * kBlockDim);
+  JXL_CHECK(in_rect.ysize() == sigma_rect.ysize() * kBlockDim);
+  JXL_CHECK(storage1->xsize() >= out_rect.xsize() + 4 * kBlockDim);
+  JXL_CHECK(storage1->ysize() >= kEpf1InputRows);
+  JXL_CHECK(storage2->xsize() >= out_rect.xsize() + 2 * kBlockDim);
+  JXL_CHECK(storage2->ysize() >= kEpf2InputRows);
+
+  const size_t xsize = in_rect.xsize();
+  const size_t ysize = in_rect.ysize();
+
+  float gab_weights[9] = {
+      1, lf.gab_x_weight1, lf.gab_x_weight2,
+      1, lf.gab_y_weight1, lf.gab_y_weight2,
+      1, lf.gab_b_weight1, lf.gab_b_weight2,
+  };
+  // Normalize
+  for (size_t c = 0; c < 3; c++) {
+    const float mul =
+        1.0f / (gab_weights[3 * c] +
+                4 * (gab_weights[3 * c + 1] + gab_weights[3 * c + 2]));
+    gab_weights[3 * c] *= mul;
+    gab_weights[3 * c + 1] *= mul;
+    gab_weights[3 * c + 2] *= mul;
+  }
+
+  const float* JXL_RESTRICT rows_sigma =
+      lf.epf ? sigma_rect.ConstRow(sigma, 0) : nullptr;
+  const size_t sigma_stride = sigma.PixelsPerRow();
+
+  // Simple case first: gaborish only, no EPF.
+  if (lf.gab && !lf.epf) {
+    PROFILER_ZONE("Gaborish");
+    const float* JXL_RESTRICT rows_in[3] = {
+        in_rect.ConstPlaneRow(in, 0, 2 * kBlockDim) + 2 * kBlockDim,
+        in_rect.ConstPlaneRow(in, 1, 2 * kBlockDim) + 2 * kBlockDim,
+        in_rect.ConstPlaneRow(in, 2, 2 * kBlockDim) + 2 * kBlockDim,
+    };
+    const size_t in_stride = in.PixelsPerRow();
+
+    float* JXL_RESTRICT rows_out[3] = {
+        out_rect.PlaneRow(out, 0, 0),
+        out_rect.PlaneRow(out, 1, 0),
+        out_rect.PlaneRow(out, 2, 0),
+    };
+    const size_t out_stride = out->PixelsPerRow();
+
+    for (size_t iy = 0; iy < ysize; iy++) {
+      GaborishRow(rows_in, in_stride, iy, 0, rows_out, out_stride, iy, 0, xsize,
+                  gab_weights);
+    }
+    return;
+  }
+  // EPF only, no gaborish.
+  if (lf.epf && !lf.gab) {
+    PROFILER_ZONE("EPF");
+
+    const float* JXL_RESTRICT rows_in[3] = {
+        in_rect.ConstPlaneRow(in, 0, 0),
+        in_rect.ConstPlaneRow(in, 1, 0),
+        in_rect.ConstPlaneRow(in, 2, 0),
+    };
+    const size_t in_stride = in.PixelsPerRow();
+
+    float* JXL_RESTRICT rows_storage[3] = {storage2->PlaneRow(0, 0),
+                                           storage2->PlaneRow(1, 0),
+                                           storage2->PlaneRow(2, 0)};
+    const size_t storage_stride = storage2->PixelsPerRow();
+    float* JXL_RESTRICT rows_out[3] = {
+        out_rect.PlaneRow(out, 0, 0),
+        out_rect.PlaneRow(out, 1, 0),
+        out_rect.PlaneRow(out, 2, 0),
+    };
+    const size_t out_stride = out->PixelsPerRow();
+
+    // First two rows.
+    Epf1Row(lf, rows_in, in_stride, 2 * kBlockDim - 1, kBlockDim - 1,
+            rows_sigma, sigma_stride, 1, rows_storage, storage_stride,
+            kBlockDim - 1, xsize);
+    Epf1Row(lf, rows_in, in_stride, 2 * kBlockDim, 0, rows_sigma, sigma_stride,
+            2, rows_storage, storage_stride, kBlockDim, xsize);
+
+    for (size_t y = 0; y < ysize; y++) {
+      size_t sy = y + 2 * kBlockDim + 1;
+      size_t dy = y + kBlockDim + 1;
+      size_t sby = sy / kBlockDim;
+      Epf1Row(lf, rows_in, in_stride, sy, sy % kBlockDim, rows_sigma,
+              sigma_stride, sby, rows_storage, storage_stride, dy, xsize);
+      size_t sy2 = y + kBlockDim;
+      size_t dy2 = y;
+      size_t sby2 = y / kBlockDim + 2;
+      Epf2Row(lf, rows_storage, storage_stride, sy2, rows_sigma, sigma_stride,
+              sby2, rows_out, out_stride, dy2, xsize);
+    }
+    return;
+  }
+
+  // Most complex case: EPF and gaborish.
+  PROFILER_ZONE("Gaborish+EPF");
+
+  // Storage areas.
+  const float* JXL_RESTRICT rows_in[3] = {
+      in_rect.ConstPlaneRow(in, 0, 0),
+      in_rect.ConstPlaneRow(in, 1, 0),
+      in_rect.ConstPlaneRow(in, 2, 0),
+  };
+  const size_t in_stride = in.PixelsPerRow();
+
+  float* JXL_RESTRICT rows_storage1[3] = {
+      storage1->PlaneRow(0, 0),
+      storage1->PlaneRow(1, 0),
+      storage1->PlaneRow(2, 0),
+  };
+  const size_t storage1_stride = storage1->PixelsPerRow();
+
+  float* JXL_RESTRICT rows_storage2[3] = {storage2->PlaneRow(0, 0),
+                                          storage2->PlaneRow(1, 0),
+                                          storage2->PlaneRow(2, 0)};
+  const size_t storage2_stride = storage2->PixelsPerRow();
+
+  float* JXL_RESTRICT rows_out[3] = {
+      out_rect.PlaneRow(out, 0, 0),
+      out_rect.PlaneRow(out, 1, 0),
+      out_rect.PlaneRow(out, 2, 0),
+  };
+  const size_t out_stride = out->PixelsPerRow();
+
+  constexpr size_t kNumXBorderPixels = DivCeil(3, df.N) * df.N;
+  constexpr size_t x0 = 2 * kBlockDim - kNumXBorderPixels;
+
+  // First rows.
+  for (size_t iy = kBlockDim + 5; iy < kBlockDim + 10; iy++) {
+    GaborishRow(rows_in, in_stride, iy, x0, rows_storage1, storage1_stride,
+                iy % kEpf1InputRows, x0, xsize + 2 * kNumXBorderPixels,
+                gab_weights);
+  }
+  Epf1Row(lf,
+          const_cast<const float * JXL_RESTRICT * JXL_RESTRICT>(rows_storage1),
+          storage1_stride, (2 * kBlockDim - 1) % kEpf1InputRows, kBlockDim - 1,
+          rows_sigma, sigma_stride, 1, rows_storage2, storage2_stride,
+          kBlockDim - 1, xsize);
+
+  GaborishRow(rows_in, in_stride, kBlockDim + 10, x0, rows_storage1,
+              storage1_stride, (kBlockDim + 10) % kEpf1InputRows, x0,
+              xsize + 2 * kNumXBorderPixels, gab_weights);
+  Epf1Row(lf,
+          const_cast<const float * JXL_RESTRICT * JXL_RESTRICT>(rows_storage1),
+          storage1_stride, (2 * kBlockDim) % kEpf1InputRows, 0, rows_sigma,
+          sigma_stride, 2, rows_storage2, storage2_stride, kBlockDim, xsize);
+
+  for (size_t y = 0; y < ysize; y++) {
+    size_t sy_gab = y + kBlockDim + 11;
+    size_t dy_gab = y + kBlockDim + 11;
+    GaborishRow(rows_in, in_stride, sy_gab, x0, rows_storage1, storage1_stride,
+                dy_gab % kEpf1InputRows, x0, xsize + 2 * kNumXBorderPixels,
+                gab_weights);
+    size_t sy = y + 2 * kBlockDim + 1;
+    size_t dy = y + kBlockDim + 1;
+    size_t sby = sy / kBlockDim;
+    Epf1Row(
+        lf,
+        const_cast<const float * JXL_RESTRICT * JXL_RESTRICT>(rows_storage1),
+        storage1_stride, sy % kEpf1InputRows, sy % kBlockDim, rows_sigma,
+        sigma_stride, sby, rows_storage2, storage2_stride, dy, xsize);
+    size_t sy2 = y + kBlockDim;
+    size_t dy2 = y;
+    size_t sby2 = y / kBlockDim + 2;
+    Epf2Row(lf, rows_storage2, storage2_stride, sy2, rows_sigma, sigma_stride,
+            sby2, rows_out, out_stride, dy2, xsize);
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/epf.h b/jxl/epf.h
new file mode 100644
index 000000000..746e450fb
--- /dev/null
+++ b/jxl/epf.h
@@ -0,0 +1,54 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EPF_H_
+#define JXL_EPF_H_
+
+// Fast SIMD edge preserving filter (adaptive, nonlinear).
+// "In-loop" filter: edge-preserving filter + quantization constraint.
+
+#include <stddef.h>
+
+#include <hwy/runtime_targets.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/dec_cache.h"
+
+namespace jxl {
+
+static_assert(kNumEpfSharpness <= LoopFilter::kEpfSharpEntries, "Mismatch");
+
+// 4 * (sqrt(0.5)-1), so that Weight(sigma) = 0.5.
+static constexpr float kInvSigmaNum = -1.1715728752538099024f;
+
+// Applies gaborish + EPF to the given `in_rect` part of the input image `in`,
+// storing the result in the `out_rect` rect of the output image and reading
+// sigma values from the `sigma_rect` portion of `sigma`. `in` and `sigma` are
+// assumed to be padded with two blocks worth of data on each side (with only
+// the innermost 3 pixels of the padding needing to be valid), and the
+// corresponding rects should ignore this padding. `storage1` should be at least
+// as wide as the output rect plus two blocks of padding on each side and should
+// have at least 7 rows, while `storage2` should be at least as wide as the
+// output rect plus one block of padding on each side and should have at least 3
+// rows.
+HWY_ATTR void EdgePreservingFilter(const LoopFilter& lf, const Rect& in_rect,
+                                   const Image3F& in, const Rect& sigma_rect,
+                                   const ImageF& sigma, const Rect& out_rect,
+                                   Image3F* JXL_RESTRICT out,
+                                   Image3F* JXL_RESTRICT storage1,
+                                   Image3F* JXL_RESTRICT storage2);
+
+}  // namespace jxl
+
+#endif  // JXL_EPF_H_
diff --git a/jxl/epf_dispatch.cc b/jxl/epf_dispatch.cc
new file mode 100644
index 000000000..4a002cdaf
--- /dev/null
+++ b/jxl/epf_dispatch.cc
@@ -0,0 +1,30 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/epf_dispatch.h"
+
+#include <hwy/runtime_dispatch.h>
+
+#include "jxl/epf.h"
+
+namespace jxl {
+
+void DoAdaptiveReconstruction(Image3F* JXL_RESTRICT out,
+                              PassesDecoderState* JXL_RESTRICT decoder_state,
+                              ThreadPool* pool) {
+  const hwy::Target target = hwy::TargetBitfield().Best();
+  Dispatch(target, AdaptiveReconstruction(), out, decoder_state, pool);
+}
+
+}  // namespace jxl
diff --git a/jxl/epf_test.cc b/jxl/epf_test.cc
new file mode 100644
index 000000000..e69de29bb
diff --git a/jxl/external_image.cc b/jxl/external_image.cc
new file mode 100644
index 000000000..317533ebe
--- /dev/null
+++ b/jxl/external_image.cc
@@ -0,0 +1,1300 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/external_image.h"
+
+#include <string.h>
+
+#include <algorithm>
+#include <array>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/byte_order.h"
+#include "jxl/base/cache_aligned.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+
+namespace jxl {
+namespace {
+
+#define JXL_EXT_VERBOSE 0
+
+#if JXL_EXT_VERBOSE >= 2
+// For printing RGB values at this X within each line.
+constexpr size_t kX = 1;
+#endif
+
+// Encoding ImageBundle using other codecs requires format conversions to their
+// "External" representation:
+// ImageBundle -[1]-> Temp01 -[CMS]-> Temp01 -[2dt]-> External
+// For External -> ImageBundle, we need only demux and rescale.
+//
+// "Temp01" and "Temp255" are interleaved and have 1 or 3 non-alpha channels.
+// Alpha is included in External but not Temp because it is neither color-
+// transformed nor included in Image3F.
+// "ImageBundle" is Image3F (range [0, 255]) + ImageU alpha.
+//
+// "Temp01" is in range float [0, 1] as required by the CMS, but cannot
+// losslessly represent 8-bit integer values [0, 255] due to floating point
+// precision, which will reflect as a loss in Image3F which uses float range
+// [0, 255] instead, which may cause effects on butteraugli score. Therefore,
+// only use Temp01 if CMS transformation to different color space is required.
+//
+// "Temp255" is in range float [0, 255] and can losslessly represent 8-bit
+// integer values [0, 255], but has floating point loss for 16-bit integer
+// values [0, 65535]. The latter is not an issue however since Image3F uses
+// float [0, 255] so has the same loss (so no butteraugli score effect), and
+// the loss is gone when outputting to external integer again.
+//
+// Summary of formats:
+// Name        |   Bits  |    Max   | Channels |   Layout    |  Alpha
+// ------------+---------+----------+----------+-------------+---------
+// External    | 8,16,32 | 2^Bits-1 |  1,2,3,4 | Interleaved | Included
+// Temp01      |    32   |     1    |    1,3   | Interleaved | Separate
+// Temp255     |    32   |    255   |    1,3   | Interleaved | Separate
+// ImageBundle |    32   |    255   |    3,4   |   Planar    |  ImageU
+
+// Number of external channels including alpha.
+struct Channels1 {
+  static const char* Name() { return "1"; }
+};
+struct Channels2 {
+  static const char* Name() { return "2"; }
+};
+struct Channels3 {
+  static const char* Name() { return "3"; }
+};
+struct Channels4 {
+  static const char* Name() { return "4"; }
+};
+
+// Step 1: interleaved <-> planar and rescale [0, 1] <-> [0, 255]
+struct Interleave {
+  static JXL_INLINE void Image3ToTemp01(Channels1 /*tag*/, const size_t y,
+                                        const Image3F& image, const Rect& rect,
+                                        float* JXL_RESTRICT row_temp) {
+    const float* JXL_RESTRICT row_image1 = rect.ConstPlaneRow(image, 1, y);
+    for (size_t x = 0; x < rect.xsize(); ++x) {
+      row_temp[x] = row_image1[x] * (1.0f / 255);
+    }
+  }
+
+  static JXL_INLINE void Image3ToTemp01(Channels3 /*tag*/, const size_t y,
+                                        const Image3F& image, const Rect& rect,
+                                        float* JXL_RESTRICT row_temp) {
+    const float* JXL_RESTRICT row_image0 = rect.ConstPlaneRow(image, 0, y);
+    const float* JXL_RESTRICT row_image1 = rect.ConstPlaneRow(image, 1, y);
+    const float* JXL_RESTRICT row_image2 = rect.ConstPlaneRow(image, 2, y);
+    for (size_t x = 0; x < rect.xsize(); ++x) {
+      row_temp[3 * x + 0] = row_image0[x] * (1.0f / 255);
+      row_temp[3 * x + 1] = row_image1[x] * (1.0f / 255);
+      row_temp[3 * x + 2] = row_image2[x] * (1.0f / 255);
+    }
+  }
+
+  // Same implementation for 2/4 because neither Image3 nor Temp have alpha.
+  static JXL_INLINE void Image3ToTemp01(Channels2 /*tag*/, const size_t y,
+                                        const Image3F& image, const Rect& rect,
+                                        float* JXL_RESTRICT row_temp) {
+    Image3ToTemp01(Channels1(), y, image, rect, row_temp);
+  }
+
+  static JXL_INLINE void Image3ToTemp01(Channels4 /*tag*/, const size_t y,
+                                        const Image3F& image, const Rect& rect,
+                                        float* JXL_RESTRICT row_temp) {
+    Image3ToTemp01(Channels3(), y, image, rect, row_temp);
+  }
+
+  static JXL_INLINE void Temp255ToImage3(Channels1 /*tag*/,
+                                         const float* JXL_RESTRICT row_temp,
+                                         size_t y,
+                                         const Image3F* JXL_RESTRICT image) {
+    const size_t xsize = image->xsize();
+    float* JXL_RESTRICT row0 = const_cast<float*>(image->PlaneRow(0, y));
+    for (size_t x = 0; x < xsize; ++x) {
+      row0[x] = row_temp[x];
+    }
+
+    for (size_t c = 1; c < 3; ++c) {
+      float* JXL_RESTRICT row = const_cast<float*>(image->PlaneRow(c, y));
+      memcpy(row, row0, xsize * sizeof(float));
+    }
+  }
+
+  static JXL_INLINE void Temp255ToImage3(Channels3 /*tag*/,
+                                         const float* JXL_RESTRICT row_temp,
+                                         size_t y,
+                                         const Image3F* JXL_RESTRICT image) {
+    float* JXL_RESTRICT row_image0 = const_cast<float*>(image->PlaneRow(0, y));
+    float* JXL_RESTRICT row_image1 = const_cast<float*>(image->PlaneRow(1, y));
+    float* JXL_RESTRICT row_image2 = const_cast<float*>(image->PlaneRow(2, y));
+    for (size_t x = 0; x < image->xsize(); ++x) {
+      row_image0[x] = row_temp[3 * x + 0];
+      row_image1[x] = row_temp[3 * x + 1];
+      row_image2[x] = row_temp[3 * x + 2];
+    }
+  }
+
+  static JXL_INLINE void Temp255ToImage3(Channels2 /*tag*/,
+                                         const float* JXL_RESTRICT row_temp,
+                                         size_t y,
+                                         const Image3F* JXL_RESTRICT image) {
+    Temp255ToImage3(Channels1(), row_temp, y, image);
+  }
+
+  static JXL_INLINE void Temp255ToImage3(Channels4 /*tag*/,
+                                         const float* JXL_RESTRICT row_temp,
+                                         size_t y,
+                                         const Image3F* JXL_RESTRICT image) {
+    Temp255ToImage3(Channels3(), row_temp, y, image);
+  }
+};
+
+// Step 2t: type conversion
+
+// Same naming convention as Image: B=u8, U=u16, F=f32. kSize enables generic
+// functions with Type and Order template arguments.
+struct TypeB {
+  static const char* Name() { return "B"; }
+  static constexpr size_t kSize = 1;
+  static constexpr uint16_t kMaxAlpha = 0xFF;
+};
+struct TypeU {
+  static const char* Name() { return "U"; }
+  static constexpr size_t kSize = 2;
+  static constexpr uint16_t kMaxAlpha = 0xFFFF;
+};
+struct TypeF {
+  static const char* Name() { return "F"; }
+  static constexpr size_t kSize = 4;
+  static constexpr uint16_t kMaxAlpha = 0xFFFF;
+};
+
+// Load/stores float "sample" (gray/color) from/to u8/u16/float.
+struct Sample {
+  template <class Order>
+  static JXL_INLINE float FromExternal(TypeB /*tag*/, const uint8_t* external) {
+    return *external;
+  }
+
+  template <class Order>
+  static JXL_INLINE float FromExternal(TypeU /*tag*/, const uint8_t* external) {
+    return Load16(Order(), external);
+  }
+
+  template <class Order>
+  static JXL_INLINE float FromExternal(TypeF /*tag*/, const uint8_t* external) {
+    const int32_t bits = Load32(Order(), external);
+    float sample;
+    memcpy(&sample, &bits, 4);
+    return sample;
+  }
+
+  template <class Order>
+  static JXL_INLINE void ToExternal(TypeB /*tag*/, const float sample,
+                                    uint8_t* external) {
+    JXL_ASSERT(0 <= sample && sample < 256);
+    // Don't need std::round since sample value is positive.
+    *external = static_cast<int>(sample + 0.5f);
+  }
+
+  template <class Order>
+  static JXL_INLINE void ToExternal(TypeU /*tag*/, const float sample,
+                                    uint8_t* external) {
+    JXL_ASSERT(0 <= sample && sample < 65536);
+    // Don't need std::round since sample value is positive.
+    Store16(Order(), static_cast<int>(sample + 0.5f), external);
+  }
+
+  template <class Order>
+  static JXL_INLINE void ToExternal(TypeF /*tag*/, const float sample,
+                                    uint8_t* external) {
+    int32_t bits;
+    memcpy(&bits, &sample, 4);
+    Store32(Order(), bits, external);
+  }
+};
+
+// Load/stores uint32_t (8/16-bit range) "alpha" from/to u8/u16. Lossless.
+struct Alpha {
+  // Per-thread alpha statistics.
+  struct Stats {
+    // Bitwise AND of all alpha values; used to detect all-opaque alpha.
+    uint32_t and_bits = 0xFFFF;
+
+    // Bitwise OR; used to detect out of bounds values (i.e. > 255 for 8-bit).
+    uint32_t or_bits = 0;
+
+    // Prevents false sharing.
+    uint8_t pad[CacheAligned::kAlignment - sizeof(and_bits) - sizeof(or_bits)];
+  };
+
+  static JXL_INLINE uint32_t FromExternal(TypeB /*tag*/, OrderLE /*tag*/,
+                                          const uint8_t* external) {
+    return *external;
+  }
+
+  // Any larger type implies 16-bit alpha. NOTE: if TypeF, the alpha is smaller
+  // than other external values (subsequent bytes are uninitialized/ignored).
+  template <typename Type, class Order>
+  static JXL_INLINE uint32_t FromExternal(Type /*tag*/, Order /*tag*/,
+                                          const uint8_t* external) {
+    const uint32_t alpha = Load16(Order(), external);
+    return alpha;
+  }
+
+  static JXL_INLINE void ToExternal(TypeB /*tag*/, OrderLE /*tag*/,
+                                    const uint32_t alpha, uint8_t* external) {
+    JXL_ASSERT(alpha < 256);
+    *external = alpha;
+  }
+
+  // Any larger type implies 16-bit alpha. NOTE: if TypeF, the alpha is smaller
+  // than other external values (subsequent bytes are uninitialized/ignored).
+  template <typename Type, class Order>
+  static JXL_INLINE void ToExternal(Type /*tag*/, Order /*tag*/,
+                                    const uint32_t alpha, uint8_t* external) {
+    Store16(Order(), alpha, external);
+  }
+};
+
+// Step 2d: demux external into separate (type-converted) color and alpha.
+// Supports Temp01 and Temp255, the Cast decides this.
+struct Demux {
+  // 1 plane - copy all.
+  template <class Type, class Order, class Cast>
+  static JXL_INLINE void ExternalToTemp(Type type, Order order,
+                                        Channels1 /*tag*/, const size_t xsize,
+                                        const uint8_t* external,
+                                        const Cast cast,
+                                        float* JXL_RESTRICT row_temp) {
+    for (size_t x = 0; x < xsize; ++x) {
+      const float rounded =
+          Sample::FromExternal<Order>(type, external + x * Type::kSize);
+      row_temp[x] = cast.FromExternal(rounded, 0);
+    }
+  }
+  template <class Type, class Order, class Cast>
+  static JXL_INLINE void TempToExternal(Type type, Order order,
+                                        Channels1 /*tag*/, const size_t xsize,
+                                        const float* JXL_RESTRICT row_temp,
+                                        const Cast cast,
+                                        uint8_t* row_external) {
+    for (size_t x = 0; x < xsize; ++x) {
+      const float sample = cast.FromTemp(row_temp[x], 0);
+      Sample::ToExternal<Order>(type, sample, row_external + x * Type::kSize);
+    }
+  }
+
+  // 2 planes - ignore alpha.
+  template <class Type, class Order, class Cast>
+  static JXL_INLINE void ExternalToTemp(Type type, Order order,
+                                        Channels2 /*tag*/, const size_t xsize,
+                                        const uint8_t* external,
+                                        const Cast cast,
+                                        float* JXL_RESTRICT row_temp) {
+    for (size_t x = 0; x < xsize; ++x) {
+      const float rounded = Sample::FromExternal<Order>(
+          type, external + (2 * x + 0) * Type::kSize);
+      row_temp[x] = cast.FromExternal(rounded, 0);
+    }
+  }
+  template <class Type, class Order, class Cast>
+  static JXL_INLINE void TempToExternal(Type type, Order order,
+                                        Channels2 /*tag*/, const size_t xsize,
+                                        const float* JXL_RESTRICT row_temp,
+                                        const Cast cast,
+                                        uint8_t* row_external) {
+    for (size_t x = 0; x < xsize; ++x) {
+      const float sample = cast.FromTemp(row_temp[x], 0);
+      Sample::ToExternal<Order>(type, sample,
+                                row_external + (2 * x + 0) * Type::kSize);
+    }
+  }
+
+  // 3 planes - copy all.
+  template <class Type, class Order, class Cast>
+  static JXL_INLINE void ExternalToTemp(Type type, Order order,
+                                        Channels3 /*tag*/, const size_t xsize,
+                                        const uint8_t* external,
+                                        const Cast cast,
+                                        float* JXL_RESTRICT row_temp) {
+    for (size_t x = 0; x < xsize; ++x) {
+      const float rounded0 = Sample::FromExternal<Order>(
+          type, external + (3 * x + 0) * Type::kSize);
+      const float rounded1 = Sample::FromExternal<Order>(
+          type, external + (3 * x + 1) * Type::kSize);
+      const float rounded2 = Sample::FromExternal<Order>(
+          type, external + (3 * x + 2) * Type::kSize);
+      row_temp[3 * x + 0] = cast.FromExternal(rounded0, 0);
+      row_temp[3 * x + 1] = cast.FromExternal(rounded1, 1);
+      row_temp[3 * x + 2] = cast.FromExternal(rounded2, 2);
+    }
+  }
+  template <class Type, class Order, class Cast>
+  static JXL_INLINE void TempToExternal(Type type, Order order,
+                                        Channels3 /*tag*/, const size_t xsize,
+                                        const float* JXL_RESTRICT row_temp,
+                                        const Cast cast,
+                                        uint8_t* row_external) {
+    for (size_t x = 0; x < xsize; ++x) {
+      const float sample0 = cast.FromTemp(row_temp[3 * x + 0], 0);
+      const float sample1 = cast.FromTemp(row_temp[3 * x + 1], 1);
+      const float sample2 = cast.FromTemp(row_temp[3 * x + 2], 2);
+      Sample::ToExternal<Order>(type, sample0,
+                                row_external + (3 * x + 0) * Type::kSize);
+      Sample::ToExternal<Order>(type, sample1,
+                                row_external + (3 * x + 1) * Type::kSize);
+      Sample::ToExternal<Order>(type, sample2,
+                                row_external + (3 * x + 2) * Type::kSize);
+    }
+  }
+
+  // 4 planes - ignore alpha.
+  template <class Type, class Order, class Cast>
+  static JXL_INLINE void ExternalToTemp(Type type, Order order,
+                                        Channels4 /*tag*/, const size_t xsize,
+                                        const uint8_t* external,
+                                        const Cast cast,
+                                        float* JXL_RESTRICT row_temp) {
+    for (size_t x = 0; x < xsize; ++x) {
+      const float rounded0 = Sample::FromExternal<Order>(
+          type, external + (4 * x + 0) * Type::kSize);
+      const float rounded1 = Sample::FromExternal<Order>(
+          type, external + (4 * x + 1) * Type::kSize);
+      const float rounded2 = Sample::FromExternal<Order>(
+          type, external + (4 * x + 2) * Type::kSize);
+      row_temp[3 * x + 0] = cast.FromExternal(rounded0, 0);
+      row_temp[3 * x + 1] = cast.FromExternal(rounded1, 1);
+      row_temp[3 * x + 2] = cast.FromExternal(rounded2, 2);
+    }
+  }
+  template <class Type, class Order, class Cast>
+  static JXL_INLINE void TempToExternal(Type type, Order order,
+                                        Channels4 /*tag*/, const size_t xsize,
+                                        const float* JXL_RESTRICT row_temp,
+                                        const Cast cast,
+                                        uint8_t* row_external) {
+    for (size_t x = 0; x < xsize; ++x) {
+      const float sample0 = cast.FromTemp(row_temp[3 * x + 0], 0);
+      const float sample1 = cast.FromTemp(row_temp[3 * x + 1], 1);
+      const float sample2 = cast.FromTemp(row_temp[3 * x + 2], 2);
+      Sample::ToExternal<Order>(type, sample0,
+                                row_external + (4 * x + 0) * Type::kSize);
+      Sample::ToExternal<Order>(type, sample1,
+                                row_external + (4 * x + 1) * Type::kSize);
+      Sample::ToExternal<Order>(type, sample2,
+                                row_external + (4 * x + 2) * Type::kSize);
+    }
+  }
+
+  // Gray only, no alpha.
+  template <class Type, class Order>
+  static JXL_INLINE void ExternalToAlpha(Type type, Order order,
+                                         Channels1 /*tag*/, const size_t xsize,
+                                         const uint8_t* external,
+                                         uint16_t* JXL_RESTRICT row_alpha,
+                                         const size_t thread,
+                                         std::vector<Alpha::Stats>* stats) {}
+  template <class Type, class Order>
+  static JXL_INLINE void AlphaToExternal(Type type, Order order,
+                                         Channels1 /*tag*/, const size_t xsize,
+                                         const uint16_t* JXL_RESTRICT row_alpha,
+                                         uint8_t* row_external) {}
+
+  // Gray + alpha.
+  template <class Type, class Order>
+  static JXL_INLINE void ExternalToAlpha(Type type, Order order,
+                                         Channels2 /*tag*/, const size_t xsize,
+                                         const uint8_t* external,
+                                         uint16_t* JXL_RESTRICT row_alpha,
+                                         const size_t thread,
+                                         std::vector<Alpha::Stats>* stats) {
+    if (row_alpha == nullptr) return;
+    uint32_t and_bits = 0xFFFF;
+    uint32_t or_bits = 0;
+    for (size_t x = 0; x < xsize; ++x) {
+      const uint32_t alpha = Alpha::FromExternal(
+          type, order, external + (2 * x + 1) * Type::kSize);
+      and_bits &= alpha;
+      or_bits |= alpha;
+      row_alpha[x] = alpha;
+    }
+    (*stats)[thread].and_bits &= and_bits;
+    (*stats)[thread].or_bits |= or_bits;
+  }
+  template <class Type, class Order>
+  static JXL_INLINE void AlphaToExternal(Type type, Order order,
+                                         Channels2 /*tag*/, const size_t xsize,
+                                         const uint16_t* JXL_RESTRICT row_alpha,
+                                         uint8_t* row_external) {
+    if (row_alpha == nullptr) {
+      for (size_t x = 0; x < xsize; ++x) {
+        Alpha::ToExternal(type, order, type.kMaxAlpha,
+                          row_external + (2 * x + 1) * Type::kSize);
+      }
+    } else {
+      for (size_t x = 0; x < xsize; ++x) {
+        Alpha::ToExternal(type, order, row_alpha[x],
+                          row_external + (2 * x + 1) * Type::kSize);
+      }
+    }
+  }
+
+  // RGB only, no alpha.
+  template <class Type, class Order>
+  static JXL_INLINE void ExternalToAlpha(Type type, Order order,
+                                         Channels3 /*tag*/, const size_t xsize,
+                                         const uint8_t* external,
+                                         uint16_t* JXL_RESTRICT row_alpha,
+                                         const size_t thread,
+                                         std::vector<Alpha::Stats>* stats) {}
+  template <class Type, class Order>
+  static JXL_INLINE void AlphaToExternal(Type type, Order order,
+                                         Channels3 /*tag*/, const size_t xsize,
+                                         const uint16_t* JXL_RESTRICT row_alpha,
+                                         uint8_t* row_external) {}
+
+  // RGBA.
+  template <class Type, class Order>
+  static JXL_INLINE void ExternalToAlpha(Type type, Order order,
+                                         Channels4 /*tag*/, const size_t xsize,
+                                         const uint8_t* external,
+                                         uint16_t* JXL_RESTRICT row_alpha,
+                                         const size_t thread,
+                                         std::vector<Alpha::Stats>* stats) {
+    if (row_alpha == nullptr) return;
+    uint32_t and_bits = 0xFFFF;
+    uint32_t or_bits = 0;
+    for (size_t x = 0; x < xsize; ++x) {
+      const uint32_t alpha = Alpha::FromExternal(
+          type, order, external + (4 * x + 3) * Type::kSize);
+      and_bits &= alpha;
+      or_bits |= alpha;
+      row_alpha[x] = alpha;
+    }
+    (*stats)[thread].and_bits &= and_bits;
+    (*stats)[thread].or_bits |= or_bits;
+  }
+  template <class Type, class Order>
+  static JXL_INLINE void AlphaToExternal(Type type, Order order,
+                                         Channels4 /*tag*/, const size_t xsize,
+                                         const uint16_t* JXL_RESTRICT row_alpha,
+                                         uint8_t* row_external) {
+    if (row_alpha == nullptr) {
+      for (size_t x = 0; x < xsize; ++x) {
+        Alpha::ToExternal(type, order, type.kMaxAlpha,
+                          row_external + (4 * x + 3) * Type::kSize);
+      }
+    } else {
+      for (size_t x = 0; x < xsize; ++x) {
+        Alpha::ToExternal(type, order, row_alpha[x],
+                          row_external + (4 * x + 3) * Type::kSize);
+      }
+    }
+  }
+};
+
+// Used to select the Transformer::DoRow overload to call.
+struct ToExternal1 {};  // first phase: store to temp and compute min/max.
+struct ToExternal2 {};  // second phase: rescale temp to external.
+struct ToExternal {};   // single-pass, only usable with CastClip.
+
+// For ToExternal - assumes known/static extents of temp values.
+struct ExtentsStatic {
+  static Status SetNumThreads(size_t /* num_threads */) { return true; }
+};
+
+// For ToExternal1 - computes extents of temp values.
+class ExtentsDynamic {
+ public:
+  ExtentsDynamic(const size_t xsize, const size_t ysize,
+                 const ColorEncoding& c_desired)
+      : temp_intervals_(c_desired.Channels()) {
+    // Store all temp pixels here, convert to external in a second phase after
+    // Finalize computes ChannelIntervals from min_max_.
+    temp_ = ImageF(xsize * temp_intervals_, ysize);
+  }
+
+  Status SetNumThreads(size_t num_threads) {
+    min_max_.resize(num_threads);
+    return true;
+  }
+
+  float* RowTemp(const size_t y) { return temp_.Row(y); }
+
+  // Row size is obtained from temp_. NOTE: clamps temp values to kMax.
+  JXL_INLINE void Update(const size_t thread, float* JXL_RESTRICT row_temp) {
+    // row_temp is interleaved - keep track of current channel.
+    size_t c = 0;
+    for (size_t i = 0; i < temp_.xsize(); ++i, ++c) {
+      if (c == temp_intervals_) c = 0;
+      if (row_temp[i] > min_max_[thread].max[c]) {
+        if (row_temp[i] > kMax) row_temp[i] = kMax;
+        min_max_[thread].max[c] = row_temp[i];
+      }
+      if (row_temp[i] < min_max_[thread].min[c]) {
+        if (row_temp[i] < -kMax) row_temp[i] = -kMax;
+        min_max_[thread].min[c] = row_temp[i];
+      }
+    }
+  }
+
+  void Finalize(CodecIntervals* temp_intervals) const {
+    // Any other ChannelInterval remains default-initialized.
+    for (size_t c = 0; c < temp_intervals_; ++c) {
+      float min = min_max_[0].min[c];
+      float max = min_max_[0].max[c];
+      for (size_t i = 1; i < min_max_.size(); ++i) {
+        min = std::min(min, min_max_[i].min[c]);
+        max = std::max(max, min_max_[i].max[c]);
+      }
+      // Update ensured these are clamped.
+      JXL_ASSERT(-kMax <= min && min <= max && max <= kMax);
+      (*temp_intervals)[c] = CodecInterval(min, max);
+    }
+  }
+
+ private:
+  // Larger values are probably invalid, so clamp to preserve some precision.
+  static constexpr float kMax = 1E10;
+
+  struct MinMax {
+    MinMax() {
+      for (size_t c = 0; c < 4; ++c) {
+        min[c] = kMax;
+        max[c] = -kMax;
+      }
+    }
+
+    float min[4];
+    float max[4];
+    // Prevents false sharing.
+    uint8_t pad[CacheAligned::kAlignment - sizeof(min) - sizeof(max)];
+  };
+
+  const size_t temp_intervals_;
+  ImageF temp_;
+  std::vector<MinMax> min_max_;
+};
+
+// For ToExternal1, which updates ExtentsDynamic without casting.
+struct CastUnused {};
+
+// Returns range of valid values for all channel.
+CodecInterval GetInterval(const size_t bits_per_sample) {
+  if (bits_per_sample == 32) {
+    // This ensures ConvertImage produces an image with the same [0, 255]
+    // range as its input, but increases round trip error by ~2x vs [0, 1].
+    return CodecInterval(0.0f, 255.0f);
+  } else {
+    const float max = (1U << bits_per_sample) - 1;
+    return CodecInterval(0, max);
+  }
+}
+
+// Lossless conversion between [0, 1] and [min, min+width]. Width is 1 or
+// > 1 ("unbounded", useful for round trip testing). This is used to scale to
+// the external type and back to the arbitrary interval.
+class CastRescale01 {
+ public:
+  static const char* Name() { return "Rescale01"; }
+  CastRescale01(const CodecIntervals& temp_intervals,
+                const CodecInterval ext_interval) {
+    for (size_t c = 0; c < 4; ++c) {
+      temp_min_[c] = temp_intervals[c].min;
+      temp_mul_[c] = ext_interval.width / temp_intervals[c].width;
+      external_min_[c] = ext_interval.min;
+      external_mul_[c] = temp_intervals[c].width / ext_interval.width;
+    }
+#if JXL_EXT_VERBOSE >= 2
+    printf("CastRescale01 min %f width %f %f\n", temp_intervals[0].min,
+           temp_intervals[0].width, ext_interval.width);
+#endif
+  }
+
+  JXL_INLINE float FromExternal(const float external, const size_t c) const {
+    return (external - external_min_[c]) * external_mul_[c] + temp_min_[c];
+  }
+  JXL_INLINE float FromTemp(const float temp, const size_t c) const {
+    return (temp - temp_min_[c]) * temp_mul_[c] + external_min_[c];
+  }
+
+ private:
+  float temp_min_[4];
+  float temp_mul_[4];
+  float external_min_[4];
+  float external_mul_[4];
+};
+
+// Lossless conversion between [0, 255] and [min, min+width]. Width is 255 or
+// > 255 ("unbounded", useful for round trip testing). This is used to scale to
+// the external type and back to the arbitrary interval.
+// NOTE: this rescaler exists to make CopyTo match the convention of
+// "temp_intervals" used by the color converting constructor. In the external to
+// IO case without color conversion, one normally does not use this parameter.
+class CastRescale255 {
+ public:
+  static const char* Name() { return "Rescale255"; }
+  CastRescale255(const CodecIntervals& temp_intervals,
+                 const CodecInterval ext_interval) {
+    for (size_t c = 0; c < 4; ++c) {
+      temp_min_[c] = 255.0f * temp_intervals[c].min;
+      temp_mul_[c] =
+          ext_interval.width / temp_intervals[c].width * (1.0f / 255);
+      external_min_[c] = ext_interval.min * (1.0f / 255);
+      external_mul_[c] = 255.0f * temp_intervals[c].width / ext_interval.width;
+    }
+#if JXL_EXT_VERBOSE >= 2
+    printf("CastRescale255 min %f width %f %f\n", temp_intervals[0].min,
+           temp_intervals[0].width, ext_interval.width);
+#endif
+  }
+
+  JXL_INLINE float FromExternal(const float external, const size_t c) const {
+    return (external - external_min_[c]) * external_mul_[c] + temp_min_[c];
+  }
+  JXL_INLINE float FromTemp(const float temp, const size_t c) const {
+    return (temp - temp_min_[c]) * temp_mul_[c] + external_min_[c];
+  }
+
+ private:
+  float temp_min_[4];
+  float temp_mul_[4];
+  float external_min_[4];
+  float external_mul_[4];
+};
+
+// Converts between [0, 1] and the external type's range. Lossy because values
+// outside [0, 1] are clamped - this is necessary for codecs that are not able
+// to store min/width metadata.
+class CastClip01 {
+ public:
+  static const char* Name() { return "Clip01"; }
+  explicit CastClip01(const CodecInterval ext_interval) {
+    for (size_t c = 0; c < 4; ++c) {
+      temp_mul_[c] = ext_interval.width;
+      external_min_[c] = ext_interval.min;
+      external_mul_[c] = 1.0f / ext_interval.width;
+    }
+#if JXL_EXT_VERBOSE >= 2
+    printf("CastClip01 width %f\n", ext_interval.width);
+#endif
+  }
+
+  JXL_INLINE float FromExternal(const float external, const size_t c) const {
+    const float temp01 = (external - external_min_[c]) * external_mul_[c];
+    return temp01;
+  }
+  JXL_INLINE float FromTemp(const float temp, const size_t c) const {
+    return Clamp01(temp) * temp_mul_[c] + external_min_[c];
+  }
+
+ private:
+  static JXL_INLINE float Clamp01(const float temp) {
+    return std::min(std::max(0.0f, temp), 1.0f);
+  }
+
+  float temp_mul_[4];
+  float external_min_[4];
+  float external_mul_[4];
+};
+
+struct CastFloat {
+  static const char* Name() { return "Float"; }
+  explicit CastFloat(const CodecInterval ext_interval) {
+    for (size_t c = 0; c < 4; ++c) {
+      JXL_CHECK(ext_interval.min == 0.0f);
+      JXL_CHECK(ext_interval.width == 255.0f);
+    }
+#if JXL_EXT_VERBOSE >= 2
+    printf("CastFloat\n");
+#endif
+  }
+
+  JXL_INLINE float FromExternal(const float external, const size_t c) const {
+    const float temp01 = external * (1.0f / 255);
+    return temp01;
+  }
+  JXL_INLINE float FromTemp(const float temp, const size_t c) const {
+    return temp * 255.0f;
+  }
+};
+
+// Converts between [0, 255] and the external type's range. Lossy because values
+// outside [0, 255] are clamped - this is necessary for codecs that are not able
+// to store min/width metadata.
+class CastClip255 {
+ public:
+  static const char* Name() { return "Clip255"; }
+  explicit CastClip255(const CodecInterval ext_interval) {
+    for (size_t c = 0; c < 4; ++c) {
+      temp_mul_[c] = ext_interval.width;
+      external_min_[c] = ext_interval.min;
+      external_mul_[c] = 255.0f / ext_interval.width;
+    }
+#if JXL_EXT_VERBOSE >= 2
+    printf("CastClip255 width %f\n", ext_interval.width);
+#endif
+  }
+
+  JXL_INLINE float FromExternal(const float external, const size_t c) const {
+    const float temp255 = (external - external_min_[c]) * external_mul_[c];
+    return temp255;
+  }
+  JXL_INLINE float FromTemp(const float temp, const size_t c) const {
+    return Clamp255(temp) * temp_mul_[c] + external_min_[c];
+  }
+
+ private:
+  static JXL_INLINE float Clamp255(const float temp) {
+    return std::min(std::max(0.0f, temp), 255.0f);
+  }
+
+  float temp_mul_[4];
+  float external_min_[4];
+  float external_mul_[4];
+};
+
+struct CastFloat01 {
+  static const char* Name() { return "Float01"; }
+  explicit CastFloat01(const CodecInterval ext_interval) {
+    for (size_t c = 0; c < 4; ++c) {
+      JXL_CHECK(ext_interval.min == 0.0f);
+      JXL_CHECK(ext_interval.width == 255.0f);
+    }
+#if JXL_EXT_VERBOSE >= 2
+    printf("CastFloat01\n");
+#endif
+  }
+
+  JXL_INLINE float FromExternal(const float external, const size_t c) const {
+    const float temp01 = external * (1.0f / 255);
+    return temp01;
+  }
+  JXL_INLINE float FromTemp(const float temp, const size_t c) const {
+    return temp * 255.0f;
+  }
+};
+
+// No-op
+struct CastFloat255 {
+  static const char* Name() { return "Float255"; }
+  explicit CastFloat255(const CodecInterval ext_interval) {
+    for (size_t c = 0; c < 4; ++c) {
+      JXL_CHECK(ext_interval.min == 0.0f);
+      JXL_CHECK(ext_interval.width == 255.0f);
+    }
+#if JXL_EXT_VERBOSE >= 2
+    printf("CastFloat255\n");
+#endif
+  }
+
+  JXL_INLINE float FromExternal(const float external, const size_t c) const {
+    return external;
+  }
+  JXL_INLINE float FromTemp(const float temp, const size_t c) const {
+    return temp;
+  }
+};
+
+// Multithreaded color space transform from IO to ExternalImage.
+class Transformer {
+ public:
+  Transformer(ThreadPool* pool, const Image3F& color, const Rect& rect,
+              const bool has_alpha, const ImageU* alpha,
+              ExternalImage* external, const ColorEncoding& c_src,
+              const ColorEncoding& c_dst)
+      : pool_(pool),
+        color_(color),
+        rect_(rect),
+        alpha_(alpha),
+        external_(external),
+        want_alpha_(has_alpha && external->HasAlpha()),
+        c_src_(c_src),
+        c_dst_(c_dst) {
+    JXL_ASSERT(rect.IsInside(color));
+    JXL_ASSERT(SameSize(rect, *external));
+  }
+
+  // Converts in the specified direction (To*).
+  template <class To, class Extent, class Cast>
+  Status Run(Extent* extents, const Cast& cast) {
+    const size_t bytes = DivCeil(external_->BitsPerSample(), kBitsPerByte);
+    const bool big_endian = external_->BigEndian();
+    if (bytes == 1) {
+      return DispatchType<To, TypeB, OrderLE>(extents, cast);
+    } else if (bytes == 2 && big_endian) {
+      return DispatchType<To, TypeU, OrderBE>(extents, cast);
+    } else if (bytes == 2) {
+      return DispatchType<To, TypeU, OrderLE>(extents, cast);
+    } else if (bytes == 4 && big_endian) {
+      return DispatchType<To, TypeF, OrderBE>(extents, cast);
+    } else if (bytes == 4) {
+      return DispatchType<To, TypeF, OrderLE>(extents, cast);
+    } else {
+      return JXL_FAILURE("Unsupported BitsPerSample");
+    }
+    return true;
+  }
+
+ private:
+  Status InitBeforeRun(size_t num_threads) {
+    if (init_called_) return true;
+    init_called_ = true;
+#if JXL_EXT_VERBOSE >= 1
+    printf("ExtImg Transformer %s->%s\n", Description(c_src_).c_str(),
+           Description(c_dst_).c_str());
+#endif
+    return transform_.Init(c_src_, c_dst_, rect_.xsize(), num_threads);
+  }
+
+  // First pass: only needed for ExtentsDynamic/CastUnused.
+  template <class Type, class Order, class Channels>
+  JXL_INLINE void DoRow(ToExternal1 /*tag*/, ExtentsDynamic* extents,
+                        const CastUnused /*unused*/, const size_t y,
+                        const size_t thread) {
+    float* JXL_RESTRICT row_temp = extents->RowTemp(y);
+
+    Interleave::Image3ToTemp01(Channels(), y, color_, rect_, row_temp);
+
+#if JXL_EXT_VERBOSE >= 2
+    const float in0 = row_temp[3 * kX + 0], in1 = row_temp[3 * kX + 1];
+    const float in2 = row_temp[3 * kX + 2];
+#endif
+
+    transform_.Run(thread, row_temp, row_temp);
+
+#if JXL_EXT_VERBOSE >= 2
+    printf("ToExt1: in %.4f %.4f %.4f; xform %.4f %.4f %.4f\n", in0, in1, in2,
+           row_temp[3 * kX + 0], row_temp[3 * kX + 1], row_temp[3 * kX + 2]);
+#endif
+
+    extents->Update(thread, row_temp);
+  }
+
+  // Second pass: only needed for ExtentsDynamic/CastRescale.
+  template <class Type, class Order, class Channels>
+  JXL_INLINE void DoRow(ToExternal2 /*tag*/, ExtentsDynamic* extents,
+                        const CastRescale01& cast, const size_t y,
+                        const size_t thread) {
+    const float* JXL_RESTRICT row_temp = extents->RowTemp(y);
+    uint8_t* JXL_RESTRICT row_external = external_->Row(y);
+    Demux::TempToExternal(Type(), Order(), Channels(), rect_.xsize(), row_temp,
+                          cast, row_external);
+
+#if JXL_EXT_VERBOSE >= 2
+    printf("ToExt2: ext %3d %3d %3d\n", row_external[3 * kX + 0],
+           row_external[3 * kX + 1], row_external[3 * kX + 2]);
+#endif
+
+    const uint16_t* JXL_RESTRICT row_alpha =
+        want_alpha_ ? alpha_->ConstRow(y) : nullptr;
+    Demux::AlphaToExternal(Type(), Order(), Channels(), rect_.xsize(),
+                           row_alpha, row_external);
+  }
+
+  // Single-pass: only works for ExtentsStatic.
+  template <class Type, class Order, class Channels, class Cast>
+  JXL_INLINE void DoRow(ToExternal /*tag*/, ExtentsStatic* /*unused*/,
+                        const Cast& cast, const size_t y, const size_t thread) {
+    float* JXL_RESTRICT row_temp = transform_.BufDst(thread);
+    Interleave::Image3ToTemp01(Channels(), y, color_, rect_, row_temp);
+
+#if JXL_EXT_VERBOSE >= 2
+    // Save inputs for printing before in-place transform overwrites them.
+    const float in0 = row_temp[3 * kX + 0];
+    const float in1 = row_temp[3 * kX + 1];
+    const float in2 = row_temp[3 * kX + 2];
+#endif
+    transform_.Run(thread, row_temp, row_temp);
+
+    uint8_t* JXL_RESTRICT row_external = external_->Row(y);
+    Demux::TempToExternal(Type(), Order(), Channels(), rect_.xsize(), row_temp,
+                          cast, row_external);
+
+#if JXL_EXT_VERBOSE >= 2
+    const float tmp0 = row_temp[3 * kX + 0];
+    const float tmp1 = row_temp[3 * kX + 1];
+    const float tmp2 = row_temp[3 * kX + 2];
+    // Convert back so we can print the external values
+    Demux::ExternalToTemp(Type(), Order(), Channels(), rect_.xsize(),
+                          row_external, cast, row_temp);
+    printf("ToExt(%s%s %s): tmp %.4f %.4f %.4f|%.4f %.4f %.4f|%.4f %.4f %.4f\n",
+           Channels::Name(), Type::Name(), Cast::Name(), in0, in1, in2, tmp0,
+           tmp1, tmp2, row_temp[3 * kX + 0], row_temp[3 * kX + 1],
+           row_temp[3 * kX + 2]);
+#endif
+
+    const uint16_t* JXL_RESTRICT row_alpha =
+        want_alpha_ ? alpha_->ConstRow(y) : nullptr;
+    Demux::AlphaToExternal(Type(), Order(), Channels(), rect_.xsize(),
+                           row_alpha, row_external);
+  }
+
+  // Closure callable by ThreadPool.
+  template <class To, class Type, class Order, class Channels, class Extent,
+            class Cast>
+  class Bind {
+   public:
+    explicit Bind(Transformer* converter, Extent* extents, const Cast& cast)
+        : xform_(converter), extents_(extents), cast_(cast) {}
+
+    JXL_INLINE void operator()(const int task, const int thread) const {
+      xform_->DoRow<Type, Order, Channels>(To(), extents_, cast_, task, thread);
+    }
+
+   private:
+    Transformer* xform_;  // not owned
+    Extent* extents_;     // not owned
+    const Cast cast_;
+  };
+
+  template <class To, class Type, class Order, class Channels, class Extent,
+            class Cast>
+  Status DoRows(Extent* extents, const Cast& cast) {
+    return RunOnPool(
+        pool_, 0, rect_.ysize(),
+        [this, extents](const size_t num_threads) {
+          return InitBeforeRun(num_threads) &&
+                 extents->SetNumThreads(num_threads);
+        },
+        Bind<To, Type, Order, Channels, Extent, Cast>(this, extents, cast),
+        "ExtImg xform");
+  }
+
+  // Calls the instantiation with the matching Type and Order.
+  template <class To, class Type, class Order, class Extent, class Cast>
+  Status DispatchType(Extent* extents, const Cast& cast) {
+    if (external_->IsGray()) {
+      if (external_->HasAlpha()) {
+        return DoRows<To, Type, Order, Channels2>(extents, cast);
+      } else {
+        return DoRows<To, Type, Order, Channels1>(extents, cast);
+      }
+    } else {
+      if (external_->HasAlpha()) {
+        return DoRows<To, Type, Order, Channels4>(extents, cast);
+      } else {
+        return DoRows<To, Type, Order, Channels3>(extents, cast);
+      }
+    }
+  }
+
+  ThreadPool* pool_;  // not owned
+  const Image3F& color_;
+  const Rect rect_;          // whence in color_ to copy, and output size.
+  const ImageU* alpha_;      // not owned
+  ExternalImage* external_;  // not owned
+
+  bool want_alpha_;
+  // Whether InitBeforeRun() was already called.
+  bool init_called_ = false;
+
+  ColorSpaceTransform transform_;
+  const ColorEncoding& c_src_;
+  const ColorEncoding& c_dst_;
+};
+
+// Multithreaded deinterleaving/conversion from ExternalImage to Image3.
+class Converter {
+ public:
+  Converter(ThreadPool* pool, const PackedImage& desc, const uint8_t* bytes)
+      : pool_(pool),
+        desc_(&desc),
+        bytes_(bytes),
+        color_(desc.xsize, desc.ysize) {}
+
+  // Run can only be called once per instance.
+  template <class Cast>
+  Status Run(const Cast& cast) {
+    const size_t bytes = DivCeil(desc_->bits_per_sample, kBitsPerByte);
+    const bool big_endian = desc_->big_endian;
+    if (bytes == 1) {
+      return DispatchType<TypeB, OrderLE>(cast);
+    } else if (bytes == 2 && big_endian) {
+      return DispatchType<TypeU, OrderBE>(cast);
+    } else if (bytes == 2) {
+      return DispatchType<TypeU, OrderLE>(cast);
+    } else if (bytes == 4 && big_endian) {
+      return DispatchType<TypeF, OrderBE>(cast);
+    } else if (bytes == 4) {
+      return DispatchType<TypeF, OrderLE>(cast);
+    } else {
+      return JXL_FAILURE("Unsupported BitsPerSample");
+    }
+    return true;
+  }
+
+  Status MoveTo(ImageBundle* ib) {
+    ib->SetFromImage(std::move(color_), desc_->c_current);
+
+    // Don't have alpha; during TransformTo, don't remove existing alpha.
+    if (alpha_stats_.empty()) return true;
+
+    const size_t max_alpha = (1 << bits_per_alpha_) - 1;
+
+    // Reduce per-thread statistics.
+    uint32_t and_bits = alpha_stats_[0].and_bits;
+    uint32_t or_bits = alpha_stats_[0].or_bits;
+    for (size_t i = 1; i < alpha_stats_.size(); ++i) {
+      and_bits &= alpha_stats_[i].and_bits;
+      or_bits |= alpha_stats_[i].or_bits;
+    }
+
+    if (or_bits > max_alpha) {
+      return JXL_FAILURE("Alpha out of range");
+    }
+
+    // Keep alpha if at least one value is (semi)transparent.
+    if (and_bits != max_alpha) {
+      ib->SetAlpha(std::move(alpha_));
+    } else {
+      ib->RemoveAlpha();
+    }
+
+    return true;
+  }
+
+ private:
+  Status InitBeforeRun(size_t num_threads) {
+    if (init_called_) return true;
+    init_called_ = true;
+
+    temp_buf_ = ImageF(desc_->xsize * desc_->c_current.Channels(), num_threads);
+
+    if (desc_->HasAlpha()) {
+      alpha_ = ImageU(desc_->xsize, desc_->ysize);
+      bits_per_alpha_ = desc_->bits_per_alpha;
+      alpha_stats_.resize(num_threads);
+    }
+    return true;
+  }
+
+  template <class Type, class Order, class Channels, class Cast>
+  JXL_INLINE void DoRow(const Cast& cast, const size_t y, const size_t thread) {
+    const size_t iy = desc_->flipped_y ? (desc_->ysize - 1 - y) : y;
+    const uint8_t* JXL_RESTRICT row_external = bytes_ + iy * desc_->row_size;
+
+    if (!alpha_stats_.empty()) {
+      // No-op if Channels1/3.
+      Demux::ExternalToAlpha(Type(), Order(), Channels(), desc_->xsize,
+                             row_external, alpha_.Row(y), thread,
+                             &alpha_stats_);
+    }
+
+    float* JXL_RESTRICT row_temp = temp_buf_.Row(thread);
+    Demux::ExternalToTemp(Type(), Order(), Channels(), desc_->xsize,
+                          row_external, cast, row_temp);
+
+#if JXL_EXT_VERBOSE >= 2
+    printf("ToIO(%s%s %s): ext %3d %3d %3d  tmp %.4f %.4f %.4f\n",
+           Channels::Name(), Type::Name(), Cast::Name(),
+           row_external[3 * kX + 0], row_external[3 * kX + 1],
+           row_external[3 * kX + 2], row_temp[3 * kX + 0], row_temp[3 * kX + 1],
+           row_temp[3 * kX + 2]);
+#endif
+
+    Interleave::Temp255ToImage3(Channels(), row_temp, y, &color_);
+  }
+
+  // Closure callable by ThreadPool.
+  template <class Type, class Order, class Channels, class Cast>
+  class Bind {
+   public:
+    explicit Bind(Converter* converter, const Cast& cast)
+        : converter_(converter), cast_(cast) {}
+
+    JXL_INLINE void operator()(const int task, const int thread) const {
+      converter_->DoRow<Type, Order, Channels>(cast_, task, thread);
+    }
+
+   private:
+    Converter* converter_;  // not owned
+    const Cast cast_;
+  };
+
+  template <class Type, class Order, class Channels, class Cast>
+  Status DoRows(const Cast& cast) {
+    return RunOnPool(
+        pool_, 0, desc_->ysize,
+        [this](const size_t num_threads) {
+          return this->InitBeforeRun(num_threads);
+        },
+        Bind<Type, Order, Channels, Cast>(this, cast), "ExtImg cvt");
+  }
+
+  // Calls the instantiation with the matching Type and Order.
+  template <class Type, class Order, class Cast>
+  Status DispatchType(const Cast& cast) {
+    if (desc_->c_current.IsGray()) {
+      if (desc_->HasAlpha()) {
+        return DoRows<Type, Order, Channels2>(cast);
+      } else {
+        return DoRows<Type, Order, Channels1>(cast);
+      }
+    } else {
+      if (desc_->HasAlpha()) {
+        return DoRows<Type, Order, Channels4>(cast);
+      } else {
+        return DoRows<Type, Order, Channels3>(cast);
+      }
+    }
+  }
+
+  ThreadPool* pool_;         // not owned
+  const PackedImage* desc_;  // not owned
+  const uint8_t* bytes_;     // not owned
+  Image3F color_;
+
+  ImageF temp_buf_;
+
+  // Only initialized if external_->HasAlpha() && want_alpha:
+  std::vector<Alpha::Stats> alpha_stats_;
+  ImageU alpha_;
+  size_t bits_per_alpha_;
+
+  // Whether InitBeforeRun() was already called.
+  bool init_called_ = false;
+};
+
+}  // namespace
+
+ExternalImage::ExternalImage(const size_t xsize, const size_t ysize,
+                             const ColorEncoding& c_current,
+                             const bool has_alpha, const size_t bits_per_alpha,
+                             const size_t bits_per_sample,
+                             const bool big_endian)
+    : desc_(xsize, ysize, c_current, has_alpha, bits_per_alpha, bits_per_sample,
+            big_endian, /*flipped_y=*/false),
+      is_healthy_(true) {
+  JXL_ASSERT(1 <= desc_.channels && desc_.channels <= 4);
+  JXL_ASSERT(1 <= bits_per_sample && bits_per_sample <= 32);
+  if (has_alpha) JXL_ASSERT(1 <= bits_per_alpha && bits_per_alpha <= 32);
+  // NOTE: codec_* are responsible for ensuring xsize/ysize are small enough.
+  bytes_.resize(desc_.ysize * desc_.row_size);
+  if (bytes_.empty()) {
+    is_healthy_ = false;
+    JXL_NOTIFY_ERROR("Zero-dimensioned image");
+  }
+}
+
+ExternalImage::ExternalImage(const size_t xsize, const size_t ysize,
+                             const ColorEncoding& c_current,
+                             const bool has_alpha, const size_t bits_per_alpha,
+                             const size_t bits_per_sample,
+                             const bool big_endian, const uint8_t* bytes,
+                             const uint8_t* end)
+    : ExternalImage(xsize, ysize, c_current, has_alpha, bits_per_alpha,
+                    bits_per_sample, big_endian) {
+  JXL_ASSERT(end > bytes);
+  if (!is_healthy_) return;
+  if (bytes + bytes_.size() > end) {
+    is_healthy_ = false;
+    JXL_NOTIFY_ERROR("Not enough bytes given to fill image");
+    return;
+  }
+  memcpy(bytes_.data(), bytes, bytes_.size());
+}
+
+ExternalImage::ExternalImage(ThreadPool* pool, const Image3F& color,
+                             const Rect& rect, const ColorEncoding& c_current,
+                             const ColorEncoding& c_desired,
+                             const bool has_alpha, const ImageU* alpha,
+                             size_t bits_per_alpha, size_t bits_per_sample,
+                             bool big_endian, CodecIntervals* temp_intervals)
+    : ExternalImage(rect.xsize(), rect.ysize(), c_desired, has_alpha,
+                    bits_per_alpha, bits_per_sample, big_endian) {
+  if (!is_healthy_) return;
+  Transformer transformer(pool, color, rect, has_alpha, alpha, this, c_current,
+                          c_desired);
+
+  const CodecInterval ext_interval = GetInterval(bits_per_sample);
+
+  if (bits_per_sample == 32) {
+    ExtentsStatic extents;
+    const CastFloat01 cast(ext_interval);  // only multiply by const
+    if (!transformer.Run<ToExternal>(&extents, cast)) {
+      is_healthy_ = false;
+      JXL_NOTIFY_ERROR("Color transform with CastFloat01 failed");
+    }
+  } else if (temp_intervals != nullptr) {
+    // Store temp to separate image and obtain per-channel intervals.
+    ExtentsDynamic extents(desc_.xsize, desc_.ysize, c_desired);
+    const CastUnused unused;
+    if (!transformer.Run<ToExternal1>(&extents, unused)) {
+      is_healthy_ = false;
+      JXL_NOTIFY_ERROR("Color transform with CastUnused failed");
+      return;
+    }
+    extents.Finalize(temp_intervals);
+
+    // Rescale based on temp_intervals.
+    const CastRescale01 cast(*temp_intervals, ext_interval);
+    if (!transformer.Run<ToExternal2>(&extents, cast)) {
+      is_healthy_ = false;
+      JXL_NOTIFY_ERROR("Color transform with CastRescale01 failed");
+    }
+  } else {
+    ExtentsStatic extents;
+    const CastClip01 cast(ext_interval);  // clip
+    if (!transformer.Run<ToExternal>(&extents, cast)) {
+      is_healthy_ = false;
+      JXL_NOTIFY_ERROR("Color transform with CastClip01 failed");
+    }
+  }
+}
+
+Status ExternalImage::CopyTo(const CodecIntervals* temp_intervals,
+                             ThreadPool* pool, ImageBundle* ib) const {
+  JXL_ASSERT(IsHealthy());  // Caller should have checked beforehand.
+
+  Converter converter(pool, desc_, bytes_.data());
+
+  const CodecInterval ext_interval = GetInterval(desc_.bits_per_sample);
+
+  if (desc_.bits_per_sample == 32) {
+    const CastFloat255 cast(ext_interval);
+    JXL_RETURN_IF_ERROR(converter.Run(cast));
+  } else if (temp_intervals != nullptr) {
+    const CastRescale255 cast(*temp_intervals, ext_interval);
+    JXL_RETURN_IF_ERROR(converter.Run(cast));
+  } else {
+    const CastClip255 cast(ext_interval);
+    JXL_RETURN_IF_ERROR(converter.Run(cast));
+  }
+
+  return converter.MoveTo(ib);
+}
+
+Status CopyTo(const PackedImage& desc, Span<const uint8_t> bytes,
+              ThreadPool* pool, ImageBundle* ib) {
+  Converter converter(pool, desc, bytes.data());
+
+  const CodecInterval ext_interval = GetInterval(desc.bits_per_sample);
+
+  if (desc.bits_per_sample == 32) {
+    const CastFloat255 cast(ext_interval);
+    JXL_RETURN_IF_ERROR(converter.Run(cast));
+  } else {
+    const CastClip255 cast(ext_interval);
+    JXL_RETURN_IF_ERROR(converter.Run(cast));
+  }
+
+  return converter.MoveTo(ib);
+}
+
+}  // namespace jxl
diff --git a/jxl/external_image.h b/jxl/external_image.h
new file mode 100644
index 000000000..5260d93e7
--- /dev/null
+++ b/jxl/external_image.h
@@ -0,0 +1,134 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EXTERNAL_IMAGE_H_
+#define JXL_EXTERNAL_IMAGE_H_
+
+// Interleaved image for color transforms and Codec.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+
+// Packed (no row padding), interleaved (RGBRGB) u8/u16/f32.
+struct PackedImage {
+  PackedImage(size_t xsize, size_t ysize, const ColorEncoding& c_current,
+              bool has_alpha, size_t bits_per_alpha, size_t bits_per_sample,
+              bool big_endian, bool flipped_y)
+      : xsize(xsize),
+        ysize(ysize),
+        c_current(c_current),
+        channels(c_current.Channels() + has_alpha),
+        bits_per_alpha(bits_per_alpha),
+        bits_per_sample(bits_per_sample),
+        row_size(xsize * channels * DivCeil(bits_per_sample, kBitsPerByte)),
+        big_endian(big_endian),
+        flipped_y(flipped_y) {}
+
+  bool HasAlpha() const { return channels == 2 || channels == 4; }
+
+  size_t xsize;
+  size_t ysize;
+  ColorEncoding c_current;
+  size_t channels;
+  // Per alpha channel value
+  size_t bits_per_alpha;
+  // Per color channel
+  size_t bits_per_sample;
+  size_t row_size;
+  bool big_endian;
+  bool flipped_y;
+};
+
+// Packed (no row padding), interleaved (RGBRGB) u8/u16/f32.
+class ExternalImage {
+ public:
+  // Copies from existing interleaved image. Called by decoders. "big_endian"
+  // only matters for bits_per_sample > 8. "end" is the STL-style end of "bytes"
+  // for range checks.
+  //
+  // DEPRECATED, use ::CopyTo instead
+  ExternalImage(size_t xsize, size_t ysize, const ColorEncoding& c_current,
+                bool has_alpha, size_t bits_per_alpha, size_t bits_per_sample,
+                bool big_endian, const uint8_t* bytes, const uint8_t* end);
+
+  // Copies pixels from rect and converts from c_current to c_desired. Called by
+  // encoders and ImageBundle::CopyTo. alpha is nullptr iff !has_alpha.
+  // If temp_intervals != null, fills them such that CopyTo can rescale to that
+  // range. Otherwise, clamps temp to [0, 1].
+  ExternalImage(ThreadPool* pool, const Image3F& color, const Rect& rect,
+                const ColorEncoding& c_current, const ColorEncoding& c_desired,
+                bool has_alpha, const ImageU* alpha, size_t bits_per_alpha,
+                size_t bits_per_sample, bool big_endian,
+                CodecIntervals* temp_intervals);
+
+  // Indicates whether the ctor succeeded; if not, do not use this instance.
+  Status IsHealthy() const { return is_healthy_; }
+
+  // Sets "ib" to a newly allocated copy with c_current color space.
+  // Uses temp_intervals for rescaling if not null (NOTE: temp_intervals is
+  // given as if a range of [0.0f-1.0f] would be used, even though it uses
+  // [0.0f-255.0f] internally, to match the same parameter given to the
+  // color converting constructor).
+  //
+  // DEPRECATED, use ::CopyTo instead
+  Status CopyTo(const CodecIntervals* temp_intervals, ThreadPool* pool,
+                ImageBundle* ib) const;
+
+  // Packed, interleaved pixels, for passing to encoders.
+  const PaddedBytes& Bytes() const { return bytes_; }
+
+  const PackedImage& Desc() const { return desc_; }
+
+  size_t xsize() const { return desc_.xsize; }
+  size_t ysize() const { return desc_.ysize; }
+  const ColorEncoding& c_current() const { return desc_.c_current; }
+  bool IsGray() const { return desc_.c_current.IsGray(); }
+  bool HasAlpha() const { return desc_.channels == 2 || desc_.channels == 4; }
+  size_t BitsPerAlpha() const { return desc_.bits_per_alpha; }
+  size_t BitsPerSample() const { return desc_.bits_per_sample; }
+  bool BigEndian() const { return desc_.big_endian; }
+
+  uint8_t* Row(size_t y) { return bytes_.data() + y * desc_.row_size; }
+  const uint8_t* ConstRow(size_t y) const {
+    return bytes_.data() + y * desc_.row_size;
+  }
+
+ private:
+  ExternalImage(size_t xsize, size_t ysize, const ColorEncoding& c_current,
+                bool has_alpha, size_t bits_per_alpha, size_t bits_per_sample,
+                bool big_endian);
+
+  PackedImage desc_;
+  PaddedBytes bytes_;
+  bool is_healthy_;
+};
+
+// Copies from packed u8/u16/f32 to floating-point, keeping the same color
+// space. No clipping; assumes the range of unsigned integers maps to [0, 1].
+Status CopyTo(const PackedImage& desc, Span<const uint8_t> bytes,
+              ThreadPool* pool, ImageBundle* ib);
+
+}  // namespace jxl
+
+#endif  // JXL_EXTERNAL_IMAGE_H_
diff --git a/jxl/external_image_test.cc b/jxl/external_image_test.cc
new file mode 100644
index 000000000..25d782016
--- /dev/null
+++ b/jxl/external_image_test.cc
@@ -0,0 +1,357 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/external_image.h"
+
+#include <stdio.h>
+
+#include <array>
+#include <new>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/image_ops.h"
+#include "jxl/image_test_utils.h"
+
+namespace jxl {
+namespace {
+
+// Small enough to be fast. If changed, must update Generate*.
+static constexpr size_t kWidth = 8;
+static constexpr size_t kHeight = 3;
+
+struct Globals {
+  Globals()
+      : pool(3)  // matches kHeight below
+  {
+    // Linear BT.2100
+    c_wide[0].SetColorSpace(ColorSpace::kRGB);
+    c_wide[0].primaries = Primaries::k2100;
+    c_wide[0].tf.SetTransferFunction(TransferFunction::kLinear);
+    JXL_CHECK(ColorManagement::CreateProfile(&c_wide[0]));
+
+    // Same, but gray
+    c_wide[1] = c_wide[0];
+    c_wide[1].SetColorSpace(ColorSpace::kGray);
+    JXL_CHECK(ColorManagement::CreateProfile(&c_wide[1]));
+
+    // Index = (channels - 1) + 8bit ? 0 : 4.
+    color[1 - 1 + 0] = MakeGray();
+    color[2 - 1 + 0] = MakeGray();
+    alpha[2 - 1 + 0] = MakeAlpha(8);
+    color[1 - 1 + 4] = MakeGray();
+    color[2 - 1 + 4] = MakeGray();
+    alpha[2 - 1 + 4] = MakeAlpha(16);
+    color[3 - 1 + 0] = MakeColor();
+    color[4 - 1 + 0] = MakeColor();
+    alpha[4 - 1 + 0] = MakeAlpha(8);
+    color[3 - 1 + 4] = MakeColor();
+    color[4 - 1 + 4] = MakeColor();
+    alpha[4 - 1 + 4] = MakeAlpha(16);
+  }
+
+ private:
+  static ImageU MakeAlpha(size_t bits_per_sample) {
+    const uint16_t max_alpha = bits_per_sample <= 8 ? 255 : 65535;
+    ImageU alpha(kWidth, kHeight);
+    RandomFillImage(&alpha, max_alpha);
+    return alpha;
+  }
+
+  static Image3F MakeGray() {
+    Image3F out(kWidth, kHeight);
+    for (size_t c = 0; c < 3; ++c) {
+      for (int32_t y = 0; y < kHeight; ++y) {
+        float* JXL_RESTRICT row = out.PlaneRow(c, y);
+        // Increasing left to right, top to bottom
+        for (int32_t x = 0; x < kWidth; ++x) {
+          row[x] = y + x * (255.0f - (kHeight - 1)) / (kWidth - 1);  // [0, 255]
+        }
+      }
+    }
+    return out;
+  }
+
+  static Image3F MakeColor() {
+    Image3F out(kWidth, kHeight);
+    ZeroFillImage(&out);
+
+    // Row 0: neutral
+    float* JXL_RESTRICT row0 = out.PlaneRow(0, 0);
+    float* JXL_RESTRICT row1 = out.PlaneRow(1, 0);
+    float* JXL_RESTRICT row2 = out.PlaneRow(2, 0);
+    for (int32_t x = 0; x < kWidth; ++x) {
+      row2[x] = row1[x] = row0[x] = x * 255.0f / (kWidth - 1);  // [0, 255]
+    }
+
+    // Row 1: pure RGB
+    for (int32_t x = 0; x < kWidth; ++x) {
+      const int32_t c = x % 3;
+      float* JXL_RESTRICT row = out.PlaneRow(c, 1);
+      row[x] = x * 255.0f / (kWidth - 1);  // [0, 255]
+    }
+
+    // Row 2: mixed
+    for (int32_t x = 0; x < kWidth; ++x) {
+      for (size_t c = 0; c < 3; ++c) {
+        out.PlaneRow(c, 2)[x] = 40.0f;
+      }
+      const int32_t c = x % 3;
+      float* JXL_RESTRICT row = out.PlaneRow(c, 2);
+      row[x] = x * 255.0f / (kWidth - 1);  // [0, 255]
+    }
+    return out;
+  }
+
+ public:
+  ThreadPoolInternal pool;
+  std::array<ColorEncoding, 2> c_wide;
+
+  // Pregenerate to reduce allocations.
+  Image3F color[8];
+  ImageU alpha[8];
+};
+static Globals* g;
+
+class ExternalImageTest : public ::testing::TestWithParam<ColorEncoding> {
+ public:
+  static void SetUpTestSuite() { g = new Globals; }
+  static void TearDownTestSuite() { delete g; }
+
+  // "Same" pixels (modulo quantization) after converting to/from ExternalImage.
+  static void VerifyPixelRoundTrip(const ColorEncoding& c_external,
+                                   const bool add_alpha, const bool big_endian,
+                                   const size_t bits_per_sample) {
+    if (big_endian && (bits_per_sample == 8)) return;
+
+    printf("sb:%zu, gr:%d al:%d be:%d\n", bits_per_sample, c_external.IsGray(),
+           add_alpha, big_endian);
+
+    const size_t channels = c_external.Channels() + add_alpha;
+    const size_t idx = channels - 1 + (bits_per_sample <= 8 ? 0 : 4);
+    const ColorEncoding& c_current = g->c_wide[c_external.IsGray()];
+
+    CodecInOut io;
+    io.metadata.bits_per_sample = bits_per_sample;
+    io.metadata.color_encoding = c_current;
+    io.SetFromImage(CopyImage(g->color[idx]), c_current);
+    ImageBundle& ib = io.Main();
+    const size_t alpha_bits = bits_per_sample <= 8 ? 8 : 16;
+    if (add_alpha) {
+      io.metadata.alpha_bits = alpha_bits;
+      ib.SetAlpha(CopyImage(g->alpha[idx]));
+    }
+
+    const ImageU* alpha = io.metadata.HasAlpha() ? &ib.alpha() : nullptr;
+    // Rescale values instead of clipping. This avoids truncating negative or
+    // > max (out of gamut) samples - happens after chromatic adaptation of pure
+    // colors (Bradford matrix has negative entries). Reduces round-trip error.
+    // Codec currently are not able to store this in metadata.
+    CodecIntervals temp_intervals;
+    const ExternalImage external(&g->pool, ib.color(), Rect(ib), ib.c_current(),
+                                 c_external, ib.HasAlpha(), alpha, alpha_bits,
+                                 bits_per_sample, big_endian, &temp_intervals);
+    ASSERT_TRUE(external.IsHealthy());
+
+    // Copy for later comparison.
+    const Image3F& prev_color = CopyImage(ib.color());
+
+    // Ensure TransformTo actually fills color.
+    ZeroFillImage(const_cast<Image3F*>(&ib.color()));
+
+    // Copy c_external pixels to ib..
+    ASSERT_TRUE(external.CopyTo(&temp_intervals, &g->pool, &ib));
+    ASSERT_TRUE(ib.c_current().SameColorSpace(c_external));
+
+    // .. and transform back to c_current
+    ASSERT_TRUE(ib.TransformTo(c_current, &g->pool));
+
+    if (add_alpha) {
+      JXL_CHECK(SamePixels(g->alpha[idx], ib.alpha()));
+    }
+
+    // => should be the same as prev_color.
+    double max_l1, max_rel;
+    const bool is2100 = c_external.tf.IsPQ() || c_external.tf.IsHLG();
+    // Different gamut or chromatic adaptation => higher error.
+    const bool needsCompute =
+        (!c_external.IsGray() &&
+         c_external.primaries != ib.c_current().primaries) ||
+        c_external.white_point != ib.c_current().white_point;
+#if JPEGXL_ENABLE_SKCMS
+    if (true) {
+#else
+    if (bits_per_sample <= 12) {
+#endif
+      if (is2100 && needsCompute) {
+        max_l1 = 8.3;  // 8 bit is just not enough for PQ + adaptation.
+        max_rel = 2E-3;
+      } else if (c_external.IsGray()) {
+        max_l1 = 0.4;
+        max_rel = is2100 ? 2E-2 : 7E-3;
+      } else if (is2100) {
+        max_l1 = 5E-4;
+        max_rel = 2E-2;
+      } else if (needsCompute) {
+        max_l1 = 2.0;
+        max_rel = 4E-2;
+      } else {
+        max_l1 = 5E-4;
+        max_rel = 1.5E-2;
+      }
+    } else if (bits_per_sample == 16) {
+      if (is2100 && needsCompute) {
+        max_l1 = 3E-2;
+        max_rel = 5E-4;
+      } else if (c_external.IsGray()) {
+        max_l1 = 7E-3;
+        max_rel = 1E-4;
+      } else if (is2100) {
+        max_l1 = 7E-5;
+        max_rel = 7E-5;
+      } else if (needsCompute) {
+        max_l1 = 1E-2;
+        max_rel = 2E-4;
+      } else {
+        max_l1 = 1E-6;
+        max_rel = 3E-5;
+      }
+    } else {
+      JXL_ASSERT(bits_per_sample == 32);
+      if (is2100 && needsCompute) {
+        max_l1 = 4E-4;
+        max_rel = 2E-5;
+      } else if (c_external.IsGray()) {
+        max_l1 = 4E-7;
+        max_rel = 2E-5;
+      } else if (is2100) {
+        max_l1 = 4E-7;
+        max_rel = 2E-5;
+      } else if (needsCompute) {
+        max_l1 = 7E-3;
+        max_rel = 2E-5;
+      } else {
+        max_l1 = 2E-8;
+#ifdef __ARM_ARCH
+        // pow() implementation in arm is a bit less precise than in x86 and
+        // therefore we need a bigger error margin in this case.
+        max_rel = 2E-5;
+#else
+        max_rel = 1E-6;
+#endif
+      }
+    }
+
+    VerifyRelativeError(prev_color, ib.color(), max_l1, max_rel);
+  }
+
+  // Copying from CodecInOut to Image3 => same pixels (modulo clipping).
+  template <typename T>
+  static void TestCopyTo() {
+    const size_t bits_per_sample = sizeof(T) * kBitsPerByte;
+    // Largest value the converted image can have.
+    const float external_max = bits_per_sample == 16 ? 65535 : 255;
+    // ExternalImage assumes input ImageF are always [0, 255] regardless of
+    // the output type.
+    const float range = 256.0f;
+    const float scale = external_max / 255;
+
+    // First and last are out of bounds.
+    const size_t xsize = 1 + 16 + 1;
+    const float step = range / (xsize - 2);
+    const size_t ysize = 2;
+
+    Image3F image(xsize, ysize);
+    for (size_t c = 0; c < 3; ++c) {
+      float* JXL_RESTRICT row0 = image.PlaneRow(c, 0);
+      row0[0] = -1.0f;
+      for (size_t x = 1; x < xsize; ++x) row0[x] = x * step - 1;
+      EXPECT_GE(row0[xsize - 1], range);
+      float* JXL_RESTRICT row1 = image.PlaneRow(c, 1);
+      row1[0] = -9.0f;
+      for (size_t x = 1; x < xsize; ++x) row1[x] = (x - 1) * step;
+      EXPECT_GE(row1[xsize - 1], range);
+    }
+
+    CodecInOut io;
+    io.metadata.bits_per_sample = bits_per_sample;
+    io.metadata.color_encoding = ColorManagement::SRGB();
+    io.SetFromImage(std::move(image), io.metadata.color_encoding);
+    const ImageBundle& ib = io.Main();
+
+    Image3<T> out;
+    ASSERT_TRUE(ib.CopyTo(Rect(ib), ColorManagement::SRGB(ib.IsGray()), &out,
+                          &g->pool));
+    for (size_t c = 0; c < 3; ++c) {
+      const T* row_out0 = out.PlaneRow(c, 0);
+      const T* row_out1 = out.PlaneRow(c, 1);
+
+      // No clipping/clamping for float.
+      if (sizeof(T) != sizeof(float)) {
+        // x = 0: clamped to min
+        EXPECT_EQ(0, row_out0[0]);
+        EXPECT_EQ(0, row_out1[0]);
+
+        // x = xsize - 1: clamped to external_max
+        EXPECT_EQ(external_max, row_out0[xsize - 1]);
+        EXPECT_EQ(external_max, row_out1[xsize - 1]);
+      }
+
+      // Interior: same values
+      const float* row_in0 = ib.color().ConstPlaneRow(c, 0);
+      const float* row_in1 = ib.color().ConstPlaneRow(c, 1);
+      for (size_t x = 1; x < xsize - 1; ++x) {
+        EXPECT_NEAR(scale * row_in0[x], row_out0[x], 5E-5) << " 0 " << x;
+        EXPECT_NEAR(scale * row_in1[x], row_out1[x], 5E-5) << " 1 " << x;
+      }
+    }
+  }
+};
+INSTANTIATE_TEST_SUITE_P(ExternalImageTestInstantiation, ExternalImageTest,
+                         ::testing::ValuesIn(AllEncodings()));
+
+TEST_F(ExternalImageTest, TestConvert) {
+  TestCopyTo<uint8_t>();
+  TestCopyTo<uint16_t>();
+  TestCopyTo<float>();
+}
+
+TEST_P(ExternalImageTest, RoundTrip) {
+  ColorEncoding c_external = GetParam();
+  // To speed up the test, ignore rendering_intent (unused here)
+  if (c_external.rendering_intent != RenderingIntent::kRelative) return;
+  JXL_CHECK(ColorManagement::CreateProfile(&c_external));
+
+  printf("%s\n", Description(c_external).c_str());
+
+  for (size_t bits_per_sample : {8, 16, 32}) {
+    for (bool add_alpha : {false, true}) {
+      for (bool big_endian : {false, true}) {
+        VerifyPixelRoundTrip(c_external, add_alpha, big_endian,
+                             bits_per_sample);
+      }
+    }
+  }
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/extras/README.md b/jxl/extras/README.md
new file mode 100644
index 000000000..06a9b5ea0
--- /dev/null
+++ b/jxl/extras/README.md
@@ -0,0 +1,5 @@
+## JPEG XL "extras"
+
+The files in this directory do not form part of the library or codec and are
+only used by tests or specific internal tools that have access to the internals
+of the library.
diff --git a/jxl/extras/codec.cc b/jxl/extras/codec.cc
new file mode 100644
index 000000000..6ec63214a
--- /dev/null
+++ b/jxl/extras/codec.cc
@@ -0,0 +1,170 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/extras/codec.h"
+
+#include "jxl/base/file_io.h"
+#include "jxl/extras/codec_apng.h"
+#if JPEGXL_ENABLE_EXR
+#include "jxl/extras/codec_exr.h"
+#endif
+#include "jxl/extras/codec_gif.h"
+#include "jxl/extras/codec_jpg.h"
+#include "jxl/extras/codec_pgx.h"
+#include "jxl/extras/codec_png.h"
+#include "jxl/extras/codec_pnm.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+namespace {
+
+// Any valid encoding is larger (ensures codecs can read the first few bytes)
+constexpr size_t kMinBytes = 9;
+
+}  // namespace
+
+std::string ExtensionFromCodec(Codec codec, const bool is_gray,
+                               const size_t bits_per_sample) {
+  switch (codec) {
+    case Codec::kJPG:
+      return ".jpg";
+    case Codec::kPGX:
+      return ".pgx";
+    case Codec::kPNG:
+      return ".png";
+    case Codec::kPNM:
+      if (is_gray) return ".pgm";
+      return (bits_per_sample == 32) ? ".pfm" : ".ppm";
+    case Codec::kGIF:
+      return ".gif";
+    case Codec::kEXR:
+      return ".exr";
+    case Codec::kUnknown:
+      return std::string();
+  }
+  JXL_UNREACHABLE;
+  return std::string();
+}
+
+Codec CodecFromExtension(const std::string& extension,
+                         size_t* JXL_RESTRICT bits_per_sample) {
+  if (extension == ".png") return Codec::kPNG;
+
+  if (extension == ".jpg") return Codec::kJPG;
+  if (extension == ".jpeg") return Codec::kJPG;
+
+  if (extension == ".pgx") return Codec::kPGX;
+
+  if (extension == ".pgm") return Codec::kPNM;
+  if (extension == ".ppm") return Codec::kPNM;
+  if (extension == ".pfm") {
+    *bits_per_sample = 32;
+    return Codec::kPNM;
+  }
+
+  if (extension == ".gif") return Codec::kGIF;
+
+  if (extension == ".exr") return Codec::kEXR;
+
+  return Codec::kUnknown;
+}
+
+Status SetFromBytes(const Span<const uint8_t> bytes, CodecInOut* io,
+                    ThreadPool* pool, const DecodeTarget decode_target) {
+  if (bytes.size() < kMinBytes) return JXL_FAILURE("Too few bytes");
+
+  io->metadata.bits_per_sample = 0;  // (For is-set check below)
+
+  if (!DecodeImagePNG(bytes, pool, io) && !DecodeImageAPNG(bytes, io) &&
+      !DecodeImagePGX(bytes, pool, io) && !DecodeImagePNM(bytes, pool, io) &&
+      !DecodeImageGIF(bytes, io) && !DecodeImageJPG(bytes, io, decode_target)
+#if JPEGXL_ENABLE_EXR
+      && !DecodeImageEXR(bytes, io)
+#endif
+  ) {
+    return JXL_FAILURE("Codecs failed to decode");
+  }
+
+  io->CheckMetadata();
+  return true;
+}
+
+Status SetFromFile(const std::string& pathname, CodecInOut* io,
+                   ThreadPool* pool, const DecodeTarget decode_target) {
+  PaddedBytes encoded;
+  JXL_RETURN_IF_ERROR(ReadFile(pathname, &encoded));
+  JXL_RETURN_IF_ERROR(
+      SetFromBytes(Span<const uint8_t>(encoded), io, pool, decode_target));
+  return true;
+}
+
+Status Encode(const CodecInOut& io, const Codec codec,
+              const ColorEncoding& c_desired, size_t bits_per_sample,
+              PaddedBytes* bytes, ThreadPool* pool) {
+  JXL_CHECK(!io.Main().c_current().icc.empty());
+  JXL_CHECK(!c_desired.icc.empty());
+  io.CheckMetadata();
+  if (io.Main().jpeg_xsize && codec != Codec::kJPG)
+    return JXL_FAILURE(
+        "Output format has to be JPEG for losslessly recompressed JPEG "
+        "reconstruction");
+
+  switch (codec) {
+    case Codec::kPNG:
+      return EncodeImagePNG(&io, c_desired, bits_per_sample, pool, bytes);
+    case Codec::kJPG:
+      return EncodeImageJPG(
+          &io, io.use_sjpeg ? JpegEncoder::kSJpeg : JpegEncoder::kLibJpeg,
+          io.jpeg_quality,
+          io.use_sjpeg ? YCbCrChromaSubsampling::kAuto
+                       : YCbCrChromaSubsampling::k444,
+          pool, bytes,
+          io.Main().jpeg_xsize ? DecodeTarget::kQuantizedCoeffs
+                               : DecodeTarget::kPixels);
+    case Codec::kPNM:
+      return EncodeImagePNM(&io, c_desired, bits_per_sample, pool, bytes);
+    case Codec::kPGX:
+      return EncodeImagePGX(&io, c_desired, bits_per_sample, pool, bytes);
+    case Codec::kGIF:
+      return JXL_FAILURE("Encoding to GIF is not implemented");
+    case Codec::kEXR:
+#if JPEGXL_ENABLE_EXR
+      return EncodeImageEXR(&io, c_desired, pool, bytes);
+#else
+      return JXL_FAILURE("JPEG XL was built without OpenEXR support");
+#endif
+    case Codec::kUnknown:
+      return JXL_FAILURE("Cannot encode using Codec::kUnknown");
+  }
+
+  return JXL_FAILURE("Invalid codec");
+}
+
+Status EncodeToFile(const CodecInOut& io, const ColorEncoding& c_desired,
+                    size_t bits_per_sample, const std::string& pathname,
+                    ThreadPool* pool) {
+  const Codec codec = CodecFromExtension(Extension(pathname), &bits_per_sample);
+
+  PaddedBytes encoded;
+  return Encode(io, codec, c_desired, bits_per_sample, &encoded, pool) &&
+         WriteFile(encoded, pathname);
+}
+
+Status EncodeToFile(const CodecInOut& io, const std::string& pathname,
+                    ThreadPool* pool) {
+  return EncodeToFile(io, io.metadata.color_encoding,
+                      io.metadata.bits_per_sample, pathname, pool);
+}
+
+}  // namespace jxl
diff --git a/jxl/extras/codec.h b/jxl/extras/codec.h
new file mode 100644
index 000000000..94278c185
--- /dev/null
+++ b/jxl/extras/codec.h
@@ -0,0 +1,99 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EXTRAS_CODEC_H_
+#define JXL_EXTRAS_CODEC_H_
+
+// Facade for image encoders/decoders (PNG, PNM, ...).
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <string>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/field_encodings.h"  // MakeBit
+
+namespace jxl {
+
+// Codecs supported by CodecInOut::Encode.
+enum class Codec : uint32_t {
+  kUnknown,  // for CodecFromExtension
+  kPNG,
+  kPNM,
+  kPGX,
+  kJPG,
+  kGIF,
+  kEXR,
+};
+
+// For Codec::kJPG, convert between JPEG and pixels or between JPEG and
+// quantized DCT coefficients
+enum class DecodeTarget {
+  kPixels,
+  kQuantizedCoeffs,
+};
+
+static inline constexpr uint64_t EnumBits(Codec /*unused*/) {
+  // Return only fully-supported codecs (kGIF is decode-only).
+  return MakeBit(Codec::kPNG) | MakeBit(Codec::kPNM) | MakeBit(Codec::kJPG)
+#if JPEGXL_ENABLE_EXR
+         | MakeBit(Codec::kEXR)
+#endif
+      ;
+}
+
+// Lower case ASCII including dot, e.g. ".png".
+std::string ExtensionFromCodec(Codec codec, bool is_gray,
+                               size_t bits_per_sample);
+
+// If and only if extension is ".pfm", *bits_per_sample is updated to 32 so
+// that Encode() would encode to PFM instead of PPM.
+Codec CodecFromExtension(const std::string& extension,
+                         size_t* JXL_RESTRICT bits_per_sample);
+
+// Decodes "bytes" and sets io->metadata.
+// dec_hints may specify the "color_space" (otherwise, defaults to sRGB).
+Status SetFromBytes(const Span<const uint8_t> bytes, CodecInOut* io,
+                    ThreadPool* pool = nullptr,
+                    const DecodeTarget decode_target = DecodeTarget::kPixels);
+
+// Reads from file and calls SetFromBytes.
+Status SetFromFile(const std::string& pathname, CodecInOut* io,
+                   ThreadPool* pool = nullptr,
+                   const DecodeTarget decode_target = DecodeTarget::kPixels);
+
+// Replaces "bytes" with an encoding of pixels transformed from c_current
+// color space to c_desired.
+Status Encode(const CodecInOut& io, Codec codec, const ColorEncoding& c_desired,
+              size_t bits_per_sample, PaddedBytes* bytes,
+              ThreadPool* pool = nullptr);
+
+// Deduces codec, calls Encode and writes to file.
+Status EncodeToFile(const CodecInOut& io, const ColorEncoding& c_desired,
+                    size_t bits_per_sample, const std::string& pathname,
+                    ThreadPool* pool = nullptr);
+// Same, but defaults to metadata.original color_encoding and bits_per_sample.
+Status EncodeToFile(const CodecInOut& io, const std::string& pathname,
+                    ThreadPool* pool = nullptr);
+
+}  // namespace jxl
+
+#endif  // JXL_EXTRAS_CODEC_H_
diff --git a/jxl/extras/codec_apng.cc b/jxl/extras/codec_apng.cc
new file mode 100644
index 000000000..ba787d55b
--- /dev/null
+++ b/jxl/extras/codec_apng.cc
@@ -0,0 +1,423 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/extras/codec_apng.h"
+
+// Parts of this code are taken from apngdis, which has the following license:
+/* APNG Disassembler 2.8
+ *
+ * Deconstructs APNG files into individual frames.
+ *
+ * http://apngdis.sourceforge.net
+ *
+ * Copyright (c) 2010-2015 Max Stepin
+ * maxst at users.sourceforge.net
+ *
+ * zlib license
+ * ------------
+ *
+ * This software is provided 'as-is', without any express or implied
+ * warranty.  In no event will the authors be held liable for any damages
+ * arising from the use of this software.
+ *
+ * Permission is granted to anyone to use this software for any purpose,
+ * including commercial applications, and to alter it and redistribute it
+ * freely, subject to the following restrictions:
+ *
+ * 1. The origin of this software must not be misrepresented; you must not
+ *    claim that you wrote the original software. If you use this software
+ *    in a product, an acknowledgment in the product documentation would be
+ *    appreciated but is not required.
+ * 2. Altered source versions must be plainly marked as such, and must not be
+ *    misrepresented as being the original software.
+ * 3. This notice may not be removed or altered from any source distribution.
+ *
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+#define NOMINMAX
+#define WIN32_LEAN_AND_MEAN
+#include <Windows.h>
+#endif
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/frame_header.h"
+#include "jxl/headers.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "png.h" /* original (unpatched) libpng is ok */
+
+namespace jxl {
+
+namespace {
+#define notabc(c) ((c) < 65 || (c) > 122 || ((c) > 90 && (c) < 97))
+
+#define id_IHDR 0x52444849
+#define id_acTL 0x4C546361
+#define id_fcTL 0x4C546366
+#define id_IDAT 0x54414449
+#define id_fdAT 0x54416466
+#define id_IEND 0x444E4549
+
+struct CHUNK {
+  unsigned char* p;
+  unsigned int size;
+};
+struct APNGFrame {
+  unsigned char *p, **rows;
+  unsigned int w, h, delay_num, delay_den;
+};
+
+const unsigned long cMaxPNGSize = 1000000UL;
+const size_t kMaxPNGChunkSize = 100000000;  // 100 MB
+
+void info_fn(png_structp png_ptr, png_infop info_ptr) {
+  png_set_expand(png_ptr);
+  png_set_strip_16(png_ptr);
+  png_set_gray_to_rgb(png_ptr);
+  png_set_add_alpha(png_ptr, 0xff, PNG_FILLER_AFTER);
+  (void)png_set_interlace_handling(png_ptr);
+  png_read_update_info(png_ptr, info_ptr);
+}
+
+void row_fn(png_structp png_ptr, png_bytep new_row, png_uint_32 row_num,
+            int pass) {
+  APNGFrame* frame = (APNGFrame*)png_get_progressive_ptr(png_ptr);
+  png_progressive_combine_row(png_ptr, frame->rows[row_num], new_row);
+}
+
+inline unsigned int read_chunk(FILE* f, CHUNK* pChunk) {
+  unsigned char len[4];
+  pChunk->size = 0;
+  pChunk->p = 0;
+  if (fread(&len, 4, 1, f) == 1) {
+    pChunk->size = png_get_uint_32(len) + 12;
+    if (pChunk->size > kMaxPNGChunkSize) {
+      JXL_WARNING("APNG chunk size is too big");
+      return 0;
+    }
+    pChunk->p = new unsigned char[pChunk->size];
+    memcpy(pChunk->p, len, 4);
+    if (fread(pChunk->p + 4, pChunk->size - 4, 1, f) == 1)
+      return *(unsigned int*)(pChunk->p + 4);
+  }
+  return 0;
+}
+
+int processing_start(png_structp& png_ptr, png_infop& info_ptr, void* frame_ptr,
+                     bool hasInfo, CHUNK& chunkIHDR,
+                     std::vector<CHUNK>& chunksInfo) {
+  unsigned char header[8] = {137, 80, 78, 71, 13, 10, 26, 10};
+
+  png_ptr = png_create_read_struct(PNG_LIBPNG_VER_STRING, NULL, NULL, NULL);
+  info_ptr = png_create_info_struct(png_ptr);
+  if (!png_ptr || !info_ptr) return 1;
+
+  if (setjmp(png_jmpbuf(png_ptr))) {
+    png_destroy_read_struct(&png_ptr, &info_ptr, 0);
+    return 1;
+  }
+
+  png_set_crc_action(png_ptr, PNG_CRC_QUIET_USE, PNG_CRC_QUIET_USE);
+  png_set_progressive_read_fn(png_ptr, frame_ptr, info_fn, row_fn, NULL);
+
+  png_process_data(png_ptr, info_ptr, header, 8);
+  png_process_data(png_ptr, info_ptr, chunkIHDR.p, chunkIHDR.size);
+
+  if (hasInfo)
+    for (unsigned int i = 0; i < chunksInfo.size(); i++)
+      png_process_data(png_ptr, info_ptr, chunksInfo[i].p, chunksInfo[i].size);
+  return 0;
+}
+
+int processing_data(png_structp png_ptr, png_infop info_ptr, unsigned char* p,
+                    unsigned int size) {
+  if (!png_ptr || !info_ptr) return 1;
+
+  if (setjmp(png_jmpbuf(png_ptr))) {
+    png_destroy_read_struct(&png_ptr, &info_ptr, 0);
+    return 1;
+  }
+
+  png_process_data(png_ptr, info_ptr, p, size);
+  return 0;
+}
+
+int processing_finish(png_structp png_ptr, png_infop info_ptr) {
+  unsigned char footer[12] = {0, 0, 0, 0, 73, 69, 78, 68, 174, 66, 96, 130};
+
+  if (!png_ptr || !info_ptr) return 1;
+
+  if (setjmp(png_jmpbuf(png_ptr))) {
+    png_destroy_read_struct(&png_ptr, &info_ptr, 0);
+    return 1;
+  }
+
+  png_process_data(png_ptr, info_ptr, footer, 12);
+  png_destroy_read_struct(&png_ptr, &info_ptr, 0);
+
+  return 0;
+}
+
+#if defined(_WIN32) || defined(_WIN64)
+FILE* fmemopen(void* buf, size_t size, const char* mode) {
+  char temp[999];
+  if (!GetTempPath(sizeof(temp), temp)) return nullptr;
+
+  char pathname[999];
+  if (!GetTempFileName(temp, "jpegxl", 0, pathname)) return nullptr;
+
+  FILE* f = fopen(pathname, "wb");
+  if (f == nullptr) return nullptr;
+  fwrite(buf, 1, size, f);
+  JXL_CHECK(fclose(f) == 0);
+
+  return fopen(pathname, mode);
+}
+
+#endif
+
+}  // namespace
+
+Status DecodeImageAPNG(Span<const uint8_t> bytes, CodecInOut* io) {
+  FILE* f;
+  unsigned int id, i, j, w, h, w0, h0, x0, y0;
+  unsigned int delay_num, delay_den, dop, bop, rowbytes, imagesize;
+  unsigned char sig[8];
+  png_structp png_ptr;
+  png_infop info_ptr;
+  CHUNK chunk;
+  CHUNK chunkIHDR;
+  std::vector<CHUNK> chunksInfo;
+  bool isAnimated = false;
+  bool skipFirst = false;
+  bool hasInfo = false;
+  APNGFrame frameRaw = {};
+
+  if (!(f = fmemopen((void*)bytes.data(), bytes.size(), "rb"))) {
+    return JXL_FAILURE("Failed to fmemopen");
+  }
+  // Not an aPNG => not an error
+  unsigned char png_signature[8] = {137, 80, 78, 71, 13, 10, 26, 10};
+  if (fread(sig, 1, 8, f) != 8 || memcmp(sig, png_signature, 8)) {
+    fclose(f);
+    return false;
+  }
+  id = read_chunk(f, &chunkIHDR);
+
+  io->frames.clear();
+  io->dec_pixels = 0;
+  io->metadata.bits_per_sample = 8;
+  io->metadata.alpha_bits = 8;
+  io->metadata.color_encoding =
+      ColorManagement::SRGB();  // todo: get data from png metadata
+  io->enc_size = bytes.size();
+  io->dec_hints.Foreach(
+      [](const std::string& key, const std::string& /*value*/) {
+        JXL_WARNING("APNG decoder ignoring %s hint", key.c_str());
+      });
+
+  if (id == id_IHDR && chunkIHDR.size == 25) {
+    w0 = w = png_get_uint_32(chunkIHDR.p + 8);
+    h0 = h = png_get_uint_32(chunkIHDR.p + 12);
+
+    if (w > cMaxPNGSize || h > cMaxPNGSize) {
+      fclose(f);
+      return false;
+    }
+
+    x0 = 0;
+    y0 = 0;
+    delay_num = 1;
+    delay_den = 10;
+    dop = 0;
+    bop = 0;
+    rowbytes = w * 4;
+    imagesize = h * rowbytes;
+
+    frameRaw.p = new unsigned char[imagesize];
+    frameRaw.rows = new png_bytep[h * sizeof(png_bytep)];
+    for (j = 0; j < h; j++) frameRaw.rows[j] = frameRaw.p + j * rowbytes;
+
+    if (!processing_start(png_ptr, info_ptr, (void*)&frameRaw, hasInfo,
+                          chunkIHDR, chunksInfo)) {
+      while (!feof(f)) {
+        id = read_chunk(f, &chunk);
+        if (!id) break;
+        JXL_ASSERT(chunk.p != nullptr);
+
+        if (id == id_acTL && !hasInfo && !isAnimated) {
+          isAnimated = true;
+          skipFirst = true;
+          io->metadata.m2.have_animation = true;
+          io->animation.tps_numerator_minus_1 = 1000 - 1;
+        } else if (id == id_IEND ||
+                   (id == id_fcTL && (!hasInfo || isAnimated))) {
+          if (hasInfo) {
+            if (!processing_finish(png_ptr, info_ptr)) {
+              AnimationFrame animation_frame;
+              animation_frame.duration = delay_num * 1000 / delay_den;
+              animation_frame.have_crop = (w != w0 || h != h0);
+              if (animation_frame.have_crop) {
+                animation_frame.x0 = x0;
+                animation_frame.y0 = y0;
+                animation_frame.xsize = w0;
+                animation_frame.ysize = h0;
+              } else {
+                if (!io->animation_frames.empty()) {
+                  io->animation_frames.back().SetNewBase(
+                      AnimationFrame::NewBase::kNone);
+                }
+              }
+              switch (dop) {
+                case 0:
+                  animation_frame.SetNewBase(
+                      AnimationFrame::NewBase::kCurrentFrame);
+                  break;
+                case 1:
+                  animation_frame.SetNewBase(AnimationFrame::NewBase::kNone);
+                  break;
+                case 2:
+                  animation_frame.SetNewBase(
+                      AnimationFrame::NewBase::kExisting);
+                  break;
+                default:
+                  animation_frame.SetNewBase(AnimationFrame::NewBase::kNone);
+                  break;
+              }
+              if (bop == 0)
+                animation_frame.SetBlendMode(
+                    AnimationFrame::BlendMode::kReplace);
+              else
+                animation_frame.SetBlendMode(AnimationFrame::BlendMode::kBlend);
+              io->animation_frames.push_back(animation_frame);
+              io->dec_pixels += w0 * h0;
+
+              ImageBundle bundle(&io->metadata);
+              Image3F sub_frame(w0, h0);
+              ImageU sub_frame_alpha(w0, h0);
+              for (size_t y = 0; y < h0; ++y) {
+                float* const JXL_RESTRICT row_r = sub_frame.PlaneRow(0, y);
+                float* const JXL_RESTRICT row_g = sub_frame.PlaneRow(1, y);
+                float* const JXL_RESTRICT row_b = sub_frame.PlaneRow(2, y);
+                uint16_t* const JXL_RESTRICT row_alpha = sub_frame_alpha.Row(y);
+                uint8_t* const f = frameRaw.rows[y];
+                for (size_t x = 0; x < w0; ++x) {
+                  if (f[4 * x + 3] == 0) {
+                    row_alpha[x] = 0;
+                    row_r[x] = 0;
+                    row_g[x] = 0;
+                    row_b[x] = 0;
+                    continue;
+                  }
+                  row_r[x] = f[4 * x + 0];
+                  row_g[x] = f[4 * x + 1];
+                  row_b[x] = f[4 * x + 2];
+                  row_alpha[x] = f[4 * x + 3];
+                }
+              }
+              bundle.SetFromImage(std::move(sub_frame),
+                                  ColorManagement::SRGB());
+              bundle.SetAlpha(std::move(sub_frame_alpha));
+              io->frames.push_back(std::move(bundle));
+            } else {
+              delete[] chunk.p;
+              break;
+            }
+          }
+
+          if (id == id_IEND) break;
+          // At this point the old frame is done. Let's start a new one.
+          w0 = png_get_uint_32(chunk.p + 12);
+          h0 = png_get_uint_32(chunk.p + 16);
+          x0 = png_get_uint_32(chunk.p + 20);
+          y0 = png_get_uint_32(chunk.p + 24);
+          delay_num = png_get_uint_16(chunk.p + 28);
+          delay_den = png_get_uint_16(chunk.p + 30);
+          dop = chunk.p[32];
+          bop = chunk.p[33];
+
+          if (w0 > cMaxPNGSize || h0 > cMaxPNGSize || x0 > cMaxPNGSize ||
+              y0 > cMaxPNGSize || x0 + w0 > w || y0 + h0 > h || dop > 2 ||
+              bop > 1) {
+            delete[] chunk.p;
+            break;
+          }
+
+          if (hasInfo) {
+            memcpy(chunkIHDR.p + 8, chunk.p + 12, 8);
+            if (processing_start(png_ptr, info_ptr, (void*)&frameRaw, hasInfo,
+                                 chunkIHDR, chunksInfo)) {
+              delete[] chunk.p;
+              break;
+            }
+          } else
+            skipFirst = false;
+
+          if (io->frames.size() == (skipFirst ? 1 : 0)) {
+            bop = 0;
+            if (dop == 2) dop = 1;
+          }
+        } else if (id == id_IDAT) {
+          hasInfo = true;
+          if (processing_data(png_ptr, info_ptr, chunk.p, chunk.size)) {
+            delete[] chunk.p;
+            break;
+          }
+        } else if (id == id_fdAT && isAnimated) {
+          png_save_uint_32(chunk.p + 4, chunk.size - 16);
+          memcpy(chunk.p + 8, "IDAT", 4);
+          if (processing_data(png_ptr, info_ptr, chunk.p + 4, chunk.size - 4)) {
+            delete[] chunk.p;
+            break;
+          }
+        } else if (notabc(chunk.p[4]) || notabc(chunk.p[5]) ||
+                   notabc(chunk.p[6]) || notabc(chunk.p[7])) {
+          delete[] chunk.p;
+          break;
+        } else if (!hasInfo) {
+          if (processing_data(png_ptr, info_ptr, chunk.p, chunk.size)) {
+            delete[] chunk.p;
+            break;
+          }
+          chunksInfo.push_back(chunk);
+          continue;
+        }
+        delete[] chunk.p;
+      }
+    }
+    delete[] frameRaw.rows;
+    delete[] frameRaw.p;
+  }
+
+  for (i = 0; i < chunksInfo.size(); i++) delete[] chunksInfo[i].p;
+
+  chunksInfo.clear();
+  delete[] chunkIHDR.p;
+
+  fclose(f);
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/extras/codec_apng.h b/jxl/extras/codec_apng.h
new file mode 100644
index 000000000..5b66e893e
--- /dev/null
+++ b/jxl/extras/codec_apng.h
@@ -0,0 +1,35 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EXTRAS_CODEC_APNG_H_
+#define JXL_EXTRAS_CODEC_APNG_H_
+
+// Decodes APNG images in memory.
+
+#include <stdint.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+
+namespace jxl {
+
+// Decodes `bytes` into `io`. io->dec_hints are ignored.
+Status DecodeImageAPNG(const Span<const uint8_t> bytes, CodecInOut* io);
+
+}  // namespace jxl
+
+#endif  // JXL_EXTRAS_CODEC_APNG_H_
diff --git a/jxl/extras/codec_exr.cc b/jxl/extras/codec_exr.cc
new file mode 100644
index 000000000..b9318e5f3
--- /dev/null
+++ b/jxl/extras/codec_exr.cc
@@ -0,0 +1,225 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/extras/codec_exr.h"
+
+#include <ImfChromaticitiesAttribute.h>
+#include <ImfIO.h>
+#include <ImfRgbaFile.h>
+
+#include <vector>
+
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+
+namespace jxl {
+
+namespace {
+
+namespace OpenEXR = OPENEXR_IMF_NAMESPACE;
+
+class InMemoryIStream : public OpenEXR::IStream {
+ public:
+  // The data pointed to by `bytes` must outlive the InMemoryIStream.
+  explicit InMemoryIStream(const Span<const uint8_t> bytes)
+      : IStream(/*fileName=*/""), bytes_(bytes) {}
+
+  bool isMemoryMapped() const override { return true; }
+  char* readMemoryMapped(const int n) override {
+    JXL_ASSERT(pos_ + n <= bytes_.size());
+    char* const result =
+        const_cast<char*>(reinterpret_cast<const char*>(bytes_.data() + pos_));
+    pos_ += n;
+    return result;
+  }
+  bool read(char c[], const int n) override {
+    std::copy_n(readMemoryMapped(n), n, c);
+    return pos_ < bytes_.size();
+  }
+
+  OpenEXR::Int64 tellg() override { return pos_; }
+  void seekg(const OpenEXR::Int64 pos) override {
+    JXL_ASSERT(pos + 1 <= bytes_.size());
+    pos_ = pos;
+  }
+
+ private:
+  const Span<const uint8_t> bytes_;
+  size_t pos_ = 0;
+};
+
+class InMemoryOStream : public OpenEXR::OStream {
+ public:
+  // `bytes` must outlive the InMemoryOStream.
+  explicit InMemoryOStream(PaddedBytes* const bytes)
+      : OStream(/*fileName=*/""), bytes_(*bytes) {}
+
+  void write(const char c[], const int n) override {
+    if (bytes_.size() < pos_ + n) {
+      bytes_.resize(pos_ + n);
+    }
+    std::copy_n(c, n, bytes_.begin() + pos_);
+    pos_ += n;
+  }
+
+  OpenEXR::Int64 tellp() override { return pos_; }
+  void seekp(const OpenEXR::Int64 pos) override {
+    if (bytes_.size() + 1 < pos) {
+      bytes_.resize(pos - 1);
+    }
+    pos_ = pos;
+  }
+
+ private:
+  PaddedBytes& bytes_;
+  size_t pos_ = 0;
+};
+
+}  // namespace
+
+Status DecodeImageEXR(Span<const uint8_t> bytes, CodecInOut* io) {
+  InMemoryIStream is(bytes);
+  OpenEXR::RgbaInputFile input(is);
+
+  if ((input.channels() & OpenEXR::RgbaChannels::WRITE_RGB) !=
+      OpenEXR::RgbaChannels::WRITE_RGB) {
+    return JXL_FAILURE("only RGB OpenEXR files are supported");
+  }
+  const bool has_alpha = (input.channels() & OpenEXR::RgbaChannels::WRITE_A) ==
+                         OpenEXR::RgbaChannels::WRITE_A;
+
+  auto image_size = input.displayWindow().size();
+  // Size is computed as max - min, but both bounds are inclusive.
+  ++image_size.x;
+  ++image_size.y;
+  Image3F image(image_size.x, image_size.y);
+  ImageU alpha;
+  if (has_alpha) {
+    alpha = ImageU(image_size.x, image_size.y);
+  }
+
+  std::vector<OpenEXR::Rgba> row_data(input.dataWindow().size().x + 1);
+  input.setFrameBuffer(row_data.data(), /*xStride=*/1, /*yStride=*/0);
+  for (int y = 0; y < image_size.y; ++y) {
+    input.readPixels(y + input.displayWindow().min.y);
+    const OpenEXR::Rgba* const JXL_RESTRICT input_row =
+        row_data.data() + input.displayWindow().min.x;
+    float* const JXL_RESTRICT rows[] = {
+        image.PlaneRow(0, y),
+        image.PlaneRow(1, y),
+        image.PlaneRow(2, y),
+    };
+    uint16_t* const JXL_RESTRICT alpha_row = has_alpha ? alpha.Row(y) : nullptr;
+    for (int x = 0; x < image_size.x; ++x) {
+      const float multiplier = 255.f / std::max<float>(input_row[x].a, 0.01f);
+      rows[0][x] = multiplier * input_row[x].r;
+      rows[1][x] = multiplier * input_row[x].g;
+      rows[2][x] = multiplier * input_row[x].b;
+      if (has_alpha) {
+        alpha_row[x] = std::numeric_limits<uint16_t>::max() * input_row[x].a;
+      }
+    }
+  }
+
+  ColorEncoding color_encoding;
+  color_encoding.tf.SetTransferFunction(TransferFunction::kLinear);
+  color_encoding.SetColorSpace(ColorSpace::kRGB);
+  PrimariesCIExy primaries = ColorManagement::SRGB().GetPrimaries();
+  CIExy white_point = ColorManagement::SRGB().GetWhitePoint();
+  const auto* const chromaticities =
+      input.header().findTypedAttribute<OpenEXR::ChromaticitiesAttribute>(
+          OpenEXR::ChromaticitiesAttribute().typeName());
+  if (chromaticities != nullptr) {
+    primaries.r.x = chromaticities->value().red.x;
+    primaries.r.y = chromaticities->value().red.y;
+    primaries.g.x = chromaticities->value().green.x;
+    primaries.g.y = chromaticities->value().green.y;
+    primaries.b.x = chromaticities->value().blue.x;
+    primaries.b.y = chromaticities->value().blue.y;
+    white_point.x = chromaticities->value().white.x;
+    white_point.y = chromaticities->value().white.y;
+  }
+  JXL_RETURN_IF_ERROR(color_encoding.SetPrimaries(primaries));
+  JXL_RETURN_IF_ERROR(color_encoding.SetWhitePoint(white_point));
+  JXL_RETURN_IF_ERROR(ColorManagement::CreateProfile(&color_encoding));
+
+  io->metadata.bits_per_sample = 32;
+  io->SetFromImage(std::move(image), color_encoding);
+  io->metadata.color_encoding = color_encoding;
+  if (has_alpha) {
+    io->metadata.alpha_bits = 16;
+    io->Main().SetAlpha(std::move(alpha));
+  }
+  return true;
+}
+
+Status EncodeImageEXR(const CodecInOut* io, const ColorEncoding& c_desired,
+                      ThreadPool* pool, PaddedBytes* bytes) {
+  ColorEncoding c_linear = c_desired;
+  c_linear.tf.SetTransferFunction(TransferFunction::kLinear);
+  JXL_RETURN_IF_ERROR(ColorManagement::CreateProfile(&c_linear));
+  ImageMetadata metadata = io->metadata;
+  ImageBundle store(&metadata);
+  const ImageBundle* linear;
+  JXL_RETURN_IF_ERROR(
+      TransformIfNeeded(io->Main(), c_linear, pool, &store, &linear));
+
+  const bool has_alpha = io->Main().HasAlpha();
+
+  InMemoryOStream os(bytes);
+  OpenEXR::Header header(io->xsize(), io->ysize());
+  OpenEXR::ChromaticitiesAttribute chromaticities;
+  const PrimariesCIExy& primaries =
+      c_linear.HasPrimaries() ? c_linear.GetPrimaries()
+                              : ColorManagement::SRGB().GetPrimaries();
+  chromaticities.value().red =
+      IMATH_NAMESPACE::V2f(primaries.r.x, primaries.r.y);
+  chromaticities.value().green =
+      IMATH_NAMESPACE::V2f(primaries.g.x, primaries.g.y);
+  chromaticities.value().blue =
+      IMATH_NAMESPACE::V2f(primaries.b.x, primaries.b.y);
+  chromaticities.value().white = IMATH_NAMESPACE::V2f(
+      c_linear.GetWhitePoint().x, c_linear.GetWhitePoint().y);
+  header.insert(chromaticities.typeName(), chromaticities);
+
+  OpenEXR::RgbaOutputFile output(
+      os, header, has_alpha ? OpenEXR::WRITE_RGBA : OpenEXR::WRITE_RGB);
+  std::vector<OpenEXR::Rgba> row_data(io->xsize());
+  output.setFrameBuffer(row_data.data(), /*xStride=*/1, /*yStride=*/0);
+
+  const float multiplier =
+      io->metadata.IntensityTarget() * kIntensityMultiplier / 255.f;
+  const float alpha_multiplier =
+      has_alpha ? 1.f / ((1 << io->metadata.alpha_bits) - 1) : 0;
+
+  for (size_t y = 0; y < io->ysize(); ++y) {
+    const float* const JXL_RESTRICT input_rows[] = {
+        linear->color().ConstPlaneRow(0, y),
+        linear->color().ConstPlaneRow(1, y),
+        linear->color().ConstPlaneRow(2, y),
+    };
+    const uint16_t* const JXL_RESTRICT alpha_row =
+        has_alpha ? io->Main().alpha().ConstRow(y) : nullptr;
+    for (size_t x = 0; x < io->xsize(); ++x) {
+      const float alpha = has_alpha ? alpha_multiplier * alpha_row[x] : 1.f;
+      row_data[x] = OpenEXR::Rgba(multiplier * alpha * input_rows[0][x],
+                                  multiplier * alpha * input_rows[1][x],
+                                  multiplier * alpha * input_rows[2][x], alpha);
+    }
+    output.writePixels(/*numScanLines=*/1);
+  }
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/extras/codec_exr.h b/jxl/extras/codec_exr.h
new file mode 100644
index 000000000..0af682faa
--- /dev/null
+++ b/jxl/extras/codec_exr.h
@@ -0,0 +1,39 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EXTRAS_CODEC_EXR_H_
+#define JXL_EXTRAS_CODEC_EXR_H_
+
+// Encodes OpenEXR images in memory.
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+
+namespace jxl {
+
+// Decodes `bytes` into `io`. io->dec_hints are ignored.
+Status DecodeImageEXR(Span<const uint8_t> bytes, CodecInOut* io);
+
+// Transforms from io->c_current to `c_desired` (with the transfer function set
+// to linear as that is the OpenEXR convention) and encodes into `bytes`.
+Status EncodeImageEXR(const CodecInOut* io, const ColorEncoding& c_desired,
+                      ThreadPool* pool, PaddedBytes* bytes);
+
+}  // namespace jxl
+
+#endif  // JXL_EXTRAS_CODEC_EXR_H_
diff --git a/jxl/extras/codec_gif.cc b/jxl/extras/codec_gif.cc
new file mode 100644
index 000000000..fb59a5102
--- /dev/null
+++ b/jxl/extras/codec_gif.cc
@@ -0,0 +1,319 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/extras/codec_gif.h"
+
+#include <gif_lib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/frame_header.h"
+#include "jxl/headers.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+
+namespace jxl {
+
+namespace {
+
+struct ReadState {
+  Span<const uint8_t> bytes;
+};
+
+struct DGifCloser {
+  void operator()(GifFileType* const ptr) const { DGifCloseFile(ptr, nullptr); }
+};
+using GifUniquePtr = std::unique_ptr<GifFileType, DGifCloser>;
+
+// Gif does not support partial transparency, so this considers anything non-0
+// as opaque.
+bool AllOpaque(const ImageU& alpha) {
+  for (size_t y = 0; y < alpha.ysize(); ++y) {
+    const uint16_t* const JXL_RESTRICT row = alpha.ConstRow(y);
+    for (size_t x = 0; x < alpha.xsize(); ++x) {
+      if (!row[x]) {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+}  // namespace
+
+Status DecodeImageGIF(Span<const uint8_t> bytes, CodecInOut* io) {
+  int error;
+  ReadState state = {bytes};
+  const auto ReadFromSpan = [](GifFileType* const gif, GifByteType* const bytes,
+                               int n) {
+    ReadState* const state = reinterpret_cast<ReadState*>(gif->UserData);
+    if (n > state->bytes.size()) {
+      n = state->bytes.size();
+    }
+    memcpy(bytes, state->bytes.data(), n);
+    state->bytes.remove_prefix(n);
+    return n;
+  };
+  GifUniquePtr gif(DGifOpen(&state, ReadFromSpan, &error));
+  if (gif == nullptr) {
+    if (error == D_GIF_ERR_NOT_GIF_FILE) {
+      // Not an error.
+      return false;
+    } else {
+      return JXL_FAILURE("Failed to read GIF: %s", GifErrorString(error));
+    }
+  }
+  error = DGifSlurp(gif.get());
+  if (error != GIF_OK) {
+    return JXL_FAILURE("Failed to read GIF: %s", GifErrorString(gif->Error));
+  }
+
+  if (gif->ImageCount > 1) {
+    io->metadata.m2.have_animation = true;
+    // Delays in GIF are specified in 100ths of a second.
+    io->animation.tps_numerator_minus_1 = 100 - 1;
+    io->animation_frames.reserve(gif->ImageCount);
+  }
+
+  io->frames.clear();
+  io->frames.reserve(gif->ImageCount);
+  io->dec_pixels = 0;
+
+  io->metadata.bits_per_sample = 8;
+  io->metadata.color_encoding = ColorManagement::SRGB();
+  io->metadata.alpha_bits = 0;
+  io->enc_size = bytes.size();
+  io->dec_hints.Foreach(
+      [](const std::string& key, const std::string& /*value*/) {
+        JXL_WARNING("GIF decoder ignoring %s hint", key.c_str());
+      });
+
+  Image3F canvas(gif->SWidth, gif->SHeight);
+  ImageU alpha(gif->SWidth, gif->SHeight);
+  GifColorType background_color;
+  if (gif->SColorMap == nullptr) {
+    background_color = {0, 0, 0};
+  } else {
+    if (gif->SBackGroundColor > gif->SColorMap->ColorCount) {
+      return JXL_FAILURE("GIF specifies out-of-bounds background color");
+    }
+    background_color = gif->SColorMap->Colors[gif->SBackGroundColor];
+  }
+  FillPlane<float>(background_color.Red,
+                   const_cast<Plane<float>*>(&canvas.Plane(0)));
+  FillPlane<float>(background_color.Green,
+                   const_cast<Plane<float>*>(&canvas.Plane(1)));
+  FillPlane<float>(background_color.Blue,
+                   const_cast<Plane<float>*>(&canvas.Plane(2)));
+  ZeroFillImage(&alpha);
+
+  Rect previous_rect_if_restore_to_background;
+
+  bool has_alpha = false;
+  bool replace = true;
+  for (int i = 0; i < gif->ImageCount; ++i) {
+    const SavedImage& image = gif->SavedImages[i];
+    const Rect image_rect(image.ImageDesc.Left, image.ImageDesc.Top,
+                          image.ImageDesc.Width, image.ImageDesc.Height);
+    io->dec_pixels += image_rect.xsize() * image_rect.ysize();
+    Rect total_rect;
+    if (previous_rect_if_restore_to_background.xsize() != 0 ||
+        previous_rect_if_restore_to_background.ysize() != 0) {
+      const size_t xbegin = std::min(
+          image_rect.x0(), previous_rect_if_restore_to_background.x0());
+      const size_t ybegin = std::min(
+          image_rect.y0(), previous_rect_if_restore_to_background.y0());
+      const size_t xend =
+          std::max(image_rect.x0() + image_rect.xsize(),
+                   previous_rect_if_restore_to_background.x0() +
+                       previous_rect_if_restore_to_background.xsize());
+      const size_t yend =
+          std::max(image_rect.y0() + image_rect.ysize(),
+                   previous_rect_if_restore_to_background.y0() +
+                       previous_rect_if_restore_to_background.ysize());
+      total_rect = Rect(xbegin, ybegin, xend - xbegin, yend - ybegin);
+      previous_rect_if_restore_to_background = Rect();
+      replace = true;
+    } else {
+      total_rect = image_rect;
+      replace = false;
+    }
+    if (!image_rect.IsInside(canvas)) {
+      return JXL_FAILURE("GIF frame extends outside of the canvas");
+    }
+    const ColorMapObject* const color_map = image.ImageDesc.ColorMap != nullptr
+                                                ? image.ImageDesc.ColorMap
+                                                : gif->SColorMap;
+    if (color_map == nullptr) {
+      return JXL_FAILURE("Missing GIF color map");
+    }
+    GraphicsControlBlock gcb;
+    DGifSavedExtensionToGCB(gif.get(), i, &gcb);
+    if (io->metadata.m2.have_animation) {
+      AnimationFrame animation_frame;
+      animation_frame.duration = gcb.DelayTime;
+      animation_frame.have_crop = total_rect.x0() != 0 ||
+                                  total_rect.y0() != 0 ||
+                                  total_rect.xsize() != canvas.xsize() ||
+                                  total_rect.ysize() != canvas.ysize();
+      if (animation_frame.have_crop) {
+        animation_frame.x0 = total_rect.x0();
+        animation_frame.y0 = total_rect.y0();
+        animation_frame.xsize = total_rect.xsize();
+        animation_frame.ysize = total_rect.ysize();
+      } else {
+        if (!io->animation_frames.empty()) {
+          io->animation_frames.back().SetNewBase(
+              AnimationFrame::NewBase::kNone);
+        }
+      }
+      switch (gcb.DisposalMode) {
+        case DISPOSE_DO_NOT:
+        case DISPOSE_BACKGROUND:
+          animation_frame.SetNewBase(AnimationFrame::NewBase::kCurrentFrame);
+          break;
+        case DISPOSE_PREVIOUS:
+          animation_frame.SetNewBase(AnimationFrame::NewBase::kExisting);
+          break;
+        case DISPOSAL_UNSPECIFIED:
+        default:
+          animation_frame.SetNewBase(AnimationFrame::NewBase::kNone);
+          break;
+      }
+      if (replace)
+        animation_frame.SetBlendMode(AnimationFrame::BlendMode::kReplace);
+      else
+        animation_frame.SetBlendMode(AnimationFrame::BlendMode::kBlend);
+      io->animation_frames.push_back(animation_frame);
+    }
+    ImageBundle bundle(&io->metadata);
+    Image3F frame = CopyImage(canvas);
+    ImageU frame_alpha = CopyImage(alpha);
+    for (size_t y = 0, byte_index = 0; y < image_rect.ysize(); ++y) {
+      float* const JXL_RESTRICT row_r =
+          image_rect.MutableRow(&frame.Plane(0), y);
+      float* const JXL_RESTRICT row_g =
+          image_rect.MutableRow(&frame.Plane(1), y);
+      float* const JXL_RESTRICT row_b =
+          image_rect.MutableRow(&frame.Plane(2), y);
+      uint16_t* const JXL_RESTRICT row_alpha =
+          image_rect.MutableRow(&frame_alpha, y);
+      for (size_t x = 0; x < image_rect.xsize(); ++x, ++byte_index) {
+        const GifByteType byte = image.RasterBits[byte_index];
+        if (byte >= color_map->ColorCount) {
+          return JXL_FAILURE("GIF color is out of bounds");
+        }
+        if (byte == gcb.TransparentColor) continue;
+        GifColorType color = color_map->Colors[byte];
+        row_alpha[x] = 255;
+        row_r[x] = color.Red;
+        row_g[x] = color.Green;
+        row_b[x] = color.Blue;
+      }
+    }
+    Image3F sub_frame(total_rect.xsize(), total_rect.ysize());
+    ImageU sub_frame_alpha(total_rect.xsize(), total_rect.ysize());
+    bool blend_alpha = false;
+    if (replace) {
+      CopyImageTo(total_rect, frame, &sub_frame);
+      CopyImageTo(total_rect, frame_alpha, &sub_frame_alpha);
+    } else {
+      for (size_t y = 0, byte_index = 0; y < image_rect.ysize(); ++y) {
+        float* const JXL_RESTRICT row_r = sub_frame.PlaneRow(0, y);
+        float* const JXL_RESTRICT row_g = sub_frame.PlaneRow(1, y);
+        float* const JXL_RESTRICT row_b = sub_frame.PlaneRow(2, y);
+        uint16_t* const JXL_RESTRICT row_alpha = sub_frame_alpha.Row(y);
+        for (size_t x = 0; x < image_rect.xsize(); ++x, ++byte_index) {
+          const GifByteType byte = image.RasterBits[byte_index];
+          if (byte > color_map->ColorCount) {
+            return JXL_FAILURE("GIF color is out of bounds");
+          }
+          if (byte == gcb.TransparentColor) {
+            row_alpha[x] = 0;
+            row_r[x] = 0;
+            row_g[x] = 0;
+            row_b[x] = 0;
+            blend_alpha =
+                true;  // need to use alpha channel if BlendMode blend is used
+            continue;
+          }
+          GifColorType color = color_map->Colors[byte];
+          row_alpha[x] = 255;
+          row_r[x] = color.Red;
+          row_g[x] = color.Green;
+          row_b[x] = color.Blue;
+        }
+      }
+    }
+    bundle.SetFromImage(std::move(sub_frame), ColorManagement::SRGB());
+    if (has_alpha || !AllOpaque(frame_alpha) || blend_alpha) {
+      if (!has_alpha) {
+        has_alpha = true;
+        io->metadata.alpha_bits = 8;
+        for (ImageBundle& previous_frame : io->frames) {
+          ImageU previous_alpha(previous_frame.xsize(), previous_frame.ysize());
+          FillImage<uint16_t>(255, &previous_alpha);
+          previous_frame.SetAlpha(std::move(previous_alpha));
+        }
+      }
+      bundle.SetAlpha(std::move(sub_frame_alpha));
+    }
+    io->frames.push_back(std::move(bundle));
+    switch (gcb.DisposalMode) {
+      case DISPOSE_DO_NOT:
+        canvas = std::move(frame);
+        alpha = std::move(frame_alpha);
+        break;
+
+      case DISPOSE_BACKGROUND:
+        FillPlane<float>(background_color.Red,
+                         const_cast<Plane<float>*>(&canvas.Plane(0)),
+                         image_rect);
+        FillPlane<float>(background_color.Green,
+                         const_cast<Plane<float>*>(&canvas.Plane(1)),
+                         image_rect);
+        FillPlane<float>(background_color.Blue,
+                         const_cast<Plane<float>*>(&canvas.Plane(2)),
+                         image_rect);
+        FillPlane<uint16_t>(0, &alpha, image_rect);
+        previous_rect_if_restore_to_background = image_rect;
+        break;
+
+      case DISPOSE_PREVIOUS:
+        break;
+
+      case DISPOSAL_UNSPECIFIED:
+      default:
+        FillPlane<float>(background_color.Red,
+                         const_cast<Plane<float>*>(&canvas.Plane(0)));
+        FillPlane<float>(background_color.Green,
+                         const_cast<Plane<float>*>(&canvas.Plane(1)));
+        FillPlane<float>(background_color.Blue,
+                         const_cast<Plane<float>*>(&canvas.Plane(2)));
+        ZeroFillImage(&alpha);
+    }
+  }
+
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/extras/codec_gif.h b/jxl/extras/codec_gif.h
new file mode 100644
index 000000000..1b5b5d378
--- /dev/null
+++ b/jxl/extras/codec_gif.h
@@ -0,0 +1,35 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EXTRAS_CODEC_GIF_H_
+#define JXL_EXTRAS_CODEC_GIF_H_
+
+// Decodes GIF images in memory.
+
+#include <stdint.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+
+namespace jxl {
+
+// Decodes `bytes` into `io`. io->dec_hints are ignored.
+Status DecodeImageGIF(const Span<const uint8_t> bytes, CodecInOut* io);
+
+}  // namespace jxl
+
+#endif  // JXL_EXTRAS_CODEC_GIF_H_
diff --git a/jxl/extras/codec_jpg.cc b/jxl/extras/codec_jpg.cc
new file mode 100644
index 000000000..4273b5913
--- /dev/null
+++ b/jxl/extras/codec_jpg.cc
@@ -0,0 +1,646 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/extras/codec_jpg.h"
+
+#include <stddef.h>
+#include <stdio.h>
+// After stddef/stdio
+#include <jpeglib.h>
+
+#include <setjmp.h>
+#include <stdint.h>
+
+#include <algorithm>
+#include <iterator>
+#include <limits>
+#include <numeric>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#if JPEGXL_ENABLE_SJPEG
+#include "sjpeg.h"
+#endif
+
+#ifdef MEMORY_SANITIZER
+#include "sanitizer/msan_interface.h"
+#endif
+
+namespace jxl {
+
+namespace {
+
+constexpr float kJPEGSampleMultiplier = 1 << (BITS_IN_JSAMPLE - 8);
+constexpr unsigned char kICCSignature[12] = {
+    0x49, 0x43, 0x43, 0x5F, 0x50, 0x52, 0x4F, 0x46, 0x49, 0x4C, 0x45, 0x00};
+constexpr int kICCMarker = JPEG_APP0 + 2;
+constexpr size_t kMaxBytesInMarker = 65533;
+
+constexpr float kJPEGSampleMin = std::numeric_limits<JSAMPLE>::min();
+constexpr float kJPEGSampleMax = std::numeric_limits<JSAMPLE>::max();
+
+bool MarkerIsICC(const jpeg_saved_marker_ptr marker) {
+  return marker->marker == kICCMarker &&
+         marker->data_length >= sizeof kICCSignature + 2 &&
+         std::equal(std::begin(kICCSignature), std::end(kICCSignature),
+                    marker->data);
+}
+
+Status ReadICCProfile(jpeg_decompress_struct* const cinfo,
+                      PaddedBytes* const icc) {
+  // Markers are 1-indexed, and we keep them that way in this vector to get a
+  // convenient 0 at the front for when we compute the offsets later.
+  std::vector<size_t> marker_lengths;
+  int num_markers = 0;
+  bool has_num_markers = false;
+  for (jpeg_saved_marker_ptr marker = cinfo->marker_list; marker != nullptr;
+       marker = marker->next) {
+#ifdef MEMORY_SANITIZER
+    // marker is initialized by libjpeg, which we are not instrumenting with
+    // msan.
+    __msan_unpoison(marker, sizeof(*marker));
+    __msan_unpoison(marker->data, marker->data_length);
+#endif
+    if (!MarkerIsICC(marker)) continue;
+
+    const int current_marker = marker->data[sizeof kICCSignature];
+    const int current_num_markers = marker->data[sizeof kICCSignature + 1];
+    if (has_num_markers) {
+      if (current_num_markers != num_markers) {
+        return JXL_FAILURE("inconsistent numbers of JPEG ICC markers");
+      }
+    } else {
+      num_markers = current_num_markers;
+      has_num_markers = true;
+      marker_lengths.resize(num_markers + 1);
+    }
+
+    if (marker_lengths[current_marker] != 0) {
+      return JXL_FAILURE("duplicate JPEG ICC marker number");
+    }
+    marker_lengths[current_marker] =
+        marker->data_length - sizeof kICCSignature - 2;
+  }
+
+  if (marker_lengths.empty()) {
+    // Not an error.
+    return false;
+  }
+
+  std::vector<size_t> offsets = std::move(marker_lengths);
+  std::partial_sum(offsets.begin(), offsets.end(), offsets.begin());
+  icc->resize(offsets.back());
+
+  for (jpeg_saved_marker_ptr marker = cinfo->marker_list; marker != nullptr;
+       marker = marker->next) {
+    if (!MarkerIsICC(marker)) continue;
+    const uint8_t* first = marker->data + sizeof kICCSignature + 2;
+    size_t count = marker->data_length - sizeof kICCSignature - 2;
+    size_t offset = offsets[marker->data[sizeof kICCSignature] - 1];
+    if (offset + count > icc->size()) {
+      // TODO(lode): catch this issue earlier at the root cause of this.
+      return JXL_FAILURE("ICC out of bounds");
+    }
+    std::copy_n(first, count, icc->data() + offset);
+  }
+
+  return true;
+}
+
+void WriteICCProfile(jpeg_compress_struct* const cinfo,
+                     const PaddedBytes& icc) {
+  constexpr size_t kMaxIccBytesInMarker =
+      kMaxBytesInMarker - sizeof kICCSignature - 2;
+  const int num_markers =
+      static_cast<int>(DivCeil(icc.size(), kMaxIccBytesInMarker));
+  size_t begin = 0;
+  for (int current_marker = 0; current_marker < num_markers; ++current_marker) {
+    const auto length = std::min(kMaxIccBytesInMarker, icc.size() - begin);
+    jpeg_write_m_header(
+        cinfo, kICCMarker,
+        static_cast<unsigned int>(length + sizeof kICCSignature + 2));
+    for (const unsigned char c : kICCSignature) {
+      jpeg_write_m_byte(cinfo, c);
+    }
+    jpeg_write_m_byte(cinfo, current_marker + 1);
+    jpeg_write_m_byte(cinfo, num_markers);
+    for (int i = 0; i < length; ++i) {
+      jpeg_write_m_byte(cinfo, icc[begin]);
+      ++begin;
+    }
+  }
+}
+
+Status SetChromaSubsampling(const YCbCrChromaSubsampling chroma_subsampling,
+                            jpeg_compress_struct* const cinfo) {
+  cinfo->comp_info[1].h_samp_factor = 1;
+  cinfo->comp_info[1].v_samp_factor = 1;
+  cinfo->comp_info[2].h_samp_factor = 1;
+  cinfo->comp_info[2].v_samp_factor = 1;
+  switch (chroma_subsampling) {
+    case YCbCrChromaSubsampling::kAuto:
+      return JXL_FAILURE(
+          "no rule for setting chroma subsampling automatically with libjpeg");
+
+    case YCbCrChromaSubsampling::k444:
+      cinfo->comp_info[0].h_samp_factor = 1;
+      cinfo->comp_info[0].v_samp_factor = 1;
+      return true;
+
+    case YCbCrChromaSubsampling::k422:
+      cinfo->comp_info[0].h_samp_factor = 2;
+      cinfo->comp_info[0].v_samp_factor = 1;
+      return true;
+
+    case YCbCrChromaSubsampling::k420:
+      cinfo->comp_info[0].h_samp_factor = 2;
+      cinfo->comp_info[0].v_samp_factor = 2;
+      return true;
+
+    case YCbCrChromaSubsampling::k411:
+      cinfo->comp_info[0].h_samp_factor = 4;
+      cinfo->comp_info[0].v_samp_factor = 1;
+      return true;
+
+    default:
+      return JXL_FAILURE("invalid chroma subsampling");
+  }
+}
+
+void MyErrorExit(j_common_ptr cinfo) {
+  jmp_buf* env = static_cast<jmp_buf*>(cinfo->client_data);
+  (*cinfo->err->output_message)(cinfo);
+  jpeg_destroy_decompress(reinterpret_cast<j_decompress_ptr>(cinfo));
+  longjmp(*env, 1);
+}
+
+void MyOutputMessage(j_common_ptr cinfo) {
+#if JXL_DEBUG_WARNING == 1
+  char buf[JMSG_LENGTH_MAX];
+  (*cinfo->err->format_message)(cinfo, buf);
+  JXL_WARNING("%s", buf);
+#endif
+}
+
+constexpr int kPlaneOrder[] = {1, 0, 2};
+constexpr int kInvPlaneOrder[] = {1, 0, 2};
+
+}  // namespace
+
+Status DecodeImageJPG(const Span<const uint8_t> bytes, CodecInOut* io,
+                      const DecodeTarget target) {
+  jpeg_decompress_struct cinfo;
+#ifdef MEMORY_SANITIZER
+  // cinfo is initialized by libjpeg, which we are not instrumenting with
+  // msan, therefore we need to initialize cinfo here.
+  memset(&cinfo, 0, sizeof(cinfo));
+#endif
+  // Setup error handling in jpeg library so we can deal with broken jpegs in
+  // the fuzzer.
+  jpeg_error_mgr jerr;
+  jmp_buf env;
+  cinfo.err = jpeg_std_error(&jerr);
+  jerr.error_exit = &MyErrorExit;
+  jerr.output_message = &MyOutputMessage;
+  if (setjmp(env)) {
+    return false;
+  }
+  cinfo.client_data = static_cast<void*>(&env);
+
+  jpeg_create_decompress(&cinfo);
+  jpeg_mem_src(&cinfo, bytes.data(), bytes.size());
+  jpeg_save_markers(&cinfo, kICCMarker, 0xFFFF);
+  jpeg_read_header(&cinfo, TRUE);
+  ColorEncoding color_encoding;
+  PaddedBytes icc;
+  if (ReadICCProfile(&cinfo, &icc)) {
+    if (!ColorManagement::SetProfile(std::move(icc), &color_encoding)) {
+      jpeg_abort_decompress(&cinfo);
+      jpeg_destroy_decompress(&cinfo);
+      return JXL_FAILURE("read an invalid ICC profile");
+    }
+  } else {
+    color_encoding = ColorManagement::SRGB(cinfo.output_components == 1);
+  }
+  io->metadata.bits_per_sample = BITS_IN_JSAMPLE;
+  io->metadata.color_encoding = color_encoding;
+  io->enc_size = bytes.size();
+  int nbcomp = cinfo.num_components;
+  if (nbcomp != 1 && nbcomp != 3) {
+    jpeg_abort_decompress(&cinfo);
+    jpeg_destroy_decompress(&cinfo);
+    return JXL_FAILURE("unsupported number of components (%d) in JPEG",
+                       cinfo.output_components);
+  }
+  io->dec_hints.Foreach(
+      [](const std::string& key, const std::string& /*value*/) {
+        JXL_WARNING("JPEG decoder ignoring %s hint", key.c_str());
+      });
+
+  if (target == DecodeTarget::kPixels) {
+    jpeg_start_decompress(&cinfo);
+    JXL_ASSERT(cinfo.output_components == nbcomp);
+    Image3F image(cinfo.image_width, cinfo.image_height);
+    std::unique_ptr<JSAMPLE[]> row(
+        new JSAMPLE[cinfo.output_components * cinfo.image_width]);
+    for (size_t y = 0; y < image.ysize(); ++y) {
+      JSAMPROW rows[] = {row.get()};
+      jpeg_read_scanlines(&cinfo, rows, 1);
+#ifdef MEMORY_SANITIZER
+      __msan_unpoison(row.get(), sizeof(JSAMPLE) * cinfo.output_components *
+                                     cinfo.image_width);
+#endif
+      float* const JXL_RESTRICT output_row[] = {
+          image.PlaneRow(0, y), image.PlaneRow(1, y), image.PlaneRow(2, y)};
+      if (cinfo.output_components == 1) {
+        for (size_t x = 0; x < image.xsize(); ++x) {
+          output_row[0][x] = output_row[1][x] = output_row[2][x] =
+              row[x] * (1.f / kJPEGSampleMultiplier);
+        }
+      } else {  // 3 components
+        for (size_t x = 0; x < image.xsize(); ++x) {
+          for (size_t c = 0; c < 3; ++c) {
+            output_row[c][x] = row[3 * x + c] * (1.f / kJPEGSampleMultiplier);
+          }
+        }
+      }
+    }
+    io->SetFromImage(std::move(image), color_encoding);
+  } else {  // (target == DecodeTarget::kQuantizedCoeffs)
+    jvirt_barray_ptr* coeffs_array = jpeg_read_coefficients(&cinfo);
+
+#ifdef MEMORY_SANITIZER
+    if (cinfo.comp_info) {
+      __msan_unpoison(cinfo.comp_info,
+                      sizeof(*cinfo.comp_info) * cinfo.num_components);
+    }
+#endif
+
+    YCbCrChromaSubsampling cs = YCbCrChromaSubsampling::k444;
+    if (nbcomp == 3 && cinfo.comp_info[0].h_samp_factor == 2 &&
+        cinfo.comp_info[0].v_samp_factor == 2 &&
+        cinfo.comp_info[1].h_samp_factor == 1 &&
+        cinfo.comp_info[1].v_samp_factor == 1 &&
+        cinfo.comp_info[2].h_samp_factor == 1 &&
+        cinfo.comp_info[2].v_samp_factor == 1) {
+      cs = YCbCrChromaSubsampling::k420;
+    } else if (nbcomp == 3 && cinfo.comp_info[0].h_samp_factor == 2 &&
+               cinfo.comp_info[0].v_samp_factor == 1 &&
+               cinfo.comp_info[1].h_samp_factor == 1 &&
+               cinfo.comp_info[1].v_samp_factor == 1 &&
+               cinfo.comp_info[2].h_samp_factor == 1 &&
+               cinfo.comp_info[2].v_samp_factor == 1) {
+      cs = YCbCrChromaSubsampling::k422;
+    } else if (nbcomp == 3 && cinfo.comp_info[0].h_samp_factor == 4 &&
+               cinfo.comp_info[0].v_samp_factor == 1 &&
+               cinfo.comp_info[1].h_samp_factor == 1 &&
+               cinfo.comp_info[1].v_samp_factor == 1 &&
+               cinfo.comp_info[2].h_samp_factor == 1 &&
+               cinfo.comp_info[2].v_samp_factor == 1) {
+      cs = YCbCrChromaSubsampling::k411;
+    } else {
+      for (int ci = 0; ci < nbcomp; ci++) {
+        jpeg_component_info* compptr = cinfo.comp_info + ci;
+        int srh = compptr->h_samp_factor;
+        int srv = compptr->v_samp_factor;
+        if (srh > 1 || srv > 1) {
+          JXL_FAILURE("Cannot handle this chroma subsampling mode");
+          return false;
+        }
+      }
+    }
+
+    io->frames.clear();
+    io->frames.reserve(1);
+    ImageBundle bundle(&io->metadata);
+
+    if (cinfo.jpeg_color_space == JCS_YCbCr || nbcomp == 1) {
+      bundle.color_transform = ColorTransform::kYCbCr;
+    } else {
+      bundle.color_transform = ColorTransform::kNone;
+    }
+
+    bundle.chroma_subsampling = cs;
+
+    Image3F coeffs(cinfo.comp_info->width_in_blocks * 8,
+                   cinfo.comp_info->height_in_blocks * 8);
+    ZeroFillImage(&coeffs);
+    for (int ci : kPlaneOrder) {
+      if (ci >= nbcomp) {
+        for (int i = 0; i < 64; i++) {
+          bundle.jpeg_quant_table.push_back(1);
+        }
+        continue;
+      }
+      jpeg_component_info* compptr = cinfo.comp_info + ci;
+#ifdef MEMORY_SANITIZER
+      if (compptr->quant_table) {
+        __msan_unpoison(compptr->quant_table, sizeof(*compptr->quant_table));
+      }
+#endif
+      int hib = compptr->height_in_blocks;
+      int wib = compptr->width_in_blocks;
+      for (int i = 0; i < 64; i++) {
+        bundle.jpeg_quant_table.push_back(compptr->quant_table->quantval[i]);
+      }
+      intptr_t onerow = coeffs.PixelsPerRow();
+      for (int by = 0; by < hib; by++) {
+#ifdef MEMORY_SANITIZER
+        if (cinfo.mem) {
+          __msan_unpoison(cinfo.mem, sizeof(*cinfo.mem));
+        }
+#endif
+        JBLOCKARRAY buffer = (cinfo.mem->access_virt_barray)(
+            (j_common_ptr)&cinfo, coeffs_array[ci], by, (JDIMENSION)1, FALSE);
+#ifdef MEMORY_SANITIZER
+        if (buffer) {
+          __msan_unpoison(buffer, sizeof(buffer));
+        }
+#endif
+
+        float* JXL_RESTRICT coeff =
+            const_cast<float*>(coeffs.PlaneRow((ci < 2 ? 1 - ci : 2), by * 8));
+        for (int bx = 0; bx < wib; bx++) {
+          JCOEFPTR blockptr = buffer[0][bx];
+          for (int i = 0; i < 64; i++) {
+            coeff[8 * bx + (i % 8) + onerow * (i / 8)] = blockptr[i];
+          }
+        }
+      }
+    }
+
+    bundle.SetFromImage(std::move(coeffs), color_encoding);
+    bundle.jpeg_xsize = cinfo.image_width;
+    bundle.jpeg_ysize = cinfo.image_height;
+    io->frames.push_back(std::move(bundle));
+  }
+  jpeg_finish_decompress(&cinfo);
+  jpeg_destroy_decompress(&cinfo);
+  io->dec_pixels = io->xsize() * io->ysize();
+  return true;
+}
+
+Status EncodeWithLibJpeg(const ImageBundle* ib, size_t quality,
+                         YCbCrChromaSubsampling chroma_subsampling,
+                         PaddedBytes* bytes, const DecodeTarget target) {
+  jpeg_compress_struct cinfo;
+  jvirt_barray_ptr coeffs_array[3] = {};
+#ifdef MEMORY_SANITIZER
+  // cinfo is initialized by libjpeg, which we are not instrumenting with
+  // msan.
+  __msan_unpoison(&cinfo, sizeof(cinfo));
+#endif
+  jpeg_error_mgr jerr;
+  cinfo.err = jpeg_std_error(&jerr);
+  jpeg_create_compress(&cinfo);
+  unsigned char* buffer = nullptr;
+  unsigned long size = 0;
+  jpeg_mem_dest(&cinfo, &buffer, &size);
+  cinfo.image_width = ib->xsize();
+  cinfo.image_height = ib->ysize();
+  if (ib->IsGray()) {
+    cinfo.input_components = 1;
+    cinfo.in_color_space = JCS_GRAYSCALE;
+  } else {
+    cinfo.input_components = 3;
+    cinfo.in_color_space = JCS_RGB;
+  }
+  jpeg_set_defaults(&cinfo);
+  cinfo.optimize_coding = TRUE;
+  if (target == DecodeTarget::kPixels) {
+    if (cinfo.input_components == 3) {
+      JXL_RETURN_IF_ERROR(SetChromaSubsampling(chroma_subsampling, &cinfo));
+    }
+    jpeg_set_quality(&cinfo, quality, TRUE);
+    jpeg_start_compress(&cinfo, TRUE);
+    if (!ib->IsSRGB()) {
+      WriteICCProfile(&cinfo, ib->c_current().icc);
+    }
+    if (cinfo.input_components > 3)
+      return JXL_FAILURE("invalid numbers of components");
+
+    std::unique_ptr<JSAMPLE[]> row(
+        new JSAMPLE[cinfo.input_components * cinfo.image_width]);
+    for (size_t y = 0; y < ib->ysize(); ++y) {
+      const float* const JXL_RESTRICT input_row[3] = {
+          ib->color().ConstPlaneRow(0, y), ib->color().ConstPlaneRow(1, y),
+          ib->color().ConstPlaneRow(2, y)};
+      for (size_t x = 0; x < ib->xsize(); ++x) {
+        for (size_t c = 0; c < cinfo.input_components; ++c) {
+          row[cinfo.input_components * x + c] = static_cast<JSAMPLE>(
+              std::max(std::min(kJPEGSampleMultiplier * input_row[c][x] + .5f,
+                                kJPEGSampleMax),
+                       kJPEGSampleMin));
+        }
+      }
+      JSAMPROW rows[] = {row.get()};
+      jpeg_write_scanlines(&cinfo, rows, 1);
+    }
+  } else {
+    cinfo.image_width = ib->xsize();
+    cinfo.image_height = ib->ysize();
+#if JPEG_LIB_VERSION >= 70
+    cinfo.jpeg_width = ib->xsize();
+    cinfo.jpeg_height = ib->ysize();
+    cinfo.scale_num = 1;
+    cinfo.scale_denom = 1;
+#endif  // JPEG_LIB_VERSION >= 70
+
+    cinfo.jpeg_color_space = JCS_RGB;
+    if (ib->color_transform == ColorTransform::kYCbCr)
+      cinfo.jpeg_color_space = JCS_YCbCr;
+    if (cinfo.input_components == 3) {
+      JXL_RETURN_IF_ERROR(SetChromaSubsampling(ib->chroma_subsampling, &cinfo));
+    }
+
+    // reconstruct the height-in-blocks (hib) and width-in-blocks (wib)
+    int hib = ib->color().ysize() / 8;
+    int wib = ib->color().xsize() / 8;
+    int chroma_hib = hib;
+    int chroma_wib = wib;
+    int luma_alloc_hib = hib;
+    int luma_alloc_wib = wib;
+    if (ib->chroma_subsampling == YCbCrChromaSubsampling::k420) {
+      chroma_hib = ((ib->ysize() + 1) / 2 + 7) / 8;
+      luma_alloc_hib = 2 * chroma_hib;
+    }
+    if (ib->chroma_subsampling == YCbCrChromaSubsampling::k420 ||
+        ib->chroma_subsampling == YCbCrChromaSubsampling::k422) {
+      chroma_wib = ((ib->xsize() + 1) / 2 + 7) / 8;
+      luma_alloc_wib = 2 * chroma_wib;
+    }
+    if (ib->chroma_subsampling == YCbCrChromaSubsampling::k411) {
+      chroma_wib = ((ib->xsize() + 3) / 4 + 7) / 8;
+      luma_alloc_wib = 4 * chroma_wib;
+    }
+
+    for (int ci = 0; ci < 3; ci++) {
+      jpeg_component_info* compptr = cinfo.comp_info + ci;
+      if (ci == 0) {
+        compptr->height_in_blocks = hib;
+        compptr->width_in_blocks = wib;
+        coeffs_array[ci] = cinfo.mem->request_virt_barray(
+            (j_common_ptr)&cinfo, JPOOL_IMAGE, TRUE, luma_alloc_wib,
+            luma_alloc_hib, luma_alloc_hib);
+      } else {
+        compptr->height_in_blocks = chroma_hib;
+        compptr->width_in_blocks = chroma_wib;
+        coeffs_array[ci] = cinfo.mem->request_virt_barray(
+            (j_common_ptr)&cinfo, JPOOL_IMAGE, TRUE, compptr->width_in_blocks,
+            compptr->height_in_blocks, compptr->height_in_blocks);
+      }
+      cinfo.quant_tbl_ptrs[ci] = jpeg_alloc_quant_table((j_common_ptr)&cinfo);
+      cinfo.quant_tbl_ptrs[ci]->sent_table = FALSE;
+      compptr->quant_tbl_no = ci;
+    }
+    cinfo.mem->realize_virt_arrays((j_common_ptr)&cinfo);
+    for (int ci = 0; ci < 3; ci++) {
+      int src_q = kInvPlaneOrder[ci] * 64;
+      for (int i = 0; i < 64; i++) {
+        cinfo.quant_tbl_ptrs[ci]->quantval[i] = ib->jpeg_quant_table[i + src_q];
+      }
+    }
+    jpeg_write_coefficients(&cinfo, coeffs_array);
+    intptr_t onerow = ib->color().PixelsPerRow();
+    for (int ci = 0; ci < 3; ci++) {
+      for (int by = 0; by < (ci == 0 ? hib : chroma_hib); by++) {
+        JBLOCKARRAY buffer = (cinfo.mem->access_virt_barray)(
+            (j_common_ptr)&cinfo, coeffs_array[ci], by, (JDIMENSION)1, TRUE);
+#ifdef MEMORY_SANITIZER
+        if (buffer) {
+          __msan_unpoison(buffer, sizeof(buffer));
+        }
+#endif
+        const float* JXL_RESTRICT coeff =
+            ib->color().ConstPlaneRow((ci < 2 ? 1 - ci : ci), by * 8);
+        for (int bx = 0; bx < (ci == 0 ? wib : chroma_wib); bx++) {
+          JCOEFPTR blockptr = buffer[0][bx];
+          for (int i = 0; i < 64; i++) {
+            blockptr[i] = coeff[8 * bx + (i % 8) + onerow * (i / 8)];
+          }
+        }
+      }
+    }
+    if (!ib->IsSRGB()) {
+      WriteICCProfile(&cinfo, ib->c_current().icc);
+    }
+  }
+  jpeg_finish_compress(&cinfo);
+  jpeg_destroy_compress(&cinfo);
+  bytes->resize(size);
+#ifdef MEMORY_SANITIZER
+  // Compressed image data is initialized by libjpeg, which we are not
+  // instrumenting with msan.
+  __msan_unpoison(buffer, size);
+#endif
+  std::copy_n(buffer, size, bytes->data());
+  std::free(buffer);
+  return true;
+}
+
+Status EncodeWithSJpeg(const ImageBundle* ib, size_t quality,
+                       YCbCrChromaSubsampling chroma_subsampling,
+                       PaddedBytes* bytes) {
+#if !JPEGXL_ENABLE_SJPEG
+  return JXL_FAILURE("JPEG XL was built without sjpeg support");
+#else
+  sjpeg::EncoderParam param(quality);
+  if (!ib->IsSRGB()) {
+    param.iccp.assign(ib->metadata()->color_encoding.icc.begin(),
+                      ib->metadata()->color_encoding.icc.end());
+  }
+  switch (chroma_subsampling) {
+    case YCbCrChromaSubsampling::kAuto:
+      param.yuv_mode = SJPEG_YUV_AUTO;
+      break;
+    case YCbCrChromaSubsampling::k444:
+      param.yuv_mode = SJPEG_YUV_444;
+      break;
+    case YCbCrChromaSubsampling::k422:
+      return JXL_FAILURE("sjpeg does not support 4:2:2 chroma subsampling");
+    case YCbCrChromaSubsampling::k420:
+      param.yuv_mode = SJPEG_YUV_SHARP;
+      break;
+    case YCbCrChromaSubsampling::k411:
+      return JXL_FAILURE("sjpeg does not support 4:1:1 chroma subsampling");
+  }
+  std::vector<uint8_t> rgb;
+  rgb.reserve(ib->xsize() * ib->ysize() * 3);
+  for (size_t y = 0; y < ib->ysize(); ++y) {
+    const float* const rows[] = {
+        ib->color().ConstPlaneRow(0, y),
+        ib->color().ConstPlaneRow(1, y),
+        ib->color().ConstPlaneRow(2, y),
+    };
+    for (size_t x = 0; x < ib->xsize(); ++x) {
+      for (const float* const row : rows) {
+        rgb.push_back(static_cast<uint8_t>(
+            std::max(0.f, std::min(255.f, std::round(row[x])))));
+      }
+    }
+  }
+  std::string output;
+  JXL_RETURN_IF_ERROR(sjpeg::Encode(rgb.data(), ib->xsize(), ib->ysize(),
+                                    ib->xsize() * 3, param, &output));
+  bytes->assign(
+      reinterpret_cast<const uint8_t*>(output.data()),
+      reinterpret_cast<const uint8_t*>(output.data() + output.size()));
+  return true;
+#endif
+}
+
+Status EncodeImageJPG(const CodecInOut* io, JpegEncoder encoder, size_t quality,
+                      YCbCrChromaSubsampling chroma_subsampling,
+                      ThreadPool* pool, PaddedBytes* bytes,
+                      const DecodeTarget target) {
+  if (io->Main().HasAlpha()) {
+    return JXL_FAILURE("alpha is not supported");
+  }
+  if (quality < 0 || quality > 100) {
+    return JXL_FAILURE("please specify a 0-100 JPEG quality");
+  }
+
+  const ImageBundle* ib;
+  ImageMetadata metadata = io->metadata;
+  ImageBundle ib_store(&metadata);
+  JXL_RETURN_IF_ERROR(TransformIfNeeded(io->Main(), io->metadata.color_encoding,
+                                        pool, &ib_store, &ib));
+
+  switch (encoder) {
+    case JpegEncoder::kLibJpeg:
+      JXL_RETURN_IF_ERROR(
+          EncodeWithLibJpeg(ib, quality, chroma_subsampling, bytes, target));
+      break;
+    case JpegEncoder::kSJpeg:
+      if (target != DecodeTarget::kPixels)
+        return JXL_FAILURE("Not implemented: SJpeg encode from DCT");
+      JXL_RETURN_IF_ERROR(
+          EncodeWithSJpeg(ib, quality, chroma_subsampling, bytes));
+      break;
+    default:
+      return JXL_FAILURE("tried to use an unknown JPEG encoder");
+  }
+
+  io->enc_size = bytes->size();
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/extras/codec_jpg.h b/jxl/extras/codec_jpg.h
new file mode 100644
index 000000000..1fd452ce7
--- /dev/null
+++ b/jxl/extras/codec_jpg.h
@@ -0,0 +1,48 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EXTRAS_CODEC_JPG_H_
+#define JXL_EXTRAS_CODEC_JPG_H_
+
+// Encodes JPG pixels and metadata in memory.
+
+#include <stdint.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/extras/codec.h"
+
+namespace jxl {
+
+enum class JpegEncoder {
+  kLibJpeg,
+  kSJpeg,
+};
+
+// Decodes `bytes` into `io`. io->dec_hints are ignored.
+Status DecodeImageJPG(const Span<const uint8_t> bytes, CodecInOut* io,
+                      const DecodeTarget target = DecodeTarget::kPixels);
+
+// Encodes into `bytes`.
+Status EncodeImageJPG(const CodecInOut* io, JpegEncoder encoder, size_t quality,
+                      YCbCrChromaSubsampling chroma_subsampling,
+                      ThreadPool* pool, PaddedBytes* bytes,
+                      const DecodeTarget target = DecodeTarget::kPixels);
+
+}  // namespace jxl
+
+#endif  // JXL_EXTRAS_CODEC_JPG_H_
diff --git a/jxl/extras/codec_pgx.cc b/jxl/extras/codec_pgx.cc
new file mode 100644
index 000000000..988348af0
--- /dev/null
+++ b/jxl/extras/codec_pgx.cc
@@ -0,0 +1,348 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/extras/codec_pgx.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/bits.h"
+#include "jxl/base/byte_order.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/color_management.h"
+#include "jxl/external_image.h"
+#include "jxl/fields.h"  // AllDefault
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+namespace {
+
+struct HeaderPGX {
+  // NOTE: PGX is always grayscale
+  size_t xsize;
+  size_t ysize;
+  size_t bits_per_sample;
+  bool big_endian;
+  bool is_signed;
+};
+
+class Parser {
+ public:
+  explicit Parser(const Span<const uint8_t> bytes)
+      : pos_(bytes.data()), end_(pos_ + bytes.size()) {}
+
+  // Sets "pos" to the first non-header byte/pixel on success.
+  Status ParseHeader(HeaderPGX* header, const uint8_t** pos) {
+    // codec_facade ensures we have at least two bytes => no range check here.
+    if (pos_[0] != 'P' || pos_[1] != 'G') return false;
+    pos_ += 2;
+    return ParseHeaderPGX(header, pos);
+  }
+
+  // Exposed for testing
+  Status ParseUnsigned(size_t* number) {
+    if (pos_ == end_) return JXL_FAILURE("PGX: reached end before number");
+    if (!IsDigit(*pos_)) return JXL_FAILURE("PGX: expected unsigned number");
+
+    *number = 0;
+    while (pos_ < end_ && *pos_ >= '0' && *pos_ <= '9') {
+      *number *= 10;
+      *number += *pos_ - '0';
+      ++pos_;
+    }
+
+    return true;
+  }
+
+ private:
+  static bool IsDigit(const uint8_t c) { return '0' <= c && c <= '9'; }
+  static bool IsLineBreak(const uint8_t c) { return c == '\r' || c == '\n'; }
+  static bool IsWhitespace(const uint8_t c) {
+    return IsLineBreak(c) || c == '\t' || c == ' ';
+  }
+
+  Status SkipSpace() {
+    if (pos_ == end_) return JXL_FAILURE("PGX: reached end before space");
+    const uint8_t c = *pos_;
+    if (c != ' ') return JXL_FAILURE("PGX: expected space");
+    ++pos_;
+    return true;
+  }
+
+  Status SkipLineBreak() {
+    if (pos_ == end_) return JXL_FAILURE("PGX: reached end before line break");
+    if (!IsLineBreak(*pos_)) return JXL_FAILURE("PGX: expected line break");
+    ++pos_;
+    // 0xd 0xa linebreak is possible
+    if (pos_ != end_ && IsLineBreak(*pos_)) ++pos_;
+    return true;
+  }
+
+  Status SkipSingleWhitespace() {
+    if (pos_ == end_) return JXL_FAILURE("PGX: reached end before whitespace");
+    if (!IsWhitespace(*pos_)) return JXL_FAILURE("PGX: expected whitespace");
+    ++pos_;
+    return true;
+  }
+
+  Status ParseHeaderPGX(HeaderPGX* header, const uint8_t** pos) {
+    JXL_RETURN_IF_ERROR(SkipSpace());
+    if (pos_ + 2 > end_) return JXL_FAILURE("PGX: header too small");
+    if (*pos_ == 'M' && *(pos_ + 1) == 'L') {
+      header->big_endian = true;
+    } else if (*pos_ == 'L' && *(pos_ + 1) == 'M') {
+      header->big_endian = false;
+    } else {
+      return JXL_FAILURE("PGX: invalid endianness");
+    }
+    pos_ += 2;
+    JXL_RETURN_IF_ERROR(SkipSpace());
+    if (pos_ == end_) return JXL_FAILURE("PGX: header too small");
+    if (*pos_ == '+') {
+      header->is_signed = false;
+    } else if (*pos_ == '-') {
+      header->is_signed = true;
+    } else {
+      return JXL_FAILURE("PGX: invalid signedness");
+    }
+    pos_++;
+    // Skip optional space
+    if (pos_ < end_ && *pos_ == ' ') pos_++;
+    JXL_RETURN_IF_ERROR(ParseUnsigned(&header->bits_per_sample));
+    JXL_RETURN_IF_ERROR(SkipSingleWhitespace());
+    JXL_RETURN_IF_ERROR(ParseUnsigned(&header->xsize));
+    JXL_RETURN_IF_ERROR(SkipSingleWhitespace());
+    JXL_RETURN_IF_ERROR(ParseUnsigned(&header->ysize));
+    // 0xa, or 0xd 0xa.
+    JXL_RETURN_IF_ERROR(SkipLineBreak());
+
+    if (header->bits_per_sample > 16) {
+      return JXL_FAILURE("PGX: >16 bits not yet supported");
+    }
+    // TODO(lode): support signed integers. This may require changing the way
+    // external_image works.
+    if (header->is_signed) {
+      return JXL_FAILURE("PGX: signed not yet supported");
+    }
+
+    size_t numpixels = header->xsize * header->ysize;
+    size_t bytes_per_pixel = header->bits_per_sample <= 8
+                                 ? 1
+                                 : header->bits_per_sample <= 16 ? 2 : 4;
+    if (pos_ + numpixels * bytes_per_pixel > end_) {
+      return JXL_FAILURE("PGX: data too small");
+    }
+
+    *pos = pos_;
+    return true;
+  }
+
+  const uint8_t* pos_;
+  const uint8_t* const end_;
+};
+
+constexpr size_t kMaxHeaderSize = 200;
+
+Status EncodeHeader(const ExternalImage& external, char* header,
+                    int* JXL_RESTRICT chars_written) {
+  if (external.HasAlpha()) return JXL_FAILURE("PGX: can't store alpha");
+  if (!external.IsGray()) return JXL_FAILURE("PGX: must be grayscale");
+  // TODO(lode): verify other bit depths: for other bit depths such as 1 or 4
+  // bits, have a test case to verify it works correctly. For bits > 16, we may
+  // need to change the way external_image works.
+  if (external.BitsPerSample() != 8 && external.BitsPerSample() != 16) {
+    return JXL_FAILURE("PGX: bits other than 8 or 16 not yet supported");
+  }
+
+  // Use ML (Big Endian), LM may not be well supported by all decoders.
+  snprintf(header, kMaxHeaderSize, "PG ML + %zu %zu %zu\n%n",
+           external.BitsPerSample(), external.xsize(), external.ysize(),
+           chars_written);
+  return true;
+}
+
+Status ApplyHints(CodecInOut* io) {
+  bool got_color_space = false;
+  Status ok = true;
+
+  io->dec_hints.Foreach([io, &got_color_space, &ok](const std::string& key,
+                                                    const std::string& value) {
+    if (key == "color_space") {
+      ColorEncoding* c_original = &io->metadata.color_encoding;
+      if (!ParseDescription(value, c_original) ||
+          !ColorManagement::CreateProfile(c_original)) {
+        JXL_WARNING("PGX: Failed to apply color_space");
+        ok = false;
+      }
+
+      if (!io->metadata.color_encoding.IsGray()) {
+        JXL_WARNING("PGX: color_space hint must be grayscale");
+        ok = false;
+      }
+
+      got_color_space = true;
+    } else {
+      JXL_WARNING("PGX decoder ignoring %s hint", key.c_str());
+    }
+  });
+
+  if (!got_color_space) {
+    JXL_WARNING("PGX: no color_space hint given, assuming sRGB");
+    io->metadata.color_encoding.SetSRGB(ColorSpace::kGray);
+    JXL_RETURN_IF_ERROR(
+        ColorManagement::CreateProfile(&io->metadata.color_encoding));
+  }
+
+  if (!ok) return JXL_FAILURE("PGX ApplyHints failed");
+  return true;
+}
+
+template <typename T>
+void ExpectNear(T a, T b, T precision) {
+  JXL_CHECK(std::abs(a - b) <= precision);
+}
+
+Span<const uint8_t> MakeSpan(const char* str) {
+  return Span<const uint8_t>(reinterpret_cast<const uint8_t*>(str),
+                             strlen(str));
+}
+
+}  // namespace
+
+Status DecodeImagePGX(const Span<const uint8_t> bytes, ThreadPool* pool,
+                      CodecInOut* io) {
+  io->enc_size = bytes.size();
+
+  Parser parser(bytes);
+  HeaderPGX header = {};
+  const uint8_t* pos;
+  JXL_RETURN_IF_ERROR(parser.ParseHeader(&header, &pos));
+  JXL_RETURN_IF_ERROR(io->VerifyDimensions(header.xsize, header.ysize));
+  if (header.bits_per_sample == 0 || header.bits_per_sample > 32) {
+    return JXL_FAILURE("PGX: bits_per_sample invalid");
+  }
+
+  JXL_RETURN_IF_ERROR(ApplyHints(io));
+  io->metadata.bits_per_sample = header.bits_per_sample;
+  io->metadata.alpha_bits = 0;
+  io->dec_pixels = header.xsize * header.ysize;
+  io->frames.clear();
+  io->frames.reserve(1);
+  ImageBundle ib(&io->metadata);
+
+  const bool has_alpha = false;
+  const bool flipped_y = false;
+  const PackedImage desc(header.xsize, header.ysize,
+                         io->metadata.color_encoding, has_alpha,
+                         io->metadata.alpha_bits, io->metadata.bits_per_sample,
+                         header.big_endian, flipped_y);
+  const Span<const uint8_t> span(pos, bytes.data() + bytes.size() - pos);
+  if (!CopyTo(desc, span, pool, &ib)) return false;
+  io->frames.push_back(std::move(ib));
+  return true;
+}
+
+Status EncodeImagePGX(const CodecInOut* io, const ColorEncoding& c_desired,
+                      size_t bits_per_sample, ThreadPool* pool,
+                      PaddedBytes* bytes) {
+  io->enc_bits_per_sample = bits_per_sample;
+  const bool big_endian = true;
+
+  if (!Bundle::AllDefault(io->metadata)) {
+    JXL_WARNING("PGX encoder ignoring metadata - use a different codec");
+  }
+  if (!c_desired.IsSRGB()) {
+    JXL_WARNING(
+        "PGX encoder cannot store custom ICC profile; decoder\n"
+        "will need hint key=color_space to get the same values");
+  }
+
+  const ImageBundle& ib = io->Main();
+  const ImageU* alpha = ib.HasAlpha() ? &ib.alpha() : nullptr;
+  const size_t alpha_bits = ib.HasAlpha() ? io->metadata.alpha_bits : 0;
+  CodecIntervals* temp_intervals = nullptr;  // Can't store min/max.
+
+  const Image3F* to_external_image = &ib.color();
+  ExternalImage external(pool, *to_external_image, Rect(ib), ib.c_current(),
+                         c_desired, ib.HasAlpha(), alpha, alpha_bits,
+                         io->enc_bits_per_sample, big_endian, temp_intervals);
+  JXL_RETURN_IF_ERROR(external.IsHealthy());
+
+  char header[kMaxHeaderSize];
+  int header_size = 0;
+  JXL_RETURN_IF_ERROR(EncodeHeader(external, header, &header_size));
+
+  const PaddedBytes& pixels = external.Bytes();
+  io->enc_size = static_cast<size_t>(header_size) + pixels.size();
+  bytes->resize(io->enc_size);
+  memcpy(bytes->data(), header, static_cast<size_t>(header_size));
+  memcpy(bytes->data() + header_size, pixels.data(), pixels.size());
+
+  return true;
+}
+
+void TestCodecPGX() {
+  {
+    std::string pgx = "PG ML + 8 2 3\npixels";
+
+    CodecInOut io;
+    ThreadPool* pool = nullptr;
+
+    Status ok = DecodeImagePGX(MakeSpan(pgx.c_str()), pool, &io);
+    JXL_CHECK(ok == true);
+
+    JXL_CHECK(io.metadata.bits_per_sample == 8);
+    JXL_CHECK(io.metadata.color_encoding.IsGray());
+    JXL_CHECK(io.xsize() == 2);
+    JXL_CHECK(io.ysize() == 3);
+    JXL_CHECK(io.Main().color().Plane(0).Row(0)[0] == 'p');
+    JXL_CHECK(io.Main().color().Plane(0).Row(0)[1] == 'i');
+    JXL_CHECK(io.Main().color().Plane(0).Row(1)[0] == 'x');
+    JXL_CHECK(io.Main().color().Plane(0).Row(1)[1] == 'e');
+    JXL_CHECK(io.Main().color().Plane(0).Row(2)[0] == 'l');
+    JXL_CHECK(io.Main().color().Plane(0).Row(2)[1] == 's');
+  }
+
+  {
+    std::string pgx = "PG ML + 16 2 3\np_i_x_e_l_s_";
+
+    CodecInOut io;
+    ThreadPool* pool = nullptr;
+
+    Status ok = DecodeImagePGX(MakeSpan(pgx.c_str()), pool, &io);
+    JXL_CHECK(ok == true);
+
+    JXL_CHECK(io.metadata.bits_per_sample == 16);
+    JXL_CHECK(io.metadata.color_encoding.IsGray());
+    JXL_CHECK(io.xsize() == 2);
+    JXL_CHECK(io.ysize() == 3);
+    float eps = 1e-7;
+    const auto& plane = io.Main().color().Plane(0);
+    ExpectNear(256.0f * 'p' + '_', plane.Row(0)[0] * 257, eps);
+    ExpectNear(256.0f * 'i' + '_', plane.Row(0)[1] * 257, eps);
+    ExpectNear(256.0f * 'x' + '_', plane.Row(1)[0] * 257, eps);
+    ExpectNear(256.0f * 'e' + '_', plane.Row(1)[1] * 257, eps);
+    ExpectNear(256.0f * 'l' + '_', plane.Row(2)[0] * 257, eps);
+    ExpectNear(256.0f * 's' + '_', plane.Row(2)[1] * 257, eps);
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/extras/codec_pgx.h b/jxl/extras/codec_pgx.h
new file mode 100644
index 000000000..6b4d94c92
--- /dev/null
+++ b/jxl/extras/codec_pgx.h
@@ -0,0 +1,45 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EXTRAS_CODEC_PGX_H_
+#define JXL_EXTRAS_CODEC_PGX_H_
+
+// Encodes/decodes PGX pixels in memory.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+
+namespace jxl {
+
+// Decodes `bytes` into `io`. io->dec_hints may specify "color_space", which
+// defaults to sRGB.
+Status DecodeImagePGX(const Span<const uint8_t> bytes, ThreadPool* pool,
+                      CodecInOut* io);
+
+// Transforms from io->c_current to `c_desired` and encodes into `bytes`.
+Status EncodeImagePGX(const CodecInOut* io, const ColorEncoding& c_desired,
+                      size_t bits_per_sample, ThreadPool* pool,
+                      PaddedBytes* bytes);
+
+void TestCodecPGX();
+}  // namespace jxl
+
+#endif  // JXL_EXTRAS_CODEC_PGX_H_
diff --git a/jxl/extras/codec_png.cc b/jxl/extras/codec_png.cc
new file mode 100644
index 000000000..ec3af3bd5
--- /dev/null
+++ b/jxl/extras/codec_png.cc
@@ -0,0 +1,771 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/extras/codec_png.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/byte_order.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/external_image.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "third_party/lodepng/lodepng.h"
+
+namespace jxl {
+namespace {
+
+#define JXL_PNG_VERBOSE 0
+
+// Retrieves XMP and EXIF/IPTC from itext and text.
+class BlobsReaderPNG {
+ public:
+  static Status Decode(const LodePNGInfo& info, Blobs* blobs) {
+    for (unsigned idx_itext = 0; idx_itext < info.itext_num; ++idx_itext) {
+      // We trust these are properly null-terminated by LodePNG.
+      const char* key = info.itext_keys[idx_itext];
+      const char* value = info.itext_strings[idx_itext];
+      if (strstr(key, "XML:com.adobe.xmp")) {
+        blobs->xmp.resize(strlen(value));  // safe, see above
+        memcpy(blobs->xmp.data(), value, blobs->xmp.size());
+      }
+    }
+
+    for (unsigned idx_text = 0; idx_text < info.text_num; ++idx_text) {
+      // We trust these are properly null-terminated by LodePNG.
+      const char* key = info.text_keys[idx_text];
+      const char* value = info.text_strings[idx_text];
+      std::string type;
+      PaddedBytes bytes;
+
+      // Handle text chunks annotated with key "Raw profile type ####", with
+      // #### a type, which may contain metadata.
+      const char* kKey = "Raw profile type ";
+      if (strncmp(key, kKey, strlen(kKey)) != 0) continue;
+
+      if (!MaybeDecodeBase16(key, value, &type, &bytes)) {
+        JXL_WARNING("Couldn't parse 'Raw format type' text chunk");
+        continue;
+      }
+      if (type == "exif") {
+        if (!blobs->exif.empty()) {
+          JXL_WARNING("overwriting EXIF (%zu bytes) with base16 (%zu bytes)",
+                      blobs->exif.size(), bytes.size());
+        }
+        blobs->exif = std::move(bytes);
+      } else if (type == "iptc") {
+        blobs->iptc = std::move(bytes);
+      } else if (type == "xmp") {
+        // Generated by ImageMagick.
+        blobs->xmp.resize(strlen(value));  // safe, see above
+        memcpy(blobs->xmp.data(), value, blobs->xmp.size());
+      } else {
+        JXL_WARNING(
+            "Unknown type in 'Raw format type' text chunk: %s: %zu bytes",
+            type.c_str(), bytes.size());
+      }
+    }
+
+    return true;
+  }
+
+ private:
+  // Returns false if invalid.
+  static JXL_INLINE Status DecodeNibble(const char c,
+                                        uint32_t* JXL_RESTRICT nibble) {
+    if ('a' <= c && c <= 'f') {
+      *nibble = 10 + c - 'a';
+    } else if ('0' <= c && c <= '9') {
+      *nibble = c - '0';
+    } else {
+      *nibble = 0;
+      return JXL_FAILURE("Invalid metadata nibble");
+    }
+    JXL_ASSERT(*nibble < 16);
+    return true;
+  }
+
+  // Parses a PNG text chunk with key of the form "Raw profile type ####", with
+  // #### a type.
+  // Returns whether it could successfully parse the content.
+  // We trust key and encoded are null-terminated because they come from
+  // LodePNG.
+  static Status MaybeDecodeBase16(const char* key, const char* encoded,
+                                  std::string* type, PaddedBytes* bytes) {
+    const char* encoded_end = encoded + strlen(encoded);
+
+    const char* kKey = "Raw profile type ";
+    if (strncmp(key, kKey, strlen(kKey)) != 0) return false;
+    *type = key + strlen(kKey);
+    const size_t kMaxTypeLen = 20;
+    if (type->length() > kMaxTypeLen) return false;  // Type too long
+
+    // Header: freeform string and number of bytes
+    unsigned long bytes_to_decode;
+    int header_len;
+    std::vector<char> description((encoded_end - encoded) + 1);
+    const int fields = sscanf(encoded, "\n%[^\n]\n%8lu%n", description.data(),
+                              &bytes_to_decode, &header_len);
+    if (fields != 2) return false;  // Failed to decode metadata header
+    JXL_ASSERT(bytes->empty());
+    bytes->reserve(bytes_to_decode);
+
+    // Encoding: base16 with newline after 72 chars.
+    const char* pos = encoded + header_len;
+    for (size_t i = 0; i < bytes_to_decode; ++i) {
+      if (i % 36 == 0) {
+        if (pos + 1 >= encoded_end) return false;  // Truncated base16 1
+        if (*pos != '\n') return false;            // Expected newline
+        ++pos;
+      }
+
+      if (pos + 2 >= encoded_end) return false;  // Truncated base16 2;
+      uint32_t nibble0, nibble1;
+      JXL_RETURN_IF_ERROR(DecodeNibble(pos[0], &nibble0));
+      JXL_RETURN_IF_ERROR(DecodeNibble(pos[1], &nibble1));
+      bytes->push_back(static_cast<uint8_t>((nibble1 << 4) + nibble0));
+      pos += 2;
+    }
+    if (pos + 1 != encoded_end) return false;  // Too many encoded bytes
+    if (pos[0] != '\n') return false;          // Incorrect metadata terminator
+    return true;
+  }
+};
+
+// Stores XMP and EXIF/IPTC into itext and text.
+class BlobsWriterPNG {
+ public:
+  static Status Encode(const Blobs& blobs, LodePNGInfo* JXL_RESTRICT info) {
+    if (!blobs.exif.empty()) {
+      JXL_RETURN_IF_ERROR(EncodeBase16("exif", blobs.exif, info));
+    }
+    if (!blobs.iptc.empty()) {
+      JXL_RETURN_IF_ERROR(EncodeBase16("iptc", blobs.iptc, info));
+    }
+
+    if (!blobs.xmp.empty()) {
+      const char* key = "XML:com.adobe.xmp";
+      const std::string text(reinterpret_cast<const char*>(blobs.xmp.data()),
+                             blobs.xmp.size());
+      if (lodepng_add_itext(info, key, "", "", text.c_str()) != 0) {
+        return JXL_FAILURE("Failed to add itext");
+      }
+    }
+
+    return true;
+  }
+
+ private:
+  static JXL_INLINE char EncodeNibble(const uint8_t nibble) {
+    JXL_ASSERT(nibble < 16);
+    return (nibble < 10) ? '0' + nibble : 'a' + nibble - 10;
+  }
+
+  static Status EncodeBase16(const std::string& type, const PaddedBytes& bytes,
+                             LodePNGInfo* JXL_RESTRICT info) {
+    // Encoding: base16 with newline after 72 chars.
+    const size_t base16_size =
+        2 * bytes.size() + DivCeil(bytes.size(), size_t(36)) + 1;
+    std::string base16;
+    base16.reserve(base16_size);
+    for (size_t i = 0; i < bytes.size(); ++i) {
+      if (i % 36 == 0) base16.push_back('\n');
+      base16.push_back(EncodeNibble(bytes[i] & 0x0F));
+      base16.push_back(EncodeNibble(bytes[i] >> 4));
+    }
+    base16.push_back('\n');
+    JXL_ASSERT(base16.length() == base16_size);
+
+    char key[30];
+    snprintf(key, sizeof(key), "Raw profile type %s", type.c_str());
+
+    char header[30];
+    snprintf(header, sizeof(header), "\n%s\n%8zu", type.c_str(), bytes.size());
+
+    const std::string& encoded = std::string(header) + base16;
+    if (lodepng_add_text(info, key, encoded.c_str()) != 0) {
+      return JXL_FAILURE("Failed to add text");
+    }
+
+    return true;
+  }
+};
+
+// Retrieves ColorEncoding from PNG chunks.
+class ColorEncodingReaderPNG {
+ public:
+  // Fills original->color_encoding or returns false.
+  Status operator()(const Span<const uint8_t> bytes, const bool is_gray,
+                    CodecInOut* io) {
+    ColorEncoding* c_original = &io->metadata.color_encoding;
+    JXL_RETURN_IF_ERROR(Decode(bytes, &io->blobs));
+
+    const ColorSpace color_space =
+        is_gray ? ColorSpace::kGray : ColorSpace::kRGB;
+
+    if (have_pq_) {
+      c_original->SetColorSpace(color_space);
+      c_original->white_point = WhitePoint::kD65;
+      c_original->primaries = Primaries::k2100;
+      c_original->tf.SetTransferFunction(TransferFunction::kPQ);
+      c_original->rendering_intent = RenderingIntent::kRelative;
+      if (ColorManagement::CreateProfile(c_original)) return true;
+      JXL_WARNING("Failed to synthesize BT.2100 PQ");
+      // Else: try the actual ICC profile.
+    }
+
+    // ICC overrides anything else if present.
+    if (ColorManagement::SetProfile(std::move(icc_), c_original)) {
+      if (have_srgb_) {
+        JXL_WARNING("Invalid PNG with both sRGB and ICC; ignoring sRGB");
+      }
+      if (is_gray != c_original->IsGray()) {
+        return JXL_FAILURE("Mismatch between ICC and PNG header grayscale");
+      }
+      return true;  // it's fine to ignore gAMA/cHRM.
+    }
+
+    // PNG requires that sRGB override gAMA/cHRM.
+    if (have_srgb_) {
+      c_original->SetSRGB(color_space);
+      c_original->rendering_intent = rendering_intent_;  // after SetSRGB!
+      return ColorManagement::CreateProfile(c_original);
+    }
+
+    // Try to create a custom profile:
+
+    c_original->SetColorSpace(color_space);
+
+    // Attempt to set whitepoint and primaries if there is a cHRM chunk, or else
+    // use default sRGB (the PNG then is device-dependent).
+    // In case of grayscale, do not attempt to set the primaries and ignore the
+    // ones the PNG image has (but still set the white point).
+    if (!have_chrm_ || !c_original->SetWhitePoint(white_point_) ||
+        (!is_gray && !c_original->SetPrimaries(primaries_))) {
+#if JXL_PNG_VERBOSE >= 1
+      JXL_WARNING("No (valid) cHRM, assuming sRGB");
+#endif
+      c_original->white_point = WhitePoint::kD65;
+      c_original->primaries = Primaries::kSRGB;
+    }
+
+    if (!have_gama_ || !c_original->tf.SetGamma(gamma_)) {
+#if JXL_PNG_VERBOSE >= 1
+      JXL_WARNING("No (valid) gAMA nor sRGB, assuming sRGB");
+#endif
+      c_original->tf.SetTransferFunction(TransferFunction::kSRGB);
+    }
+
+    c_original->rendering_intent = RenderingIntent::kRelative;
+    if (ColorManagement::CreateProfile(c_original)) return true;
+
+    JXL_WARNING(
+        "DATA LOSS: unable to create an ICC profile for PNG gAMA/cHRM.\n"
+        "Image pixels will be interpreted as sRGB. Please add an ICC \n"
+        "profile to the input image");
+    c_original->SetSRGB(color_space);
+    return ColorManagement::CreateProfile(c_original);
+  }
+
+ private:
+  Status DecodeICC(const unsigned char* const payload,
+                   const size_t payload_size) {
+    if (payload_size == 0) return JXL_FAILURE("Empty ICC payload");
+    const unsigned char* pos = payload;
+    const unsigned char* end = payload + payload_size;
+
+    // Profile name
+    if (*pos == '\0') return JXL_FAILURE("Expected ICC name");
+    for (size_t i = 0;; ++i) {
+      if (i == 80) return JXL_FAILURE("ICC profile name too long");
+      if (pos == end) return JXL_FAILURE("Not enough bytes for ICC name");
+      if (*pos++ == '\0') break;
+    }
+
+    // Special case for BT.2100 PQ (https://w3c.github.io/png-hdr-pq/) - try to
+    // synthesize the profile because table-based curves are less accurate.
+    // strcmp is safe because we already verified the string is 0-terminated.
+    if (!strcmp(reinterpret_cast<const char*>(payload), "ITUR_2100_PQ_FULL")) {
+      have_pq_ = true;
+    }
+
+    // Skip over compression method (only one is allowed)
+    if (pos == end) return JXL_FAILURE("Not enough bytes for ICC method");
+    if (*pos++ != 0) return JXL_FAILURE("Unsupported ICC method");
+
+    // Decompress
+    unsigned char* icc_buf = nullptr;
+    size_t icc_size = 0;
+    LodePNGDecompressSettings settings;
+    lodepng_decompress_settings_init(&settings);
+    const unsigned err = lodepng_zlib_decompress(
+        &icc_buf, &icc_size, pos, payload_size - (pos - payload), &settings);
+    if (err == 0) {
+      icc_.resize(icc_size);
+      memcpy(icc_.data(), icc_buf, icc_size);
+    }
+    free(icc_buf);
+    return true;
+  }
+
+  // Returns floating-point value from the PNG encoding (times 10^5).
+  static double F64FromU32(const uint32_t x) {
+    return static_cast<int32_t>(x) * 1E-5;
+  }
+
+  Status DecodeSRGB(const unsigned char* payload, const size_t payload_size) {
+    if (payload_size != 1) return JXL_FAILURE("Wrong sRGB size");
+    // (PNG uses the same values as ICC.)
+    rendering_intent_ = static_cast<RenderingIntent>(payload[0]);
+    have_srgb_ = true;
+    return true;
+  }
+
+  Status DecodeGAMA(const unsigned char* payload, const size_t payload_size) {
+    if (payload_size != 4) return JXL_FAILURE("Wrong gAMA size");
+    gamma_ = F64FromU32(LoadBE32(payload));
+    have_gama_ = true;
+    return true;
+  }
+
+  Status DecodeCHRM(const unsigned char* payload, const size_t payload_size) {
+    if (payload_size != 32) return JXL_FAILURE("Wrong cHRM size");
+    white_point_.x = F64FromU32(LoadBE32(payload + 0));
+    white_point_.y = F64FromU32(LoadBE32(payload + 4));
+    primaries_.r.x = F64FromU32(LoadBE32(payload + 8));
+    primaries_.r.y = F64FromU32(LoadBE32(payload + 12));
+    primaries_.g.x = F64FromU32(LoadBE32(payload + 16));
+    primaries_.g.y = F64FromU32(LoadBE32(payload + 20));
+    primaries_.b.x = F64FromU32(LoadBE32(payload + 24));
+    primaries_.b.y = F64FromU32(LoadBE32(payload + 28));
+    have_chrm_ = true;
+    return true;
+  }
+
+  Status DecodeEXIF(const unsigned char* payload, const size_t payload_size,
+                    Blobs* blobs) {
+    // If we already have EXIF, keep the larger one.
+    if (blobs->exif.size() > payload_size) return true;
+    blobs->exif.resize(payload_size);
+    memcpy(blobs->exif.data(), payload, payload_size);
+    return true;
+  }
+
+  Status Decode(const Span<const uint8_t> bytes, Blobs* blobs) {
+    // Look for colorimetry and text chunks in the PNG image. The PNG chunks
+    // begin after the PNG magic header of 8 bytes.
+    const unsigned char* chunk = bytes.data() + 8;
+    const unsigned char* end = bytes.data() + bytes.size();
+    for (;;) {
+      // chunk points to the first field of a PNG chunk. The chunk has
+      // respectively 4 bytes of length, 4 bytes type, length bytes of data,
+      // 4 bytes CRC.
+      if (chunk + 4 >= end) {
+        break;  // Regular end reached.
+      }
+
+      char type_char[5];
+      lodepng_chunk_type(type_char, chunk);
+      std::string type = type_char;
+
+      if (type == "acTL" || type == "fcTL" || type == "fdAT") {
+        // this is an APNG file, without proper handling we would just return
+        // the first frame, so for now codec_apng handles animation until the
+        // animation chunk handling is added here
+        return false;
+      }
+      if (type == "eXIf" || type == "iCCP" || type == "sRGB" ||
+          type == "gAMA" || type == "cHRM") {
+        const unsigned char* payload = lodepng_chunk_data_const(chunk);
+        const size_t payload_size = lodepng_chunk_length(chunk);
+        // The entire chunk needs also 4 bytes of CRC after the payload.
+        if (payload + payload_size + 4 >= end) {
+          JXL_NOTIFY_ERROR("PNG: truncated chunk");
+          break;
+        }
+        if (lodepng_chunk_check_crc(chunk) != 0) {
+          JXL_NOTIFY_ERROR("CRC mismatch in unknown PNG chunk");
+          continue;
+        }
+
+        if (type == "eXIf") {
+          JXL_RETURN_IF_ERROR(DecodeEXIF(payload, payload_size, blobs));
+        } else if (type == "iCCP") {
+          JXL_RETURN_IF_ERROR(DecodeICC(payload, payload_size));
+        } else if (type == "sRGB") {
+          JXL_RETURN_IF_ERROR(DecodeSRGB(payload, payload_size));
+        } else if (type == "gAMA") {
+          JXL_RETURN_IF_ERROR(DecodeGAMA(payload, payload_size));
+        } else if (type == "cHRM") {
+          JXL_RETURN_IF_ERROR(DecodeCHRM(payload, payload_size));
+        }
+      }
+
+      chunk = lodepng_chunk_next_const(chunk);
+    }
+    return true;
+  }
+
+  PaddedBytes icc_;
+
+  bool have_pq_ = false;
+  bool have_srgb_ = false;
+  bool have_gama_ = false;
+  bool have_chrm_ = false;
+
+  // Only valid if have_srgb_:
+  RenderingIntent rendering_intent_;
+
+  // Only valid if have_gama_:
+  double gamma_;
+
+  // Only valid if have_chrm_:
+  CIExy white_point_;
+  PrimariesCIExy primaries_;
+};
+
+// Stores ColorEncoding into PNG chunks.
+class ColorEncodingWriterPNG {
+ public:
+  static Status Encode(const ColorEncoding& c, LodePNGInfo* JXL_RESTRICT info) {
+    // Prefer to only write sRGB - smaller.
+    if (c.IsSRGB()) {
+      JXL_RETURN_IF_ERROR(AddSRGB(c, info));
+      // PNG recommends not including both sRGB and iCCP, so skip the latter.
+    } else {
+      JXL_ASSERT(!c.icc.empty());
+      JXL_RETURN_IF_ERROR(AddICC(c.icc, info));
+    }
+
+    // gAMA and cHRM are always allowed but will be overridden by sRGB/iCCP.
+    JXL_RETURN_IF_ERROR(MaybeAddGAMA(c, info));
+    JXL_RETURN_IF_ERROR(MaybeAddCHRM(c, info));
+    return true;
+  }
+
+ private:
+  static Status AddChunk(const char* type, const PaddedBytes& payload,
+                         LodePNGInfo* JXL_RESTRICT info) {
+    // Ignore original location/order of chunks; place them in the first group.
+    if (lodepng_chunk_create(&info->unknown_chunks_data[0],
+                             &info->unknown_chunks_size[0], payload.size(),
+                             type, payload.data()) != 0) {
+      return JXL_FAILURE("Failed to add chunk");
+    }
+    return true;
+  }
+
+  static Status AddICC(const PaddedBytes& icc, LodePNGInfo* JXL_RESTRICT info) {
+    LodePNGCompressSettings settings;
+    lodepng_compress_settings_init(&settings);
+    unsigned char* out = nullptr;
+    size_t out_size = 0;
+    if (lodepng_zlib_compress(&out, &out_size, icc.data(), icc.size(),
+                              &settings) != 0) {
+      return JXL_FAILURE("Failed to compress ICC");
+    }
+
+    PaddedBytes payload;
+    payload.resize(3 + out_size);
+    // TODO(janwas): use special name if PQ
+    payload[0] = '1';  // profile name
+    payload[1] = '\0';
+    payload[2] = 0;  // compression method (zlib)
+    memcpy(&payload[3], out, out_size);
+    free(out);
+
+    return AddChunk("iCCP", payload, info);
+  }
+
+  static Status AddSRGB(const ColorEncoding& c,
+                        LodePNGInfo* JXL_RESTRICT info) {
+    PaddedBytes payload;
+    payload.push_back(static_cast<uint8_t>(c.rendering_intent));
+    return AddChunk("sRGB", payload, info);
+  }
+
+  // Returns PNG encoding of floating-point value (times 10^5).
+  static uint32_t U32FromF64(const double x) {
+    return static_cast<int32_t>(std::round(x * 1E5));
+  }
+
+  static Status MaybeAddGAMA(const ColorEncoding& c,
+                             LodePNGInfo* JXL_RESTRICT info) {
+    if (!c.tf.IsGamma()) return true;
+    const double gamma = c.tf.GetGamma();
+
+    PaddedBytes payload(4);
+    StoreBE32(U32FromF64(gamma), payload.data());
+    return AddChunk("gAMA", payload, info);
+  }
+
+  static Status MaybeAddCHRM(const ColorEncoding& c,
+                             LodePNGInfo* JXL_RESTRICT info) {
+    // TODO(lode): remove this, PNG can also have cHRM for P3, sRGB, ...
+    if (c.white_point != WhitePoint::kCustom &&
+        c.primaries != Primaries::kCustom) {
+      return true;
+    }
+
+    const CIExy white_point = c.GetWhitePoint();
+    // A PNG image stores both whitepoint and primaries in the cHRM chunk, but
+    // for grayscale images we don't have primaries. It does not matter what
+    // values are stored in the PNG though (all colors are a multiple of the
+    // whitepoint), so choose default ones. See
+    // http://www.libpng.org/pub/png/spec/1.2/PNG-Chunks.html section 4.2.2.1.
+    const PrimariesCIExy primaries =
+        c.IsGray() ? ColorEncoding().GetPrimaries() : c.GetPrimaries();
+
+    PaddedBytes payload(32);
+    StoreBE32(U32FromF64(white_point.x), &payload[0]);
+    StoreBE32(U32FromF64(white_point.y), &payload[4]);
+    StoreBE32(U32FromF64(primaries.r.x), &payload[8]);
+    StoreBE32(U32FromF64(primaries.r.y), &payload[12]);
+    StoreBE32(U32FromF64(primaries.g.x), &payload[16]);
+    StoreBE32(U32FromF64(primaries.g.y), &payload[20]);
+    StoreBE32(U32FromF64(primaries.b.x), &payload[24]);
+    StoreBE32(U32FromF64(primaries.b.y), &payload[28]);
+    return AddChunk("cHRM", payload, info);
+  }
+};
+
+// RAII - ensures state is freed even if returning early.
+struct PNGState {
+  PNGState() { lodepng_state_init(&s); }
+  ~PNGState() { lodepng_state_cleanup(&s); }
+
+  LodePNGState s;
+};
+
+Status CheckGray(const LodePNGColorMode& mode, bool has_icc, bool* is_gray) {
+  switch (mode.colortype) {
+    case LCT_GREY:
+    case LCT_GREY_ALPHA:
+      *is_gray = true;
+      return true;
+
+    case LCT_RGB:
+    case LCT_RGBA:
+      *is_gray = false;
+      return true;
+
+    case LCT_PALETTE: {
+      if (has_icc) {
+        // If an ICC profile is present, the PNG specification requires
+        // palette to be intepreted as RGB colored, not grayscale, so we must
+        // output color in that case and unfortunately can't optimize it to
+        // gray if the palette only has gray entries.
+        *is_gray = false;
+        return true;
+      } else {
+        *is_gray = true;
+        for (size_t i = 0; i < mode.palettesize; i++) {
+          if (mode.palette[i * 4] != mode.palette[i * 4 + 1] ||
+              mode.palette[i * 4] != mode.palette[i * 4 + 2]) {
+            *is_gray = false;
+            break;
+          }
+        }
+        return true;
+      }
+    }
+
+    default:
+      *is_gray = false;
+      return JXL_FAILURE("Unexpected PNG color type");
+  }
+}
+
+Status CheckAlpha(const LodePNGColorMode& mode, bool* has_alpha) {
+  if (mode.key_defined) {
+    // Color key marks a single color as transparent.
+    *has_alpha = true;
+    return true;
+  }
+
+  switch (mode.colortype) {
+    case LCT_GREY:
+    case LCT_RGB:
+      *has_alpha = false;
+      return true;
+
+    case LCT_GREY_ALPHA:
+    case LCT_RGBA:
+      *has_alpha = true;
+      return true;
+
+    case LCT_PALETTE: {
+      *has_alpha = false;
+      for (size_t i = 0; i < mode.palettesize; i++) {
+        // PNG palettes are always 8-bit.
+        if (mode.palette[i * 4 + 3] != 255) {
+          *has_alpha = true;
+          break;
+        }
+      }
+      return true;
+    }
+
+    default:
+      *has_alpha = false;
+      return JXL_FAILURE("Unexpected PNG color type");
+  }
+}
+
+LodePNGColorType MakeType(const bool is_gray, const bool has_alpha) {
+  if (is_gray) {
+    return has_alpha ? LCT_GREY_ALPHA : LCT_GREY;
+  }
+  return has_alpha ? LCT_RGBA : LCT_RGB;
+}
+
+// Inspects first chunk of the given type and updates state with the information
+// when the chunk is relevant and present in the file.
+Status InspectChunkType(const Span<const uint8_t> bytes,
+                        const std::string& type, LodePNGState* state) {
+  const unsigned char* chunk = lodepng_chunk_find_const(
+      bytes.data(), bytes.data() + bytes.size(), type.c_str());
+  if (chunk && lodepng_inspect_chunk(state, chunk - bytes.data(), bytes.data(),
+                                     bytes.size()) != 0) {
+    return JXL_FAILURE("Invalid chunk \"%s\" in PNG image", type.c_str());
+  }
+  return true;
+}
+
+}  // namespace
+
+Status DecodeImagePNG(const Span<const uint8_t> bytes, ThreadPool* pool,
+                      CodecInOut* io) {
+  unsigned w, h;
+  PNGState state;
+  if (lodepng_inspect(&w, &h, &state.s, bytes.data(), bytes.size()) != 0) {
+    return false;  // not an error - just wrong format
+  }
+  JXL_RETURN_IF_ERROR(io->VerifyDimensions(w, h));
+  // Palette RGB values
+  if (!InspectChunkType(bytes, "PLTE", &state.s)) {
+    return false;
+  }
+  // Transparent color key, or palette transparency
+  if (!InspectChunkType(bytes, "tRNS", &state.s)) {
+    return false;
+  }
+  // ICC profile
+  if (!InspectChunkType(bytes, "iCCP", &state.s)) {
+    return false;
+  }
+  const LodePNGColorMode& color_mode = state.s.info_png.color;
+  bool has_icc = state.s.info_png.iccp_defined;
+
+  bool is_gray, has_alpha;
+  JXL_RETURN_IF_ERROR(CheckGray(color_mode, has_icc, &is_gray));
+  JXL_RETURN_IF_ERROR(CheckAlpha(color_mode, &has_alpha));
+  // We want LodePNG to promote 1/2/4 bit pixels to 8.
+  size_t bits_per_sample = std::max(color_mode.bitdepth, 8u);
+  if (bits_per_sample != 8 && bits_per_sample != 16) {
+    return JXL_FAILURE("Unexpected PNG bit depth");
+  }
+  io->metadata.bits_per_sample = static_cast<uint32_t>(bits_per_sample);
+  io->metadata.alpha_bits = has_alpha ? io->metadata.bits_per_sample : 0;
+
+  io->enc_size = bytes.size();
+  io->dec_hints.Foreach(
+      [](const std::string& key, const std::string& /*value*/) {
+        JXL_WARNING("PNG decoder ignoring %s hint", key.c_str());
+      });
+
+  // Always decode to 8/16-bit RGB/RGBA, not LCT_PALETTE.
+  state.s.info_raw.bitdepth = static_cast<unsigned>(bits_per_sample);
+  state.s.info_raw.colortype = MakeType(is_gray, has_alpha);
+  unsigned char* out;
+  if (lodepng_decode(&out, &w, &h, &state.s, bytes.data(), bytes.size()) != 0) {
+    return JXL_FAILURE("PNG decode failed");
+  }
+
+  if (!BlobsReaderPNG::Decode(state.s.info_png, &io->blobs)) {
+    JXL_WARNING("PNG metadata may be incomplete");
+  }
+  ColorEncodingReaderPNG reader;
+  JXL_RETURN_IF_ERROR(reader(bytes, is_gray, io));
+#if JXL_PNG_VERBOSE >= 1
+  printf("PNG read %s\n", Description(io->metadata.color_encoding).c_str());
+#endif
+
+  const size_t num_channels = (is_gray ? 1 : 3) + has_alpha;
+  const size_t out_size = w * h * num_channels * bits_per_sample / kBitsPerByte;
+
+  const bool big_endian = true;  // PNG requirement
+  const PackedImage desc(w, h, io->metadata.color_encoding, has_alpha,
+                         io->metadata.alpha_bits, io->metadata.bits_per_sample,
+                         big_endian, /*flipped_y=*/false);
+  const Span<const uint8_t> span(out, out_size);
+  const bool ok = CopyTo(desc, span, pool, &io->Main());
+  free(out);
+  io->dec_pixels = w * h;
+  io->metadata.bits_per_sample = io->Main().DetectRealBitdepth();
+  return ok;
+}
+
+Status EncodeImagePNG(const CodecInOut* io, const ColorEncoding& c_desired,
+                      size_t bits_per_sample, ThreadPool* pool,
+                      PaddedBytes* bytes) {
+  io->enc_bits_per_sample = bits_per_sample == 8 ? 8 : 16;
+
+  const ImageBundle& ib = io->Main();
+  const ImageU* alpha = ib.HasAlpha() ? &ib.alpha() : nullptr;
+  const size_t alpha_bits = ib.HasAlpha() ? io->metadata.alpha_bits : 0;
+  const bool big_endian = true;              // PNG requirement
+  CodecIntervals* temp_intervals = nullptr;  // Can't store min/max.
+  const ExternalImage external(
+      pool, ib.color(), Rect(ib), ib.c_current(), c_desired, ib.HasAlpha(),
+      alpha, alpha_bits, io->enc_bits_per_sample, big_endian, temp_intervals);
+  JXL_RETURN_IF_ERROR(external.IsHealthy());
+
+  PNGState state;
+  // For maximum compatibility, still store 8-bit even if pixels are all zero.
+  state.s.encoder.auto_convert = 0;
+
+  LodePNGInfo* info = &state.s.info_png;
+  info->color.bitdepth = io->enc_bits_per_sample;
+  info->color.colortype = MakeType(ib.IsGray(), ib.HasAlpha());
+  state.s.info_raw = info->color;
+
+  JXL_RETURN_IF_ERROR(ColorEncodingWriterPNG::Encode(c_desired, info));
+  JXL_RETURN_IF_ERROR(BlobsWriterPNG::Encode(io->blobs, info));
+
+  unsigned char* out = nullptr;
+  size_t out_size = 0;
+  if (lodepng_encode(&out, &out_size, external.Bytes().data(), ib.xsize(),
+                     ib.ysize(), &state.s) != 0) {
+    return JXL_FAILURE("Failed to encode PNG");
+  }
+  bytes->resize(out_size);
+  memcpy(bytes->data(), out, out_size);
+  free(out);
+
+  io->enc_size = out_size;
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/extras/codec_png.h b/jxl/extras/codec_png.h
new file mode 100644
index 000000000..f990826c6
--- /dev/null
+++ b/jxl/extras/codec_png.h
@@ -0,0 +1,43 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EXTRAS_CODEC_PNG_H_
+#define JXL_EXTRAS_CODEC_PNG_H_
+
+// Encodes/decodes PNG pixels and metadata in memory.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+
+namespace jxl {
+
+// Decodes `bytes` into `io`. io->dec_hints are ignored.
+Status DecodeImagePNG(const Span<const uint8_t> bytes, ThreadPool* pool,
+                      CodecInOut* io);
+
+// Transforms from io->c_current to `c_desired` and encodes into `bytes`.
+Status EncodeImagePNG(const CodecInOut* io, const ColorEncoding& c_desired,
+                      size_t bits_per_sample, ThreadPool* pool,
+                      PaddedBytes* bytes);
+
+}  // namespace jxl
+
+#endif  // JXL_EXTRAS_CODEC_PNG_H_
diff --git a/jxl/extras/codec_pnm.cc b/jxl/extras/codec_pnm.cc
new file mode 100644
index 000000000..7f47961f8
--- /dev/null
+++ b/jxl/extras/codec_pnm.cc
@@ -0,0 +1,417 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/extras/codec_pnm.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/bits.h"
+#include "jxl/base/byte_order.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/color_management.h"
+#include "jxl/external_image.h"
+#include "jxl/fields.h"  // AllDefault
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+namespace {
+
+struct HeaderPNM {
+  size_t xsize;
+  size_t ysize;
+  bool is_gray;
+  size_t bits_per_sample;
+  bool big_endian;
+};
+
+class Parser {
+ public:
+  explicit Parser(const Span<const uint8_t> bytes)
+      : pos_(bytes.data()), end_(pos_ + bytes.size()) {}
+
+  // Sets "pos" to the first non-header byte/pixel on success.
+  Status ParseHeader(HeaderPNM* header, const uint8_t** pos) {
+    // codec_facade ensures we have at least two bytes => no range check here.
+    if (pos_[0] != 'P') return false;
+    const uint8_t type = pos_[1];
+    pos_ += 2;
+
+    switch (type) {
+      case '5':
+        header->is_gray = true;
+        return ParseHeaderPNM(header, pos);
+
+      case '6':
+        header->is_gray = false;
+        return ParseHeaderPNM(header, pos);
+
+      case 'F':
+        header->is_gray = false;
+        return ParseHeaderPFM(header, pos);
+
+      case 'f':
+        header->is_gray = true;
+        return ParseHeaderPFM(header, pos);
+    }
+    return false;
+  }
+
+  // Exposed for testing
+  Status ParseUnsigned(size_t* number) {
+    if (pos_ == end_) return JXL_FAILURE("PNM: reached end before number");
+    if (!IsDigit(*pos_)) return JXL_FAILURE("PNM: expected unsigned number");
+
+    *number = 0;
+    while (pos_ < end_ && *pos_ >= '0' && *pos_ <= '9') {
+      *number *= 10;
+      *number += *pos_ - '0';
+      ++pos_;
+    }
+
+    return true;
+  }
+
+  Status ParseSigned(double* number) {
+    if (pos_ == end_) return JXL_FAILURE("PNM: reached end before signed");
+
+    if (*pos_ != '-' && *pos_ != '+' && !IsDigit(*pos_)) {
+      return JXL_FAILURE("PNM: expected signed number");
+    }
+
+    // Skip sign
+    const bool is_neg = *pos_ == '-';
+    if (is_neg || *pos_ == '+') {
+      ++pos_;
+      if (pos_ == end_) return JXL_FAILURE("PNM: reached end before digits");
+    }
+
+    // Leading digits
+    *number = 0.0;
+    while (pos_ < end_ && *pos_ >= '0' && *pos_ <= '9') {
+      *number *= 10;
+      *number += *pos_ - '0';
+      ++pos_;
+    }
+
+    // Decimal places?
+    if (pos_ < end_ && *pos_ == '.') {
+      ++pos_;
+      double place = 0.1;
+      while (pos_ < end_ && *pos_ >= '0' && *pos_ <= '9') {
+        *number += (*pos_ - '0') * place;
+        place *= 0.1;
+        ++pos_;
+      }
+    }
+
+    if (is_neg) *number = -*number;
+    return true;
+  }
+
+ private:
+  static bool IsDigit(const uint8_t c) { return '0' <= c && c <= '9'; }
+  static bool IsLineBreak(const uint8_t c) { return c == '\r' || c == '\n'; }
+  static bool IsWhitespace(const uint8_t c) {
+    return IsLineBreak(c) || c == '\t' || c == ' ';
+  }
+
+  Status SkipBlank() {
+    if (pos_ == end_) return JXL_FAILURE("PNM: reached end before blank");
+    const uint8_t c = *pos_;
+    if (c != ' ' && c != '\n') return JXL_FAILURE("PNM: expected blank");
+    ++pos_;
+    return true;
+  }
+
+  Status SkipSingleWhitespace() {
+    if (pos_ == end_) return JXL_FAILURE("PNM: reached end before whitespace");
+    if (!IsWhitespace(*pos_)) return JXL_FAILURE("PNM: expected whitespace");
+    ++pos_;
+    return true;
+  }
+
+  Status SkipWhitespace() {
+    if (pos_ == end_) return JXL_FAILURE("PNM: reached end before whitespace");
+    if (!IsWhitespace(*pos_) && *pos_ != '#') {
+      return JXL_FAILURE("PNM: expected whitespace/comment");
+    }
+
+    while (pos_ < end_ && IsWhitespace(*pos_)) {
+      ++pos_;
+    }
+
+    // Comment(s)
+    while (pos_ != end_ && *pos_ == '#') {
+      while (pos_ != end_ && !IsLineBreak(*pos_)) {
+        ++pos_;
+      }
+      // Newline(s)
+      while (pos_ != end_ && IsLineBreak(*pos_)) pos_++;
+    }
+
+    while (pos_ < end_ && IsWhitespace(*pos_)) {
+      ++pos_;
+    }
+    return true;
+  }
+
+  Status ParseHeaderPNM(HeaderPNM* header, const uint8_t** pos) {
+    JXL_RETURN_IF_ERROR(SkipWhitespace());
+    JXL_RETURN_IF_ERROR(ParseUnsigned(&header->xsize));
+
+    JXL_RETURN_IF_ERROR(SkipWhitespace());
+    JXL_RETURN_IF_ERROR(ParseUnsigned(&header->ysize));
+
+    JXL_RETURN_IF_ERROR(SkipWhitespace());
+    size_t max_val;
+    JXL_RETURN_IF_ERROR(ParseUnsigned(&max_val));
+    if (max_val == 0 || max_val >= 65536) return JXL_FAILURE("PNM: bad MaxVal");
+    header->bits_per_sample = CeilLog2Nonzero(max_val);
+    header->big_endian = true;
+
+    JXL_RETURN_IF_ERROR(SkipSingleWhitespace());
+
+    *pos = pos_;
+    return true;
+  }
+
+  Status ParseHeaderPFM(HeaderPNM* header, const uint8_t** pos) {
+    JXL_RETURN_IF_ERROR(SkipSingleWhitespace());
+    JXL_RETURN_IF_ERROR(ParseUnsigned(&header->xsize));
+
+    JXL_RETURN_IF_ERROR(SkipBlank());
+    JXL_RETURN_IF_ERROR(ParseUnsigned(&header->ysize));
+
+    JXL_RETURN_IF_ERROR(SkipSingleWhitespace());
+    double scale;
+    JXL_RETURN_IF_ERROR(ParseSigned(&scale));
+    header->big_endian = scale >= 0.0;
+    header->bits_per_sample = 32;
+
+    JXL_RETURN_IF_ERROR(SkipSingleWhitespace());
+
+    *pos = pos_;
+    return true;
+  }
+
+  const uint8_t* pos_;
+  const uint8_t* const end_;
+};
+
+constexpr size_t kMaxHeaderSize = 200;
+
+Status EncodeHeader(const ExternalImage& external, char* header,
+                    int* JXL_RESTRICT chars_written) {
+  if (external.HasAlpha()) return JXL_FAILURE("PNM: can't store alpha");
+
+  if (external.BitsPerSample() == 32) {  // PFM
+    const char type = external.IsGray() ? 'f' : 'F';
+    const double scale = external.BigEndian() ? 1.0 : -1.0;
+    snprintf(header, kMaxHeaderSize, "P%c %zu %zu\n%f\n%n", type,
+             external.xsize(), external.ysize(), scale, chars_written);
+  } else {  // PGM/PPM
+    const uint32_t max_val = (1U << external.BitsPerSample()) - 1;
+    if (max_val >= 65536) return JXL_FAILURE("PNM cannot have > 16 bits");
+    const char type = external.IsGray() ? '5' : '6';
+    snprintf(header, kMaxHeaderSize, "P%c\n%zu %zu\n%u\n%n", type,
+             external.xsize(), external.ysize(), max_val, chars_written);
+  }
+  return true;
+}
+
+Status ApplyHints(const bool is_gray, CodecInOut* io) {
+  bool got_color_space = false;
+  Status ok = true;
+
+  io->dec_hints.Foreach([is_gray, io, &got_color_space, &ok](
+                            const std::string& key, const std::string& value) {
+    if (key == "color_space") {
+      ColorEncoding* c_original = &io->metadata.color_encoding;
+      if (!ParseDescription(value, c_original) ||
+          !ColorManagement::CreateProfile(c_original)) {
+        JXL_WARNING("PNM: Failed to apply color_space");
+        ok = false;
+      }
+
+      if (is_gray != io->metadata.color_encoding.IsGray()) {
+        JXL_WARNING("PNM: mismatch between file and color_space hint");
+        ok = false;
+      }
+
+      got_color_space = true;
+    } else {
+      JXL_WARNING("PNM decoder ignoring %s hint", key.c_str());
+    }
+  });
+
+  if (!got_color_space) {
+    JXL_WARNING("PNM: no color_space hint given, assuming sRGB");
+    io->metadata.color_encoding.SetSRGB(is_gray ? ColorSpace::kGray
+                                                : ColorSpace::kRGB);
+    JXL_RETURN_IF_ERROR(
+        ColorManagement::CreateProfile(&io->metadata.color_encoding));
+  }
+
+  if (!ok) return JXL_FAILURE("PNM ApplyHints failed");
+  return true;
+}
+
+Span<const uint8_t> MakeSpan(const char* str) {
+  return Span<const uint8_t>(reinterpret_cast<const uint8_t*>(str),
+                             strlen(str));
+}
+
+// Flip the image vertically for loading/saving PFM files which have the
+// scanlines inverted.
+Image3F VerticallyFlipImage(const Image3F& image) {
+  Image3F flipped = Image3F(image.xsize(), image.ysize());
+  for (int c = 0; c < 3; c++) {
+    for (size_t y = 0; y < image.ysize(); y++) {
+      float* row_out = flipped.PlaneRow(c, y);
+      const float* row_in = image.PlaneRow(c, image.ysize() - y - 1);
+      memcpy(row_out, row_in, image.xsize() * sizeof(float));
+    }
+  }
+  return flipped;
+}
+
+}  // namespace
+
+Status DecodeImagePNM(const Span<const uint8_t> bytes, ThreadPool* pool,
+                      CodecInOut* io) {
+  io->enc_size = bytes.size();
+
+  Parser parser(bytes);
+  HeaderPNM header = {};
+  const uint8_t* pos;
+  JXL_RETURN_IF_ERROR(parser.ParseHeader(&header, &pos));
+  JXL_RETURN_IF_ERROR(io->VerifyDimensions(header.xsize, header.ysize));
+  if (header.bits_per_sample == 0 || header.bits_per_sample > 32) {
+    return JXL_FAILURE("PNM: bits_per_sample invalid");
+  }
+
+  JXL_RETURN_IF_ERROR(ApplyHints(header.is_gray, io));
+  io->metadata.bits_per_sample = header.bits_per_sample;
+  io->metadata.alpha_bits = 0;
+  io->dec_pixels = header.xsize * header.ysize;
+  io->frames.clear();
+  io->frames.reserve(1);
+  ImageBundle ib(&io->metadata);
+
+  const bool has_alpha = false;
+  const bool flipped_y = header.bits_per_sample == 32;  // PFMs are flipped
+  const PackedImage desc(header.xsize, header.ysize,
+                         io->metadata.color_encoding, has_alpha,
+                         io->metadata.alpha_bits, io->metadata.bits_per_sample,
+                         header.big_endian, flipped_y);
+  const Span<const uint8_t> span(pos, bytes.data() + bytes.size() - pos);
+  if (!CopyTo(desc, span, pool, &ib)) return false;
+  if (header.bits_per_sample != 32)
+    io->metadata.bits_per_sample = ib.DetectRealBitdepth();
+  io->frames.push_back(std::move(ib));
+  return true;
+}
+
+Status EncodeImagePNM(const CodecInOut* io, const ColorEncoding& c_desired,
+                      size_t bits_per_sample, ThreadPool* pool,
+                      PaddedBytes* bytes) {
+  io->enc_bits_per_sample = bits_per_sample <= 16 ? bits_per_sample : 32;
+  // Choose native for PFM; PGM/PPM require big-endian.
+  const bool big_endian = (bits_per_sample == 32) ? !IsLittleEndian() : true;
+
+  if (!Bundle::AllDefault(io->metadata)) {
+    JXL_WARNING("PNM encoder ignoring metadata - use a different codec");
+  }
+  if (!c_desired.IsSRGB()) {
+    JXL_WARNING(
+        "PNM encoder cannot store custom ICC profile; decoder\n"
+        "will need hint key=color_space to get the same values");
+  }
+
+  const ImageBundle& ib = io->Main();
+  const ImageU* alpha = ib.HasAlpha() ? &ib.alpha() : nullptr;
+  const size_t alpha_bits = ib.HasAlpha() ? io->metadata.alpha_bits : 0;
+  CodecIntervals* temp_intervals = nullptr;  // Can't store min/max.
+
+  // TODO(lode): Implement the flipping in external_image.cc instead.
+  // In case of PFM the image must be flipped upside down since that format
+  // is designed that way.
+  const Image3F* to_external_image = &ib.color();
+  Image3F flipped;
+  if (bits_per_sample == 32) {
+    flipped = VerticallyFlipImage(ib.color());
+    to_external_image = &flipped;
+  }
+  ExternalImage external(pool, *to_external_image, Rect(ib), ib.c_current(),
+                         c_desired, ib.HasAlpha(), alpha, alpha_bits,
+                         io->enc_bits_per_sample, big_endian, temp_intervals);
+  JXL_RETURN_IF_ERROR(external.IsHealthy());
+
+  char header[kMaxHeaderSize];
+  int header_size = 0;
+  JXL_RETURN_IF_ERROR(EncodeHeader(external, header, &header_size));
+
+  const PaddedBytes& pixels = external.Bytes();
+  io->enc_size = static_cast<size_t>(header_size) + pixels.size();
+  bytes->resize(io->enc_size);
+  memcpy(bytes->data(), header, static_cast<size_t>(header_size));
+  memcpy(bytes->data() + header_size, pixels.data(), pixels.size());
+
+  return true;
+}
+
+void TestCodecPNM() {
+  size_t u = 77777;  // Initialized to wrong value.
+  double d = 77.77;
+// Failing to parse invalid strings results in a crash if `JXL_CRASH_ON_ERROR`
+// is defined and hence the tests fail. Therefore we only run these tests if
+// `JXL_CRASH_ON_ERROR` is not defined.
+#ifndef JXL_CRASH_ON_ERROR
+  JXL_CHECK(false == Parser(MakeSpan("")).ParseUnsigned(&u));
+  JXL_CHECK(false == Parser(MakeSpan("+")).ParseUnsigned(&u));
+  JXL_CHECK(false == Parser(MakeSpan("-")).ParseUnsigned(&u));
+  JXL_CHECK(false == Parser(MakeSpan("A")).ParseUnsigned(&u));
+
+  JXL_CHECK(false == Parser(MakeSpan("")).ParseSigned(&d));
+  JXL_CHECK(false == Parser(MakeSpan("+")).ParseSigned(&d));
+  JXL_CHECK(false == Parser(MakeSpan("-")).ParseSigned(&d));
+  JXL_CHECK(false == Parser(MakeSpan("A")).ParseSigned(&d));
+#endif
+  JXL_CHECK(true == Parser(MakeSpan("1")).ParseUnsigned(&u));
+  JXL_CHECK(u == 1);
+
+  JXL_CHECK(true == Parser(MakeSpan("32")).ParseUnsigned(&u));
+  JXL_CHECK(u == 32);
+
+  JXL_CHECK(true == Parser(MakeSpan("1")).ParseSigned(&d));
+  JXL_CHECK(d == 1.0);
+  JXL_CHECK(true == Parser(MakeSpan("+2")).ParseSigned(&d));
+  JXL_CHECK(d == 2.0);
+  JXL_CHECK(true == Parser(MakeSpan("-3")).ParseSigned(&d));
+  JXL_CHECK(std::abs(d - -3.0) < 1E-15);
+  JXL_CHECK(true == Parser(MakeSpan("3.141592")).ParseSigned(&d));
+  JXL_CHECK(std::abs(d - 3.141592) < 1E-15);
+  JXL_CHECK(true == Parser(MakeSpan("-3.141592")).ParseSigned(&d));
+  JXL_CHECK(std::abs(d - -3.141592) < 1E-15);
+}
+
+}  // namespace jxl
diff --git a/jxl/extras/codec_pnm.h b/jxl/extras/codec_pnm.h
new file mode 100644
index 000000000..ce6f1983c
--- /dev/null
+++ b/jxl/extras/codec_pnm.h
@@ -0,0 +1,46 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_EXTRAS_CODEC_PNM_H_
+#define JXL_EXTRAS_CODEC_PNM_H_
+
+// Encodes/decodes PGM/PPM/PFM pixels in memory.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+
+namespace jxl {
+
+// Decodes `bytes` into `io`. io->dec_hints may specify "color_space", which
+// defaults to sRGB.
+Status DecodeImagePNM(const Span<const uint8_t> bytes, ThreadPool* pool,
+                      CodecInOut* io);
+
+// Transforms from io->c_current to `c_desired` and encodes into `bytes`.
+Status EncodeImagePNM(const CodecInOut* io, const ColorEncoding& c_desired,
+                      size_t bits_per_sample, ThreadPool* pool,
+                      PaddedBytes* bytes);
+
+void TestCodecPNM();
+
+}  // namespace jxl
+
+#endif  // JXL_EXTRAS_CODEC_PNM_H_
diff --git a/jxl/extras/codec_test.cc b/jxl/extras/codec_test.cc
new file mode 100644
index 000000000..6578b9648
--- /dev/null
+++ b/jxl/extras/codec_test.cc
@@ -0,0 +1,362 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/extras/codec.h"
+
+#include <stddef.h>
+#include <stdio.h>
+
+#include <algorithm>
+#include <random>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/color_management.h"
+#include "jxl/extras/codec_pgx.h"
+#include "jxl/extras/codec_pnm.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_test_utils.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+
+CodecInOut CreateTestImage(const size_t xsize, const size_t ysize,
+                           const bool is_gray, const bool add_alpha,
+                           const size_t bits_per_sample,
+                           const ColorEncoding& c_native) {
+  Image3F image(xsize, ysize);
+  std::mt19937_64 rng(129);
+  std::uniform_real_distribution<float> dist(0.0f, 255.0f);
+  if (is_gray) {
+    for (size_t y = 0; y < ysize; ++y) {
+      float* JXL_RESTRICT row0 = image.PlaneRow(0, y);
+      float* JXL_RESTRICT row1 = image.PlaneRow(1, y);
+      float* JXL_RESTRICT row2 = image.PlaneRow(2, y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row0[x] = row1[x] = row2[x] = dist(rng);
+      }
+    }
+  } else {
+    RandomFillImage(&image, 255.0f);
+  }
+  CodecInOut io;
+  io.metadata.bits_per_sample = bits_per_sample;
+  io.metadata.color_encoding = c_native;
+  io.SetFromImage(std::move(image), c_native);
+  if (add_alpha) {
+    ImageU alpha(xsize, ysize);
+    const size_t alpha_bits = bits_per_sample <= 8 ? 8 : 16;
+    const uint16_t max = (1U << alpha_bits) - 1;
+    RandomFillImage(&alpha, max);
+    io.metadata.alpha_bits = alpha_bits;
+    io.Main().SetAlpha(std::move(alpha));
+  }
+  return io;
+}
+
+// Ensures reading a newly written file leads to the same image pixels.
+void TestRoundTrip(Codec codec, const size_t xsize, const size_t ysize,
+                   const bool is_gray, const bool add_alpha,
+                   const size_t bits_per_sample, ThreadPool* pool) {
+  // JPEG encoding is not lossless.
+  if (codec == Codec::kJPG) return;
+  if (codec == Codec::kPNM && add_alpha) return;
+  printf("Codec %s bps:%zu gr:%d al:%d\n",
+         ExtensionFromCodec(codec, is_gray, bits_per_sample).c_str(),
+         bits_per_sample, is_gray, add_alpha);
+
+  ColorEncoding c_native;
+  c_native.SetColorSpace(is_gray ? ColorSpace::kGray : ColorSpace::kRGB);
+  // Note: this must not be wider than c_external, otherwise gamut clipping
+  // will cause large round-trip errors.
+  c_native.primaries = Primaries::kP3;
+  c_native.tf.SetTransferFunction(TransferFunction::kLinear);
+  JXL_CHECK(ColorManagement::CreateProfile(&c_native));
+
+  // Generally store same color space to reduce round trip errors..
+  ColorEncoding c_external = c_native;
+  // .. unless we have enough precision for some transforms.
+  if (bits_per_sample >= 16) {
+    c_external.white_point = WhitePoint::kE;
+    c_external.primaries = Primaries::k2100;
+    c_external.tf.SetTransferFunction(TransferFunction::kSRGB);
+  }
+  JXL_CHECK(ColorManagement::CreateProfile(&c_external));
+
+  const CodecInOut io = CreateTestImage(xsize, ysize, is_gray, add_alpha,
+                                        bits_per_sample, c_native);
+  const ImageBundle& ib1 = io.Main();
+
+  PaddedBytes encoded;
+  JXL_CHECK(Encode(io, codec, c_external, bits_per_sample, &encoded, pool));
+
+  CodecInOut io2;
+  // Only for PNM because PNG will warn about ignoring them.
+  if (codec == Codec::kPNM) {
+    io2.dec_hints.Add("color_space", Description(c_external));
+  }
+  JXL_CHECK(SetFromBytes(Span<const uint8_t>(encoded), &io2, pool));
+  ImageBundle& ib2 = io2.Main();
+
+  EXPECT_EQ(io.enc_bits_per_sample, io2.metadata.bits_per_sample);
+  EXPECT_EQ(Description(c_external), Description(io2.metadata.color_encoding));
+
+  // See c_external above - for low bits_per_sample the encoded space is
+  // already the same.
+  if (bits_per_sample < 16) {
+    EXPECT_EQ(Description(ib1.c_current()), Description(ib2.c_current()));
+  }
+
+  if (add_alpha) {
+    EXPECT_TRUE(SamePixels(ib1.alpha(), ib2.alpha()));
+  }
+
+  JXL_CHECK(ib2.TransformTo(ib1.c_current(), pool));
+
+  double max_l1, max_rel;
+  // Round-trip tolerances must be higher than in external_image_test because
+  // codecs do not support unbounded ranges.
+#if JPEGXL_ENABLE_SKCMS
+  if (true) {
+#else
+  if (io.enc_bits_per_sample <= 12) {
+#endif
+    max_l1 = 0.5;
+    max_rel = 6E-3;
+  } else if (io.enc_bits_per_sample == 16) {
+    max_l1 = 3E-3;
+    max_rel = 1E-4;
+  } else {
+#ifdef __ARM_ARCH
+    // pow() implementation in arm is a bit less precise than in x86 and
+    // therefore we need a bigger error margin in this case.
+    max_l1 = 1E-7;
+    max_rel = 1E-4;
+#else
+    max_l1 = 1E-7;
+    max_rel = 1E-5;
+#endif
+  }
+
+  VerifyRelativeError(ib1.color(), ib2.color(), max_l1, max_rel);
+}
+
+TEST(CodecTest, TestRoundTrip) {
+  ThreadPoolInternal pool(12);
+
+  const size_t xsize = 7;
+  const size_t ysize = 4;
+
+  for (Codec codec : Values<Codec>()) {
+    for (int bits_per_sample : {8, 10, 12, 16, 32}) {
+      for (bool is_gray : {false, true}) {
+        for (bool add_alpha : {false, true}) {
+          TestRoundTrip(codec, xsize, ysize, is_gray, add_alpha,
+                        static_cast<size_t>(bits_per_sample), &pool);
+        }
+      }
+    }
+  }
+}
+
+CodecInOut DecodeRoundtrip(const std::string& pathname, Codec expected_codec,
+                           ThreadPool* pool,
+                           const DecoderHints& dec_hints = DecoderHints()) {
+  CodecInOut io;
+  io.dec_hints = dec_hints;
+  JXL_CHECK(SetFromFile(pathname, &io, pool));
+  const ImageBundle& ib1 = io.Main();
+
+  // Encode/Decode again to make sure Encode carries through all metadata.
+  PaddedBytes encoded;
+  JXL_CHECK(Encode(io, expected_codec, io.metadata.color_encoding,
+                   io.metadata.bits_per_sample, &encoded, pool));
+
+  CodecInOut io2;
+  io2.dec_hints = dec_hints;
+  JXL_CHECK(SetFromBytes(Span<const uint8_t>(encoded), &io2, pool));
+  const ImageBundle& ib2 = io2.Main();
+  EXPECT_EQ(Description(ib1.metadata()->color_encoding),
+            Description(ib2.metadata()->color_encoding));
+  EXPECT_EQ(Description(ib1.c_current()), Description(ib2.c_current()));
+
+  // "Same" pixels?
+  double max_l1 = io.enc_bits_per_sample <= 12 ? 1.3 : 2E-3;
+  double max_rel = io.enc_bits_per_sample <= 12 ? 6E-3 : 1E-4;
+  if (ib1.metadata()->color_encoding.IsGray()) {
+    max_rel *= 2.0;
+  } else if (ib1.metadata()->color_encoding.primaries != Primaries::kSRGB) {
+    // Need more tolerance for large gamuts (anything but sRGB)
+    max_l1 *= 1.5;
+    max_rel *= 3.0;
+  }
+  VerifyRelativeError(ib1.color(), ib2.color(), max_l1, max_rel);
+
+  // Simulate the encoder removing profile and decoder restoring it.
+  if (!ib2.metadata()->color_encoding.opaque_icc) {
+    io2.metadata.color_encoding.icc.clear();
+    EXPECT_TRUE(ColorManagement::CreateProfile(&io2.metadata.color_encoding));
+  }
+
+  return io2;
+}
+
+TEST(CodecTest, TestMetadataSRGB) {
+  ThreadPoolInternal pool(12);
+
+  const char* paths[] = {"raw.pixls/DJI-FC6310-16bit_srgb8_v4_krita.png",
+                         "raw.pixls/Google-Pixel2XL-16bit_srgb8_v4_krita.png",
+                         "raw.pixls/HUAWEI-EVA-L09-16bit_srgb8_dt.png",
+                         "raw.pixls/Nikon-D300-12bit_srgb8_dt.png",
+                         "raw.pixls/Sony-DSC-RX1RM2-14bit_srgb8_v4_krita.png"};
+  for (const char* relative_pathname : paths) {
+    const CodecInOut io =
+        DecodeRoundtrip(GetTestDataPath(relative_pathname), Codec::kPNG, &pool);
+    EXPECT_EQ(8, io.metadata.bits_per_sample);
+
+    EXPECT_EQ(64, io.xsize());
+    EXPECT_EQ(64, io.ysize());
+    EXPECT_FALSE(io.metadata.HasAlpha());
+
+    const ColorEncoding& c_original = io.metadata.color_encoding;
+    EXPECT_FALSE(c_original.icc.empty());
+    EXPECT_EQ(ColorSpace::kRGB, c_original.GetColorSpace());
+    EXPECT_EQ(WhitePoint::kD65, c_original.white_point);
+    EXPECT_EQ(Primaries::kSRGB, c_original.primaries);
+    EXPECT_TRUE(c_original.tf.IsSRGB());
+  }
+}
+
+TEST(CodecTest, TestMetadataLinear) {
+  ThreadPoolInternal pool(12);
+
+  const char* paths[3] = {
+      "raw.pixls/Google-Pixel2XL-16bit_acescg_g1_v4_krita.png",
+      "raw.pixls/HUAWEI-EVA-L09-16bit_709_g1_dt.png",
+      "raw.pixls/Nikon-D300-12bit_2020_g1_dt.png",
+  };
+  const WhitePoint white_points[3] = {WhitePoint::kCustom, WhitePoint::kD65,
+                                      WhitePoint::kD65};
+  const Primaries primaries[3] = {Primaries::kCustom, Primaries::kSRGB,
+                                  Primaries::k2100};
+
+  for (size_t i = 0; i < 3; ++i) {
+    const CodecInOut io =
+        DecodeRoundtrip(GetTestDataPath(paths[i]), Codec::kPNG, &pool);
+    EXPECT_EQ(16, io.metadata.bits_per_sample);
+
+    EXPECT_EQ(64, io.xsize());
+    EXPECT_EQ(64, io.ysize());
+    EXPECT_FALSE(io.metadata.HasAlpha());
+
+    const ColorEncoding& c_original = io.metadata.color_encoding;
+    EXPECT_FALSE(c_original.icc.empty());
+    EXPECT_EQ(ColorSpace::kRGB, c_original.GetColorSpace());
+    EXPECT_EQ(white_points[i], c_original.white_point);
+    EXPECT_EQ(primaries[i], c_original.primaries);
+    EXPECT_TRUE(c_original.tf.IsLinear());
+  }
+}
+
+TEST(CodecTest, TestMetadataICC) {
+  ThreadPoolInternal pool(12);
+
+  const char* paths[] = {
+      "raw.pixls/DJI-FC6310-16bit_709_v4_krita.png",
+      "raw.pixls/Sony-DSC-RX1RM2-14bit_709_v4_krita.png",
+  };
+  for (const char* relative_pathname : paths) {
+    const CodecInOut io =
+        DecodeRoundtrip(GetTestDataPath(relative_pathname), Codec::kPNG, &pool);
+    EXPECT_GE(16, io.metadata.bits_per_sample);
+    EXPECT_LE(14, io.metadata.bits_per_sample);
+
+    EXPECT_EQ(64, io.xsize());
+    EXPECT_EQ(64, io.ysize());
+    EXPECT_FALSE(io.metadata.HasAlpha());
+
+    const ColorEncoding& c_original = io.metadata.color_encoding;
+    EXPECT_FALSE(c_original.icc.empty());
+    EXPECT_EQ(RenderingIntent::kPerceptual, c_original.rendering_intent);
+    EXPECT_EQ(ColorSpace::kRGB, c_original.GetColorSpace());
+    EXPECT_EQ(WhitePoint::kD65, c_original.white_point);
+    EXPECT_EQ(Primaries::kSRGB, c_original.primaries);
+    EXPECT_EQ(TransferFunction::k709, c_original.tf.GetTransferFunction());
+  }
+}
+
+TEST(CodecTest, TestPNGSuite) {
+  ThreadPoolInternal pool(12);
+
+  // Ensure we can load PNG with text, japanese UTF-8, compressed text.
+  (void)DecodeRoundtrip(GetTestDataPath("pngsuite/ct1n0g04.png"), Codec::kPNG,
+                        &pool);
+  (void)DecodeRoundtrip(GetTestDataPath("pngsuite/ctjn0g04.png"), Codec::kPNG,
+                        &pool);
+  (void)DecodeRoundtrip(GetTestDataPath("pngsuite/ctzn0g04.png"), Codec::kPNG,
+                        &pool);
+
+  // Extract gAMA
+  const CodecInOut b1 = DecodeRoundtrip(
+      GetTestDataPath("pngsuite/g10n3p04.png"), Codec::kPNG, &pool);
+  EXPECT_TRUE(b1.metadata.color_encoding.tf.IsLinear());
+
+  // Extract cHRM
+  const CodecInOut b_p = DecodeRoundtrip(
+      GetTestDataPath("pngsuite/ccwn2c08.png"), Codec::kPNG, &pool);
+  EXPECT_EQ(Primaries::kSRGB, b_p.metadata.color_encoding.primaries);
+  EXPECT_EQ(WhitePoint::kD65, b_p.metadata.color_encoding.white_point);
+
+  // Extract EXIF from (new-style) dedicated chunk
+  const CodecInOut b_exif = DecodeRoundtrip(
+      GetTestDataPath("pngsuite/exif2c08.png"), Codec::kPNG, &pool);
+  EXPECT_EQ(978, b_exif.blobs.exif.size());
+}
+
+void VerifyWideGamutMetadata(const std::string& relative_pathname,
+                             const Primaries primaries, ThreadPool* pool) {
+  const CodecInOut io =
+      DecodeRoundtrip(GetTestDataPath(relative_pathname), Codec::kPNG, pool);
+
+  EXPECT_EQ(8, io.metadata.bits_per_sample);
+
+  const ColorEncoding& c_original = io.metadata.color_encoding;
+  EXPECT_FALSE(c_original.icc.empty());
+  EXPECT_EQ(RenderingIntent::kAbsolute, c_original.rendering_intent);
+  EXPECT_EQ(ColorSpace::kRGB, c_original.GetColorSpace());
+  EXPECT_EQ(WhitePoint::kD65, c_original.white_point);
+  EXPECT_EQ(primaries, c_original.primaries);
+}
+
+TEST(CodecTest, TestWideGamut) {
+  ThreadPoolInternal pool(12);
+  VerifyWideGamutMetadata("wide-gamut-tests/P3-sRGB-color-bars.png",
+                          Primaries::kP3, &pool);
+  VerifyWideGamutMetadata("wide-gamut-tests/P3-sRGB-color-ring.png",
+                          Primaries::kP3, &pool);
+  VerifyWideGamutMetadata("wide-gamut-tests/R2020-sRGB-color-bars.png",
+                          Primaries::k2100, &pool);
+  VerifyWideGamutMetadata("wide-gamut-tests/R2020-sRGB-color-ring.png",
+                          Primaries::k2100, &pool);
+}
+
+TEST(CodecTest, TestPNM) { TestCodecPNM(); }
+TEST(CodecTest, TestPGX) { TestCodecPGX(); }
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/field_encodings.h b/jxl/field_encodings.h
new file mode 100644
index 000000000..54ac236e5
--- /dev/null
+++ b/jxl/field_encodings.h
@@ -0,0 +1,145 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_FIELD_ENCODINGS_H_
+#define JXL_FIELD_ENCODINGS_H_
+
+// Constants needed to encode/decode fields; avoids including the full fields.h.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/base/bits.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Distribution of U32 values for one particular selector. Represents either a
+// power of two-sized range, or a single value. A separate type ensures this is
+// only passed to the U32Enc ctor.
+struct U32Distr {
+  // No need to validate - all `d` are legitimate.
+  constexpr explicit U32Distr(uint32_t d) : d(d) {}
+
+  static constexpr uint32_t kDirect = 0x80000000u;
+
+  constexpr bool IsDirect() const { return (d & kDirect) != 0; }
+
+  // Only call if IsDirect().
+  constexpr uint32_t Direct() const { return d & (kDirect - 1); }
+
+  // Only call if !IsDirect().
+  constexpr size_t ExtraBits() const { return (d & 0x1F) + 1; }
+  uint32_t Offset() const { return (d >> 5) & 0x3FFFFFF; }
+
+  uint32_t d;
+};
+
+// A direct-coded 31-bit value occupying 2 bits in the bitstream.
+constexpr U32Distr Val(uint32_t value) {
+  return U32Distr(value | U32Distr::kDirect);
+}
+
+// Value - `offset` will be signaled in `bits` extra bits.
+constexpr U32Distr BitsOffset(uint32_t bits, uint32_t offset) {
+  return U32Distr(((bits - 1) & 0x1F) + ((offset & 0x3FFFFFF) << 5));
+}
+
+// Value will be signaled in `bits` extra bits.
+constexpr U32Distr Bits(uint32_t bits) { return BitsOffset(bits, 0); }
+
+// See U32Coder documentation in fields.h.
+class U32Enc {
+ public:
+  constexpr U32Enc(const U32Distr d0, const U32Distr d1, const U32Distr d2,
+                   const U32Distr d3)
+      : d_{d0, d1, d2, d3} {}
+
+  // Returns the U32Distr at `selector` = 0..3, least-significant first.
+  U32Distr GetDistr(const uint32_t selector) const {
+    JXL_ASSERT(selector < 4);
+    return d_[selector];
+  }
+
+ private:
+  U32Distr d_[4];
+};
+
+// Convert between two's complement and sign bit in LSB so S64 can use
+// U64Coder etc.
+static inline constexpr uint32_t U32FromS32(const int32_t v) {
+  return v < 0 ? (static_cast<uint32_t>(-(v + 1)) << 1) + 1
+               : (static_cast<uint32_t>(v) << 1);
+}
+
+static inline constexpr uint64_t U64FromS64(const int64_t v) {
+  return v < 0 ? (static_cast<uint64_t>(-(v + 1)) << 1) + 1
+               : (static_cast<uint64_t>(v) << 1);
+}
+
+static inline constexpr int32_t S32FromU32(const uint32_t u) {
+  return (u & 1) ? -static_cast<int32_t>((u >> 1) + 1)
+                 : static_cast<int32_t>(u >> 1);
+}
+
+static inline constexpr int64_t S64FromU64(const uint64_t u) {
+  return (u & 1) ? -static_cast<int64_t>((u >> 1) + 1)
+                 : static_cast<int64_t>(u >> 1);
+}
+
+// Returns bit with the given `index` (0 = least significant).
+template <typename T>
+static inline constexpr uint64_t MakeBit(T index) {
+  return 1ULL << static_cast<uint32_t>(index);
+}
+
+// Returns vector of all possible values of an Enum type. Relies on each Enum
+// providing an overload of EnumBits() that returns a bit array of its values,
+// which implies values must be in [0, 64).
+template <typename Enum>
+std::vector<Enum> Values() {
+  uint64_t bits = EnumBits(Enum());
+
+  std::vector<Enum> values;
+  values.reserve(PopCount(bits));
+
+  // For each 1-bit in bits: add its index as value
+  while (bits != 0) {
+    const int index = NumZeroBitsBelowLSBNonzero(bits);
+    values.push_back(static_cast<Enum>(index));
+    bits &= bits - 1;  // clear least-significant bit
+  }
+  return values;
+}
+
+// Returns true if value is one of Values<Enum>().
+template <class Enum>
+Status EnumValid(const Enum value) {
+  if (static_cast<uint32_t>(value) >= 64) {
+    return JXL_FAILURE("Value %u too large for %s\n",
+                       static_cast<uint32_t>(value), EnumName(Enum()));
+  }
+  const uint64_t bit = MakeBit(value);
+  if ((EnumBits(Enum()) & bit) == 0) {
+    return JXL_FAILURE("Invalid value %u for %s\n",
+                       static_cast<uint32_t>(value), EnumName(Enum()));
+  }
+  return true;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_FIELD_ENCODINGS_H_
diff --git a/jxl/fields.cc b/jxl/fields.cc
new file mode 100644
index 000000000..83cd3cbc7
--- /dev/null
+++ b/jxl/fields.cc
@@ -0,0 +1,289 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/fields.h"
+
+#include <stddef.h>
+
+#include <algorithm>
+#include <cmath>
+
+#include "jxl/base/bits.h"
+
+namespace jxl {
+
+size_t U32Coder::MaxEncodedBits(const U32Enc enc) {
+  size_t extra_bits = 0;
+  for (uint32_t selector = 0; selector < 4; ++selector) {
+    const U32Distr d = enc.GetDistr(selector);
+    if (d.IsDirect()) {
+      continue;
+    } else {
+      extra_bits = std::max<size_t>(extra_bits, d.ExtraBits());
+    }
+  }
+  return 2 + extra_bits;
+}
+
+Status U32Coder::CanEncode(const U32Enc enc, const uint32_t value,
+                           size_t* JXL_RESTRICT encoded_bits) {
+  uint32_t selector;
+  size_t total_bits;
+  const Status ok = ChooseSelector(enc, value, &selector, &total_bits);
+  *encoded_bits = ok ? total_bits : 0;
+  return ok;
+}
+
+HWY_ATTR uint32_t U32Coder::Read(const U32Enc enc,
+                                 BitReader* JXL_RESTRICT reader) {
+  const uint32_t selector = reader->ReadFixedBits<2>();
+  const U32Distr d = enc.GetDistr(selector);
+  if (d.IsDirect()) {
+    return d.Direct();
+  } else {
+    return reader->ReadBits(d.ExtraBits()) + d.Offset();
+  }
+}
+
+// Returns false if the value is too large to encode.
+Status U32Coder::Write(const U32Enc enc, const uint32_t value,
+                       BitWriter* JXL_RESTRICT writer) {
+  uint32_t selector;
+  size_t total_bits;
+  JXL_RETURN_IF_ERROR(ChooseSelector(enc, value, &selector, &total_bits));
+
+  writer->Write(2, selector);
+
+  const U32Distr d = enc.GetDistr(selector);
+  if (!d.IsDirect()) {  // Nothing more to write for direct encoding
+    const uint32_t offset = d.Offset();
+    JXL_ASSERT(value >= offset);
+    writer->Write(total_bits - 2, value - offset);
+  }
+
+  return true;
+}
+
+Status U32Coder::ChooseSelector(const U32Enc enc, const uint32_t value,
+                                uint32_t* JXL_RESTRICT selector,
+                                size_t* JXL_RESTRICT total_bits) {
+#if JXL_ENABLE_ASSERT
+  const size_t bits_required = 32 - NumZeroBitsAboveMSB(value);
+#endif  // JXL_ENABLE_ASSERT
+  JXL_ASSERT(bits_required <= 32);
+
+  *selector = 0;
+  *total_bits = 0;
+
+  // It is difficult to verify whether Dist32Byte are sorted, so check all
+  // selectors and keep the one with the fewest total_bits.
+  *total_bits = 64;  // more than any valid encoding
+  for (uint32_t s = 0; s < 4; ++s) {
+    const U32Distr d = enc.GetDistr(s);
+    if (d.IsDirect()) {
+      if (d.Direct() == value) {
+        *selector = s;
+        *total_bits = 2;
+        return true;  // Done, direct is always the best possible.
+      }
+      continue;
+    }
+    const size_t extra_bits = d.ExtraBits();
+    const uint32_t offset = d.Offset();
+    if (value < offset || value >= offset + (1ULL << extra_bits)) continue;
+
+    // Better than prior encoding, remember it:
+    if (2 + extra_bits < *total_bits) {
+      *selector = s;
+      *total_bits = 2 + extra_bits;
+    }
+  }
+
+  if (*total_bits == 64) {
+    return JXL_FAILURE("No feasible selector for %u", value);
+  }
+
+  return true;
+}
+
+HWY_ATTR uint64_t U64Coder::Read(BitReader* JXL_RESTRICT reader) {
+  uint64_t selector = reader->ReadFixedBits<2>();
+  if (selector == 0) {
+    return 0;
+  }
+  if (selector == 1) {
+    return 1 + reader->ReadFixedBits<4>();
+  }
+  if (selector == 2) {
+    return 17 + reader->ReadFixedBits<8>();
+  }
+
+  // selector 3, varint, groups have first 12, then 8, and last 4 bits.
+  uint64_t result = reader->ReadFixedBits<12>();
+
+  uint64_t shift = 12;
+  while (reader->ReadFixedBits<1>()) {
+    if (shift == 60) {
+      result |= static_cast<uint64_t>(reader->ReadFixedBits<4>()) << shift;
+      break;
+    }
+    result |= static_cast<uint64_t>(reader->ReadFixedBits<8>()) << shift;
+    shift += 8;
+  }
+
+  return result;
+}
+
+// Returns false if the value is too large to encode.
+Status U64Coder::Write(uint64_t value, BitWriter* JXL_RESTRICT writer) {
+  if (value == 0) {
+    // Selector: use 0 bits, value 0
+    writer->Write(2, 0);
+  } else if (value <= 16) {
+    // Selector: use 4 bits, value 1..16
+    writer->Write(2, 1);
+    writer->Write(4, value - 1);
+  } else if (value <= 272) {
+    // Selector: use 8 bits, value 17..272
+    writer->Write(2, 2);
+    writer->Write(8, value - 17);
+  } else {
+    // Selector: varint, first a 12-bit group, after that per 8-bit group.
+    writer->Write(2, 3);
+    writer->Write(12, value & 4095);
+    value >>= 12;
+    int shift = 12;
+    while (value > 0 && shift < 60) {
+      // Indicate varint not done
+      writer->Write(1, 1);
+      writer->Write(8, value & 255);
+      value >>= 8;
+      shift += 8;
+    }
+    if (value > 0) {
+      // This only could happen if shift == N - 4.
+      writer->Write(1, 1);
+      writer->Write(4, value & 15);
+      // Implicitly closed sequence, no extra stop bit is required.
+    } else {
+      // Indicate end of varint
+      writer->Write(1, 0);
+    }
+  }
+
+  return true;
+}
+
+// Can always encode, but useful because it also returns bit size.
+Status U64Coder::CanEncode(uint64_t value, size_t* JXL_RESTRICT encoded_bits) {
+  if (value == 0) {
+    *encoded_bits = 2;  // 2 selector bits
+  } else if (value <= 16) {
+    *encoded_bits = 2 + 4;  // 2 selector bits + 4 payload bits
+  } else if (value <= 272) {
+    *encoded_bits = 2 + 8;  // 2 selector bits + 8 payload bits
+  } else {
+    *encoded_bits = 2 + 12;  // 2 selector bits + 12 payload bits
+    value >>= 12;
+    int shift = 12;
+    while (value > 0 && shift < 60) {
+      *encoded_bits += 1 + 8;  // 1 continuation bit + 8 payload bits
+      value >>= 8;
+      shift += 8;
+    }
+    if (value > 0) {
+      // This only could happen if shift == N - 4.
+      *encoded_bits += 1 + 4;  // 1 continuation bit + 4 payload bits
+    } else {
+      *encoded_bits += 1;  // 1 stop bit
+    }
+  }
+
+  return true;
+}
+
+HWY_ATTR Status F16Coder::Read(BitReader* JXL_RESTRICT reader,
+                               float* JXL_RESTRICT value) {
+  const uint32_t bits16 = reader->ReadFixedBits<16>();
+  const uint32_t sign = bits16 >> 15;
+  const uint32_t biased_exp = (bits16 >> 10) & 0x1F;
+  const uint32_t mantissa = bits16 & 0x3FF;
+
+  if (JXL_UNLIKELY(biased_exp == 31)) {
+    return JXL_FAILURE("F16 infinity or NaN are not supported");
+  }
+
+  // Subnormal or zero
+  if (JXL_UNLIKELY(biased_exp == 0)) {
+    *value = (1.0f / 16384) * (mantissa * (1.0f / 1024));
+    return true;
+  }
+
+  // Normalized: convert the representation directly (faster than ldexp/tables).
+  const uint32_t biased_exp32 = biased_exp + (127 - 15);
+  const uint32_t mantissa32 = mantissa << (23 - 10);
+  const uint32_t bits32 = (sign << 31) | (biased_exp32 << 23) | mantissa32;
+  memcpy(value, &bits32, sizeof(bits32));
+  return true;
+}
+
+Status F16Coder::Write(float value, BitWriter* JXL_RESTRICT writer) {
+  uint32_t bits32;
+  memcpy(&bits32, &value, sizeof(bits32));
+  const uint32_t sign = bits32 >> 31;
+  const uint32_t biased_exp32 = (bits32 >> 23) & 0xFF;
+  const uint32_t mantissa32 = bits32 & 0x7FFFFF;
+
+  const int32_t exp = static_cast<int32_t>(biased_exp32) - 127;
+  if (JXL_UNLIKELY(exp > 15)) {
+    return JXL_FAILURE("Too big to encode, CanEncode should return false");
+  }
+
+  // Tiny or zero => zero.
+  if (exp < -24) {
+    writer->Write(16, 0);
+    return true;
+  }
+
+  uint32_t biased_exp16, mantissa16;
+
+  // exp = [-24, -15] => subnormal
+  if (JXL_UNLIKELY(exp < -14)) {
+    biased_exp16 = 0;
+    const uint32_t sub_exp = static_cast<uint32_t>(-14 - exp);
+    JXL_ASSERT(1 <= sub_exp && sub_exp < 11);
+    mantissa16 = (1 << (10 - sub_exp)) + (mantissa32 >> (13 + sub_exp));
+  } else {
+    // exp = [-14, 15]
+    biased_exp16 = static_cast<uint32_t>(exp + 15);
+    JXL_ASSERT(1 <= biased_exp16 && biased_exp16 < 31);
+    mantissa16 = mantissa32 >> 13;
+  }
+
+  JXL_ASSERT(mantissa16 < 1024);
+  const uint32_t bits16 = (sign << 15) | (biased_exp16 << 10) | mantissa16;
+  JXL_ASSERT(bits16 < 0x10000);
+  writer->Write(16, bits16);
+  return true;
+}
+
+Status F16Coder::CanEncode(float value, size_t* JXL_RESTRICT encoded_bits) {
+  *encoded_bits = MaxEncodedBits();
+  if (std::isnan(value) || std::isinf(value)) {
+    return JXL_FAILURE("Should not attempt to store NaN and infinity");
+  }
+  return std::abs(value) <= 65504.0f;
+}
+
+}  // namespace jxl
diff --git a/jxl/fields.h b/jxl/fields.h
new file mode 100644
index 000000000..7e7c8d7e7
--- /dev/null
+++ b/jxl/fields.h
@@ -0,0 +1,825 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_FIELDS_H_
+#define JXL_FIELDS_H_
+
+// Forward/backward-compatible 'bundles' with auto-serialized 'fields'.
+
+#include <inttypes.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <cmath>  // abs
+#include <cstdarg>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/field_encodings.h"
+
+namespace jxl {
+
+// Integer coders: BitsCoder (raw), U32Coder (table), U64Coder (varint).
+
+// Reads/writes a given (fixed) number of bits <= 32.
+class BitsCoder {
+ public:
+  static size_t MaxEncodedBits(const size_t bits) { return bits; }
+
+  static Status CanEncode(const size_t bits, const uint32_t value,
+                          size_t* JXL_RESTRICT encoded_bits) {
+    *encoded_bits = bits;
+    if (value >= (1ULL << bits)) {
+      return JXL_FAILURE("Value %u too large for %zu bits", value, bits);
+    }
+    return true;
+  }
+
+  HWY_ATTR static uint32_t Read(const size_t bits,
+                                BitReader* JXL_RESTRICT reader) {
+    return reader->ReadBits(bits);
+  }
+
+  // Returns false if the value is too large to encode.
+  static Status Write(const size_t bits, const uint32_t value,
+                      BitWriter* JXL_RESTRICT writer) {
+    if (value >= (1ULL << bits)) {
+      return JXL_FAILURE("Value %d too large to encode in %zu bits", value,
+                         bits);
+    }
+    writer->Write(bits, value);
+    return true;
+  }
+};
+
+// Encodes u32 using a lookup table and/or extra bits, governed by a per-field
+// encoding `enc` which consists of four distributions `d` chosen via a 2-bit
+// selector (least significant = 0). Each d may have two modes:
+// - direct: if d.IsDirect(), the value is d.Direct();
+// - offset: the value is derived from d.ExtraBits() extra bits plus d.Offset();
+// This encoding is denser than Exp-Golomb or Gamma codes when both small and
+// large values occur.
+//
+// Examples:
+// Direct: U32Enc(Val(8), Val(16), Val(32), Bits(6)), value 32 => 10b.
+// Offset: U32Enc(Val(0), BitsOffset(1, 1), BitsOffset(2, 3), BitsOffset(8, 8))
+//   defines the following prefix code:
+//   00 -> 0
+//   01x -> 1..2
+//   10xx -> 3..7
+//   11xxxxxxxx -> 8..263
+class U32Coder {
+ public:
+  static size_t MaxEncodedBits(U32Enc enc);
+  static Status CanEncode(U32Enc enc, uint32_t value,
+                          size_t* JXL_RESTRICT encoded_bits);
+  static uint32_t Read(U32Enc enc, BitReader* JXL_RESTRICT reader);
+
+  // Returns false if the value is too large to encode.
+  static Status Write(U32Enc enc, uint32_t value,
+                      BitWriter* JXL_RESTRICT writer);
+
+ private:
+  static Status ChooseSelector(U32Enc enc, uint32_t value,
+                               uint32_t* JXL_RESTRICT selector,
+                               size_t* JXL_RESTRICT total_bits);
+};
+
+// Encodes 64-bit unsigned integers with a fixed distribution, taking 2 bits
+// to encode 0, 6 bits to encode 1 to 16, 10 bits to encode 17 to 272, 15 bits
+// to encode up to 4095, and on the order of log2(value) * 1.125 bits for
+// larger values.
+class U64Coder {
+ public:
+  static constexpr size_t MaxEncodedBits() {
+    return 2 + 12 + 6 * (8 + 1) + (4 + 1);
+  }
+
+  static uint64_t Read(BitReader* JXL_RESTRICT reader);
+
+  // Returns false if the value is too large to encode.
+  static Status Write(uint64_t value, BitWriter* JXL_RESTRICT writer);
+
+  // Can always encode, but useful because it also returns bit size.
+  static Status CanEncode(uint64_t value, size_t* JXL_RESTRICT encoded_bits);
+};
+
+// IEEE 754 half-precision (binary16). Refuses to read/write NaN/Inf.
+class F16Coder {
+ public:
+  static constexpr size_t MaxEncodedBits() { return 16; }
+
+  // Returns false if the bit representation is NaN or infinity
+  static Status Read(BitReader* JXL_RESTRICT reader, float* JXL_RESTRICT value);
+
+  // Returns false if the value is too large to encode.
+  static Status Write(float value, BitWriter* JXL_RESTRICT writer);
+  static Status CanEncode(float value, size_t* JXL_RESTRICT encoded_bits);
+};
+
+// A "bundle" is a forward- and backward compatible collection of fields.
+// They are used for SizeHeader/FrameHeader/GroupHeader. Bundles can be
+// extended by appending(!) fields. Optional fields may be omitted from the
+// bitstream by conditionally visiting them. When reading new bitstreams with
+// old code, we skip unknown fields at the end of the bundle. This requires
+// storing the amount of extra appended bits, and that fields are visited in
+// chronological order of being added to the format, because old decoders
+// cannot skip some future fields and resume reading old fields. Similarly,
+// new readers query bits in an "extensions" field to skip (groups of) fields
+// not present in old bitstreams. Note that each bundle must include an
+// "extensions" field prior to freezing the format, otherwise it cannot be
+// extended.
+//
+// To ensure interoperability, there will be no opaque fields.
+//
+// HOWTO:
+// - basic usage: define a struct with member variables ("fields") and a
+//   VisitFields(v) member function that calls v->U32/Bool etc. for each
+//   field, specifying their default values. The ctor must call
+//   Bundle::Init(this).
+//
+// - print a trace of visitors: ensure each bundle has a static Name() member
+//   function, and change Bundle::Print* to return true.
+//
+// - optional fields: in VisitFields, add if (v->Conditional(your_condition))
+//   { v->Bool(default, &field); }. This prevents reading/writing field
+//   if !your_condition, which is typically computed from a prior field.
+//   WARNING: to ensure all fields are initialized, do not add an else branch;
+//   instead add another if (v->Conditional(!your_condition)).
+//
+// - repeated fields: for dynamic sizes, use e.g. std::vector and in
+//   VisitFields, if (v->IfReading()) field.resize(size) before accessing field.
+//   For static or bounded sizes, use an array or std::array. In all cases,
+//   simply visit each array element as if it were a normal field.
+//
+// - nested bundles: add a bundle as a normal field and in VisitFields call
+//   JXL_RETURN_IF_ERROR(v->VisitNested(&nested));
+//
+// - allow future extensions: define a "uint64_t extensions" field and call
+//   v->BeginExtensions(&extensions) after visiting all non-extension fields,
+//   and `return v->EndExtensions();` after the last extension field.
+//
+// - encode an entire bundle in one bit if ALL its fields equal their default
+//   values: add a "mutable bool all_default" field and as the first visitor:
+//   if (v->AllDefault(*this, &all_default)) return true;
+//   Note: if extensions are present, AllDefault() == false.
+
+class Bundle {
+ public:
+  // Print the type of each visitor called.
+  static constexpr bool PrintVisitors() { return false; }
+  // Print default value for each field and AllDefault result.
+  static constexpr bool PrintAllDefault() { return false; }
+  // Print values decoded for each field in Read.
+  static constexpr bool PrintRead() { return false; }
+  // Print size for each field and CanEncode total_bits.
+  static constexpr bool PrintSizes() { return false; }
+
+  template <class T>
+  static void Init(T* JXL_RESTRICT t) {
+    InitVisitor visitor;
+    if (!visitor.Visit(t, PrintVisitors() ? "-- Init\n" : "")) {
+      JXL_ASSERT(false);  // Init should never fail.
+    }
+  }
+
+  // Returns whether ALL fields (including `extensions`, if present) are equal
+  // to their default value.
+  template <class T>
+  static bool AllDefault(const T& t) {
+    AllDefaultVisitor visitor;
+    const char* name =
+        (PrintVisitors() || PrintAllDefault()) ? "[[AllDefault\n" : "";
+    if (!visitor.VisitConst(t, name)) {
+      JXL_ASSERT(false);  // AllDefault should never fail.
+    }
+
+    if (PrintAllDefault()) printf("  %d]]\n", visitor.AllDefault());
+    return visitor.AllDefault();
+  }
+
+  // Returns max number of bits required to encode a T.
+  template <class T>
+  static size_t MaxBits(const T& t) {
+    MaxBitsVisitor visitor;
+#if JXL_ENABLE_ASSERT
+    Status ret =
+#else
+    (void)
+#endif  // JXL_ENABLE_ASSERT
+        visitor.VisitConst(t, PrintVisitors() ? "-- MaxBits\n" : "");
+    JXL_ASSERT(ret);
+    return visitor.MaxBits();
+  }
+
+  // Returns whether a header's fields can all be encoded, i.e. they have a
+  // valid representation. If so, "*total_bits" is the exact number of bits
+  // required. Called by Write.
+  template <class T>
+  static Status CanEncode(const T& t, size_t* JXL_RESTRICT extension_bits,
+                          size_t* JXL_RESTRICT total_bits) {
+    CanEncodeVisitor visitor;
+    const char* name = (PrintVisitors() || PrintSizes()) ? "[[CanEncode\n" : "";
+    JXL_RETURN_IF_ERROR(visitor.VisitConst(t, name));
+    JXL_RETURN_IF_ERROR(visitor.GetSizes(extension_bits, total_bits));
+    if (PrintSizes()) printf("  %zu]]\n", *total_bits);
+    return true;
+  }
+
+  template <class T>
+  static Status Read(BitReader* reader, T* JXL_RESTRICT t) {
+    ReadVisitor visitor(reader);
+    JXL_RETURN_IF_ERROR(visitor.Visit(t, PrintVisitors() ? "-- Read\n" : ""));
+    return visitor.OK();
+  }
+
+  template <class T>
+  static Status Write(const T& t, BitWriter* JXL_RESTRICT writer, size_t layer,
+                      AuxOut* aux_out) {
+    size_t extension_bits, total_bits;
+    JXL_RETURN_IF_ERROR(CanEncode(t, &extension_bits, &total_bits));
+
+    BitWriter::Allotment allotment(writer, total_bits);
+    WriteVisitor visitor(extension_bits, writer);
+    JXL_RETURN_IF_ERROR(
+        visitor.VisitConst(t, PrintVisitors() ? "-- Write\n" : ""));
+    JXL_RETURN_IF_ERROR(visitor.OK());
+    ReclaimAndCharge(writer, &allotment, layer, aux_out);
+    return true;
+  }
+
+ private:
+  // A bundle can be in one of three states concerning extensions: not-begun,
+  // active, ended. Bundles may be nested, so we need a stack of states.
+  class ExtensionStates {
+   public:
+    static constexpr size_t kMaxDepth = 64;
+
+    void Push() {
+      // Initial state = not-begun.
+      begun_ <<= 1;
+      ended_ <<= 1;
+    }
+
+    // Clears current state; caller must check IsEnded beforehand.
+    void Pop() {
+      begun_ >>= 1;
+      ended_ >>= 1;
+    }
+
+    // Returns true if state == active || state == ended.
+    Status IsBegun() const { return (begun_ & 1) != 0; }
+    // Returns true if state != not-begun && state != active.
+    Status IsEnded() const { return (ended_ & 1) != 0; }
+
+    void Begin() {
+      JXL_ASSERT(!IsBegun());
+      JXL_ASSERT(!IsEnded());
+      begun_ += 1;
+    }
+
+    void End() {
+      JXL_ASSERT(IsBegun());
+      JXL_ASSERT(!IsEnded());
+      ended_ += 1;
+    }
+
+   private:
+    // Current state := least-significant bit of begun_ and ended_.
+    uint64_t begun_ = 0;
+    uint64_t ended_ = 0;
+  };
+
+  // Visitors generate Init/AllDefault/Read/Write logic for all fields. Each
+  // bundle's VisitFields member function calls visitor->U32 etc. We do not
+  // overload operator() because a function name is easier to search for.
+
+  template <class Derived>
+  class VisitorBase {
+   public:
+    explicit VisitorBase(bool print_bundles = false)
+        : print_bundles_(print_bundles) {}
+    ~VisitorBase() { JXL_ASSERT(depth_ == 0); }
+
+    // This is the only call site of T::VisitFields. Adds tracing and ensures
+    // EndExtensions was called.
+    template <class T>
+    Status Visit(T* t, const char* visitor_name) {
+      fputs(visitor_name, stdout);  // No newline; no effect if empty
+      if (print_bundles_) {
+        Trace("%s\n", print_bundles_ ? T::Name() : "");
+      }
+
+      depth_ += 1;
+      JXL_ASSERT(depth_ <= ExtensionStates::kMaxDepth);
+      extension_states_.Push();
+
+      Derived* self = static_cast<Derived*>(this);
+      const Status ok = t->VisitFields(self);
+
+      if (ok) {
+        // If VisitFields called BeginExtensions, must also call
+        // EndExtensions.
+        JXL_ASSERT(!extension_states_.IsBegun() || extension_states_.IsEnded());
+      } else {
+        // Failed, undefined state: don't care whether EndExtensions was
+        // called.
+      }
+
+      extension_states_.Pop();
+      JXL_ASSERT(depth_ != 0);
+      depth_ -= 1;
+
+      return ok;
+    }
+
+    // For visitors accepting a const T, need to const-cast so we can call the
+    // non-const T::VisitFields. NOTE: T is not modified except the
+    // `all_default` field by CanEncodeVisitor.
+    template <class T>
+    Status VisitConst(const T& t, const char* message) {
+      return Visit(const_cast<T*>(&t), message);
+    }
+
+    // Helper to construct U32Enc from U32Distr.
+    void U32(const U32Distr d0, const U32Distr d1, const U32Distr d2,
+             const U32Distr d3, const uint32_t default_value,
+             uint32_t* JXL_RESTRICT value) {
+      Derived* self = static_cast<Derived*>(this);
+      // The rarely-used function that takes a U32Enc must have a different
+      // name, else it would HIDE the base class U32() member.
+      self->U32WithEnc(U32Enc(d0, d1, d2, d3), default_value, value);
+    }
+
+    // Derived types (overridden by InitVisitor because it is unsafe to read
+    // from *value there)
+
+    void Bool(bool default_value, bool* JXL_RESTRICT value) {
+      uint32_t bits = *value ? 1 : 0;
+      Derived* self = static_cast<Derived*>(this);
+      self->Bits(1, static_cast<uint32_t>(default_value), &bits);
+      JXL_DASSERT(bits <= 1);
+      *value = bits == 1;
+    }
+
+    template <typename EnumT>
+    Status Enum(const EnumT default_value, EnumT* JXL_RESTRICT value) {
+      Derived* self = static_cast<Derived*>(this);
+      uint32_t u32 = static_cast<uint32_t>(*value);
+      // 00 -> 0
+      // 01 -> 1
+      // 10xxxx -> 2..17
+      // 11yyyyyy -> 18..81
+      self->U32(Val(0), Val(1), BitsOffset(4, 2), BitsOffset(6, 18),
+                static_cast<uint32_t>(default_value), &u32);
+      *value = static_cast<EnumT>(u32);
+      return EnumValid(*value);
+    }
+
+    void S32(const U32Distr d0, const U32Distr d1, const U32Distr d2,
+             const U32Distr d3, const int32_t default_value,
+             int32_t* JXL_RESTRICT value) {
+      uint32_t u32 = U32FromS32(*value);
+      Derived* self = static_cast<Derived*>(this);
+      self->U32(d0, d1, d2, d3, U32FromS32(default_value), &u32);
+      *value = S32FromU32(u32);
+    }
+
+    void S64(const int64_t default_value, int64_t* JXL_RESTRICT value) {
+      uint64_t u64 = U64FromS64(*value);
+      Derived* self = static_cast<Derived*>(this);
+      self->U64(U64FromS64(default_value), &u64);
+      *value = S64FromU64(u64);
+    }
+
+    // Returns whether VisitFields should visit some subsequent fields.
+    // "condition" is typically from prior fields, e.g. flags.
+    // Overridden by InitVisitor and MaxBitsVisitor.
+    Status Conditional(bool condition) { return condition; }
+
+    // Overridden by InitVisitor, AllDefaultVisitor and CanEncodeVisitor.
+    template <class Fields>
+    Status AllDefault(const Fields& fields, bool* JXL_RESTRICT all_default) {
+      Derived* self = static_cast<Derived*>(this);
+      self->Bool(true, all_default);
+      return *all_default;
+    }
+
+    // Returns the result of visiting a nested Bundle.
+    // Overridden by InitVisitor.
+    template <class Fields>
+    Status VisitNested(Fields* fields) {
+      Derived* self = static_cast<Derived*>(this);
+      return self->Visit(fields, "");
+    }
+
+    // Overridden by ReadVisitor. Enables dynamically-sized fields.
+    Status IsReading() const { return false; }
+
+    // Overriden by ReadVisitor and WriteVisitor.
+    // Called before any conditional visit based on "extensions".
+    // Overridden by ReadVisitor, CanEncodeVisitor and WriteVisitor.
+    void BeginExtensions(uint64_t* JXL_RESTRICT extensions) {
+      Derived* self = static_cast<Derived*>(this);
+      self->U64(0, extensions);
+
+      extension_states_.Begin();
+    }
+
+    // Called after all extension fields (if any). Although non-extension
+    // fields could be visited afterward, we prefer the convention that
+    // extension fields are always the last to be visited. Overridden by
+    // ReadVisitor.
+    Status EndExtensions() {
+      extension_states_.End();
+      return true;
+    }
+
+   protected:
+    // Prints indentation, <format_in>.
+    JXL_FORMAT(2, 3)  // 1-based plus one because member function
+    void Trace(const char* format_in, ...) const {
+      // New format string with indentation included
+      char format[200];
+      memset(format, ' ', depth_ * 2);  // indentation
+      const size_t pos = depth_ * 2;
+      strncpy(format + pos, format_in, sizeof(format) - pos - 1);
+      format[sizeof(format) - 1] = '\0';
+
+      char buf[2000];
+      va_list args;
+      va_start(args, format_in);
+      vsnprintf(buf, sizeof(buf), format, args);
+      va_end(args);
+      fputs(buf, stdout);  // unlike puts, does not add newline
+    }
+
+   private:
+    size_t depth_ = 0;  // for indentation.
+    ExtensionStates extension_states_;
+    const bool print_bundles_;
+  };
+
+  struct InitVisitor : public VisitorBase<InitVisitor> {
+    void Bits(const size_t /*unused*/, const uint32_t default_value,
+              uint32_t* JXL_RESTRICT value) {
+      *value = default_value;
+    }
+
+    void U32WithEnc(const U32Enc /*unused*/, const uint32_t default_value,
+                    uint32_t* JXL_RESTRICT value) {
+      *value = default_value;
+    }
+
+    void U64(const uint64_t default_value, uint64_t* JXL_RESTRICT value) {
+      *value = default_value;
+    }
+
+    void Bool(bool default_value, bool* JXL_RESTRICT value) {
+      *value = default_value;
+    }
+
+    template <typename T>
+    Status Enum(const T default_value, T* JXL_RESTRICT value) {
+      *value = default_value;
+      return EnumValid(*value);
+    }
+
+    void S32(U32Distr /*unused*/, U32Distr /*unused*/, U32Distr /*unused*/,
+             U32Distr /*unused*/, const int32_t default_value,
+             int32_t* JXL_RESTRICT value) {
+      *value = default_value;
+    }
+
+    void S64(const int64_t default_value, int64_t* JXL_RESTRICT value) {
+      *value = default_value;
+    }
+
+    void F16(const float default_value, float* JXL_RESTRICT value) {
+      *value = default_value;
+    }
+
+    // Always visit conditional fields to ensure they are initialized.
+    Status Conditional(bool condition) { return true; }
+
+    template <class Fields>
+    Status AllDefault(const Fields& fields, bool* JXL_RESTRICT all_default) {
+      // Just initialize this field and don't skip initializing others.
+      Bool(true, all_default);
+      return false;
+    }
+
+    template <class Fields>
+    Status VisitNested(Fields* fields) {
+      // Avoid re-initializing nested bundles (their ctors already called
+      // Bundle::Init for their fields).
+      return true;
+    }
+  };
+
+  class AllDefaultVisitor : public VisitorBase<AllDefaultVisitor> {
+   public:
+    AllDefaultVisitor() : VisitorBase<AllDefaultVisitor>(PrintAllDefault()) {}
+
+    void Bits(const size_t bits, const uint32_t default_value,
+              const uint32_t* JXL_RESTRICT value) {
+      if (PrintAllDefault()) {
+        Trace("  u(%zu) = %u, default %u\n", bits, *value, default_value);
+      }
+
+      all_default_ &= *value == default_value;
+    }
+
+    void U32WithEnc(const U32Enc /*unused*/, const uint32_t default_value,
+                    const uint32_t* JXL_RESTRICT value) {
+      if (PrintAllDefault()) {
+        Trace("  U32 = %u, default %u\n", *value, default_value);
+      }
+
+      all_default_ &= *value == default_value;
+    }
+
+    void U64(const uint64_t default_value, const uint64_t* JXL_RESTRICT value) {
+      if (PrintAllDefault()) {
+        Trace("  U64 = %" PRIu64 ", default %" PRIu64 "\n", *value,
+              default_value);
+      }
+
+      all_default_ &= *value == default_value;
+    }
+
+    void F16(const float default_value, const float* JXL_RESTRICT value) {
+      if (PrintAllDefault()) {
+        Trace("  F16 = %.6f, default %.6f\n", *value, default_value);
+      }
+      all_default_ &= std::abs(*value - default_value) < 1E-6f;
+    }
+
+    template <class Fields>
+    Status AllDefault(const Fields& fields, bool* JXL_RESTRICT all_default) {
+      // Visit all fields so we can compute the actual all_default_ value.
+      return false;
+    }
+
+    bool AllDefault() const { return all_default_; }
+
+   private:
+    bool all_default_ = true;
+  };
+
+  class ReadVisitor : public VisitorBase<ReadVisitor> {
+   public:
+    explicit ReadVisitor(BitReader* reader)
+        : VisitorBase(PrintRead()), reader_(reader) {}
+
+    void Bits(const size_t bits, const uint32_t default_value,
+              uint32_t* JXL_RESTRICT value) {
+      *value = BitsCoder::Read(bits, reader_);
+      if (PrintRead()) Trace("  u(%zu) = %u\n", bits, *value);
+    }
+
+    void U32WithEnc(const U32Enc dist, const uint32_t default_value,
+                    uint32_t* JXL_RESTRICT value) {
+      *value = U32Coder::Read(dist, reader_);
+      if (PrintRead()) Trace("  U32 = %u\n", *value);
+    }
+
+    void U64(const uint64_t default_value, uint64_t* JXL_RESTRICT value) {
+      *value = U64Coder::Read(reader_);
+      if (PrintRead()) Trace("  U64 = %" PRIu64 "\n", *value);
+    }
+
+    void F16(const float default_value, float* JXL_RESTRICT value) {
+      ok_ &= F16Coder::Read(reader_, value);
+      if (PrintRead()) Trace("  F16 = %f\n", *value);
+    }
+
+    Status IsReading() const { return true; }
+
+    void BeginExtensions(uint64_t* JXL_RESTRICT extensions) {
+      VisitorBase<ReadVisitor>::BeginExtensions(extensions);
+      if (*extensions != 0) {
+        // Read the additional U64 indicating the number of extension bits
+        // (more compact than sending the total size).
+        extension_bits_ = U64Coder::Read(reader_);  // >= 0
+        // Used by EndExtensions to skip past any _remaining_ extensions.
+        pos_after_ext_size_ = reader_->TotalBitsConsumed();
+        JXL_ASSERT(pos_after_ext_size_ != 0);
+      }
+    }
+
+    Status EndExtensions() {
+      JXL_RETURN_IF_ERROR(VisitorBase<ReadVisitor>::EndExtensions());
+      // Happens if extensions == 0: don't read size, done.
+      if (pos_after_ext_size_ == 0) return true;
+
+      // Skip new fields this (old?) decoder didn't know about, if any.
+      const size_t bits_read = reader_->TotalBitsConsumed();
+      const uint64_t end = pos_after_ext_size_ + extension_bits_;
+      if (bits_read > end) {
+        return JXL_FAILURE("Read more extension bits than budgeted");
+      }
+      const size_t remaining_bits = end - bits_read;
+      if (remaining_bits != 0) {
+        JXL_WARNING("Skipping %zu-bit extension(s)", remaining_bits);
+        reader_->SkipBits(remaining_bits);
+      }
+      return true;
+    }
+
+    Status OK() const { return ok_; }
+
+   private:
+    bool ok_ = true;
+    BitReader* const reader_;
+    uint64_t extension_bits_ = 0;    // May be 0 even if extensions present.
+    size_t pos_after_ext_size_ = 0;  // 0 iff extensions == 0.
+  };
+
+  class MaxBitsVisitor : public VisitorBase<MaxBitsVisitor> {
+   public:
+    void Bits(const size_t bits, const uint32_t default_value,
+              const uint32_t* JXL_RESTRICT value) {
+      max_bits_ += BitsCoder::MaxEncodedBits(bits);
+    }
+
+    void U32WithEnc(const U32Enc enc, const uint32_t default_value,
+                    const uint32_t* JXL_RESTRICT value) {
+      max_bits_ += U32Coder::MaxEncodedBits(enc);
+    }
+
+    void U64(const uint64_t default_value, const uint64_t* JXL_RESTRICT value) {
+      max_bits_ += U64Coder::MaxEncodedBits();
+    }
+
+    void F16(const float default_value, const float* JXL_RESTRICT value) {
+      max_bits_ += F16Coder::MaxEncodedBits();
+    }
+
+    template <class Fields>
+    Status AllDefault(const Fields& fields, bool* JXL_RESTRICT all_default) {
+      Bool(true, all_default);
+      return false;
+    }
+
+    // Always visit conditional fields to get a (loose) upper bound.
+    Status Conditional(bool condition) { return true; }
+
+    void BeginExtensions(uint64_t* JXL_RESTRICT extensions) {
+      // Skip - extensions are not included in "MaxBits" because their length
+      // is potentially unbounded.
+    }
+
+    Status EndExtensions() { return true; }
+
+    size_t MaxBits() const { return max_bits_; }
+
+   private:
+    size_t max_bits_ = 0;
+  };
+
+  class CanEncodeVisitor : public VisitorBase<CanEncodeVisitor> {
+   public:
+    CanEncodeVisitor() : VisitorBase<CanEncodeVisitor>(PrintSizes()) {}
+
+    void Bits(const size_t bits, const uint32_t default_value,
+              const uint32_t* JXL_RESTRICT value) {
+      size_t encoded_bits = 0;
+      ok_ &= BitsCoder::CanEncode(bits, *value, &encoded_bits);
+      if (PrintSizes()) Trace("u(%zu) = %u\n", bits, *value);
+      encoded_bits_ += encoded_bits;
+    }
+
+    void U32WithEnc(const U32Enc enc, const uint32_t default_value,
+                    const uint32_t* JXL_RESTRICT value) {
+      size_t encoded_bits = 0;
+      ok_ &= U32Coder::CanEncode(enc, *value, &encoded_bits);
+      if (PrintSizes()) Trace("U32(%zu) = %u\n", encoded_bits, *value);
+      encoded_bits_ += encoded_bits;
+    }
+
+    void U64(const uint64_t default_value, const uint64_t* JXL_RESTRICT value) {
+      size_t encoded_bits = 0;
+      ok_ &= U64Coder::CanEncode(*value, &encoded_bits);
+      if (PrintSizes()) {
+        Trace("U64(%zu) = %" PRIu64 "\n", encoded_bits, *value);
+      }
+      encoded_bits_ += encoded_bits;
+    }
+
+    void F16(const float default_value, const float* JXL_RESTRICT value) {
+      size_t encoded_bits = 0;
+      ok_ &= F16Coder::CanEncode(*value, &encoded_bits);
+      if (PrintSizes()) {
+        Trace("F16(%zu) = %.6f\n", encoded_bits, *value);
+      }
+      encoded_bits_ += encoded_bits;
+    }
+
+    template <class Fields>
+    Status AllDefault(const Fields& fields, bool* JXL_RESTRICT all_default) {
+      *all_default = Bundle::AllDefault(fields);
+      Bool(true, all_default);
+      return *all_default;
+    }
+
+    void BeginExtensions(uint64_t* JXL_RESTRICT extensions) {
+      VisitorBase<CanEncodeVisitor>::BeginExtensions(extensions);
+      if (*extensions != 0) {
+        JXL_ASSERT(pos_after_ext_ == 0);
+        pos_after_ext_ = encoded_bits_;
+        JXL_ASSERT(pos_after_ext_ != 0);  // visited "extensions"
+      }
+    }
+    // EndExtensions = default.
+
+    Status GetSizes(size_t* JXL_RESTRICT extension_bits,
+                    size_t* JXL_RESTRICT total_bits) {
+      JXL_RETURN_IF_ERROR(ok_);
+      *extension_bits = 0;
+      *total_bits = encoded_bits_;
+      // Only if extension field was nonzero will we encode the size.
+      if (pos_after_ext_ != 0) {
+        JXL_ASSERT(encoded_bits_ >= pos_after_ext_);
+        *extension_bits = encoded_bits_ - pos_after_ext_;
+        // Also need to encode *extension_bits and bill it to *total_bits.
+        size_t encoded_bits = 0;
+        ok_ &= U64Coder::CanEncode(*extension_bits, &encoded_bits);
+        *total_bits += encoded_bits;
+      }
+      return true;
+    }
+
+   private:
+    bool ok_ = true;
+    size_t encoded_bits_ = 0;
+    // Snapshot of encoded_bits_ after visiting the extension field, but NOT
+    // including the hidden "extension_bits" u64.
+    uint64_t pos_after_ext_ = 0;
+  };
+
+  class WriteVisitor : public VisitorBase<WriteVisitor> {
+   public:
+    WriteVisitor(const size_t extension_bits, BitWriter* JXL_RESTRICT writer)
+        : extension_bits_(extension_bits), writer_(writer) {}
+
+    void Bits(const size_t bits, const uint32_t default_value,
+              const uint32_t* JXL_RESTRICT value) {
+      ok_ &= BitsCoder::Write(bits, *value, writer_);
+    }
+    void U32WithEnc(const U32Enc enc, const uint32_t default_value,
+                    const uint32_t* JXL_RESTRICT value) {
+      ok_ &= U32Coder::Write(enc, *value, writer_);
+    }
+
+    void U64(const uint64_t default_value, const uint64_t* JXL_RESTRICT value) {
+      ok_ &= U64Coder::Write(*value, writer_);
+    }
+
+    void F16(const float default_value, const float* JXL_RESTRICT value) {
+      ok_ &= F16Coder::Write(*value, writer_);
+    }
+
+    void BeginExtensions(uint64_t* JXL_RESTRICT extensions) {
+      VisitorBase<WriteVisitor>::BeginExtensions(extensions);
+      if (*extensions == 0) {
+        JXL_ASSERT(extension_bits_ == 0);
+      } else {
+        // NOTE: extension_bits_ can be zero if the extensions do not require
+        // any additional fields.
+        ok_ &= U64Coder::Write(extension_bits_, writer_);
+      }
+    }
+    // EndExtensions = default.
+
+    Status OK() const { return ok_; }
+
+   private:
+    const size_t extension_bits_;
+    BitWriter* JXL_RESTRICT writer_;
+    bool ok_ = true;
+  };
+};
+
+}  // namespace jxl
+
+#endif  // JXL_FIELDS_H_
diff --git a/jxl/fields_test.cc b/jxl/fields_test.cc
new file mode 100644
index 000000000..b3af964db
--- /dev/null
+++ b/jxl/fields_test.cc
@@ -0,0 +1,531 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/fields.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <array>
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/span.h"
+#include "jxl/common.h"
+#include "jxl/frame_header.h"
+#include "jxl/headers.h"
+
+namespace jxl {
+namespace {
+
+// Ensures `value` round-trips and in exactly `expected_bits_written`.
+void TestU32Coder(const uint32_t value, const size_t expected_bits_written) {
+  U32Coder coder;
+  const U32Enc enc(Val(0), Bits(4), Val(0x7FFFFFFF), Bits(32));
+
+  BitWriter writer;
+  BitWriter::Allotment allotment(
+      &writer, RoundUpBitsToByteMultiple(U32Coder::MaxEncodedBits(enc)));
+
+  size_t precheck_pos;
+  EXPECT_TRUE(coder.CanEncode(enc, value, &precheck_pos));
+  EXPECT_EQ(expected_bits_written, precheck_pos);
+
+  EXPECT_TRUE(coder.Write(enc, value, &writer));
+  EXPECT_EQ(expected_bits_written, writer.BitsWritten());
+  writer.ZeroPadToByte();
+  ReclaimAndCharge(&writer, &allotment, 0, nullptr);
+
+  BitReader reader(writer.GetSpan());
+  const uint32_t decoded_value = coder.Read(enc, &reader);
+  EXPECT_EQ(value, decoded_value);
+  EXPECT_TRUE(reader.Close());
+}
+
+TEST(FieldsTest, U32CoderTest) {
+  TestU32Coder(0, 2);
+  TestU32Coder(1, 6);
+  TestU32Coder(15, 6);
+  TestU32Coder(0x7FFFFFFF, 2);
+  TestU32Coder(128, 34);
+  TestU32Coder(0x7FFFFFFEu, 34);
+  TestU32Coder(0x80000000u, 34);
+  TestU32Coder(0xFFFFFFFFu, 34);
+}
+
+void TestU64Coder(const uint64_t value, const size_t expected_bits_written) {
+  U64Coder coder;
+
+  BitWriter writer;
+  BitWriter::Allotment allotment(
+      &writer, RoundUpBitsToByteMultiple(U64Coder::MaxEncodedBits()));
+
+  size_t precheck_pos;
+  EXPECT_TRUE(coder.CanEncode(value, &precheck_pos));
+  EXPECT_EQ(expected_bits_written, precheck_pos);
+
+  EXPECT_TRUE(coder.Write(value, &writer));
+  EXPECT_EQ(expected_bits_written, writer.BitsWritten());
+
+  writer.ZeroPadToByte();
+  ReclaimAndCharge(&writer, &allotment, 0, nullptr);
+
+  BitReader reader(writer.GetSpan());
+  const uint64_t decoded_value = coder.Read(&reader);
+  EXPECT_EQ(value, decoded_value);
+  EXPECT_TRUE(reader.Close());
+}
+
+TEST(FieldsTest, U64CoderTest) {
+  // Values that should take 2 bits (selector 00): 0
+  TestU64Coder(0, 2);
+
+  // Values that should take 6 bits (2 for selector, 4 for value): 1..16
+  TestU64Coder(1, 6);
+  TestU64Coder(2, 6);
+  TestU64Coder(8, 6);
+  TestU64Coder(15, 6);
+  TestU64Coder(16, 6);
+
+  // Values that should take 10 bits (2 for selector, 8 for value): 17..272
+  TestU64Coder(17, 10);
+  TestU64Coder(18, 10);
+  TestU64Coder(100, 10);
+  TestU64Coder(271, 10);
+  TestU64Coder(272, 10);
+
+  // Values that should take 15 bits (2 for selector, 12 for value, 1 for varint
+  // end): (0)..273..4095
+  TestU64Coder(273, 15);
+  TestU64Coder(274, 15);
+  TestU64Coder(1000, 15);
+  TestU64Coder(4094, 15);
+  TestU64Coder(4095, 15);
+
+  // Take 24 bits (of which 20 actual value): (0)..4096..1048575
+  TestU64Coder(4096, 24);
+  TestU64Coder(4097, 24);
+  TestU64Coder(10000, 24);
+  TestU64Coder(1048574, 24);
+  TestU64Coder(1048575, 24);
+
+  // Take 33 bits (of which 28 actual value): (0)..1048576..268435455
+  TestU64Coder(1048576, 33);
+  TestU64Coder(1048577, 33);
+  TestU64Coder(10000000, 33);
+  TestU64Coder(268435454, 33);
+  TestU64Coder(268435455, 33);
+
+  // Take 42 bits (of which 36 actual value): (0)..268435456..68719476735
+  TestU64Coder(268435456ull, 42);
+  TestU64Coder(268435457ull, 42);
+  TestU64Coder(1000000000ull, 42);
+  TestU64Coder(68719476734ull, 42);
+  TestU64Coder(68719476735ull, 42);
+
+  // Take 51 bits (of which 44 actual value): (0)..68719476736..17592186044415
+  TestU64Coder(68719476736ull, 51);
+  TestU64Coder(68719476737ull, 51);
+  TestU64Coder(1000000000000ull, 51);
+  TestU64Coder(17592186044414ull, 51);
+  TestU64Coder(17592186044415ull, 51);
+
+  // Take 60 bits (of which 52 actual value):
+  // (0)..17592186044416..4503599627370495
+  TestU64Coder(17592186044416ull, 60);
+  TestU64Coder(17592186044417ull, 60);
+  TestU64Coder(100000000000000ull, 60);
+  TestU64Coder(4503599627370494ull, 60);
+  TestU64Coder(4503599627370495ull, 60);
+
+  // Take 69 bits (of which 60 actual value):
+  // (0)..4503599627370496..1152921504606846975
+  TestU64Coder(4503599627370496ull, 69);
+  TestU64Coder(4503599627370497ull, 69);
+  TestU64Coder(10000000000000000ull, 69);
+  TestU64Coder(1152921504606846974ull, 69);
+  TestU64Coder(1152921504606846975ull, 69);
+
+  // Take 73 bits (of which 64 actual value):
+  // (0)..1152921504606846976..18446744073709551615
+  TestU64Coder(1152921504606846976ull, 73);
+  TestU64Coder(1152921504606846977ull, 73);
+  TestU64Coder(10000000000000000000ull, 73);
+  TestU64Coder(18446744073709551614ull, 73);
+  TestU64Coder(18446744073709551615ull, 73);
+}
+
+struct S32Bundle {
+  S32Bundle() { Bundle::Init(this); }
+  static const char* Name() { return "S32Bundle"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->S32(Val(U32FromS32(-1)), Val(U32FromS32(-64)), Bits(8), Bits(10),
+                 -1, &s);
+    return true;
+  }
+  int32_t s;
+};
+
+struct S64Bundle {
+  S64Bundle() { Bundle::Init(this); }
+  static const char* Name() { return "S64Bundle"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->S64(-1, &s);
+    return true;
+  }
+  int64_t s;
+};
+
+template <class BundleT, class T>
+void TestSigned(const T value, size_t expected_bits_written) {
+  BundleT bundle;
+  bundle.s = value;
+  size_t extension_bits = 999, total_bits = 999;  // Initialize as garbage.
+  EXPECT_TRUE(Bundle::CanEncode(bundle, &extension_bits, &total_bits));
+  EXPECT_EQ(0, extension_bits);
+  EXPECT_EQ(expected_bits_written, total_bits);
+
+  BitWriter writer;
+
+  AuxOut aux_out;
+  ASSERT_TRUE(Bundle::Write(bundle, &writer, 0, &aux_out));
+  EXPECT_EQ(expected_bits_written, aux_out.layers[0].total_bits);
+
+  BitWriter::Allotment allotment(&writer, 8);
+  writer.ZeroPadToByte();
+  ReclaimAndCharge(&writer, &allotment, kLayerHeader, nullptr);
+
+  BitReader reader(writer.GetSpan());
+  BundleT bundle2;
+  EXPECT_TRUE(Bundle::Read(&reader, &bundle2));
+  EXPECT_EQ(value, bundle2.s);
+  EXPECT_TRUE(reader.Close());
+}
+
+TEST(FieldsTest, S32Test) {
+  TestSigned<S32Bundle>(0, 10);
+  TestSigned<S32Bundle>(-1, 2);
+  TestSigned<S32Bundle>(-64, 2);
+  TestSigned<S32Bundle>(1, 10);
+  TestSigned<S32Bundle>(127, 10);
+  TestSigned<S32Bundle>(-128, 10);
+  TestSigned<S32Bundle>(128, 12);
+  TestSigned<S32Bundle>(-129, 12);
+  TestSigned<S32Bundle>(511, 12);
+  TestSigned<S32Bundle>(-512, 12);
+}
+
+TEST(FieldsTest, S64Test) {
+  TestSigned<S64Bundle>(0, 2);
+  TestSigned<S64Bundle>(-1, 6);
+  TestSigned<S64Bundle>(1, 6);
+  TestSigned<S64Bundle>(-8, 6);
+  TestSigned<S64Bundle>(8, 6);
+  TestSigned<S64Bundle>(-9, 10);
+  TestSigned<S64Bundle>(9, 10);
+  TestSigned<S64Bundle>(-136, 10);
+  TestSigned<S64Bundle>(136, 10);
+  TestSigned<S64Bundle>(-137, 15);
+  TestSigned<S64Bundle>(137, 15);
+  TestSigned<S64Bundle>(2047, 15);
+  TestSigned<S64Bundle>(-2048, 15);
+  TestSigned<S64Bundle>(2048, 24);
+  TestSigned<S64Bundle>(-2049, 24);
+  TestSigned<S64Bundle>(524287, 24);
+  TestSigned<S64Bundle>(-524288, 24);
+  TestSigned<S64Bundle>(524288, 33);
+  TestSigned<S64Bundle>(-524289, 33);
+  TestSigned<S64Bundle>(-134217727, 33);
+  TestSigned<S64Bundle>(134217727, 33);
+  TestSigned<S64Bundle>(-134217728, 33);
+  TestSigned<S64Bundle>(134217728, 42);
+  TestSigned<S64Bundle>(-134217729, 42);
+  TestSigned<S64Bundle>(2147483647, 42);
+  TestSigned<S64Bundle>(-2147483648, 42);
+}
+
+Status TestF16Coder(const float value) {
+  F16Coder coder;
+
+  size_t max_encoded_bits;
+  JXL_RETURN_IF_ERROR(coder.CanEncode(value, &max_encoded_bits));
+  EXPECT_EQ(F16Coder::MaxEncodedBits(), max_encoded_bits);
+
+  BitWriter writer;
+  BitWriter::Allotment allotment(&writer,
+                                 RoundUpBitsToByteMultiple(max_encoded_bits));
+
+  EXPECT_TRUE(coder.Write(value, &writer));
+  EXPECT_EQ(F16Coder::MaxEncodedBits(), writer.BitsWritten());
+  writer.ZeroPadToByte();
+  ReclaimAndCharge(&writer, &allotment, 0, nullptr);
+
+  BitReader reader(writer.GetSpan());
+  float decoded_value;
+  EXPECT_TRUE(coder.Read(&reader, &decoded_value));
+  // All values we test can be represented exactly.
+  EXPECT_EQ(value, decoded_value);
+  EXPECT_TRUE(reader.Close());
+  return true;
+}
+
+TEST(FieldsTest, F16CoderTest) {
+  for (float sign : {-1.0f, 1.0f}) {
+    // (anything less than 1E-3 are subnormals)
+    for (float mag : {0.0f, 0.5f, 1.0f, 2.0f, 2.5f, 16.015625f, 1.0f / 4096,
+                      1.0f / 16384, 65504.0f}) {
+      EXPECT_TRUE(TestF16Coder(sign * mag));
+    }
+  }
+
+  // Out of range
+  EXPECT_FALSE(TestF16Coder(65504.01f));
+  EXPECT_FALSE(TestF16Coder(-65505.0f));
+}
+
+// Ensures Read(Write()) returns the same fields.
+TEST(FieldsTest, TestRoundtripSize) {
+  for (int i = 0; i < 8; i++) {
+    SizeHeader size;
+    ASSERT_TRUE(size.Set(123 + 77 * i, 7 + i));
+
+    size_t extension_bits = 999, total_bits = 999;  // Initialize as garbage.
+    ASSERT_TRUE(Bundle::CanEncode(size, &extension_bits, &total_bits));
+    EXPECT_EQ(0, extension_bits);
+
+    BitWriter writer;
+    ASSERT_TRUE(WriteSizeHeader(size, &writer, 0, nullptr));
+    EXPECT_EQ(total_bits, writer.BitsWritten());
+    writer.ZeroPadToByte();
+
+    SizeHeader size2;
+    BitReader reader(writer.GetSpan());
+    ASSERT_TRUE(ReadSizeHeader(&reader, &size2));
+    EXPECT_EQ(total_bits, reader.TotalBitsConsumed());
+    EXPECT_TRUE(reader.Close());
+
+    EXPECT_EQ(size.xsize(), size2.xsize());
+    EXPECT_EQ(size.ysize(), size2.ysize());
+  }
+}
+
+// Ensure all values can be reached by the encoding.
+TEST(FieldsTest, TestCropRect) {
+  for (uint32_t i = 0; i < 19000; ++i) {
+    AnimationFrame f;
+    f.have_crop = true;
+    f.x0 = i;
+    f.y0 = i;
+    f.xsize = i;
+    f.ysize = i;
+    size_t extension_bits = 0, total_bits = 0;
+    ASSERT_TRUE(Bundle::CanEncode(f, &extension_bits, &total_bits));
+    EXPECT_EQ(0, extension_bits);
+    EXPECT_GE(total_bits, 9);
+  }
+}
+TEST(FieldsTest, TestPreview) {
+  // (div8 cannot represent 4360, but !div8 can go a little higher)
+  for (uint32_t i = 1; i < 4360; ++i) {
+    PreviewHeader p;
+    ASSERT_TRUE(p.Set(i, i));
+    size_t extension_bits = 0, total_bits = 0;
+    ASSERT_TRUE(Bundle::CanEncode(p, &extension_bits, &total_bits));
+    EXPECT_EQ(0, extension_bits);
+    EXPECT_GE(total_bits, 6);
+  }
+}
+
+// Ensures Read(Write()) returns the same fields.
+TEST(FieldsTest, TestRoundtripFrame) {
+  FrameHeader h;
+  h.animation_frame.nonserialized_have_timecode = false;
+  h.extensions = 0x800;
+
+  size_t extension_bits = 999, total_bits = 999;  // Initialize as garbage.
+  ASSERT_TRUE(Bundle::CanEncode(h, &extension_bits, &total_bits));
+  EXPECT_EQ(0, extension_bits);
+  BitWriter writer;
+  ASSERT_TRUE(WriteFrameHeader(h, &writer, nullptr));
+  EXPECT_EQ(total_bits, writer.BitsWritten());
+  writer.ZeroPadToByte();
+
+  FrameHeader h2;
+  h2.animation_frame.nonserialized_have_timecode = false;
+  BitReader reader(writer.GetSpan());
+  ASSERT_TRUE(ReadFrameHeader(&reader, &h2));
+  EXPECT_EQ(total_bits, reader.TotalBitsConsumed());
+  EXPECT_TRUE(reader.Close());
+
+  EXPECT_EQ(h.extensions, h2.extensions);
+  EXPECT_EQ(h.flags, h2.flags);
+}
+
+#ifndef JXL_CRASH_ON_ERROR
+// Ensure out-of-bounds values cause an error.
+TEST(FieldsTest, TestOutOfRange) {
+  SizeHeader h;
+  ASSERT_TRUE(h.Set(0xFFFFFFFFull, 0xFFFFFFFFull));
+  size_t extension_bits = 999, total_bits = 999;  // Initialize as garbage.
+  ASSERT_FALSE(Bundle::CanEncode(h, &extension_bits, &total_bits));
+}
+#endif
+
+struct OldBundle {
+  OldBundle() { Bundle::Init(this); }
+  static const char* Name() { return "OldBundle"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->U32(Val(1), Bits(2), Bits(3), Bits(4), 1, &old_small);
+    visitor->F16(1.125f, &old_f);
+    visitor->U32(Bits(7), Bits(12), Bits(16), Bits(32), 0, &old_large);
+
+    visitor->BeginExtensions(&extensions);
+    return visitor->EndExtensions();
+  }
+
+  uint32_t old_small;
+  float old_f;
+  uint32_t old_large;
+  uint64_t extensions;
+};
+
+struct NewBundle {
+  NewBundle() { Bundle::Init(this); }
+  static const char* Name() { return "NewBundle"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->U32(Val(1), Bits(2), Bits(3), Bits(4), 1, &old_small);
+    visitor->F16(1.125f, &old_f);
+    visitor->U32(Bits(7), Bits(12), Bits(16), Bits(32), 0, &old_large);
+
+    visitor->BeginExtensions(&extensions);
+    if (extensions & 1) {
+      visitor->U32(Val(2), Bits(2), Bits(3), Bits(4), 2, &new_small);
+      visitor->F16(-2.0f, &new_f);
+      visitor->U32(Bits(9), Bits(12), Bits(16), Bits(32), 0, &new_large);
+    }
+    return visitor->EndExtensions();
+  }
+
+  uint32_t old_small;
+  float old_f;
+  uint32_t old_large;
+  uint64_t extensions;
+
+  // If extensions & 1
+  uint32_t new_small = 2;
+  float new_f = -2.0f;
+  uint32_t new_large = 0;
+};
+
+HWY_ATTR void FieldsTest_TestNewDecoderOldData() {
+  OldBundle old_bundle;
+  old_bundle.old_large = 123;
+  old_bundle.old_f = 3.75f;
+  old_bundle.extensions = 0;
+
+  // Write to bit stream
+  const size_t kMaxOutBytes = 999;
+  BitWriter writer;
+  // Make sure values are initialized by code under test.
+  size_t extension_bits = 12345, total_bits = 12345;
+  ASSERT_TRUE(Bundle::CanEncode(old_bundle, &extension_bits, &total_bits));
+  ASSERT_LE(total_bits, kMaxOutBytes * kBitsPerByte);
+  EXPECT_EQ(0, extension_bits);
+  AuxOut aux_out;
+  ASSERT_TRUE(Bundle::Write(old_bundle, &writer, kLayerHeader, &aux_out));
+
+  BitWriter::Allotment allotment(&writer,
+                                 kMaxOutBytes * kBitsPerByte - total_bits);
+  writer.Write(20, 0xA55A);  // sentinel
+  writer.ZeroPadToByte();
+  ReclaimAndCharge(&writer, &allotment, kLayerHeader, nullptr);
+
+  ASSERT_LE(writer.GetSpan().size(), kMaxOutBytes);
+  BitReader reader(writer.GetSpan());
+  NewBundle new_bundle;
+  ASSERT_TRUE(Bundle::Read(&reader, &new_bundle));
+  EXPECT_EQ(reader.TotalBitsConsumed(),
+            aux_out.layers[kLayerHeader].total_bits);
+  EXPECT_EQ(reader.ReadBits(20), 0xA55A);
+  EXPECT_TRUE(reader.Close());
+
+  // Old fields are the same in both
+  EXPECT_EQ(old_bundle.extensions, new_bundle.extensions);
+  EXPECT_EQ(old_bundle.old_small, new_bundle.old_small);
+  EXPECT_EQ(old_bundle.old_f, new_bundle.old_f);
+  EXPECT_EQ(old_bundle.old_large, new_bundle.old_large);
+  // New fields match their defaults
+  EXPECT_EQ(2, new_bundle.new_small);
+  EXPECT_EQ(-2.0f, new_bundle.new_f);
+  EXPECT_EQ(0, new_bundle.new_large);
+}
+TEST(FieldsTest, TestNewDecoderOldData) { FieldsTest_TestNewDecoderOldData(); }
+
+HWY_ATTR void FieldsTest_TestOldDecoderNewData() {
+  NewBundle new_bundle;
+  new_bundle.old_large = 123;
+  new_bundle.extensions = 1;
+  new_bundle.new_f = 999.0f;
+  new_bundle.new_large = 456;
+
+  // Write to bit stream
+  constexpr size_t kMaxOutBytes = 999;
+  BitWriter writer;
+  // Make sure values are initialized by code under test.
+  size_t extension_bits = 12345, total_bits = 12345;
+  ASSERT_TRUE(Bundle::CanEncode(new_bundle, &extension_bits, &total_bits));
+  EXPECT_NE(0, extension_bits);
+  AuxOut aux_out;
+  ASSERT_TRUE(Bundle::Write(new_bundle, &writer, kLayerHeader, &aux_out));
+  ASSERT_LE(aux_out.layers[kLayerHeader].total_bits,
+            kMaxOutBytes * kBitsPerByte);
+
+  BitWriter::Allotment allotment(
+      &writer,
+      kMaxOutBytes * kBitsPerByte - aux_out.layers[kLayerHeader].total_bits);
+  // Ensure Read skips the additional fields
+  writer.Write(20, 0xA55A);  // sentinel
+  writer.ZeroPadToByte();
+  ReclaimAndCharge(&writer, &allotment, kLayerHeader, nullptr);
+
+  BitReader reader(writer.GetSpan());
+  OldBundle old_bundle;
+  ASSERT_TRUE(Bundle::Read(&reader, &old_bundle));
+  EXPECT_EQ(reader.TotalBitsConsumed(),
+            aux_out.layers[kLayerHeader].total_bits);
+  EXPECT_EQ(reader.ReadBits(20), 0xA55A);
+  EXPECT_TRUE(reader.Close());
+
+  // Old fields are the same in both
+  EXPECT_EQ(new_bundle.extensions, old_bundle.extensions);
+  EXPECT_EQ(new_bundle.old_small, old_bundle.old_small);
+  EXPECT_EQ(new_bundle.old_f, old_bundle.old_f);
+  EXPECT_EQ(new_bundle.old_large, old_bundle.old_large);
+  // (Can't check new fields because old decoder doesn't know about them)
+}
+
+TEST(FieldsTest, TestOldDecoderNewData) { FieldsTest_TestOldDecoderNewData(); }
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/frame_header.cc b/jxl/frame_header.cc
new file mode 100644
index 000000000..48a62905f
--- /dev/null
+++ b/jxl/frame_header.cc
@@ -0,0 +1,36 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/frame_header.h"
+
+#include "jxl/aux_out.h"
+#include "jxl/fields.h"
+
+namespace jxl {
+
+AnimationFrame::AnimationFrame() { Bundle::Init(this); }
+Passes::Passes() { Bundle::Init(this); }
+FrameHeader::FrameHeader() { Bundle::Init(this); }
+
+Status ReadFrameHeader(BitReader* JXL_RESTRICT reader,
+                       FrameHeader* JXL_RESTRICT frame) {
+  return Bundle::Read(reader, frame);
+}
+
+Status WriteFrameHeader(const FrameHeader& frame,
+                        BitWriter* JXL_RESTRICT writer, AuxOut* aux_out) {
+  return Bundle::Write(frame, writer, kLayerHeader, aux_out);
+}
+
+}  // namespace jxl
diff --git a/jxl/frame_header.h b/jxl/frame_header.h
new file mode 100644
index 000000000..b806b8a10
--- /dev/null
+++ b/jxl/frame_header.h
@@ -0,0 +1,376 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_FRAME_HEADER_H_
+#define JXL_FRAME_HEADER_H_
+
+// Frame header with backward and forward-compatible extension capability and
+// compressed integer fields.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/override.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/common.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/field_encodings.h"
+#include "jxl/gaborish.h"
+
+namespace jxl {
+
+enum class FrameEncoding : uint32_t {
+  kPasses,
+  kModularGroup,
+  kJpegGroup,
+};
+
+enum class ColorTransform : uint32_t { kXYB, kNone, kYCbCr };
+
+enum class YCbCrChromaSubsampling : uint32_t { k444, k420, k422, k411, kAuto };
+
+static inline const char* EnumName(FrameEncoding /*unused*/) {
+  return "FrameEncoding";
+}
+
+static inline constexpr uint64_t EnumBits(FrameEncoding /*unused*/) {
+  return MakeBit(FrameEncoding::kPasses) |
+         MakeBit(FrameEncoding::kModularGroup) |
+         MakeBit(FrameEncoding::kJpegGroup);
+}
+
+static inline const char* EnumName(ColorTransform) { return "ColorTransform"; }
+static inline constexpr uint64_t EnumBits(ColorTransform) {
+  return MakeBit(ColorTransform::kXYB) | MakeBit(ColorTransform::kNone) |
+         MakeBit(ColorTransform::kYCbCr);
+}
+static inline const char* EnumName(YCbCrChromaSubsampling) {
+  return "YCbCrChromaSubsampling";
+}
+static inline constexpr uint64_t EnumBits(YCbCrChromaSubsampling) {
+  return MakeBit(YCbCrChromaSubsampling::k444) |
+         MakeBit(YCbCrChromaSubsampling::k420) |
+         MakeBit(YCbCrChromaSubsampling::k422) |
+         MakeBit(YCbCrChromaSubsampling::k411);
+}
+
+struct AnimationFrame {
+  AnimationFrame();
+  static const char* Name() { return "AnimationFrame"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->U32(Val(0), Val(1), Bits(8), Bits(32), 0, &duration);
+
+    if (visitor->Conditional(duration > 0)) {
+      visitor->U32(Val(static_cast<uint32_t>(NewBase::kExisting)),
+                   Val(static_cast<uint32_t>(NewBase::kCurrentFrame)),
+                   Val(static_cast<uint32_t>(NewBase::kNone)), Val(3),
+                   static_cast<uint32_t>(NewBase::kCurrentFrame),
+                   &new_base_int_);
+    }
+    if (new_base_int_ == 3) {
+      return JXL_FAILURE("AnimationFrame with invalid new_base");
+    }
+    visitor->U32(Val(static_cast<uint32_t>(BlendMode::kReplace)),
+                 Val(static_cast<uint32_t>(BlendMode::kAdd)),
+                 Val(static_cast<uint32_t>(BlendMode::kBlend)), Val(3),
+                 static_cast<uint32_t>(BlendMode::kReplace), &blend_mode_int_);
+    if (blend_mode_int_ == 3) {
+      return JXL_FAILURE("AnimationFrame with invalid blend_mode");
+    }
+
+    if (visitor->Conditional(nonserialized_have_timecode)) {
+      visitor->Bits(32, 0, &timecode);
+    }
+
+    visitor->Bool(false, &have_crop);
+    if (visitor->Conditional(have_crop)) {
+      const U32Enc enc(Bits(8), BitsOffset(11, 256), BitsOffset(14, 2304),
+                       BitsOffset(30, 18688));
+      visitor->U32WithEnc(enc, 0, &x0);
+      visitor->U32WithEnc(enc, 0, &y0);
+      visitor->U32WithEnc(enc, 0, &xsize);
+      visitor->U32WithEnc(enc, 0, &ysize);
+    }
+
+    visitor->Bool(true, &is_last);
+
+    return true;
+  }
+
+  // How long to wait [in ticks, see Animation{}] after rendering.
+  // May be 0 if the current frame serves as a foundation for a frame with crop.
+  uint32_t duration;
+
+  // Indicates what the next frame will be "based" on.
+  // A full frame (have_crop = false) can be based on a frame if and only if the
+  // frame and the base are lossy. The rendered frame will then be the sum of
+  // the two. A cropped frame can be based on any kind of frame. The rendered
+  // frame will be obtained by blitting.
+  enum class NewBase {
+    // The next frame will be based on the same frame as the current one.
+    kExisting,
+    // The next frame will be based on the current one.
+    kCurrentFrame,
+    // The next frame will be a full frame (have_crop = false) and will not be
+    // based on any frame.
+    kNone,
+  };
+  NewBase new_base() const { return static_cast<NewBase>(new_base_int_); }
+  void SetNewBase(const NewBase new_base) {
+    JXL_DASSERT(new_base == NewBase::kExisting ||
+                new_base == NewBase::kCurrentFrame ||
+                new_base == NewBase::kNone);
+    new_base_int_ = static_cast<uint32_t>(new_base);
+  }
+
+  // update tools/djxl.cc if blend modes change
+  enum class BlendMode {
+    // The new values (in the crop) replace the old ones
+    kReplace,
+    // The new values (in the crop) get added to the old ones
+    kAdd,
+    // The new values (in the crop) replace the old ones if alpha>0
+    kBlend,
+  };
+  BlendMode blend_mode() const {
+    return static_cast<BlendMode>(blend_mode_int_);
+  }
+  void SetBlendMode(const BlendMode blend_mode) {
+    JXL_DASSERT(blend_mode == BlendMode::kReplace ||
+                blend_mode == BlendMode::kAdd ||
+                blend_mode == BlendMode::kBlend);
+    blend_mode_int_ = static_cast<uint32_t>(blend_mode);
+  }
+
+ private:
+  uint32_t new_base_int_;
+  uint32_t blend_mode_int_;
+
+ public:
+  bool nonserialized_have_timecode = false;
+  uint32_t timecode;  // 0xHHMMSSFF
+
+  bool have_crop;  // If false, origin/size are zero == "full frame"
+  uint32_t x0;
+  uint32_t y0;
+  uint32_t xsize;
+  uint32_t ysize;
+
+  bool is_last;
+};
+
+// For decoding to lower resolutions. Cannot mix with animation.
+struct Passes {
+  Passes();
+  static const char* Name() { return "Passes"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->U32(Val(1), Val(2), Val(3), BitsOffset(3, 4), 1, &num_passes);
+    JXL_ASSERT(num_passes <= kMaxNumPasses);  // Cannot happen when reading
+
+    if (visitor->Conditional(num_passes != 1)) {
+      visitor->U32(Val(0), Val(1), Val(2), BitsOffset(1, 3), 0,
+                   &num_downsample);
+      JXL_ASSERT(num_downsample <= 4);  // 1,2,4,8
+      if (num_downsample > num_passes) {
+        return JXL_FAILURE("num_downsample %u > num_passes %u", num_downsample,
+                           num_passes);
+      }
+
+      for (uint32_t i = 0; i < num_passes - 1; i++) {
+        visitor->Bits(2, 0, &shift[i]);
+      }
+      shift[num_passes - 1] = 0;
+
+      for (uint32_t i = 0; i < num_downsample; ++i) {
+        visitor->U32(Val(1), Val(2), Val(4), Val(8), 1, &downsample[i]);
+      }
+      for (uint32_t i = 0; i < num_downsample; ++i) {
+        visitor->U32(Val(0), Val(1), Val(2), Bits(3), 0, &last_pass[i]);
+        if (last_pass[i] >= num_passes) {
+          return JXL_FAILURE("last_pass %u >= num_passes %u", last_pass[i],
+                             num_passes);
+        }
+      }
+    }
+
+    return true;
+  }
+
+  uint32_t num_passes;      // <= kMaxNumPasses
+  uint32_t num_downsample;  // <= num_passes
+
+  // Array of num_downsample pairs. downsample=1/last_pass=num_passes-1 and
+  // downsample=8/last_pass=0 need not be specified; they are implicit.
+  uint32_t downsample[kMaxNumPasses];
+  uint32_t last_pass[kMaxNumPasses];
+  // Array of shift values for each pass. It is implicitly assumed to be 0 for
+  // the last pass.
+  uint32_t shift[kMaxNumPasses];
+};
+
+// Image/frame := one of more of these, where the last has is_last = true.
+// Starts at a byte-aligned address "a"; the next pass starts at "a + size".
+struct FrameHeader {
+  // Optional postprocessing steps. These flags are the source of truth;
+  // Override must set/clear them rather than change their meaning. Values
+  // chosen such that typical flags == 0 (encoded in only two bits).
+  enum Flags {
+    // Often but not always off => low bit value:
+
+    // Inject noise into decoded output.
+    kNoise = 1,
+
+    // Overlay patches.
+    kPatches = 2,
+
+    // 4, 8 = reserved for future sometimes-off
+
+    // Overlay splines.
+    kSplines = 16,
+
+    kUseDcFrame = 32,  // Implies kSkipAdaptiveDCSmoothing.
+
+    // 64 = reserved for future often-off
+
+    // Almost always on => negated:
+
+    kSkipAdaptiveDCSmoothing = 128,
+  };
+
+  FrameHeader();
+  static const char* Name() { return "FrameHeader"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    if (visitor->AllDefault(*this, &all_default)) return true;
+
+    // Up to 3 pyramid levels - for up to 4096x downsampling.
+    visitor->U32(Val(0), Val(1), Val(2), Val(3), 0, &dc_level);
+
+    if (visitor->Conditional(dc_level == 0)) {
+      visitor->Bool(false, &has_animation);
+      if (visitor->Conditional(has_animation))
+        JXL_RETURN_IF_ERROR(visitor->VisitNested(&animation_frame));
+    }
+
+    JXL_RETURN_IF_ERROR(visitor->Enum(FrameEncoding::kPasses, &encoding));
+
+    JXL_RETURN_IF_ERROR(visitor->Enum(ColorTransform::kXYB, &color_transform));
+    if (visitor->Conditional(color_transform == ColorTransform::kYCbCr &&
+                             encoding == FrameEncoding::kJpegGroup)) {
+      JXL_RETURN_IF_ERROR(
+          visitor->Enum(YCbCrChromaSubsampling::k444, &chroma_subsampling));
+    }
+    if (visitor->Conditional(IsLossy() ||
+                             encoding == FrameEncoding::kModularGroup)) {
+      JXL_RETURN_IF_ERROR(visitor->VisitNested(&passes));
+      visitor->U64(0, &flags);
+    }
+    // This field only makes sense for kPasses:
+    if (visitor->Conditional(IsLossy())) {
+      visitor->U32(Val(0), Val(1), Val(2), Val(3), 1, &x_qm_scale);
+    }
+
+    // Save frame as a reference frame.
+    if (visitor->Conditional(!animation_frame.is_last && dc_level == 0)) {
+      visitor->U32(Val(0), Val(1), Val(2), Val(3), 0, &save_as_reference);
+    }
+    if (visitor->Conditional(save_as_reference == 0 && dc_level == 0)) {
+      visitor->Bool(false, &is_displayed);
+    }
+
+    if (visitor->Conditional(dc_level == 0)) {
+      visitor->Bool(false, &has_alpha);
+    }
+
+    visitor->BeginExtensions(&extensions);
+    // Extensions: in chronological order of being added to the format.
+    return visitor->EndExtensions();
+  }
+
+  bool IsLossy() const { return encoding == FrameEncoding::kPasses; }
+  bool IsJpeg() const { return encoding == FrameEncoding::kJpegGroup; }
+
+  // Sets/clears `flag` based upon `condition`.
+  void UpdateFlag(const bool condition, const uint64_t flag) {
+    if (condition) {
+      flags |= flag;
+    } else {
+      flags &= ~flag;
+    }
+  }
+
+  // Returns whether this frame is a 'normal' frame that will be displayed on
+  // its own (as opposed to a reference/patch or DC level).
+  bool IsDisplayed() const {
+    return (save_as_reference == 0 || is_displayed) && dc_level == 0;
+  }
+
+  bool HasAlpha() const { return has_alpha; }
+
+  mutable bool all_default;
+
+  uint32_t dc_level;
+
+  // Per-frame animation flag - may differ between reference and displayed
+  // frames. Default false enables all_default for non-animation frames.
+  bool has_animation;
+  AnimationFrame animation_frame;
+
+  FrameEncoding encoding;
+
+  ColorTransform color_transform;
+  YCbCrChromaSubsampling chroma_subsampling;
+
+  Passes passes;  // only if IsLossy()
+
+  uint64_t flags;
+
+  uint32_t x_qm_scale;  // only if IsLossy()
+
+  // 0 = don't save. Otherwise, save as reference frame of ID
+  // `save_as_reference`-1.
+  uint32_t
+      save_as_reference;  // only if !animation_frame.is_last && dc_level == 0.
+  bool is_displayed;      // only if dc_level == 0 && save_as_reference == 0
+
+  // Per-frame alpha flag - may differ between reference and displayed frames.
+  bool has_alpha;  // only if dc_level == 0
+
+  uint64_t extensions;
+};
+
+Status ReadFrameHeader(BitReader* JXL_RESTRICT reader,
+                       FrameHeader* JXL_RESTRICT frame);
+
+Status WriteFrameHeader(const FrameHeader& frame,
+                        BitWriter* JXL_RESTRICT writer, AuxOut* aux_out);
+
+// Shared by enc/dec. 5F and 13 are by far the most common for d1/2/4/8, 0
+// ensures low overhead for small images.
+static constexpr U32Enc kOrderEnc =
+    U32Enc(Val(0x5F), Val(0x13), Val(0), Bits(kNumOrders));
+
+}  // namespace jxl
+
+#endif  // JXL_FRAME_HEADER_H_
diff --git a/jxl/gaborish.cc b/jxl/gaborish.cc
new file mode 100644
index 000000000..5d53d316d
--- /dev/null
+++ b/jxl/gaborish.cc
@@ -0,0 +1,105 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/gaborish.h"
+
+#include <stddef.h>
+
+#include <hwy/static_targets.h>
+
+#include "jxl/base/status.h"
+#include "jxl/convolve.h"
+#include "jxl/image_ops.h"
+
+namespace jxl {
+
+class GaborishKernel {
+ public:
+  // weight1,2 need not be normalized.
+  GaborishKernel(float weight1, float weight2) {
+    constexpr float weight0 = 1.0f;
+
+    // Normalize
+    const float mul = 1.0 / (weight0 + 4 * (weight1 + weight2));
+    const float w0 = weight0 * mul;
+    const float w1 = weight1 * mul;
+    const float w2 = weight2 * mul;
+
+    // Fill all lanes
+    for (size_t i = 0; i < 4; ++i) {
+      // clang-format off
+      weights_.tl[i] = w2; weights_.tc[i] = w1; weights_.tr[i] = w2;
+      weights_.ml[i] = w1; weights_.mc[i] = w0; weights_.mr[i] = w1;
+      weights_.bl[i] = w2; weights_.bc[i] = w1; weights_.br[i] = w2;
+      // clang-format on
+    }
+  }
+
+  JXL_INLINE const Weights3x3& Weights() const { return weights_; }
+
+ private:
+  Weights3x3 weights_;
+};
+
+Image3F GaborishInverse(const Image3F& in, double mul, ThreadPool* pool) {
+  JXL_ASSERT(mul >= 0.0);
+
+  // Only an approximation. One or even two 3x3, and rank-1 (separable) 5x5
+  // are insufficient.
+  static const double kGaborish[5] = {
+      -0.092359145662814029,  -0.039253623634014627, 0.016176494530216929,
+      0.00083458437774987476, 0.004512465323949319,
+  };
+  static const double kCenter = 1.0;
+  /*
+    better would be:
+      1.0 - mul * (4 * (kGaborish[0] + kGaborish[1] +
+                        kGaborish[2] + kGaborish[4]) +
+                   8 * (kGaborish[3]));
+  */
+
+  const float sharpen_weights5[9] = {
+      static_cast<float>(kCenter),
+      static_cast<float>(mul * kGaborish[0]),
+      static_cast<float>(mul * kGaborish[2]),
+
+      static_cast<float>(mul * kGaborish[0]),
+      static_cast<float>(mul * kGaborish[1]),
+      static_cast<float>(mul * kGaborish[3]),
+
+      static_cast<float>(mul * kGaborish[2]),
+      static_cast<float>(mul * kGaborish[3]),
+      static_cast<float>(mul * kGaborish[4]),
+  };
+  Image3F sharpened(in.xsize(), in.ysize());
+  slow::SymmetricConvolution<2, WrapClamp>::Run(in, Rect(in), sharpen_weights5,
+                                                pool, &sharpened);
+  return sharpened;
+}
+
+HWY_ATTR void ConvolveGaborish(const ImageF& in, float weight1, float weight2,
+                               ThreadPool* pool, ImageF* JXL_RESTRICT out) {
+  JXL_CHECK(SameSize(in, *out));
+
+  const GaborishKernel gaborish(weight1, weight2);
+  if (in.xsize() < kConvolveMinWidth) {
+    using Convolution = slow::General3x3Convolution<1, WrapMirror>;
+    Convolution::Run(in, Rect(in), gaborish, out);
+  } else {
+    using Conv3 = ConvolveT<strategy::Symmetric3>;
+    Conv3::Run(in, Rect(in), gaborish, pool, out);
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/gaborish.h b/jxl/gaborish.h
new file mode 100644
index 000000000..c7b7cc999
--- /dev/null
+++ b/jxl/gaborish.h
@@ -0,0 +1,39 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_GABORISH_H_
+#define JXL_GABORISH_H_
+
+// Linear smoothing (3x3 convolution) for deblocking without too much blur.
+
+#include <stdint.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/image.h"
+
+namespace jxl {
+
+// Used in encoder to reduce the impact of the decoder's smoothing.
+// This is approximate and slow (unoptimized 5x5 convolution).
+Image3F GaborishInverse(const Image3F& opsin, double mul, ThreadPool* pool);
+
+// Caller must ensure LoopFilter.gab == true. Otherwise, it is faster and
+// simpler to skip the convolution. out must be preallocated.
+void ConvolveGaborish(const ImageF& in, float weight1, float weight2,
+                      ThreadPool* pool, ImageF* JXL_RESTRICT out);
+
+}  // namespace jxl
+
+#endif  // JXL_GABORISH_H_
diff --git a/jxl/gaborish_test.cc b/jxl/gaborish_test.cc
new file mode 100644
index 000000000..c66be36ad
--- /dev/null
+++ b/jxl/gaborish_test.cc
@@ -0,0 +1,60 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/gaborish.h"
+
+#include "gtest/gtest.h"
+#include "jxl/image_ops.h"
+#include "jxl/image_test_utils.h"
+
+namespace jxl {
+namespace {
+
+void TestRoundTrip(const Image3F& in, double max_l1) {
+  Image3F fwd(in.xsize(), in.ysize());
+  ThreadPool* null_pool = nullptr;
+  ConvolveGaborish(in.Plane(0), 0, 0, null_pool,
+                   const_cast<ImageF*>(&fwd.Plane(0)));
+  ConvolveGaborish(in.Plane(1), 0, 0, null_pool,
+                   const_cast<ImageF*>(&fwd.Plane(1)));
+  ConvolveGaborish(in.Plane(2), 0, 0, null_pool,
+                   const_cast<ImageF*>(&fwd.Plane(2)));
+  const Image3F rev = GaborishInverse(fwd, 0.92718927264540152, null_pool);
+  VerifyRelativeError(in, rev, max_l1, 1E-4);
+}
+
+TEST(GaborishTest, TestZero) {
+  Image3F in(20, 20);
+  ZeroFillImage(&in);
+  TestRoundTrip(in, 0.0);
+}
+
+// Disabled: large difference.
+#if 0
+TEST(GaborishTest, TestDirac) {
+  Image3F in(20, 20);
+  ZeroFillImage(&in);
+  in.PlaneRow(1, 10)[10] = 10.0f;
+  TestRoundTrip(in, 0.26);
+}
+#endif
+
+TEST(GaborishTest, TestFlat) {
+  Image3F in(20, 20);
+  FillImage(1.0f, &in);
+  TestRoundTrip(in, 1E-5);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/gamma_correct_test.cc b/jxl/gamma_correct_test.cc
new file mode 100644
index 000000000..221790b77
--- /dev/null
+++ b/jxl/gamma_correct_test.cc
@@ -0,0 +1,47 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdlib.h>
+
+#include <algorithm>
+
+#include "gtest/gtest.h"
+#include "jxl/enc_gamma_correct.h"
+
+namespace jxl {
+namespace {
+
+TEST(GammaCorrectTest, TestLinearToSrgbEdgeCases) {
+  EXPECT_EQ(0, LinearToSrgb8Direct(0.0));
+  EXPECT_NEAR(0, LinearToSrgb8Direct(1E-6f), 2E-5);
+  EXPECT_EQ(0, LinearToSrgb8Direct(-1E-6f));
+  EXPECT_EQ(0, LinearToSrgb8Direct(-1E6));
+  EXPECT_NEAR(255, LinearToSrgb8Direct(255 - 1E-6f), 1E-5);
+  EXPECT_EQ(255, LinearToSrgb8Direct(255 + 1E-6f));
+  EXPECT_EQ(255, LinearToSrgb8Direct(1E6));
+}
+
+TEST(GammaCorrectTest, TestRoundTrip) {
+  double max_err = 0.0;
+  // NOLINTNEXTLINE(clang-analyzer-security.FloatLoopCounter)
+  for (double linear = 0.0; linear <= 255.0; linear += 1E-4) {
+    const double srgb = LinearToSrgb8Direct(linear);
+    const double linear2 = Srgb8ToLinearDirect(srgb);
+    max_err = std::max(max_err, std::abs(linear - linear2));
+  }
+  EXPECT_LT(max_err, 2E-13);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/gauss_blur.cc b/jxl/gauss_blur.cc
new file mode 100644
index 000000000..af62224aa
--- /dev/null
+++ b/jxl/gauss_blur.cc
@@ -0,0 +1,104 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/gauss_blur.h"
+
+#include <string.h>
+
+#include <algorithm>
+
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+
+inline void ExtrapolateBorders(const float* const JXL_RESTRICT row_in,
+                               float* const JXL_RESTRICT row_out,
+                               const int xsize, const int radius) {
+  const int lastcol = xsize - 1;
+  for (int x = 1; x <= radius; ++x) {
+    row_out[-x] = row_in[std::min(x, xsize - 1)];
+  }
+  memcpy(row_out, row_in, xsize * sizeof(row_out[0]));
+  for (int x = 1; x <= radius; ++x) {
+    row_out[lastcol + x] = row_in[std::max(0, lastcol - x)];
+  }
+}
+
+ImageF ConvolveXSampleAndTranspose(const ImageF& in,
+                                   const std::vector<float>& kernel,
+                                   const size_t res) {
+  JXL_ASSERT(kernel.size() % 2 == 1);
+  JXL_ASSERT(in.xsize() % res == 0);
+  const size_t offset = res / 2;
+  const size_t out_xsize = in.xsize() / res;
+  ImageF out(in.ysize(), out_xsize);
+  const int r = kernel.size() / 2;
+  std::vector<float> row_tmp(in.xsize() + 2 * r);
+  float* const JXL_RESTRICT rowp = &row_tmp[r];
+  const float* const kernelp = &kernel[r];
+  for (size_t y = 0; y < in.ysize(); ++y) {
+    ExtrapolateBorders(in.Row(y), rowp, in.xsize(), r);
+    for (size_t x = offset, ox = 0; x < in.xsize(); x += res, ++ox) {
+      float sum = 0.0f;
+      for (int i = -r; i <= r; ++i) {
+        sum += rowp[std::max<int>(
+                   0, std::min<int>(static_cast<int>(x) + i, in.xsize()))] *
+               kernelp[i];
+      }
+      out.Row(ox)[y] = sum;
+    }
+  }
+  return out;
+}
+
+Image3F ConvolveXSampleAndTranspose(const Image3F& in,
+                                    const std::vector<float>& kernel,
+                                    const size_t res) {
+  return Image3F(ConvolveXSampleAndTranspose(in.Plane(0), kernel, res),
+                 ConvolveXSampleAndTranspose(in.Plane(1), kernel, res),
+                 ConvolveXSampleAndTranspose(in.Plane(2), kernel, res));
+}
+
+ImageF ConvolveAndSample(const ImageF& in, const std::vector<float>& kernel_x,
+                         const std::vector<float>& kernel_y, const size_t res) {
+  ImageF tmp = ConvolveXSampleAndTranspose(in, kernel_x, res);
+  return ConvolveXSampleAndTranspose(tmp, kernel_y, res);
+}
+
+ImageF Convolve(const ImageF& in, const std::vector<float>& kernel_x,
+                const std::vector<float>& kernel_y) {
+  return ConvolveAndSample(in, kernel_x, kernel_y, 1);
+}
+
+Image3F Convolve(const Image3F& in, const std::vector<float>& kernel_x,
+                 const std::vector<float>& kernel_y) {
+  return Image3F(Convolve(in.Plane(0), kernel_x, kernel_y),
+                 Convolve(in.Plane(1), kernel_x, kernel_y),
+                 Convolve(in.Plane(2), kernel_x, kernel_y));
+}
+
+ImageF ConvolveAndSample(const ImageF& in, const std::vector<float>& kernel,
+                         const size_t res) {
+  return ConvolveAndSample(in, kernel, kernel, res);
+}
+
+ImageF Convolve(const ImageF& in, const std::vector<float>& kernel) {
+  return ConvolveAndSample(in, kernel, 1);
+}
+
+Image3F Convolve(const Image3F& in, const std::vector<float>& kernel) {
+  return Convolve(in, kernel, kernel);
+}
+
+}  // namespace jxl
diff --git a/jxl/gauss_blur.h b/jxl/gauss_blur.h
new file mode 100644
index 000000000..c49abc292
--- /dev/null
+++ b/jxl/gauss_blur.h
@@ -0,0 +1,80 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_GAUSS_BLUR_H_
+#define JXL_GAUSS_BLUR_H_
+
+#include <math.h>
+#include <stddef.h>
+
+#include <vector>
+
+#include "jxl/base/status.h"
+#include "jxl/image.h"
+
+namespace jxl {
+
+template <typename T>
+std::vector<T> GaussianKernel(int radius, T sigma) {
+  JXL_ASSERT(sigma > 0.0);
+  std::vector<T> kernel(2 * radius + 1);
+  const T scaler = -1.0 / (2 * sigma * sigma);
+  double sum = 0.0;
+  for (int i = -radius; i <= radius; ++i) {
+    const T val = std::exp(scaler * i * i);
+    kernel[i + radius] = val;
+    sum += val;
+  }
+  for (size_t i = 0; i < kernel.size(); ++i) {
+    kernel[i] /= sum;
+  }
+  return kernel;
+}
+
+// All convolution functions below apply mirroring of the input on the borders
+// in the following way:
+//
+//     input: [a0 a1 a2 ...  aN]
+//     mirrored input: [aR ... a1 | a0 a1 a2 .... aN | aN-1 ... aN-R]
+//
+// where R is the radius of the kernel (i.e. kernel size is 2*R+1).
+
+// TODO(janwas): Deprecated, use ConvolveT instead (if |kernel| <= 5).
+ImageF Convolve(const ImageF& in, const std::vector<float>& kernel);
+Image3F Convolve(const Image3F& in, const std::vector<float>& kernel);
+
+// TODO(janwas): Deprecated, use ConvolveT instead (if |kernel| <= 5).
+ImageF Convolve(const ImageF& in, const std::vector<float>& kernel_x,
+                const std::vector<float>& kernel_y);
+Image3F Convolve(const Image3F& in, const std::vector<float>& kernel_x,
+                 const std::vector<float>& kernel_y);
+
+// TODO(janwas): Use ConvolveT instead (if |kernel| <= 5 and res == 1).
+// REQUIRES: in.xsize() and in.ysize() are integer multiples of res.
+ImageF ConvolveAndSample(const ImageF& in, const std::vector<float>& kernel,
+                         const size_t res);
+ImageF ConvolveAndSample(const ImageF& in, const std::vector<float>& kernel_x,
+                         const std::vector<float>& kernel_y, const size_t res);
+
+// TODO(janwas): Use ConvolveT instead (if |kernel| <= 5 and res == 1).
+ImageF ConvolveXSampleAndTranspose(const ImageF& in,
+                                   const std::vector<float>& kernel,
+                                   const size_t res);
+Image3F ConvolveXSampleAndTranspose(const Image3F& in,
+                                    const std::vector<float>& kernel,
+                                    const size_t res);
+
+}  // namespace jxl
+
+#endif  // JXL_GAUSS_BLUR_H_
diff --git a/jxl/gradient_test.cc b/jxl/gradient_test.cc
new file mode 100644
index 000000000..b829a39c2
--- /dev/null
+++ b/jxl/gradient_test.cc
@@ -0,0 +1,214 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <math.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include <algorithm>
+#include <array>
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/override.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_file.h"
+#include "jxl/enc_params.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+
+namespace jxl {
+namespace {
+
+// Returns distance of point p to line p0..p1, the result is signed and is not
+// normalized.
+double PointLineDist(double x0, double y0, double x1, double y1, double x,
+                     double y) {
+  return (y1 - y0) * x - (x1 - x0) * y + x1 * y0 - y1 * x0;
+}
+
+// Generates a test image with a gradient from one color to another.
+// Angle in degrees, colors can be given in hex as 0xRRGGBB. The angle is the
+// angle in which the change direction happens.
+Image3F GenerateTestGradient(uint32_t color0, uint32_t color1, double angle,
+                             size_t xsize, size_t ysize) {
+  Image3F image(xsize, ysize);
+
+  double x0 = xsize / 2;
+  double y0 = ysize / 2;
+  double x1 = x0 + std::sin(angle / 360.0 * 2.0 * kPi);
+  double y1 = y0 + std::cos(angle / 360.0 * 2.0 * kPi);
+
+  double maxdist =
+      std::max<double>(fabs(PointLineDist(x0, y0, x1, y1, 0, 0)),
+                       fabs(PointLineDist(x0, y0, x1, y1, xsize, 0)));
+
+  for (size_t c = 0; c < 3; ++c) {
+    float c0 = ((color0 >> (8 * (2 - c))) & 255);
+    float c1 = ((color1 >> (8 * (2 - c))) & 255);
+    for (size_t y = 0; y < ysize; ++y) {
+      float* row = image.PlaneRow(c, y);
+      for (size_t x = 0; x < xsize; ++x) {
+        double dist = PointLineDist(x0, y0, x1, y1, x, y);
+        double v = ((dist / maxdist) + 1.0) / 2.0;
+        float color = c0 * (1.0 - v) + c1 * v;
+        row[x] = color;
+      }
+    }
+  }
+
+  return image;
+}
+
+// Computes the max of the horizontal and vertical second derivative for each
+// pixel, where second derivative means absolute value of difference of left
+// delta and right delta (top/bottom for vertical direction).
+// The radius over which the derivative is computed is only 1 pixel and it only
+// checks two angles (hor and ver), but this approximation works well enough.
+static ImageF Gradient2(const ImageF& image) {
+  size_t xsize = image.xsize();
+  size_t ysize = image.ysize();
+  ImageF image2(image.xsize(), image.ysize());
+  for (size_t y = 1; y + 1 < ysize; y++) {
+    const auto* JXL_RESTRICT row0 = image.Row(y - 1);
+    const auto* JXL_RESTRICT row1 = image.Row(y);
+    const auto* JXL_RESTRICT row2 = image.Row(y + 1);
+    auto* row_out = image2.Row(y);
+    for (int x = 1; x + 1 < xsize; x++) {
+      float ddx = (row1[x] - row1[x - 1]) - (row1[x + 1] - row1[x]);
+      float ddy = (row1[x] - row0[x]) - (row2[x] - row1[x]);
+      row_out[x] = std::max(fabsf(ddx), fabsf(ddy));
+    }
+  }
+  // Copy to the borders
+  if (ysize > 2) {
+    auto* JXL_RESTRICT row0 = image2.Row(0);
+    const auto* JXL_RESTRICT row1 = image2.Row(1);
+    const auto* JXL_RESTRICT row2 = image2.Row(ysize - 2);
+    auto* JXL_RESTRICT row3 = image2.Row(ysize - 1);
+    for (size_t x = 1; x + 1 < xsize; x++) {
+      row0[x] = row1[x];
+      row3[x] = row2[x];
+    }
+  } else {
+    const auto* row0_in = image.Row(0);
+    const auto* row1_in = image.Row(ysize - 1);
+    auto* row0_out = image2.Row(0);
+    auto* row1_out = image2.Row(ysize - 1);
+    for (size_t x = 1; x + 1 < xsize; x++) {
+      // Image too narrow, take first derivative instead
+      row0_out[x] = row1_out[x] = fabsf(row0_in[x] - row1_in[x]);
+    }
+  }
+  if (xsize > 2) {
+    for (size_t y = 0; y < ysize; y++) {
+      auto* row = image2.Row(y);
+      row[0] = row[1];
+      row[xsize - 1] = row[xsize - 2];
+    }
+  } else {
+    for (size_t y = 0; y < ysize; y++) {
+      const auto* JXL_RESTRICT row_in = image.Row(y);
+      auto* row_out = image2.Row(y);
+      // Image too narrow, take first derivative instead
+      row_out[0] = row_out[xsize - 1] = fabsf(row_in[0] - row_in[xsize - 1]);
+    }
+  }
+  return image2;
+}
+
+static Image3F Gradient2(const Image3F& image) {
+  return Image3F(Gradient2(image.Plane(0)), Gradient2(image.Plane(1)),
+                 Gradient2(image.Plane(2)));
+}
+
+/*
+Tests if roundtrip with jxl on a gradient image doesn't cause banding.
+Only tests if use_gradient is true. Set to false for debugging to see the
+distance values.
+Angle in degrees, colors can be given in hex as 0xRRGGBB.
+*/
+void TestGradient(ThreadPool* pool, uint32_t color0, uint32_t color1,
+                  size_t xsize, size_t ysize, float angle, bool fast_mode,
+                  float butteraugli_distance, bool use_gradient = true) {
+  CompressParams cparams;
+  cparams.butteraugli_distance = butteraugli_distance;
+  if (fast_mode) {
+    cparams.speed_tier = SpeedTier::kSquirrel;
+  }
+  DecompressParams dparams;
+
+  Image3F gradient = GenerateTestGradient(color0, color1, angle, xsize, ysize);
+
+  CodecInOut io;
+  io.metadata.bits_per_sample = 8;
+  io.metadata.color_encoding = ColorManagement::SRGB();
+  io.SetFromImage(std::move(gradient), io.metadata.color_encoding);
+
+  CodecInOut io2;
+
+  PaddedBytes compressed;
+  AuxOut* aux_out = nullptr;
+  PassesEncoderState enc_state;
+  EXPECT_TRUE(EncodeFile(cparams, &io, &enc_state, &compressed, aux_out, pool));
+  EXPECT_TRUE(DecodeFile(dparams, compressed, &io2, aux_out, pool));
+  EXPECT_TRUE(io2.Main().TransformTo(io2.metadata.color_encoding, pool));
+
+  if (use_gradient) {
+    // Test that the gradient map worked. For that, we take a second derivative
+    // of the image with Gradient2 to measure how linear the change is in x and
+    // y direction. For a well handled gradient, we expect max values around
+    // 0.1, while if there is noticeable banding, which means the gradient map
+    // failed, the values are around 0.5-1.0 (regardless of
+    // butteraugli_distance).
+    Image3F gradient2 = Gradient2(io2.Main().color());
+
+    std::array<float, 3> image_max;
+    Image3Max(gradient2, &image_max);
+
+    // TODO(jyrki): These values used to work with 0.2, 0.2, 0.2.
+    EXPECT_LE(image_max[0], 3.15);
+    EXPECT_LE(image_max[1], 1.72);
+    EXPECT_LE(image_max[2], 5.05);
+  }
+}
+
+static constexpr bool fast_mode = true;
+
+TEST(GradientTest, SteepGradient) {
+  ThreadPoolInternal pool(8);
+  // Relatively steep gradients, colors from the sky of stp.png
+  TestGradient(&pool, 0xd99d58, 0x889ab1, 512, 512, 90, fast_mode, 3.0);
+}
+
+TEST(GradientTest, SubtleGradient) {
+  ThreadPoolInternal pool(8);
+  // Very subtle gradient
+  TestGradient(&pool, 0xb89b7b, 0xa89b8d, 512, 512, 90, fast_mode, 4.0);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/headers.cc b/jxl/headers.cc
new file mode 100644
index 000000000..9106f3ce9
--- /dev/null
+++ b/jxl/headers.cc
@@ -0,0 +1,177 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/headers.h"
+
+#include "jxl/common.h"
+#include "jxl/fields.h"
+
+namespace jxl {
+namespace {
+
+struct Rational {
+  constexpr explicit Rational(uint32_t num, uint32_t den)
+      : num(num), den(den) {}
+
+  // Returns floor(multiplicand * rational).
+  constexpr uint32_t MulTruncate(uint32_t multiplicand) const {
+    return uint64_t(multiplicand) * num / den;
+  }
+
+  uint32_t num;
+  uint32_t den;
+};
+
+Rational FixedAspectRatios(uint32_t ratio) {
+  JXL_ASSERT(0 != ratio && ratio < 8);
+  // Other candidates: 5/4, 7/5, 14/9, 16/10, 5/3, 21/9, 12/5
+  constexpr Rational kRatios[7] = {Rational(1, 1),    // square
+                                   Rational(12, 10),  //
+                                   Rational(4, 3),    // camera
+                                   Rational(3, 2),    // mobile camera
+                                   Rational(16, 9),   // camera/display
+                                   Rational(5, 4),    //
+                                   Rational(2, 1)};   //
+  return kRatios[ratio - 1];
+}
+
+uint32_t FindAspectRatio(uint32_t xsize, uint32_t ysize) {
+  for (uint32_t r = 1; r < 8; ++r) {
+    if (xsize == FixedAspectRatios(r).MulTruncate(ysize)) {
+      return r;
+    }
+  }
+  return 0;  // Must send xsize instead
+}
+
+}  // namespace
+
+size_t SizeHeader::xsize() const {
+  if (ratio_ != 0) {
+    return FixedAspectRatios(ratio_).MulTruncate(
+        static_cast<uint32_t>(ysize()));
+  }
+  return small_ ? ((xsize_div8_minus_1_ + 1) * 8) : xsize_minus_1_ + 1;
+}
+
+Status SizeHeader::Set(size_t xsize64, size_t ysize64) {
+  if (xsize64 > 0xFFFFFFFFull || ysize64 > 0xFFFFFFFFull) {
+    return JXL_FAILURE("Image too large");
+  }
+  const uint32_t xsize32 = static_cast<uint32_t>(xsize64);
+  const uint32_t ysize32 = static_cast<uint32_t>(ysize64);
+  if (xsize64 == 0 || ysize64 == 0) return JXL_FAILURE("Empty image");
+  small_ = xsize64 <= kGroupDim && ysize64 <= kGroupDim &&
+           (xsize64 % kBlockDim) == 0 && (ysize64 % kBlockDim) == 0;
+  if (small_) {
+    ysize_div8_minus_1_ = ysize32 / 8 - 1;
+  } else {
+    ysize_minus_1_ = ysize32 - 1;
+  }
+
+  ratio_ = FindAspectRatio(xsize32, ysize32);
+  if (ratio_ == 0) {
+    if (small_) {
+      xsize_div8_minus_1_ = xsize32 / 8 - 1;
+    } else {
+      xsize_minus_1_ = xsize32 - 1;
+    }
+  }
+  JXL_ASSERT(xsize() == xsize64);
+  JXL_ASSERT(ysize() == ysize64);
+  return true;
+}
+
+Status PreviewHeader::Set(size_t xsize64, size_t ysize64) {
+  const uint32_t xsize32 = static_cast<uint32_t>(xsize64);
+  const uint32_t ysize32 = static_cast<uint32_t>(ysize64);
+  if (xsize64 == 0 || ysize64 == 0) return JXL_FAILURE("Empty preview");
+  div8_ = (xsize64 % kBlockDim) == 0 && (ysize64 % kBlockDim) == 0;
+  if (div8_) {
+    ysize_div8_minus_1_ = ysize32 / 8 - 1;
+  } else {
+    ysize_minus_1_ = ysize32 - 1;
+  }
+
+  ratio_ = FindAspectRatio(xsize32, ysize32);
+  if (ratio_ == 0) {
+    if (div8_) {
+      xsize_div8_minus_1_ = xsize32 / 8 - 1;
+    } else {
+      xsize_minus_1_ = xsize32 - 1;
+    }
+  }
+  JXL_ASSERT(xsize() == xsize64);
+  JXL_ASSERT(ysize() == ysize64);
+  return true;
+}
+
+size_t PreviewHeader::xsize() const {
+  if (ratio_ != 0) {
+    return FixedAspectRatios(ratio_).MulTruncate(
+        static_cast<uint32_t>(ysize()));
+  }
+  return div8_ ? ((xsize_div8_minus_1_ + 1) * 8) : xsize_minus_1_ + 1;
+}
+
+SizeHeader::SizeHeader() { Bundle::Init(this); }
+PreviewHeader::PreviewHeader() { Bundle::Init(this); }
+AnimationHeader::AnimationHeader() { Bundle::Init(this); }
+
+Status ReadSizeHeader(BitReader* JXL_RESTRICT reader,
+                      SizeHeader* JXL_RESTRICT size) {
+  return Bundle::Read(reader, size);
+}
+
+Status ReadPreviewHeader(BitReader* JXL_RESTRICT reader,
+                         PreviewHeader* JXL_RESTRICT preview) {
+  return Bundle::Read(reader, preview);
+}
+
+Status ReadAnimationHeader(BitReader* JXL_RESTRICT reader,
+                           AnimationHeader* JXL_RESTRICT animation) {
+  return Bundle::Read(reader, animation);
+}
+
+Status WriteSizeHeader(const SizeHeader& size, BitWriter* JXL_RESTRICT writer,
+                       size_t layer, AuxOut* aux_out) {
+  const size_t max_bits = Bundle::MaxBits(size);
+  if (max_bits != SizeHeader::kMaxBits) {
+    JXL_ABORT("Please update SizeHeader::kMaxBits from %zu to %zu\n",
+              SizeHeader::kMaxBits, max_bits);
+  }
+
+  // Only check the number of non-extension bits (extensions are unbounded).
+  // (Bundle::Write will call CanEncode again, but it is fast because SizeHeader
+  // is tiny.)
+  size_t extension_bits, total_bits;
+  JXL_RETURN_IF_ERROR(Bundle::CanEncode(size, &extension_bits, &total_bits));
+  JXL_ASSERT(total_bits - extension_bits < SizeHeader::kMaxBits);
+
+  return Bundle::Write(size, writer, layer, aux_out);
+}
+
+Status WritePreviewHeader(const PreviewHeader& preview,
+                          BitWriter* JXL_RESTRICT writer, size_t layer,
+                          AuxOut* aux_out) {
+  return Bundle::Write(preview, writer, layer, aux_out);
+}
+
+Status WriteAnimationHeader(const AnimationHeader& animation,
+                            BitWriter* JXL_RESTRICT writer, size_t layer,
+                            AuxOut* aux_out) {
+  return Bundle::Write(animation, writer, layer, aux_out);
+}
+
+}  // namespace jxl
diff --git a/jxl/headers.h b/jxl/headers.h
new file mode 100644
index 000000000..9ffe58ecb
--- /dev/null
+++ b/jxl/headers.h
@@ -0,0 +1,189 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_HEADERS_H_
+#define JXL_HEADERS_H_
+
+// Codestream headers, also stored in CodecInOut.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/field_encodings.h"
+
+namespace jxl {
+
+// Reserved by ISO/IEC 10918-1. LF causes files opened in text mode to be
+// rejected because the marker changes to 0x0D instead. The 0xFF prefix also
+// ensures there were no 7-bit transmission limitations.
+static constexpr uint8_t kMarkerShort = 0x0A;
+static constexpr uint8_t kMarkerFlexible = 0x58;  // PLT
+
+// Compact representation of image dimensions (best case: 9 bits) so decoders
+// can preallocate early.
+class SizeHeader {
+ public:
+  // All fields are valid after reading at most this many bits. WriteSizeHeader
+  // verifies this matches Bundle::MaxBits(SizeHeader).
+  static constexpr size_t kMaxBits = 78;
+
+  SizeHeader();
+  static const char* Name() { return "SizeHeader"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->Bool(false, &small_);
+
+    if (visitor->Conditional(small_)) {
+      visitor->Bits(5, 0, &ysize_div8_minus_1_);
+    }
+    if (visitor->Conditional(!small_)) {
+      // (Could still be small, but non-multiple of 8.)
+      visitor->U32(Bits(9), Bits(13), Bits(18), Bits(30), 0, &ysize_minus_1_);
+    }
+
+    visitor->Bits(3, 0, &ratio_);
+    if (visitor->Conditional(ratio_ == 0 && small_)) {
+      visitor->Bits(5, 0, &xsize_div8_minus_1_);
+    }
+    if (visitor->Conditional(ratio_ == 0 && !small_)) {
+      visitor->U32(Bits(9), Bits(13), Bits(18), Bits(30), 0, &xsize_minus_1_);
+    }
+
+    return true;
+  }
+
+  Status Set(size_t xsize, size_t ysize);
+
+  size_t xsize() const;
+  size_t ysize() const {
+    return small_ ? ((ysize_div8_minus_1_ + 1) * 8) : ysize_minus_1_ + 1;
+  }
+
+ private:
+  bool small_;  // xsize and ysize <= 256 and divisible by 8.
+
+  uint32_t ysize_div8_minus_1_;
+  uint32_t ysize_minus_1_;
+
+  uint32_t ratio_;
+  uint32_t xsize_div8_minus_1_;
+  uint32_t xsize_minus_1_;
+};
+
+// (Similar to SizeHeader but different encoding because previews are smaller)
+class PreviewHeader {
+ public:
+  PreviewHeader();
+  static const char* Name() { return "PreviewHeader"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->Bool(false, &div8_);
+
+    if (visitor->Conditional(div8_)) {
+      visitor->U32(Val(15), Val(31), Bits(5), BitsOffset(9, 32), 0,
+                   &ysize_div8_minus_1_);
+    }
+    if (visitor->Conditional(!div8_)) {
+      visitor->U32(Bits(6), BitsOffset(8, 64), BitsOffset(10, 320),
+                   BitsOffset(12, 1344), 0, &ysize_minus_1_);
+    }
+
+    visitor->Bits(3, 0, &ratio_);
+    if (visitor->Conditional(ratio_ == 0 && div8_)) {
+      visitor->U32(Val(15), Val(31), Bits(5), BitsOffset(9, 32), 0,
+                   &xsize_div8_minus_1_);
+    }
+    if (visitor->Conditional(ratio_ == 0 && !div8_)) {
+      visitor->U32(Bits(6), BitsOffset(8, 64), BitsOffset(10, 320),
+                   BitsOffset(12, 1344), 0, &xsize_minus_1_);
+    }
+
+    return true;
+  }
+
+  Status Set(size_t xsize, size_t ysize);
+
+  size_t xsize() const;
+  size_t ysize() const {
+    return div8_ ? ((ysize_div8_minus_1_ + 1) * 8) : ysize_minus_1_ + 1;
+  }
+
+ private:
+  bool div8_;  // xsize and ysize divisible by 8.
+
+  uint32_t ysize_div8_minus_1_;
+  uint32_t ysize_minus_1_;
+
+  uint32_t ratio_;
+  uint32_t xsize_div8_minus_1_;
+  uint32_t xsize_minus_1_;
+};
+
+struct AnimationHeader {
+  AnimationHeader();
+  static const char* Name() { return "AnimationHeader"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->Bool(false, &composite_still);
+    if (visitor->Conditional(!composite_still)) {
+      visitor->U32(Val(99), Val(999), Bits(6), Bits(18), 0,
+                 &tps_numerator_minus_1);
+      visitor->U32(Val(0), Val(1000), Bits(8), Bits(10), 0,
+                 &tps_denominator_minus_1);
+
+      visitor->U32(Val(0), Bits(3), Bits(16), Bits(32), 0, &num_loops);
+
+      visitor->Bool(false, &have_timecodes);
+    }
+    return true;
+  }
+
+  bool composite_still;
+
+  // Ticks per second (expressed as rational number to support NTSC)
+  uint32_t tps_numerator_minus_1;
+  uint32_t tps_denominator_minus_1;
+
+  uint32_t num_loops;  // 0 means to repeat infinitely.
+
+  bool have_timecodes;
+};
+
+Status ReadSizeHeader(BitReader* JXL_RESTRICT reader,
+                      SizeHeader* JXL_RESTRICT size);
+Status ReadPreviewHeader(BitReader* JXL_RESTRICT reader,
+                         PreviewHeader* JXL_RESTRICT preview);
+Status ReadAnimationHeader(BitReader* JXL_RESTRICT reader,
+                           AnimationHeader* JXL_RESTRICT animation);
+
+Status WriteSizeHeader(const SizeHeader& size, BitWriter* JXL_RESTRICT writer,
+                       size_t layer, AuxOut* aux_out);
+Status WritePreviewHeader(const PreviewHeader& preview,
+                          BitWriter* JXL_RESTRICT writer, size_t layer,
+                          AuxOut* aux_out);
+Status WriteAnimationHeader(const AnimationHeader& animation,
+                            BitWriter* JXL_RESTRICT writer, size_t layer,
+                            AuxOut* aux_out);
+
+}  // namespace jxl
+
+#endif  // JXL_HEADERS_H_
diff --git a/jxl/iaca_test.cc b/jxl/iaca_test.cc
new file mode 100644
index 000000000..b8bff15de
--- /dev/null
+++ b/jxl/iaca_test.cc
@@ -0,0 +1,32 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/iaca.h"
+
+#include "gtest/gtest.h"
+
+namespace jxl {
+namespace {
+
+TEST(IacaTest, MarkersDefaultToDisabledAndDoNotCrash) {
+  BeginIACA();
+  EndIACA();
+}
+
+TEST(IacaTest, ScopeDefaultToDisabledAndDoNotCrash) {
+  ScopeIACA iaca;
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/icc_codec.cc b/jxl/icc_codec.cc
new file mode 100644
index 000000000..36ce02fbb
--- /dev/null
+++ b/jxl/icc_codec.cc
@@ -0,0 +1,841 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/icc_codec.h"
+
+#include <stdint.h>
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/byte_order.h"
+#include "jxl/brotli.h"
+#include "jxl/common.h"
+
+namespace jxl {
+namespace {
+
+bool EncodeVarInt(uint64_t value, size_t output_size, size_t* output_pos,
+                  uint8_t* output) {
+  // While more than 7 bits of data are left,
+  // store 7 bits and set the next byte flag
+  while (value > 127) {
+    if (*output_pos > output_size) return false;
+    // |128: Set the next byte flag
+    output[(*output_pos)++] = ((uint8_t)(value & 127)) | 128;
+    // Remove the seven bits we just wrote
+    value >>= 7;
+  }
+  if (*output_pos > output_size) return false;
+  output[(*output_pos)++] = ((uint8_t)value) & 127;
+  return true;
+}
+
+void EncodeVarInt(uint64_t value, PaddedBytes* data) {
+  size_t pos = data->size();
+  data->resize(data->size() + 9);
+  JXL_CHECK(EncodeVarInt(value, data->size(), &pos, data->data()));
+  data->resize(pos);
+}
+
+size_t EncodeVarInt(uint64_t value, uint8_t* output) {
+  size_t pos = 0;
+  EncodeVarInt(value, 9, &pos, output);
+  return pos;
+}
+
+uint64_t DecodeVarInt(const uint8_t* input, size_t inputSize, size_t* pos) {
+  size_t i;
+  uint64_t ret = 0;
+  for (i = 0; *pos + i < inputSize && i < 10; ++i) {
+    ret |= uint64_t(input[*pos + i] & 127) << uint64_t(7 * i);
+    // If the next-byte flag is not set, stop
+    if ((input[*pos + i] & 128) == 0) break;
+  }
+  // TODO: Return a decoding error if i == 10.
+  *pos += i + 1;
+  return ret;
+}
+
+uint32_t DecodeUint32(const uint8_t* data, size_t size, size_t pos) {
+  return pos + 4 > size ? 0 : LoadBE32(data + pos);
+}
+
+void EncodeUint32(size_t pos, uint32_t value, PaddedBytes* data) {
+  if (pos + 4 > data->size()) return;
+  StoreBE32(value, data->data() + pos);
+}
+
+void AppendUint32(uint32_t value, PaddedBytes* data) {
+  data->resize(data->size() + 4);
+  EncodeUint32(data->size() - 4, value, data);
+}
+
+std::string DecodeKeyword(const uint8_t* data, size_t size, size_t pos) {
+  if (pos + 4 > size) return "    ";
+  std::string result;
+  result += data[pos + 0];
+  result += data[pos + 1];
+  result += data[pos + 2];
+  result += data[pos + 3];
+  return result;
+}
+
+void EncodeKeyword(const std::string& keyword, uint8_t* data, size_t size,
+                   size_t pos) {
+  if (keyword.size() != 4 || pos + 3 >= size) return;
+  data[pos + 0] = keyword[0];
+  data[pos + 1] = keyword[1];
+  data[pos + 2] = keyword[2];
+  data[pos + 3] = keyword[3];
+}
+
+void AppendKeyword(const std::string& keyword, PaddedBytes* data) {
+  data->resize(data->size() + 4);
+  EncodeKeyword(keyword, data->data(), data->size(), data->size() - 4);
+}
+
+PaddedBytes Transpose(const uint8_t* data, size_t size, size_t width) {
+  PaddedBytes result(size);
+  size_t s = 0, j = 0;
+  for (size_t i = 0; i < size; i++) {
+    result[j] = data[i];
+    j += width;
+    if (j >= size) j = ++s;
+  }
+  return result;
+}
+
+void Shuffle(uint8_t* data, size_t size, size_t width) {
+  PaddedBytes result = Transpose(data, size, width);
+  for (size_t i = 0; i < size; i++) {
+    data[i] = result[i];
+  }
+}
+
+void Unshuffle(uint8_t* data, size_t size, size_t width) {
+  PaddedBytes result = Transpose(data, size, (size + width - 1) / width);
+  for (size_t i = 0; i < size; i++) {
+    data[i] = result[i];
+  }
+}
+
+static constexpr size_t kICCHeaderSize = 128;
+
+// Tag names focused on RGB and GRAY monitor profiles
+static const char* kTagStrings[] = {
+    "cprt", "wtpt", "bkpt", "rXYZ", "gXYZ", "bXYZ", "kXYZ", "rTRC", "gTRC",
+    "bTRC", "kTRC", "chad", "desc", "chrm", "dmnd", "dmdd", "lumi",
+};
+
+static constexpr size_t kCommandTagUnknown = 1;
+static constexpr size_t kCommandTagTRC = 2;
+static constexpr size_t kCommandTagXYZ = 3;
+static constexpr size_t kCommandTagStringFirst = 4;
+const size_t kNumTagStrings = sizeof(kTagStrings) / sizeof(*kTagStrings);
+
+// Tag types focused on RGB and GRAY monitor profiles
+static const char* kTypeStrings[] = {
+    "XYZ ", "desc", "text", "mluc", "para", "curv", "sf32", "gbd ",
+};
+
+static constexpr size_t kCommandInsert = 1;
+static constexpr size_t kCommandShuffle2 = 2;
+static constexpr size_t kCommandShuffle4 = 3;
+static constexpr size_t kCommandPredict = 4;
+static constexpr size_t kCommandXYZ = 10;
+static constexpr size_t kCommandTypeStartFirst = 16;
+
+static constexpr size_t kFlagBitOffset = 64;
+static constexpr size_t kFlagBitSize = 128;
+
+static constexpr size_t kNumTypeStrings =
+    sizeof(kTypeStrings) / sizeof(*kTypeStrings);
+
+PaddedBytes InitHeaderPred() {
+  PaddedBytes result(kICCHeaderSize);
+  for (size_t i = 0; i < kICCHeaderSize; i++) {
+    result[i] = 0;
+  }
+  result[8] = 4;
+  EncodeKeyword("mntr", result.data(), result.size(), 12);
+  EncodeKeyword("RGB ", result.data(), result.size(), 16);
+  EncodeKeyword("XYZ ", result.data(), result.size(), 20);
+  EncodeKeyword("acsp", result.data(), result.size(), 36);
+  result[68] = 0;
+  result[69] = 0;
+  result[70] = 246;
+  result[71] = 214;
+  result[72] = 0;
+  result[73] = 1;
+  result[74] = 0;
+  result[75] = 0;
+  result[76] = 0;
+  result[77] = 0;
+  result[78] = 211;
+  result[79] = 45;
+  return result;
+}
+
+void PredictHeader(const uint8_t* icc, size_t size, uint8_t* header,
+                   size_t pos) {
+  if (pos == 8 && size >= 8) {
+    header[80] = icc[4];
+    header[81] = icc[5];
+    header[82] = icc[6];
+    header[83] = icc[7];
+  }
+  if (pos == 41 && size >= 41) {
+    if (icc[40] == 'A') {
+      header[41] = 'P';
+      header[42] = 'P';
+      header[43] = 'L';
+    }
+    if (icc[40] == 'M') {
+      header[41] = 'S';
+      header[42] = 'F';
+      header[43] = 'T';
+    }
+  }
+  if (pos == 42 && size >= 42) {
+    if (icc[40] == 'S' && icc[41] == 'G') {
+      header[42] = 'I';
+      header[43] = ' ';
+    }
+    if (icc[40] == 'S' && icc[41] == 'U') {
+      header[42] = 'N';
+      header[43] = 'W';
+    }
+  }
+}
+
+template <typename T>
+T PredictValue(T p1, T p2, T p3, int order) {
+  if (order == 0) return p1;
+  if (order == 1) return 2 * p1 - p2;
+  if (order == 2) return 3 * p1 - 3 * p2 + p3;
+  return 0;
+}
+
+// Predicts a value with linear prediction of given order (0-2), for integers
+// with width bytes and given stride in bytes between values.
+// The start position is at start + i, and the relevant modulus of i describes
+// which byte of the multi-byte integer is being handled.
+// The value start + i must be at least stride * 4.
+uint8_t PredictValue(const uint8_t* data, size_t start, size_t i, size_t stride,
+                     size_t width, int order) {
+  size_t pos = start + i;
+  if (width == 1) {
+    uint8_t p1 = data[pos - stride];
+    uint8_t p2 = data[pos - stride * 2];
+    uint8_t p3 = data[pos - stride * 3];
+    return PredictValue(p1, p2, p3, order);
+  } else if (width == 2) {
+    size_t p = start + (i & ~1);
+    uint16_t p1 = (data[p - stride * 1] << 8) + data[p - stride * 1 + 1];
+    uint16_t p2 = (data[p - stride * 2] << 8) + data[p - stride * 2 + 1];
+    uint16_t p3 = (data[p - stride * 3] << 8) + data[p - stride * 3 + 1];
+    uint16_t pred = PredictValue(p1, p2, p3, order);
+    return (i & 1) ? (pred & 255) : ((pred >> 8) & 255);
+  } else {
+    size_t p = start + (i & ~3);
+    uint32_t p1 = DecodeUint32(data, pos, p - stride);
+    uint32_t p2 = DecodeUint32(data, pos, p - stride * 2);
+    uint32_t p3 = DecodeUint32(data, pos, p - stride * 3);
+    uint32_t pred = PredictValue(p1, p2, p3, order);
+    unsigned shiftbytes = 3 - (i & 3);
+    return (pred >> (shiftbytes * 8)) & 255;
+  }
+}
+
+// Checks if a + b > size, taking possible integer flow into account.
+bool OutOfBounds(size_t a, size_t b, size_t size) {
+  size_t pos = a + b;
+  if (pos > size) return true;
+  if (pos < a) return true;  // overflow happened
+  return false;
+}
+
+// Return value is only advisory.
+bool PredictAndShuffle(size_t stride, size_t width, int order, size_t num,
+                       const uint8_t* data, size_t size, size_t* pos,
+                       PaddedBytes* result) {
+  if (OutOfBounds(*pos, num, size)) return false;
+  if (*pos < stride * 4) return false;
+  size_t start = result->size();
+  for (size_t i = 0; i < num; i++) {
+    uint8_t predicted = PredictValue(data, *pos, i, stride, width, order);
+    result->push_back(data[*pos + i] - predicted);
+  }
+  *pos += num;
+  if (width > 1) Unshuffle(result->data() + start, num, width);
+  return true;
+}
+
+const int kBrotliEffort = 9;
+
+}  // namespace
+
+// Decodes the result of PredictICC back to a valid ICC profile.
+Status UnpredictICC(const uint8_t* enc, size_t size, PaddedBytes* result) {
+  if (!result->empty()) return JXL_FAILURE("result must be empty initially");
+  size_t pos = 0;
+  // TODO(lode): technically speaking we need to check that the entire varint
+  // decoding never goes out of bounds, not just the first byte. This requires
+  // a DecodeVarInt function that returns an error code. It is safe to use
+  // DecodeVarInt with out of bounds values, it silently returns, but the
+  // specification requires an error. Idem for all DecodeVarInt below.
+  if (pos >= size) return JXL_FAILURE("Out of bounds");
+  size_t osize = DecodeVarInt(enc, size, &pos);  // Output size
+  if (pos >= size) return JXL_FAILURE("Out of bounds");
+  size_t csize = DecodeVarInt(enc, size, &pos);  // Commands size
+  if (OutOfBounds(pos, csize, size)) return JXL_FAILURE("Out of bounds");
+  size_t cpos = pos;  // pos in commands stream
+  size_t commands_end = cpos + csize;
+  pos = commands_end;  // pos in data stream
+
+  // Header
+  PaddedBytes header = InitHeaderPred();
+  EncodeUint32(0, osize, &header);
+  for (size_t i = 0; i <= kICCHeaderSize; i++) {
+    if (result->size() == osize) {
+      if (cpos != commands_end) return JXL_FAILURE("Not all commands used");
+      if (pos != size) return JXL_FAILURE("Not all data used");
+      return true;  // Valid end
+    }
+    if (i == kICCHeaderSize) break;  // Done
+    PredictHeader(result->data(), result->size(), header.data(), i);
+    if (pos >= size) return JXL_FAILURE("Out of bounds");
+    result->push_back(enc[pos++] + header[i]);
+  }
+
+  // Tag list
+  if (cpos >= commands_end) return JXL_FAILURE("Out of bounds");
+  uint64_t numtags = DecodeVarInt(enc, size, &cpos);
+
+  if (numtags != 0) {
+    numtags--;
+    AppendUint32(numtags, result);
+    size_t prevtagstart = kICCHeaderSize + numtags * 12;
+    size_t prevtagsize = 0;
+    for (;;) {
+      if (cpos > commands_end) return JXL_FAILURE("Out of bounds");
+      if (cpos == commands_end) break;  // Valid end
+      uint8_t command = enc[cpos++];
+      uint8_t tagcode = command & 63;
+      std::string tag;
+      if (tagcode == 0) {
+        break;
+      } else if (tagcode == kCommandTagUnknown) {
+        if (OutOfBounds(pos, 4, size)) return JXL_FAILURE("Out of bounds");
+        tag = DecodeKeyword(enc, size, pos);
+        pos += 4;
+      } else if (tagcode == kCommandTagTRC) {
+        tag = "rTRC";
+      } else if (tagcode == kCommandTagXYZ) {
+        tag = "rXYZ";
+      } else {
+        if (tagcode - kCommandTagStringFirst >= kNumTagStrings) {
+          return JXL_FAILURE("Unknown tagcode");
+        }
+        tag = kTagStrings[tagcode - kCommandTagStringFirst];
+      }
+      AppendKeyword(tag, result);
+
+      size_t tagstart = prevtagstart + prevtagsize;
+      size_t tagsize = prevtagsize;
+      if (tag == "rXYZ" || tag == "gXYZ" || tag == "bXYZ" || tag == "kXYZ" ||
+          tag == "wtpt" || tag == "bkpt" || tag == "lumi") {
+        tagsize = 20;
+      }
+
+      if (command & kFlagBitOffset) {
+        if (cpos >= commands_end) return JXL_FAILURE("Out of bounds");
+        tagstart = DecodeVarInt(enc, size, &cpos);
+      }
+      AppendUint32(tagstart, result);
+      if (command & kFlagBitSize) {
+        if (cpos >= commands_end) return JXL_FAILURE("Out of bounds");
+        tagsize = DecodeVarInt(enc, size, &cpos);
+      }
+      AppendUint32(tagsize, result);
+      prevtagstart = tagstart;
+      prevtagsize = tagsize;
+
+      if (tagcode == kCommandTagTRC) {
+        AppendKeyword("gTRC", result);
+        AppendUint32(tagstart, result);
+        AppendUint32(tagsize, result);
+        AppendKeyword("bTRC", result);
+        AppendUint32(tagstart, result);
+        AppendUint32(tagsize, result);
+      }
+
+      if (tagcode == kCommandTagXYZ) {
+        AppendKeyword("gXYZ", result);
+        AppendUint32(tagstart + tagsize, result);
+        AppendUint32(tagsize, result);
+        AppendKeyword("bXYZ", result);
+        AppendUint32(tagstart + tagsize * 2, result);
+        AppendUint32(tagsize, result);
+      }
+    }
+  }
+
+  // Main Content
+  for (;;) {
+    if (cpos > commands_end) return JXL_FAILURE("Out of bounds");
+    if (cpos == commands_end) break;  // Valid end
+    uint8_t command = enc[cpos++];
+    if (command == kCommandInsert) {
+      if (cpos >= commands_end) return JXL_FAILURE("Out of bounds");
+      uint64_t num = DecodeVarInt(enc, size, &cpos);
+      if (OutOfBounds(pos, num, size)) return JXL_FAILURE("Out of bounds");
+      for (size_t i = 0; i < num; i++) {
+        result->push_back(enc[pos++]);
+      }
+    } else if (command == kCommandShuffle2 || command == kCommandShuffle4) {
+      if (cpos >= commands_end) return JXL_FAILURE("Out of bounds");
+      uint64_t num = DecodeVarInt(enc, size, &cpos);
+      if (OutOfBounds(pos, num, size)) return JXL_FAILURE("Out of bounds");
+      PaddedBytes shuffled(num);
+      for (size_t i = 0; i < num; i++) {
+        shuffled[i] = enc[pos + i];
+      }
+      if (command == kCommandShuffle2) {
+        Shuffle(shuffled.data(), num, 2);
+      } else if (command == kCommandShuffle4) {
+        Shuffle(shuffled.data(), num, 4);
+      }
+      for (size_t i = 0; i < num; i++) {
+        result->push_back(shuffled[i]);
+        pos++;
+      }
+    } else if (command == kCommandPredict) {
+      if (OutOfBounds(cpos, 2, commands_end)) {
+        return JXL_FAILURE("Out of bounds");
+      }
+      uint8_t flags = enc[cpos++];
+
+      size_t width = (flags & 3) + 1;
+      if (width == 3) return JXL_FAILURE("Invalid width");
+
+      int order = (flags & 12) >> 2;
+      if (order == 3) return JXL_FAILURE("Invalid order");
+
+      size_t stride = width;
+      if (flags & 16) {
+        if (cpos >= commands_end) return JXL_FAILURE("Out of bounds");
+        stride = DecodeVarInt(enc, size, &cpos);
+        if (stride < width) {
+          return JXL_FAILURE("Invalid stride");
+        }
+      }
+      // If stride * 4 >= result->size(), return failure. The check
+      // "size == 0 || ((size - 1) >> 2) < stride" corresponds to
+      // "stride * 4 >= size", but does not suffer from integer overflow.
+      // TODO(lode): this check is more strict than necessary, returning
+      // failure if stride * (order + 1) > result->size() is sufficient, but
+      // changing this requires a comment to the spec and to PredictValue.
+      if (result->empty() || ((result->size() - 1u) >> 2u) < stride) {
+        return JXL_FAILURE("Invalid stride");
+      }
+
+      if (cpos >= commands_end) return JXL_FAILURE("Out of bounds");
+      uint64_t num = DecodeVarInt(enc, size, &cpos);  // in bytes
+      if (OutOfBounds(pos, num, size)) return JXL_FAILURE("Out of bounds");
+
+      PaddedBytes shuffled(num);
+      for (size_t i = 0; i < num; i++) {
+        shuffled[i] = enc[pos + i];
+      }
+      if (width > 1) Shuffle(shuffled.data(), num, width);
+
+      size_t start = result->size();
+      for (size_t i = 0; i < num; i++) {
+        uint8_t predicted =
+            PredictValue(result->data(), start, i, stride, width, order);
+        result->push_back(predicted + shuffled[i]);
+      }
+      pos += num;
+    } else if (command == kCommandXYZ) {
+      AppendKeyword("XYZ ", result);
+      for (int i = 0; i < 4; i++) result->push_back(0);
+      if (OutOfBounds(pos, 12, size)) return JXL_FAILURE("Out of bounds");
+      for (size_t i = 0; i < 12; i++) {
+        result->push_back(enc[pos++]);
+      }
+    } else if (command >= kCommandTypeStartFirst &&
+               command < kCommandTypeStartFirst + kNumTypeStrings) {
+      AppendKeyword(kTypeStrings[command - kCommandTypeStartFirst], result);
+      for (size_t i = 0; i < 4; i++) {
+        result->push_back(0);
+      }
+    } else {
+      JXL_FAILURE("Unknown command");
+    }
+  }
+
+  if (pos != size) return JXL_FAILURE("Not all data used");
+  if (result->size() != osize) return JXL_FAILURE("Invalid result size");
+
+  return true;
+}
+
+// Outputs a transformed form of the given icc profile. The result itself is
+// not particularly smaller than the input data in bytes, but it will be in a
+// form that is easier to compress (more zeroes, ...) and will compress better
+// with brotli.
+Status PredictICC(const uint8_t* icc, size_t size, PaddedBytes* result) {
+  PaddedBytes commands;
+  PaddedBytes data;
+
+  EncodeVarInt(size, result);
+
+  // Header
+  PaddedBytes header = InitHeaderPred();
+  EncodeUint32(0, size, &header);
+  for (size_t i = 0; i < kICCHeaderSize && i < size; i++) {
+    PredictHeader(icc, size, header.data(), i);
+    data.push_back(icc[i] - header[i]);
+  }
+  if (size <= kICCHeaderSize) {
+    EncodeVarInt(0, result);  // 0 commands
+    for (size_t i = 0; i < data.size(); i++) {
+      result->push_back(data[i]);
+    }
+    return true;
+  }
+
+  std::vector<std::string> tags;
+  std::vector<size_t> tagstarts;
+  std::vector<size_t> tagsizes;
+  std::map<size_t, size_t> tagmap;
+
+  // Tag list
+  size_t pos = kICCHeaderSize;
+  if (pos + 4 <= size) {
+    uint32_t numtags = DecodeUint32(icc, size, pos);
+    pos += 4;
+    EncodeVarInt(numtags + 1, &commands);
+    uint32_t prevtagstart = kICCHeaderSize + numtags * 12;
+    uint32_t prevtagsize = 0;
+    for (size_t i = 0; i < numtags; i++) {
+      if (pos + 12 > size) break;
+
+      std::string tag = DecodeKeyword(icc, size, pos + 0);
+      uint32_t tagstart = DecodeUint32(icc, size, pos + 4);
+      uint32_t tagsize = DecodeUint32(icc, size, pos + 8);
+      pos += 12;
+
+      tags.push_back(tag);
+      tagstarts.push_back(tagstart);
+      tagsizes.push_back(tagsize);
+      tagmap[tagstart] = tags.size() - 1;
+
+      uint8_t tagcode = kCommandTagUnknown;
+      for (size_t j = 0; j < kNumTagStrings; j++) {
+        if (tag == kTagStrings[j]) {
+          tagcode = j + kCommandTagStringFirst;
+          break;
+        }
+      }
+
+      if (tag == "rTRC" && pos + 24 < size) {
+        bool ok = true;
+        ok &= DecodeKeyword(icc, size, pos + 0) == "gTRC";
+        ok &= DecodeKeyword(icc, size, pos + 12) == "bTRC";
+        if (ok) {
+          for (size_t i = 0; i < 8; i++) {
+            if (icc[pos - 8 + i] != icc[pos + 4 + i]) ok = false;
+            if (icc[pos - 8 + i] != icc[pos + 16 + i]) ok = false;
+          }
+        }
+        if (ok) {
+          tagcode = kCommandTagTRC;
+          pos += 24;
+          i += 2;
+        }
+      }
+
+      if (tag == "rXYZ" && pos + 24 < size) {
+        bool ok = true;
+        ok &= DecodeKeyword(icc, size, pos + 0) == "gXYZ";
+        ok &= DecodeKeyword(icc, size, pos + 12) == "bXYZ";
+        uint32_t offsetr = tagstart;
+        uint32_t offsetg = DecodeUint32(icc, size, pos + 4);
+        uint32_t offsetb = DecodeUint32(icc, size, pos + 16);
+        uint32_t sizer = tagsize;
+        uint32_t sizeg = DecodeUint32(icc, size, pos + 8);
+        uint32_t sizeb = DecodeUint32(icc, size, pos + 20);
+        ok &= sizer == 20;
+        ok &= sizeg == 20;
+        ok &= sizeb == 20;
+        ok &= (offsetg == offsetr + 20);
+        ok &= (offsetb == offsetr + 40);
+        if (ok) {
+          tagcode = kCommandTagXYZ;
+          pos += 24;
+          i += 2;
+        }
+      }
+
+      uint8_t command = tagcode;
+      size_t predicted_tagstart = prevtagstart + prevtagsize;
+      if (predicted_tagstart != tagstart) command |= kFlagBitOffset;
+      size_t predicted_tagsize = prevtagsize;
+      if (tag == "rXYZ" || tag == "gXYZ" || tag == "bXYZ" || tag == "kXYZ" ||
+          tag == "wtpt" || tag == "bkpt" || tag == "lumi") {
+        predicted_tagsize = 20;
+      }
+      if (predicted_tagsize != tagsize) command |= kFlagBitSize;
+      commands.push_back(command);
+      if (tagcode == 1) {
+        AppendKeyword(tag, &data);
+      }
+      if (command & kFlagBitOffset) EncodeVarInt(tagstart, &commands);
+      if (command & kFlagBitSize) EncodeVarInt(tagsize, &commands);
+
+      prevtagstart = tagstart;
+      prevtagsize = tagsize;
+    }
+  }
+  // Indicate end of tag list or varint indicating there's none
+  commands.push_back(0);
+
+  // Main content
+  // The main content in a valid ICC profile contains tagged elements, with the
+  // tag types (4 letter names) given by the tag list above, and the tag list
+  // pointing to the start and indicating the size of each tagged element. It is
+  // allowed for tagged elements to overlap, e.g. the curve for R, G and B could
+  // all point to the same one.
+  std::string tagtype;
+  size_t tagstart = 0, tagsize = 0, clutstart = 0;
+
+  size_t last0 = pos;
+  // This loop appends commands to the output, processing some sub-section of a
+  // current tagged element each time. We need to keep track of the tagtype of
+  // the current element, and update it when we encounter the boundary of a
+  // next one.
+  // It is not required that the input data is a valid ICC profile, if the
+  // encoder does not recognize the data it will still be able to output bytes
+  // but will not predict as well.
+  while (pos <= size) {
+    size_t last1 = pos;
+    PaddedBytes commands_add;
+    PaddedBytes data_add;
+
+    // This means the loop brought the position beyond the
+    if (pos > tagstart + tagsize && !tagtype.empty()) {
+      tagtype = "";
+    }
+
+    if (commands_add.empty() && data_add.empty() && tagmap.count(pos) &&
+        pos + 4 <= size) {
+      size_t index = tagmap[pos];
+      tagtype = "";
+      for (size_t i = 0; i < 4; i++) tagtype += icc[pos + i];
+      tagstart = tagstarts[index];
+      tagsize = tagsizes[index];
+
+      if (tagtype == "mluc" && pos + tagsize <= size && tagsize > 8 &&
+          icc[pos + 4] == 0 && icc[pos + 5] == 0 && icc[pos + 6] == 0 &&
+          icc[pos + 7] == 0) {
+        size_t num = tagsize - 8;
+        commands_add.push_back(kCommandTypeStartFirst + 3);
+        pos += 8;
+        commands_add.push_back(kCommandShuffle2);
+        EncodeVarInt(num, &commands_add);
+        size_t start = data_add.size();
+        for (size_t i = 0; i < num; i++) {
+          data_add.push_back(icc[pos]);
+          pos++;
+        }
+        Unshuffle(data_add.data() + start, num, 2);
+      }
+
+      if (tagtype == "curv" && pos + tagsize <= size && tagsize > 8 &&
+          icc[pos + 4] == 0 && icc[pos + 5] == 0 && icc[pos + 6] == 0 &&
+          icc[pos + 7] == 0) {
+        size_t num = tagsize - 8;
+        if (num > 16 && num < (1 << 28) && pos + num <= size) {
+          commands_add.push_back(kCommandTypeStartFirst + 5);
+          pos += 8;
+          commands_add.push_back(kCommandPredict);
+          int order = 1, width = 2, stride = width;
+          commands_add.push_back((order << 2) | (width - 1));
+          EncodeVarInt(num, &commands_add);
+          PredictAndShuffle(stride, width, order, num, icc, size, &pos,
+                            &data_add);
+        }
+      }
+    }
+
+    if (tagtype == "mAB " || tagtype == "mBA ") {
+      if (pos + 12 < size &&
+          (DecodeKeyword(icc, size, pos) == "curv" ||
+           DecodeKeyword(icc, size, pos) == "vcgt") &&
+          DecodeUint32(icc, size, pos + 4) == 0) {
+        uint32_t num = DecodeUint32(icc, size, pos + 8) * 2;
+        if (num > 16 && num < (1 << 28) && pos + num <= size) {
+          pos += 12;
+          last1 = pos;
+          commands_add.push_back(kCommandPredict);
+          int order = 1, width = 2, stride = width;
+          commands_add.push_back((order << 2) | (width - 1));
+          EncodeVarInt(num, &commands_add);
+          PredictAndShuffle(stride, width, order, num, icc, size, &pos,
+                            &data_add);
+        }
+      }
+
+      if (pos == tagstart + 24 && pos + 4 < size) {
+        // Note that this value can be remembered for next iterations of the
+        // loop, so the "pos == clutstart" if below can trigger during a later
+        // iteration.
+        clutstart = tagstart + DecodeUint32(icc, size, pos);
+      }
+
+      if (pos == clutstart && clutstart + 16 < size) {
+        size_t numi = icc[tagstart + 8];
+        size_t numo = icc[tagstart + 9];
+        size_t width = icc[clutstart + 16];
+        size_t stride = width * numo;
+        size_t num = width * numo;
+        for (size_t i = 0; i < numi && clutstart + i < size; i++) {
+          num *= icc[clutstart + i];
+        }
+        if ((width == 1 || width == 2) && num > 64 && num < (1 << 28) &&
+            pos + num <= size) {
+          commands_add.push_back(kCommandPredict);
+          int order = 1;
+          uint8_t flags =
+              (order << 2) | (width - 1) | (stride == width ? 0 : 16);
+          commands_add.push_back(flags);
+          if (flags & 16) EncodeVarInt(stride, &commands_add);
+          EncodeVarInt(num, &commands_add);
+          PredictAndShuffle(stride, width, order, num, icc, size, &pos,
+                            &data_add);
+        }
+      }
+    }
+
+    if (commands_add.empty() && data_add.empty() && tagtype == "gbd " &&
+        pos == tagstart + 8 && pos + tagsize - 8 <= size) {
+      size_t width = 4, order = 0, stride = width;
+      size_t num = tagsize - 8;
+      uint8_t flags = (order << 2) | (width - 1) | (stride == width ? 0 : 16);
+      commands_add.push_back(kCommandPredict);
+      commands_add.push_back(flags);
+      if (flags & 16) EncodeVarInt(stride, &commands_add);
+      EncodeVarInt(num, &commands_add);
+      PredictAndShuffle(stride, width, order, num, icc, size, &pos, &data_add);
+    }
+
+    if (commands_add.empty() && data_add.empty() && pos + 20 <= size) {
+      if (DecodeKeyword(icc, size, pos) == "XYZ " &&
+          DecodeUint32(icc, size, pos + 4) == 0) {
+        commands_add.push_back(kCommandXYZ);
+        pos += 8;
+        for (size_t j = 0; j < 12; j++) data_add.push_back(icc[pos++]);
+      }
+    }
+
+    if (commands_add.empty() && data_add.empty() && pos + 8 <= size) {
+      for (size_t i = 0; i < kNumTypeStrings; i++) {
+        bool eq = true;
+        for (size_t j = 0; j < 8; j++) {
+          if (j < 4 && icc[pos + j] != kTypeStrings[i][j]) {
+            eq = false;
+            break;
+          }
+          if (j >= 4 && icc[pos + j] != 0) {
+            eq = false;
+            break;
+          }
+        }
+        if (eq) {
+          commands_add.push_back(kCommandTypeStartFirst + i);
+          pos += 8;
+          break;
+        }
+      }
+    }
+
+    if (!(commands_add.empty() && data_add.empty()) || pos == size) {
+      if (last0 < last1) {
+        commands.push_back(kCommandInsert);
+        EncodeVarInt(last1 - last0, &commands);
+        while (last0 < last1) {
+          data.push_back(icc[last0++]);
+        }
+      }
+      for (size_t i = 0; i < commands_add.size(); i++) {
+        commands.push_back(commands_add[i]);
+      }
+      for (size_t i = 0; i < data_add.size(); i++) {
+        data.push_back(data_add[i]);
+      }
+      last0 = pos;
+    }
+    if (commands_add.empty() && data_add.empty()) {
+      pos++;
+    }
+  }
+
+  EncodeVarInt(commands.size(), result);
+  for (size_t i = 0; i < commands.size(); i++) {
+    result->push_back(commands[i]);
+  }
+  for (size_t i = 0; i < data.size(); i++) {
+    result->push_back(data[i]);
+  }
+
+  return true;
+}
+
+Status WriteICC(const PaddedBytes& icc, BitWriter* JXL_RESTRICT writer,
+                size_t layer, AuxOut* JXL_RESTRICT aux_out) {
+  if (icc.empty()) return JXL_FAILURE("ICC must be non-empty");
+  PaddedBytes enc;
+  JXL_RETURN_IF_ERROR(PredictICC(icc.data(), icc.size(), &enc));
+  PaddedBytes compressed;
+  JXL_RETURN_IF_ERROR(BrotliCompress(kBrotliEffort, enc, &compressed));
+  // Extra byte for padding so Brotli can call BitReader::GetSpan.
+  const size_t max_bits = kBitsPerByte + compressed.size() * kBitsPerByte;
+  BitWriter::Allotment allotment(writer, max_bits);
+  writer->ZeroPadToByte();
+  *writer += compressed;
+  ReclaimAndCharge(writer, &allotment, layer, aux_out);
+  return true;
+}
+
+Status ReadICC(BitReader* JXL_RESTRICT reader, PaddedBytes* JXL_RESTRICT icc) {
+  icc->clear();
+  JXL_RETURN_IF_ERROR(reader->JumpToByteBoundary());
+  const size_t kMaxOutput = 1ULL << 30;
+  size_t bytes_read = 0;
+  PaddedBytes decompressed;
+  JXL_RETURN_IF_ERROR(BrotliDecompress(reader->GetSpan(), kMaxOutput,
+                                       &bytes_read, &decompressed));
+  reader->SkipBits(bytes_read * kBitsPerByte);
+  JXL_RETURN_IF_ERROR(
+      UnpredictICC(decompressed.data(), decompressed.size(), icc));
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/icc_codec.h b/jxl/icc_codec.h
new file mode 100644
index 000000000..29f1e1ad5
--- /dev/null
+++ b/jxl/icc_codec.h
@@ -0,0 +1,49 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_ICC_CODEC_H_
+#define JXL_ICC_CODEC_H_
+
+// Compressed representation of ICC profiles.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+
+namespace jxl {
+
+// Should still be called if `icc.empty()` - if so, writes only 1 bit.
+Status WriteICC(const PaddedBytes& icc, BitWriter* JXL_RESTRICT writer,
+                size_t layer, AuxOut* JXL_RESTRICT aux_out);
+
+// `icc` may be empty afterwards - if so, call CreateProfile. Does not append,
+// clears any original data that was in icc.
+Status ReadICC(BitReader* JXL_RESTRICT reader, PaddedBytes* JXL_RESTRICT icc);
+
+// Exposed only for testing
+Status PredictICC(const uint8_t* icc, size_t size, PaddedBytes* result);
+
+// Exposed only for testing
+Status UnpredictICC(const uint8_t* enc, size_t size, PaddedBytes* result);
+
+}  // namespace jxl
+
+#endif  // JXL_ICC_CODEC_H_
diff --git a/jxl/icc_codec_test.cc b/jxl/icc_codec_test.cc
new file mode 100644
index 000000000..32a04a552
--- /dev/null
+++ b/jxl/icc_codec_test.cc
@@ -0,0 +1,161 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/icc_codec.h"
+
+#include <string>
+
+#include "gtest/gtest.h"
+#include "jxl/base/span.h"
+
+namespace jxl {
+namespace {
+
+void TestProfile(const PaddedBytes& icc) {
+  BitWriter writer;
+  EXPECT_TRUE(WriteICC(icc, &writer, 0, nullptr));
+  PaddedBytes dec;
+  BitReader reader(writer.GetSpan());
+  EXPECT_TRUE(ReadICC(&reader, &dec));
+  EXPECT_TRUE(reader.Close());
+  EXPECT_EQ(icc.size(), dec.size());
+  if (icc.size() == dec.size()) {
+    for (size_t i = 0; i < icc.size(); i++) {
+      EXPECT_EQ(icc[i], dec[i]);
+      if (icc[i] != dec[i]) break;  // One output is enough
+    }
+  }
+}
+
+void TestProfile(const std::string& icc) {
+  PaddedBytes bytes(icc.size());
+  for (size_t i = 0; i < icc.size(); i++) {
+    bytes[i] = icc[i];
+  }
+  TestProfile(bytes);
+}
+
+// Valid profile from one of the images output by the decoder.
+static const unsigned char kTestProfile[] = {
+    0x00, 0x00, 0x03, 0x80, 0x6c, 0x63, 0x6d, 0x73, 0x04, 0x30, 0x00, 0x00,
+    0x6d, 0x6e, 0x74, 0x72, 0x52, 0x47, 0x42, 0x20, 0x58, 0x59, 0x5a, 0x20,
+    0x07, 0xe3, 0x00, 0x04, 0x00, 0x1d, 0x00, 0x0f, 0x00, 0x32, 0x00, 0x2e,
+    0x61, 0x63, 0x73, 0x70, 0x41, 0x50, 0x50, 0x4c, 0x00, 0x00, 0x00, 0x01,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0xf6, 0xd6,
+    0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x2d, 0x6c, 0x63, 0x6d, 0x73,
+    0x5f, 0x07, 0x0d, 0x3e, 0x4d, 0x32, 0xf2, 0x6e, 0x5d, 0x77, 0x26, 0xcc,
+    0x23, 0xb0, 0x6a, 0x15, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d,
+    0x64, 0x65, 0x73, 0x63, 0x00, 0x00, 0x01, 0x20, 0x00, 0x00, 0x00, 0x42,
+    0x63, 0x70, 0x72, 0x74, 0x00, 0x00, 0x01, 0x64, 0x00, 0x00, 0x01, 0x00,
+    0x77, 0x74, 0x70, 0x74, 0x00, 0x00, 0x02, 0x64, 0x00, 0x00, 0x00, 0x14,
+    0x63, 0x68, 0x61, 0x64, 0x00, 0x00, 0x02, 0x78, 0x00, 0x00, 0x00, 0x2c,
+    0x72, 0x58, 0x59, 0x5a, 0x00, 0x00, 0x02, 0xa4, 0x00, 0x00, 0x00, 0x14,
+    0x62, 0x58, 0x59, 0x5a, 0x00, 0x00, 0x02, 0xb8, 0x00, 0x00, 0x00, 0x14,
+    0x67, 0x58, 0x59, 0x5a, 0x00, 0x00, 0x02, 0xcc, 0x00, 0x00, 0x00, 0x14,
+    0x72, 0x54, 0x52, 0x43, 0x00, 0x00, 0x02, 0xe0, 0x00, 0x00, 0x00, 0x20,
+    0x67, 0x54, 0x52, 0x43, 0x00, 0x00, 0x02, 0xe0, 0x00, 0x00, 0x00, 0x20,
+    0x62, 0x54, 0x52, 0x43, 0x00, 0x00, 0x02, 0xe0, 0x00, 0x00, 0x00, 0x20,
+    0x63, 0x68, 0x72, 0x6d, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x00, 0x24,
+    0x64, 0x6d, 0x6e, 0x64, 0x00, 0x00, 0x03, 0x24, 0x00, 0x00, 0x00, 0x28,
+    0x64, 0x6d, 0x64, 0x64, 0x00, 0x00, 0x03, 0x4c, 0x00, 0x00, 0x00, 0x32,
+    0x6d, 0x6c, 0x75, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+    0x00, 0x00, 0x00, 0x0c, 0x65, 0x6e, 0x55, 0x53, 0x00, 0x00, 0x00, 0x26,
+    0x00, 0x00, 0x00, 0x1c, 0x00, 0x52, 0x00, 0x47, 0x00, 0x42, 0x00, 0x5f,
+    0x00, 0x44, 0x00, 0x36, 0x00, 0x35, 0x00, 0x5f, 0x00, 0x53, 0x00, 0x52,
+    0x00, 0x47, 0x00, 0x5f, 0x00, 0x52, 0x00, 0x65, 0x00, 0x6c, 0x00, 0x5f,
+    0x00, 0x37, 0x00, 0x30, 0x00, 0x39, 0x00, 0x00, 0x6d, 0x6c, 0x75, 0x63,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0c,
+    0x65, 0x6e, 0x55, 0x53, 0x00, 0x00, 0x00, 0xe4, 0x00, 0x00, 0x00, 0x1c,
+    0x00, 0x43, 0x00, 0x6f, 0x00, 0x70, 0x00, 0x79, 0x00, 0x72, 0x00, 0x69,
+    0x00, 0x67, 0x00, 0x68, 0x00, 0x74, 0x00, 0x20, 0x00, 0x32, 0x00, 0x30,
+    0x00, 0x31, 0x00, 0x38, 0x00, 0x20, 0x00, 0x47, 0x00, 0x6f, 0x00, 0x6f,
+    0x00, 0x67, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x20, 0x00, 0x4c, 0x00, 0x4c,
+    0x00, 0x43, 0x00, 0x2c, 0x00, 0x20, 0x00, 0x43, 0x00, 0x43, 0x00, 0x2d,
+    0x00, 0x42, 0x00, 0x59, 0x00, 0x2d, 0x00, 0x53, 0x00, 0x41, 0x00, 0x20,
+    0x00, 0x33, 0x00, 0x2e, 0x00, 0x30, 0x00, 0x20, 0x00, 0x55, 0x00, 0x6e,
+    0x00, 0x70, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x74, 0x00, 0x65, 0x00, 0x64,
+    0x00, 0x20, 0x00, 0x6c, 0x00, 0x69, 0x00, 0x63, 0x00, 0x65, 0x00, 0x6e,
+    0x00, 0x73, 0x00, 0x65, 0x00, 0x28, 0x00, 0x68, 0x00, 0x74, 0x00, 0x74,
+    0x00, 0x70, 0x00, 0x73, 0x00, 0x3a, 0x00, 0x2f, 0x00, 0x2f, 0x00, 0x63,
+    0x00, 0x72, 0x00, 0x65, 0x00, 0x61, 0x00, 0x74, 0x00, 0x69, 0x00, 0x76,
+    0x00, 0x65, 0x00, 0x63, 0x00, 0x6f, 0x00, 0x6d, 0x00, 0x6d, 0x00, 0x6f,
+    0x00, 0x6e, 0x00, 0x73, 0x00, 0x2e, 0x00, 0x6f, 0x00, 0x72, 0x00, 0x67,
+    0x00, 0x2f, 0x00, 0x6c, 0x00, 0x69, 0x00, 0x63, 0x00, 0x65, 0x00, 0x6e,
+    0x00, 0x73, 0x00, 0x65, 0x00, 0x73, 0x00, 0x2f, 0x00, 0x62, 0x00, 0x79,
+    0x00, 0x2d, 0x00, 0x73, 0x00, 0x61, 0x00, 0x2f, 0x00, 0x33, 0x00, 0x2e,
+    0x00, 0x30, 0x00, 0x2f, 0x00, 0x6c, 0x00, 0x65, 0x00, 0x67, 0x00, 0x61,
+    0x00, 0x6c, 0x00, 0x63, 0x00, 0x6f, 0x00, 0x64, 0x00, 0x65, 0x00, 0x29,
+    0x58, 0x59, 0x5a, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xf6, 0xd6,
+    0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x2d, 0x73, 0x66, 0x33, 0x32,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x0c, 0x42, 0x00, 0x00, 0x05, 0xde,
+    0xff, 0xff, 0xf3, 0x25, 0x00, 0x00, 0x07, 0x93, 0x00, 0x00, 0xfd, 0x90,
+    0xff, 0xff, 0xfb, 0xa1, 0xff, 0xff, 0xfd, 0xa2, 0x00, 0x00, 0x03, 0xdc,
+    0x00, 0x00, 0xc0, 0x6e, 0x58, 0x59, 0x5a, 0x20, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x6f, 0xa0, 0x00, 0x00, 0x38, 0xf5, 0x00, 0x00, 0x03, 0x90,
+    0x58, 0x59, 0x5a, 0x20, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x9f,
+    0x00, 0x00, 0x0f, 0x84, 0x00, 0x00, 0xb6, 0xc4, 0x58, 0x59, 0x5a, 0x20,
+    0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x97, 0x00, 0x00, 0xb7, 0x87,
+    0x00, 0x00, 0x18, 0xd9, 0x70, 0x61, 0x72, 0x61, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x03, 0x00, 0x00, 0x00, 0x02, 0x38, 0xe4, 0x00, 0x00, 0xe8, 0xf0,
+    0x00, 0x00, 0x17, 0x10, 0x00, 0x00, 0x38, 0xe4, 0x00, 0x00, 0x14, 0xbc,
+    0x63, 0x68, 0x72, 0x6d, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x00, 0x00,
+    0x00, 0x00, 0xa3, 0xd7, 0x00, 0x00, 0x54, 0x7c, 0x00, 0x00, 0x4c, 0xcd,
+    0x00, 0x00, 0x99, 0x9a, 0x00, 0x00, 0x26, 0x67, 0x00, 0x00, 0x0f, 0x5c,
+    0x6d, 0x6c, 0x75, 0x63, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01,
+    0x00, 0x00, 0x00, 0x0c, 0x65, 0x6e, 0x55, 0x53, 0x00, 0x00, 0x00, 0x0c,
+    0x00, 0x00, 0x00, 0x1c, 0x00, 0x47, 0x00, 0x6f, 0x00, 0x6f, 0x00, 0x67,
+    0x00, 0x6c, 0x00, 0x65, 0x6d, 0x6c, 0x75, 0x63, 0x00, 0x00, 0x00, 0x00,
+    0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x0c, 0x65, 0x6e, 0x55, 0x53,
+    0x00, 0x00, 0x00, 0x16, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x49, 0x00, 0x6d,
+    0x00, 0x61, 0x00, 0x67, 0x00, 0x65, 0x00, 0x20, 0x00, 0x63, 0x00, 0x6f,
+    0x00, 0x64, 0x00, 0x65, 0x00, 0x63, 0x00, 0x00,
+};
+
+}  // namespace
+
+TEST(IccCodecTest, Icc) {
+  // Empty string cannot be tested, encoder checks against writing it.
+  TestProfile("a");
+  TestProfile("ab");
+  TestProfile("aaaa");
+
+  {
+    // Exactly the ICC header size
+    PaddedBytes profile(128);
+    for (size_t i = 0; i < 128; i++) {
+      profile[i] = 0;
+    }
+    TestProfile(profile);
+  }
+
+  {
+    PaddedBytes profile;
+    profile.append(kTestProfile, kTestProfile + sizeof(kTestProfile));
+    TestProfile(profile);
+  }
+
+  // Test substrings of full profile
+  {
+    PaddedBytes profile;
+    for (size_t i = 0; i <= 256; i++) {
+      profile.push_back(kTestProfile[i]);
+      TestProfile(profile);
+    }
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/image.cc b/jxl/image.cc
new file mode 100644
index 000000000..5f7fc71ea
--- /dev/null
+++ b/jxl/image.cc
@@ -0,0 +1,151 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/image.h"
+
+#include <stdint.h>
+
+#include <utility>
+
+#include "jxl/base/profiler.h"
+#include "jxl/common.h"
+
+namespace jxl {
+
+CacheAlignedUniquePtr AllocateImageBytes(size_t size, size_t xsize,
+                                         size_t ysize) {
+  // (Can't profile CacheAligned itself because it is used by profiler.h)
+  PROFILER_FUNC;
+
+  // Note: size may be zero.
+  CacheAlignedUniquePtr bytes = AllocateArray(size);
+  JXL_ASSERT(reinterpret_cast<uintptr_t>(bytes.get()) % kImageAlign == 0);
+  return bytes;
+}
+
+ImageB ImageFromPacked(const uint8_t* packed, const size_t xsize,
+                       const size_t ysize, const size_t bytes_per_row) {
+  JXL_ASSERT(bytes_per_row >= xsize);
+  ImageB image(xsize, ysize);
+  PROFILER_FUNC;
+  for (size_t y = 0; y < ysize; ++y) {
+    uint8_t* const JXL_RESTRICT row = image.Row(y);
+    const uint8_t* const JXL_RESTRICT packed_row = packed + y * bytes_per_row;
+    memcpy(row, packed_row, xsize);
+  }
+  return image;
+}
+
+// Note that using mirroring here gives slightly worse results.
+ImageF PadImage(const ImageF& in, const size_t xsize, const size_t ysize) {
+  JXL_ASSERT(xsize >= in.xsize());
+  JXL_ASSERT(ysize >= in.ysize());
+  ImageF out(xsize, ysize);
+  size_t y = 0;
+  for (; y < in.ysize(); ++y) {
+    const float* JXL_RESTRICT row_in = in.ConstRow(y);
+    float* JXL_RESTRICT row_out = out.Row(y);
+    memcpy(row_out, row_in, in.xsize() * sizeof(row_in[0]));
+    const int lastcol = in.xsize() - 1;
+    const float lastval = row_out[lastcol];
+    for (size_t x = in.xsize(); x < xsize; ++x) {
+      row_out[x] = lastval;
+    }
+  }
+
+  // TODO(janwas): no need to copy if we can 'extend' image: if rows are
+  // pointers to any memory? Or allocate larger image before IO?
+  const int lastrow = in.ysize() - 1;
+  for (; y < ysize; ++y) {
+    const float* JXL_RESTRICT row_in = out.ConstRow(lastrow);
+    float* JXL_RESTRICT row_out = out.Row(y);
+    memcpy(row_out, row_in, xsize * sizeof(row_out[0]));
+  }
+  return out;
+}
+
+Image3F PadImageSymmetric(const Image3F& in, const size_t border) {
+  size_t xsize = in.xsize();
+  size_t ysize = in.ysize();
+  Image3F out(xsize + 2 * border, ysize + 2 * border);
+  CopyImageTo(in, Rect(border, border, xsize, ysize), &out);
+  for (size_t c = 0; c < 3; c++) {
+    // Horizontal pad.
+    for (size_t y = 0; y < ysize; y++) {
+      for (size_t x = 0; x < border; x++) {
+        out.PlaneRow(c, y + border)[x] = in.ConstPlaneRow(c, y)[0];
+        out.PlaneRow(c, y + border)[x + xsize + border] =
+            in.ConstPlaneRow(c, y)[xsize - 1];
+      }
+    }
+    // Vertical pad.
+    for (size_t y = 0; y < border; y++) {
+      memcpy(out.PlaneRow(c, y), out.ConstPlaneRow(c, border),
+             out.xsize() * sizeof(float));
+      memcpy(out.PlaneRow(c, y + ysize + border),
+             out.ConstPlaneRow(c, ysize + border - 1),
+             out.xsize() * sizeof(float));
+    }
+  }
+  return out;
+}
+
+Image3F PadImageToMultiple(const Image3F& in, const size_t N) {
+  PROFILER_FUNC;
+  const size_t xsize_blocks = DivCeil(in.xsize(), N);
+  const size_t ysize_blocks = DivCeil(in.ysize(), N);
+  const size_t xsize = N * xsize_blocks;
+  const size_t ysize = N * ysize_blocks;
+  ImageF out[3];
+  for (size_t c = 0; c < 3; ++c) {
+    out[c] = PadImage(in.Plane(c), xsize, ysize);
+  }
+  return Image3F(std::move(out[0]), std::move(out[1]), std::move(out[2]));
+}
+
+void PadImageToBlockMultipleInPlace(Image3F* JXL_RESTRICT in) {
+  PROFILER_FUNC;
+  const size_t xsize_orig = in->xsize();
+  const size_t ysize_orig = in->ysize();
+  const size_t xsize = RoundUpToBlockDim(xsize_orig);
+  const size_t ysize = RoundUpToBlockDim(ysize_orig);
+  // Expands image size to the originally-allocated size.
+  in->ShrinkTo(xsize, ysize);
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t y = 0; y < ysize_orig; y++) {
+      float* JXL_RESTRICT row = in->PlaneRow(c, y);
+      for (size_t x = xsize_orig; x < xsize; x++) {
+        row[x] = row[xsize_orig - 1];
+      }
+    }
+    const float* JXL_RESTRICT row_src = in->ConstPlaneRow(c, ysize_orig - 1);
+    for (size_t y = ysize_orig; y < ysize; y++) {
+      memcpy(in->PlaneRow(c, y), row_src, xsize * sizeof(float));
+    }
+  }
+}
+
+float DotProduct(const ImageF& a, const ImageF& b) {
+  double sum = 0.0;
+  for (size_t y = 0; y < a.ysize(); ++y) {
+    const float* const JXL_RESTRICT row_a = a.ConstRow(y);
+    const float* const JXL_RESTRICT row_b = b.ConstRow(y);
+    for (size_t x = 0; x < a.xsize(); ++x) {
+      sum += row_a[x] * row_b[x];
+    }
+  }
+  return sum;
+}
+
+}  // namespace jxl
diff --git a/jxl/image.h b/jxl/image.h
new file mode 100644
index 000000000..c8041e9d3
--- /dev/null
+++ b/jxl/image.h
@@ -0,0 +1,618 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_IMAGE_H_
+#define JXL_IMAGE_H_
+
+// SIMD/multicore-friendly planar image representation with row accessors.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <sys/types.h>
+
+#include <algorithm>
+#include <array>
+#include <cmath>
+#include <limits>
+#include <random>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/cache_aligned.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/robust_statistics.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Each row address is a multiple of this - enables aligned loads.
+static constexpr size_t kImageAlign = CacheAligned::kAlignment;
+static_assert(kImageAlign >= kMaxVectorSize, "Insufficient alignment");
+
+// Returns distance [bytes] between the start of two consecutive rows, a
+// multiple of kAlign but NOT CacheAligned::kAlias - see below.
+//
+// Differing "kAlign" make sense for:
+// - Image: 128 to avoid false sharing/RFOs between multiple threads processing
+//   rows independently;
+// - TileFlow: no cache line alignment needed because buffers are per-thread;
+//   just need kMaxVectorSize=16..64 for SIMD.
+//
+// "valid_bytes" is xsize * sizeof(T).
+template <size_t kAlign>
+static inline size_t BytesPerRow(size_t valid_bytes) {
+  static_assert((kAlign & (kAlign - 1)) == 0, "kAlign should be power of two");
+
+  // Extra two vectors allow *writing* a partial or full vector on the right AND
+  // left border (for convolve.h) without disturbing the next/previous row.
+  const size_t row_size = valid_bytes + 2 * kMaxVectorSize;
+
+  // Round up.
+  size_t bytes_per_row = (row_size + kAlign - 1) & ~(kAlign - 1);
+
+  // During the lengthy window before writes are committed to memory, CPUs
+  // guard against read after write hazards by checking the address, but
+  // only the lower 11 bits. We avoid a false dependency between writes to
+  // consecutive rows by ensuring their sizes are not multiples of 2 KiB.
+  // Avoid2K prevents the same problem for the planes of an Image3.
+  if (bytes_per_row % CacheAligned::kAlias == 0) {
+    bytes_per_row += kImageAlign;
+  }
+
+  return bytes_per_row;
+}
+
+// Factored out of Image<> to avoid dependency on profiler.h and <atomic>.
+CacheAlignedUniquePtr AllocateImageBytes(size_t size, size_t xsize,
+                                         size_t ysize);
+
+// Single channel, aligned rows separated by padding. T must be POD.
+//
+// Rationale: vectorization benefits from aligned operands - unaligned loads and
+// especially stores are expensive when the address crosses cache line
+// boundaries. Introducing padding after each row ensures the start of a row is
+// aligned, and that row loops can process entire vectors (writes to the padding
+// are allowed and ignored).
+//
+// We prefer a planar representation, where channels are stored as separate
+// 2D arrays, because that simplifies vectorization (repeating the same
+// operation on multiple adjacent components) without the complexity of a
+// hybrid layout (8 R, 8 G, 8 B, ...). In particular, clients can easily iterate
+// over all components in a row and Image requires no knowledge of the pixel
+// format beyond the component type "T".
+//
+// This image layout could also be achieved with a vector and a row accessor
+// function, but a class wrapper with support for "deleter" allows wrapping
+// existing memory allocated by clients without copying the pixels. It also
+// provides convenient accessors for xsize/ysize, which shortens function
+// argument lists. Supports move-construction so it can be stored in containers.
+template <typename ComponentType>
+class Plane {
+ public:
+  using T = ComponentType;
+  static constexpr size_t kNumPlanes = 1;
+
+  Plane() : xsize_(0), ysize_(0), bytes_per_row_(0), bytes_(nullptr) {}
+
+  Plane(const size_t xsize, const size_t ysize)
+      : xsize_(static_cast<uint32_t>(xsize)),
+        ysize_(static_cast<uint32_t>(ysize)),
+        bytes_per_row_(BytesPerRow<kImageAlign>(xsize * sizeof(T))),
+        bytes_(nullptr) {
+    JXL_ASSERT(bytes_per_row_ % kImageAlign == 0);
+    // xsize and/or ysize can legitimately be zero, in which case we don't
+    // want to allocate.
+    if (xsize != 0 && ysize != 0) {
+      bytes_ = AllocateImageBytes(bytes_per_row_ * ysize + kMaxVectorSize,
+                                  xsize, ysize);
+    }
+
+#ifdef MEMORY_SANITIZER
+    // Only in MSAN builds: ensure full vectors are initialized.
+    if (xsize != 0 && ysize != 0) {
+      const size_t partial = (xsize_ * sizeof(T)) % kMaxVectorSize;
+      const size_t remainder = (partial == 0) ? 0 : (kMaxVectorSize - partial);
+      for (size_t y = 0; y < ysize_; ++y) {
+#if defined(__clang__) && (__clang_major__ <= 6)
+        // There's a bug in msan in clang-6 when handling AVX2 operations. This
+        // workaround allows pass tests on msan, although it is slower and
+        // prevents msan warnings from uninitialized images.
+        memset(Row(y), 0, (xsize_ * sizeof(T)) + remainder);
+#else
+        memset(Row(y) + xsize_, 0, remainder);
+#endif
+      }
+    }
+#endif
+  }
+
+  // Copy construction/assignment is forbidden to avoid inadvertent copies,
+  // which can be very expensive. Use CopyImageTo() instead.
+  Plane(const Plane& other) = delete;
+  Plane& operator=(const Plane& other) = delete;
+
+  // Move constructor (required for returning Image from function)
+  Plane(Plane&& other) noexcept = default;
+
+  // Move assignment (required for std::vector)
+  Plane& operator=(Plane&& other) noexcept = default;
+
+  void Swap(Plane& other) {
+    std::swap(xsize_, other.xsize_);
+    std::swap(ysize_, other.ysize_);
+    std::swap(bytes_per_row_, other.bytes_per_row_);
+    std::swap(bytes_, other.bytes_);
+  }
+
+  // Useful for pre-allocating image with some padding for alignment purposes
+  // and later reporting the actual valid dimensions. Caller is responsible
+  // for ensuring xsize/ysize are <= the original dimensions.
+  void ShrinkTo(const size_t xsize, const size_t ysize) {
+    xsize_ = static_cast<uint32_t>(xsize);
+    ysize_ = static_cast<uint32_t>(ysize);
+    // NOTE: we can't recompute bytes_per_row for more compact storage and
+    // better locality because that would invalidate the image contents.
+  }
+
+  // How many pixels.
+  JXL_INLINE size_t xsize() const { return xsize_; }
+  JXL_INLINE size_t ysize() const { return ysize_; }
+
+  // Returns pointer to the start of a row, with at least xsize (rounded up to
+  // kImageAlign bytes) accessible values.
+  JXL_INLINE T* JXL_RESTRICT Row(const size_t y) {
+    RowBoundsCheck(y);
+    void* row = bytes_.get() + y * bytes_per_row_;
+    return static_cast<T*>(JXL_ASSUME_ALIGNED(row, 64));
+  }
+
+  // Returns pointer to non-const - required for writing to individual planes
+  // of an Image3.
+  JXL_INLINE T* JXL_RESTRICT MutableRow(const size_t y) const {
+    RowBoundsCheck(y);
+    void* row = bytes_.get() + y * bytes_per_row_;
+    return static_cast<T*>(JXL_ASSUME_ALIGNED(row, 64));
+  }
+
+  // Returns pointer to const (see above).
+  JXL_INLINE const T* JXL_RESTRICT Row(const size_t y) const {
+    RowBoundsCheck(y);
+    const void* row = bytes_.get() + y * bytes_per_row_;
+    return static_cast<const T*>(JXL_ASSUME_ALIGNED(row, 64));
+  }
+
+  // Returns pointer to const (see above), even if called on a non-const Image.
+  JXL_INLINE const T* JXL_RESTRICT ConstRow(const size_t y) const {
+    RowBoundsCheck(y);
+    const void* row = bytes_.get() + y * bytes_per_row_;
+    return static_cast<const T*>(JXL_ASSUME_ALIGNED(row, 64));
+  }
+
+  // Raw access to byte contents, for interfacing with other libraries.
+  // Unsigned char instead of char to avoid surprises (sign extension).
+  JXL_INLINE uint8_t* bytes() {
+    void* p = bytes_.get();
+    return static_cast<uint8_t * JXL_RESTRICT>(JXL_ASSUME_ALIGNED(p, 64));
+  }
+  JXL_INLINE const uint8_t* bytes() const {
+    const void* p = bytes_.get();
+    return static_cast<const uint8_t * JXL_RESTRICT>(JXL_ASSUME_ALIGNED(p, 64));
+  }
+
+  // NOTE: do not use this for copying rows - the valid xsize may be much less.
+  JXL_INLINE size_t bytes_per_row() const { return bytes_per_row_; }
+
+  // Returns number of pixels (some of which are padding) per row. Useful for
+  // computing other rows via pointer arithmetic. WARNING: this must
+  // NOT be used to determine xsize. NOTE: this is less efficient than
+  // ByteOffset(row, bytes_per_row).
+  JXL_INLINE intptr_t PixelsPerRow() const {
+    static_assert(kImageAlign % sizeof(T) == 0,
+                  "Padding must be divisible by the pixel size.");
+    return static_cast<intptr_t>(bytes_per_row_ / sizeof(T));
+  }
+
+  // Extra metadata about this plane.
+  struct Extra {};
+
+  Extra& GetExtra() {
+    if (extra_ == nullptr) {
+      extra_.reset(new Extra);
+    }
+    return *extra_;
+  }
+
+ private:
+  void RowBoundsCheck(const size_t y) const {
+#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \
+    defined(THREAD_SANITIZER)
+    if (y >= ysize_) {
+      Abort(__FILE__, __LINE__, "Row(%zu) >= %u\n", y, ysize_);
+    }
+#endif
+  }
+
+  // (Members are non-const to enable assignment during move-assignment.)
+  uint32_t xsize_;  // In valid pixels, not including any padding.
+  uint32_t ysize_;
+  size_t bytes_per_row_;  // Includes padding.
+  CacheAlignedUniquePtr bytes_;
+  std::unique_ptr<Extra> extra_;
+};
+
+using ImageB = Plane<uint8_t>;
+using ImageS = Plane<int16_t>;  // signed integer or half-float
+using ImageU = Plane<uint16_t>;
+using ImageI = Plane<int32_t>;
+using ImageF = Plane<float>;
+using ImageD = Plane<double>;
+
+// We omit unnecessary fields and choose smaller representations to reduce L1
+// cache pollution.
+#pragma pack(push, 1)
+
+// Size of an image in pixels. POD.
+struct ImageSize {
+  static ImageSize Make(const size_t xsize, const size_t ysize) {
+    ImageSize ret;
+    ret.xsize = static_cast<uint32_t>(xsize);
+    ret.ysize = static_cast<uint32_t>(ysize);
+    return ret;
+  }
+
+  bool operator==(const ImageSize& other) const {
+    return xsize == other.xsize && ysize == other.ysize;
+  }
+
+  uint32_t xsize;
+  uint32_t ysize;
+};
+
+#pragma pack(pop)
+
+template <typename T>
+void CopyImageTo(const Plane<T>& from, Plane<T>* JXL_RESTRICT to) {
+  PROFILER_ZONE("CopyImage1");
+  JXL_ASSERT(SameSize(from, *to));
+  for (size_t y = 0; y < from.ysize(); ++y) {
+    const T* JXL_RESTRICT row_from = from.ConstRow(y);
+    T* JXL_RESTRICT row_to = to->Row(y);
+    memcpy(row_to, row_from, from.xsize() * sizeof(T));
+  }
+}
+
+// DEPRECATED - prefer to preallocate result.
+template <typename T>
+Plane<T> CopyImage(const Plane<T>& from) {
+  Plane<T> to(from.xsize(), from.ysize());
+  CopyImageTo(from, &to);
+  return to;
+}
+
+// Also works for Image3 and mixed argument types.
+template <class Image1, class Image2>
+bool SameSize(const Image1& image1, const Image2& image2) {
+  return image1.xsize() == image2.xsize() && image1.ysize() == image2.ysize();
+}
+
+template <typename T>
+class Image3;
+
+// Rectangular region in image(s). Factoring this out of Image instead of
+// shifting the pointer by x0/y0 allows this to apply to multiple images with
+// different resolutions (e.g. color transform and quantization field).
+// Can compare using SameSize(rect1, rect2).
+class Rect {
+ public:
+  // Most windows are xsize_max * ysize_max, except those on the borders where
+  // begin + size_max > end.
+  constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize_max,
+                 size_t ysize_max, size_t xend, size_t yend)
+      : x0_(xbegin),
+        y0_(ybegin),
+        xsize_(ClampedSize(xbegin, xsize_max, xend)),
+        ysize_(ClampedSize(ybegin, ysize_max, yend)) {}
+
+  // Construct with origin and known size (typically from another Rect).
+  constexpr Rect(size_t xbegin, size_t ybegin, size_t xsize, size_t ysize)
+      : x0_(xbegin), y0_(ybegin), xsize_(xsize), ysize_(ysize) {}
+
+  // Construct a rect that covers a whole image/plane/ImageBundle etc.
+  template <typename Image>
+  explicit Rect(const Image& image)
+      : Rect(0, 0, image.xsize(), image.ysize()) {}
+
+  Rect() : Rect(0, 0, 0, 0) {}
+
+  Rect(const Rect&) = default;
+  Rect& operator=(const Rect&) = default;
+
+  Rect Subrect(size_t xbegin, size_t ybegin, size_t xsize_max,
+               size_t ysize_max) {
+    return Rect(x0_ + xbegin, y0_ + ybegin, xsize_max, ysize_max, x0_ + xsize_,
+                y0_ + ysize_);
+  }
+
+  template <typename T>
+  T* Row(Plane<T>* image, size_t y) const {
+    return image->Row(y + y0_) + x0_;
+  }
+
+  template <typename T>
+  T* MutableRow(const Plane<T>* image, size_t y) const {
+    return image->MutableRow(y + y0_) + x0_;
+  }
+
+  template <typename T>
+  T* PlaneRow(Image3<T>* image, const size_t c, size_t y) const {
+    return image->PlaneRow(c, y + y0_) + x0_;
+  }
+
+  template <typename T>
+  const T* ConstRow(const Plane<T>& image, size_t y) const {
+    return image.ConstRow(y + y0_) + x0_;
+  }
+
+  template <typename T>
+  const T* ConstPlaneRow(const Image3<T>& image, size_t c, size_t y) const {
+    return image.ConstPlaneRow(c, y + y0_) + x0_;
+  }
+
+  // Returns true if this Rect fully resides in the given image. ImageT could be
+  // Image<T> or Image3<T>; however if ImageT is Rect, results are nonsensical.
+  template <class ImageT>
+  bool IsInside(const ImageT& image) const {
+    return (x0_ + xsize_ <= image.xsize()) && (y0_ + ysize_ <= image.ysize());
+  }
+
+  size_t x0() const { return x0_; }
+  size_t y0() const { return y0_; }
+  size_t xsize() const { return xsize_; }
+  size_t ysize() const { return ysize_; }
+
+ private:
+  // Returns size_max, or whatever is left in [begin, end).
+  static constexpr size_t ClampedSize(size_t begin, size_t size_max,
+                                      size_t end) {
+    return (begin + size_max <= end) ? size_max
+                                     : (end > begin ? end - begin : 0);
+  }
+
+  size_t x0_;
+  size_t y0_;
+
+  size_t xsize_;
+  size_t ysize_;
+};
+
+// Copies `from:rect_from` to `to`.
+template <typename T>
+void CopyImageTo(const Rect& rect_from, const Plane<T>& from,
+                 const Plane<T>* JXL_RESTRICT to) {
+  PROFILER_ZONE("CopyImageR");
+  JXL_ASSERT(SameSize(rect_from, *to));
+  for (size_t y = 0; y < rect_from.ysize(); ++y) {
+    const T* JXL_RESTRICT row_from = rect_from.ConstRow(from, y);
+    T* JXL_RESTRICT row_to = to->MutableRow(y);
+    memcpy(row_to, row_from, rect_from.xsize() * sizeof(T));
+  }
+}
+
+template <typename T>
+void CopyImageTo(const Plane<T>& from, const Rect& rect_to,
+                 const Plane<T>* JXL_RESTRICT to) {
+  PROFILER_ZONE("CopyImageR");
+  JXL_ASSERT(SameSize(from, rect_to));
+  for (size_t y = 0; y < rect_to.ysize(); ++y) {
+    const T* JXL_RESTRICT row_from = from.Row(y);
+    T* JXL_RESTRICT row_to = rect_to.MutableRow(to, y);
+    memcpy(row_to, row_from, rect_to.xsize() * sizeof(T));
+  }
+}
+
+// DEPRECATED - Returns a copy of the "image" pixels that lie in "rect".
+template <typename T>
+Plane<T> CopyImage(const Rect& rect, const Plane<T>& image) {
+  Plane<T> copy(rect.xsize(), rect.ysize());
+  CopyImageTo(rect, image, &copy);
+  return copy;
+}
+
+// Currently, we abuse Image to either refer to an image that owns its storage
+// or one that doesn't. In similar vein, we abuse Image* function parameters to
+// either mean "assign to me" or "fill the provided image with data".
+// Hopefully, the "assign to me" meaning will go away and most images in the
+// codebase will not be backed by own storage. When this happens we can redesign
+// Image to be a non-storage-holding view class and introduce BackedImage in
+// those places that actually need it.
+
+// NOTE: we can't use Image as a view because invariants are violated
+// (alignment and the presence of padding before/after each "row").
+
+// A bundle of 3 same-sized images. Typically constructed by moving from three
+// rvalue references to Image. To overwrite an existing Image3 using
+// single-channel producers, we also need access to Image*. Constructing
+// temporary non-owning Image pointing to one plane of an existing Image3 risks
+// dangling references, especially if the wrapper is moved. Therefore, we
+// store an array of Image (which are compact enough that size is not a concern)
+// and provide a Plane+MutableRow accessors.
+template <typename ComponentType>
+class Image3 {
+ public:
+  using T = ComponentType;
+  using PlaneT = Plane<T>;
+  static constexpr size_t kNumPlanes = 3;
+
+  Image3() : planes_{PlaneT(), PlaneT(), PlaneT()} {}
+
+  Image3(const size_t xsize, const size_t ysize)
+      : planes_{PlaneT(xsize, ysize), PlaneT(xsize, ysize),
+                PlaneT(xsize, ysize)} {}
+
+  Image3(Image3&& other) noexcept {
+    for (size_t i = 0; i < kNumPlanes; i++) {
+      planes_[i] = std::move(other.planes_[i]);
+    }
+  }
+
+  Image3(PlaneT&& plane0, PlaneT&& plane1, PlaneT&& plane2) {
+    JXL_CHECK(SameSize(plane0, plane1));
+    JXL_CHECK(SameSize(plane0, plane2));
+    planes_[0] = std::move(plane0);
+    planes_[1] = std::move(plane1);
+    planes_[2] = std::move(plane2);
+  }
+
+  // Copy construction/assignment is forbidden to avoid inadvertent copies,
+  // which can be very expensive. Use CopyImageTo instead.
+  Image3(const Image3& other) = delete;
+  Image3& operator=(const Image3& other) = delete;
+
+  Image3& operator=(Image3&& other) noexcept {
+    for (size_t i = 0; i < kNumPlanes; i++) {
+      planes_[i] = std::move(other.planes_[i]);
+    }
+    return *this;
+  }
+
+  // Returns row pointer; usage: PlaneRow(idx_plane, y)[x] = val.
+  JXL_INLINE T* JXL_RESTRICT PlaneRow(const size_t c, const size_t y) {
+    // Custom implementation instead of calling planes_[c].Row ensures only a
+    // single multiplication is needed for PlaneRow(0..2, y).
+    PlaneRowBoundsCheck(c, y);
+    const size_t row_offset = y * planes_[0].bytes_per_row();
+    void* row = planes_[c].bytes() + row_offset;
+    return static_cast<T*>(JXL_ASSUME_ALIGNED(row, 64));
+  }
+
+  // Returns const row pointer; usage: val = PlaneRow(idx_plane, y)[x].
+  JXL_INLINE const T* JXL_RESTRICT PlaneRow(const size_t c,
+                                            const size_t y) const {
+    PlaneRowBoundsCheck(c, y);
+    const size_t row_offset = y * planes_[0].bytes_per_row();
+    const void* row = planes_[c].bytes() + row_offset;
+    return static_cast<const T*>(JXL_ASSUME_ALIGNED(row, 64));
+  }
+
+  // Returns const row pointer, even if called from a non-const Image3.
+  JXL_INLINE const T* JXL_RESTRICT ConstPlaneRow(const size_t c,
+                                                 const size_t y) const {
+    return PlaneRow(c, y);
+  }
+
+  JXL_INLINE const PlaneT& Plane(size_t idx) const { return planes_[idx]; }
+
+  void Swap(Image3& other) {
+    for (size_t c = 0; c < 3; ++c) {
+      other.planes_[c].Swap(planes_[c]);
+    }
+  }
+
+  void ShrinkTo(const size_t xsize, const size_t ysize) {
+    for (PlaneT& plane : planes_) {
+      plane.ShrinkTo(xsize, ysize);
+    }
+  }
+
+  // Sizes of all three images are guaranteed to be equal.
+  JXL_INLINE size_t xsize() const { return planes_[0].xsize(); }
+  JXL_INLINE size_t ysize() const { return planes_[0].ysize(); }
+  // Returns offset [bytes] from one row to the next row of the same plane.
+  // WARNING: this must NOT be used to determine xsize, nor for copying rows -
+  // the valid xsize may be much less.
+  JXL_INLINE size_t bytes_per_row() const { return planes_[0].bytes_per_row(); }
+  // Returns number of pixels (some of which are padding) per row. Useful for
+  // computing other rows via pointer arithmetic. WARNING: this must NOT be used
+  // to determine xsize. NOTE: this is less efficient than
+  // ByteOffset(row, bytes_per_row).
+  JXL_INLINE intptr_t PixelsPerRow() const { return planes_[0].PixelsPerRow(); }
+
+ private:
+  void PlaneRowBoundsCheck(const size_t c, const size_t y) const {
+#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \
+    defined(THREAD_SANITIZER)
+    if (c >= kNumPlanes || y >= ysize()) {
+      Abort(__FILE__, __LINE__, "PlaneRow(%zu, %zu) >= %zu\n", c, y, ysize());
+    }
+#endif
+  }
+
+ private:
+  PlaneT planes_[kNumPlanes];
+};
+
+using Image3B = Image3<uint8_t>;
+using Image3S = Image3<int16_t>;
+using Image3U = Image3<uint16_t>;
+using Image3I = Image3<int32_t>;
+using Image3F = Image3<float>;
+using Image3D = Image3<double>;
+
+// Copies `from:rect_from` to `to:rect_to`.
+template <typename T>
+void CopyImageTo(const Rect& rect_from, const Image3<T>& from,
+                 const Rect& rect_to, Image3<T>* JXL_RESTRICT to) {
+  PROFILER_ZONE("CopyImageR");
+  JXL_ASSERT(SameSize(rect_from, rect_to));
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t y = 0; y < rect_to.ysize(); ++y) {
+      const T* JXL_RESTRICT row_from = rect_from.ConstPlaneRow(from, c, y);
+      T* JXL_RESTRICT row_to = rect_to.PlaneRow(to, c, y);
+      memcpy(row_to, row_from, rect_to.xsize() * sizeof(T));
+    }
+  }
+}
+
+template <typename T>
+void CopyImageTo(const Image3<T>& from, Image3<T>* JXL_RESTRICT to) {
+  return CopyImageTo(Rect(from), from, Rect(*to), to);
+}
+
+template <typename T>
+void CopyImageTo(const Rect& rect_from, const Image3<T>& from,
+                 Image3<T>* JXL_RESTRICT to) {
+  return CopyImageTo(rect_from, from, Rect(*to), to);
+}
+
+// Copies `from` to `to:rect_to`.
+template <typename T>
+void CopyImageTo(const Image3<T>& from, const Rect& rect_to,
+                 Image3<T>* JXL_RESTRICT to) {
+  return CopyImageTo(Rect(from), from, rect_to, to);
+}
+
+// DEPRECATED - prefer to preallocate result.
+template <typename T>
+Image3<T> CopyImage(const Image3<T>& from) {
+  Image3<T> copy(from.xsize(), from.ysize());
+  CopyImageTo(from, &copy);
+  return copy;
+}
+
+// DEPRECATED - prefer to preallocate result.
+template <typename T>
+Image3<T> CopyImage(const Rect& rect, const Image3<T>& from) {
+  Image3<T> to(rect.xsize(), rect.ysize());
+  CopyImageTo(rect, from.Plane(0), const_cast<ImageF*>(&to.Plane(0)));
+  CopyImageTo(rect, from.Plane(1), const_cast<ImageF*>(&to.Plane(1)));
+  CopyImageTo(rect, from.Plane(2), const_cast<ImageF*>(&to.Plane(2)));
+  return to;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_IMAGE_H_
diff --git a/jxl/image_bundle.cc b/jxl/image_bundle.cc
new file mode 100644
index 000000000..723a3214e
--- /dev/null
+++ b/jxl/image_bundle.cc
@@ -0,0 +1,370 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/image_bundle.h"
+
+#include <utility>
+
+#include "jxl/base/byte_order.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_management.h"
+#include "jxl/external_image.h"
+#include "jxl/fields.h"
+
+namespace jxl {
+namespace {
+
+Status FromSRGB(const size_t xsize, const size_t ysize, const bool is_gray,
+                const bool has_alpha, const bool is_16bit,
+                const bool big_endian, const uint8_t* pixels,
+                const uint8_t* end, ThreadPool* pool, ImageBundle* ib) {
+  const ColorEncoding& c = ColorManagement::SRGB(is_gray);
+  const size_t bits_per_sample = (is_16bit ? 2 : 1) * kBitsPerByte;
+  const PackedImage desc(xsize, ysize, c, has_alpha,
+                         /*alpha_bits=*/bits_per_sample, bits_per_sample,
+                         big_endian, /*flipped_y=*/false);
+  const Span<const uint8_t> span(pixels, end - pixels);
+  return CopyTo(desc, span, pool, ib);
+}
+
+// Copies interleaved external color; skips any alpha. Caller ensures
+// bits_per_sample matches T, and byte order=native.
+template <typename T>
+void AllocateAndFill(const ExternalImage& external, Image3<T>* out) {
+  JXL_ASSERT(external.IsHealthy());  // Callers must check beforehand.
+
+  // Here we just copy bytes for simplicity; for conversion/byte swapping, use
+  // ExternalImage::CopyTo instead.
+  JXL_CHECK(external.BitsPerSample() == sizeof(T) * kBitsPerByte);
+  JXL_CHECK(external.BigEndian() == !IsLittleEndian());
+
+  const size_t xsize = external.xsize();
+  const size_t ysize = external.ysize();
+  *out = Image3<T>(xsize, ysize);
+  if (external.IsGray()) {
+    if (external.HasAlpha()) {
+      for (size_t y = 0; y < ysize; ++y) {
+        const T* JXL_RESTRICT row =
+            reinterpret_cast<const T*>(external.ConstRow(y));
+        T* JXL_RESTRICT row0 = out->PlaneRow(0, y);
+        T* JXL_RESTRICT row1 = out->PlaneRow(1, y);
+        T* JXL_RESTRICT row2 = out->PlaneRow(2, y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row0[x] = row[2 * x + 0];
+          row1[x] = row[2 * x + 0];
+          row2[x] = row[2 * x + 0];
+        }
+      }
+    } else {
+      for (size_t y = 0; y < ysize; ++y) {
+        const T* JXL_RESTRICT row =
+            reinterpret_cast<const T*>(external.ConstRow(y));
+        T* JXL_RESTRICT row0 = out->PlaneRow(0, y);
+        T* JXL_RESTRICT row1 = out->PlaneRow(1, y);
+        T* JXL_RESTRICT row2 = out->PlaneRow(2, y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row0[x] = row[x];
+          row1[x] = row[x];
+          row2[x] = row[x];
+        }
+      }
+    }
+  } else {
+    if (external.HasAlpha()) {
+      for (size_t y = 0; y < ysize; ++y) {
+        const T* JXL_RESTRICT row =
+            reinterpret_cast<const T*>(external.ConstRow(y));
+        T* JXL_RESTRICT row0 = out->PlaneRow(0, y);
+        T* JXL_RESTRICT row1 = out->PlaneRow(1, y);
+        T* JXL_RESTRICT row2 = out->PlaneRow(2, y);
+
+        for (size_t x = 0; x < xsize; ++x) {
+          row0[x] = row[4 * x + 0];
+          row1[x] = row[4 * x + 1];
+          row2[x] = row[4 * x + 2];
+        }
+      }
+    } else {
+      for (size_t y = 0; y < ysize; ++y) {
+        const T* JXL_RESTRICT row =
+            reinterpret_cast<const T*>(external.ConstRow(y));
+        T* JXL_RESTRICT row0 = out->PlaneRow(0, y);
+        T* JXL_RESTRICT row1 = out->PlaneRow(1, y);
+        T* JXL_RESTRICT row2 = out->PlaneRow(2, y);
+        for (size_t x = 0; x < xsize; ++x) {
+          row0[x] = row[3 * x + 0];
+          row1[x] = row[3 * x + 1];
+          row2[x] = row[3 * x + 2];
+        }
+      }
+    }
+  }
+}
+
+// Copies ib:rect, converts, and copies into out.
+template <typename T>
+Status CopyToT(const ImageMetadata* metadata, const ImageBundle* ib,
+               const Rect& rect, const ColorEncoding& c_desired,
+               ThreadPool* pool, Image3<T>* out) {
+  PROFILER_FUNC;
+  // Changing IsGray is probably a bug.
+  JXL_CHECK(ib->IsGray() == c_desired.IsGray());
+
+  const ImageU* alpha = ib->HasAlpha() ? &ib->alpha() : nullptr;
+  const size_t bits_per_sample = sizeof(T) * kBitsPerByte;
+  const bool big_endian = !IsLittleEndian();
+  CodecIntervals* temp_intervals = nullptr;  // Don't need min/max.
+  const ExternalImage external(
+      pool, ib->color(), rect, ib->c_current(), c_desired, ib->HasAlpha(),
+      alpha, metadata->alpha_bits, bits_per_sample, big_endian, temp_intervals);
+  JXL_RETURN_IF_ERROR(external.IsHealthy());
+  AllocateAndFill(external, out);
+  return true;
+}
+
+}  // namespace
+
+ImageMetadata::ImageMetadata() { Bundle::Init(this); }
+ImageMetadata2::ImageMetadata2() { Bundle::Init(this); }
+OpsinInverseMatrix::OpsinInverseMatrix() { Bundle::Init(this); }
+
+Status ReadImageMetadata(BitReader* JXL_RESTRICT reader,
+                         ImageMetadata* JXL_RESTRICT metadata) {
+  return Bundle::Read(reader, metadata);
+}
+
+Status WriteImageMetadata(const ImageMetadata& metadata,
+                          BitWriter* JXL_RESTRICT writer, size_t layer,
+                          AuxOut* aux_out) {
+  return Bundle::Write(metadata, writer, layer, aux_out);
+}
+
+void ImageBundle::ShrinkTo(size_t xsize, size_t ysize) {
+  color_.ShrinkTo(xsize, ysize);
+  if (HasAlpha()) alpha_.ShrinkTo(xsize, ysize);
+  if (HasDepth()) {
+    depth_.ShrinkTo(DepthSize(xsize), DepthSize(ysize));
+  }
+  for (ImageU& plane : extra_channels_) {
+    plane.ShrinkTo(xsize, ysize);
+  }
+}
+
+// Called by all other SetFrom*.
+void ImageBundle::SetFromImage(Image3F&& color,
+                               const ColorEncoding& c_current) {
+  JXL_CHECK(color.xsize() != 0 && color.ysize() != 0);
+  JXL_CHECK(metadata_->color_encoding.IsGray() == c_current.IsGray());
+  color_ = std::move(color);
+  c_current_ = c_current;
+  VerifySizes();
+}
+
+Status ImageBundle::SetFromSRGB(size_t xsize, size_t ysize, bool is_gray,
+                                bool has_alpha, const uint8_t* pixels,
+                                const uint8_t* end, ThreadPool* pool) {
+  const bool big_endian = false;  // don't care since each sample is a byte
+  return FromSRGB(xsize, ysize, is_gray, has_alpha, /*is_16bit=*/false,
+                  big_endian, pixels, end, pool, this);
+}
+
+Status ImageBundle::SetFromSRGB(size_t xsize, size_t ysize, bool is_gray,
+                                bool has_alpha, const uint16_t* pixels,
+                                const uint16_t* end, ThreadPool* pool) {
+  const uint8_t* bytes = reinterpret_cast<const uint8_t*>(pixels);
+  const uint8_t* bytes_end = reinterpret_cast<const uint8_t*>(end);
+  // Given as uint16_t, so is in native order.
+  const bool big_endian = !IsLittleEndian();
+  return FromSRGB(xsize, ysize, is_gray, has_alpha, /*is_16bit=*/true,
+                  big_endian, bytes, bytes_end, pool, this);
+}
+
+Status ImageBundle::SetFromSRGB(size_t xsize, size_t ysize, bool is_gray,
+                                bool has_alpha, bool is_16bit, bool big_endian,
+                                const uint8_t* pixels, const uint8_t* end,
+                                ThreadPool* pool) {
+  return FromSRGB(xsize, ysize, is_gray, has_alpha, is_16bit, big_endian,
+                  pixels, end, pool, this);
+}
+
+Status ImageBundle::TransformTo(const ColorEncoding& c_desired,
+                                ThreadPool* pool) {
+  PROFILER_FUNC;
+  // Changing IsGray is probably a bug.
+  JXL_CHECK(IsGray() == c_desired.IsGray());
+
+  const ImageU* alpha = HasAlpha() ? &alpha_ : nullptr;
+  const size_t alpha_bits = HasAlpha() ? metadata_->alpha_bits : 0;
+  const bool big_endian = !IsLittleEndian();
+  CodecIntervals temp_intervals;
+  const ExternalImage external(pool, color_, Rect(color_), c_current_,
+                               c_desired, HasAlpha(), alpha, alpha_bits, 32,
+                               big_endian, &temp_intervals);
+  return external.IsHealthy() && external.CopyTo(&temp_intervals, pool, this);
+}
+
+Status ImageBundle::CopyTo(const Rect& rect, const ColorEncoding& c_desired,
+                           Image3B* out, ThreadPool* pool) const {
+  return CopyToT(metadata_, this, rect, c_desired, pool, out);
+}
+Status ImageBundle::CopyTo(const Rect& rect, const ColorEncoding& c_desired,
+                           Image3U* out, ThreadPool* pool) const {
+  return CopyToT(metadata_, this, rect, c_desired, pool, out);
+}
+Status ImageBundle::CopyTo(const Rect& rect, const ColorEncoding& c_desired,
+                           Image3F* out, ThreadPool* pool) const {
+  return CopyToT(metadata_, this, rect, c_desired, pool, out);
+}
+
+Status ImageBundle::CopyToSRGB(const Rect& rect, Image3B* out,
+                               ThreadPool* pool) const {
+  return CopyTo(rect, ColorManagement::SRGB(IsGray()), out, pool);
+}
+
+void ImageBundle::VerifyMetadata() const {
+  JXL_CHECK(!c_current_.icc.empty());
+  JXL_CHECK(metadata_->color_encoding.IsGray() == IsGray());
+
+  if (metadata_->HasAlpha() != HasAlpha()) {
+    JXL_ABORT("MD alpha_bits %u IB alpha %zu x %zu\n", metadata_->alpha_bits,
+              alpha_.xsize(), alpha_.ysize());
+  }
+
+  const uint32_t alpha_bits = metadata_->alpha_bits;
+  JXL_CHECK(alpha_bits <= 16);
+
+  JXL_CHECK(metadata_->m2.HasDepth() == HasDepth());
+
+  JXL_CHECK(metadata_->m2.num_extra_channels == extra_channels_.size());
+}
+
+void ImageBundle::VerifySizes() const {
+  const size_t xs = xsize();
+  const size_t ys = ysize();
+  // Can only verify alpha/depth after SetFrom* or SetPlanes is called.
+  if (xs != 0 && ys != 0) {
+    if (HasAlpha()) {
+      JXL_CHECK(DivCeil(alpha_.xsize(), kBlockDim) == DivCeil(xs, kBlockDim));
+      JXL_CHECK(DivCeil(alpha_.ysize(), kBlockDim) == DivCeil(ys, kBlockDim));
+    }
+
+    if (HasDepth()) {
+      JXL_CHECK(depth_.xsize() == DepthSize(xs) &&
+                depth_.ysize() == DepthSize(ys));
+    }
+  }
+
+  if (HasExtraChannels()) {
+    JXL_CHECK(xs != 0 && ys != 0);
+    for (const ImageU& plane : extra_channels_) {
+      JXL_CHECK(plane.xsize() == xs && plane.ysize() == ys);
+    }
+  }
+}
+
+size_t ImageBundle::DetectRealBitdepth() const {
+  const size_t orig_d = metadata_->bits_per_sample;
+  const size_t maxval = (1 << orig_d) - 1;
+  const double factor = maxval / 255.;
+  size_t real_d = 1;
+  double f = maxval;
+
+  // Start with assuming that 1-bit is enough, then go over all pixels and check
+  // if the original value can be reconstructed (within a margin of 0.6) at that
+  // bit depth (it won't, unless it's a pure black & white image). As soon as
+  // you find a pixel that needs more bit depth, you increment the assumed
+  // bit-depth and try again. You don't need to revisit the pixels that were
+  // already OK at the lower bit depth (you can assume they will still be OK at
+  // the higher bit depth), but you do need to revisit the current pixel.
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t y = 0; y < ysize(); y++) {
+      const float* JXL_RESTRICT row = color_.PlaneRow(c, y);
+      for (size_t x = 0; x < xsize(); x++) {
+        int32_t po = row[x] * factor + 0.5;
+        // allow a rescaling error of +- 0.6
+        // if you set this to 1.0 or more, the 'real' bit depth will be too low
+        // (e.g. 7-bit becomes enough to encode 8-bit) if you set this to 0.5 or
+        // less, it will fail to find the real bit depth
+        if (std::abs(row[x] * factor - (po >> (orig_d - real_d)) * f) > 0.6) {
+          real_d++;
+          if (real_d == orig_d) return orig_d;
+          f = ((double)maxval) / ((1 << real_d) - 1);
+          x--;
+        }
+      }
+    }
+  }
+  JXL_DEBUG_V(0, "Interpreting input as %zu-bit (nominally it is %zu-bit)",
+              real_d, orig_d);
+  return real_d;
+}
+
+void ImageBundle::RemoveAlpha() {
+  alpha_ = ImageU();
+  JXL_ASSERT(!HasAlpha());
+  metadata_->alpha_bits = 0;  // maintain invariant for VerifyMetadata
+}
+
+void ImageBundle::SetAlpha(ImageU&& alpha) {
+  JXL_CHECK(alpha.xsize() != 0 && alpha.ysize() != 0);
+  JXL_CHECK(metadata_->alpha_bits != 0);
+  alpha_ = std::move(alpha);
+  VerifySizes();
+}
+
+void ImageBundle::SetDepth(ImageU&& depth) {
+  JXL_CHECK(depth.xsize() != 0 && depth.ysize() != 0);
+  JXL_CHECK(metadata_->m2.depth_bits != 0);
+  depth_ = std::move(depth);
+  VerifySizes();
+}
+
+void ImageBundle::SetExtraChannels(std::vector<ImageU>&& extra_channels) {
+  JXL_CHECK(!extra_channels.empty());
+  for (const ImageU& plane : extra_channels) {
+    JXL_CHECK(plane.xsize() != 0 && plane.ysize() != 0);
+  }
+  extra_channels_ = std::move(extra_channels);
+  VerifySizes();
+}
+
+Status TransformIfNeeded(const ImageBundle& in, const ColorEncoding& c_desired,
+                         ThreadPool* pool, ImageBundle* store,
+                         const ImageBundle** out) {
+  if (in.c_current().SameColorEncoding(c_desired)) {
+    *out = &in;
+    return true;
+  }
+  // TODO(janwas): avoid copying via createExternal+copyBackToIO
+  // instead of copy+createExternal+copyBackToIO
+  store->SetFromImage(CopyImage(in.color()), in.c_current());
+  if (!store->TransformTo(c_desired, pool)) {
+    return false;
+  }
+  *out = store;
+  return true;
+}
+
+float ChooseDefaultIntensityTarget(const ImageMetadata& metadata) {
+  if (metadata.color_encoding.tf.IsPQ() || metadata.color_encoding.tf.IsHLG()) {
+    // HDR
+    return 4000;
+  }
+  // SDR
+  return kDefaultIntensityTarget;
+}
+
+}  // namespace jxl
diff --git a/jxl/image_bundle.h b/jxl/image_bundle.h
new file mode 100644
index 000000000..195319de0
--- /dev/null
+++ b/jxl/image_bundle.h
@@ -0,0 +1,445 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_IMAGE_BUNDLE_H_
+#define JXL_IMAGE_BUNDLE_H_
+
+// The main image or frame consists of a bundle of associated images.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/color_encoding.h"
+#include "jxl/common.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/dec_xyb.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/field_encodings.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/opsin_params.h"
+#include "jxl/quantizer.h"
+
+namespace jxl {
+
+// EXIF orientation of the image. This field overrides any field present in
+// actual EXIF metadata. The value tells which transformation the decoder must
+// apply after decoding to display the image with the correct orientation.
+enum class Orientation : uint32_t {
+  // Values 1..8 match the EXIF definitions.
+  kIdentity = 1,
+  kFlipHorizontal,
+  kRotate180,
+  kFlipVertical,
+  kTranspose,
+  kRotate90,
+  kAntiTranspose,
+  kRotate270,
+};
+// Don't need an EnumBits because Orientation is not read via Enum().
+
+struct ExtraChannelInfo {
+  uint32_t rendered;  // 0 = no, 1 = as spot color, other values undefined
+  float color[4];     // spot color in linear RGBA
+};
+
+struct OpsinInverseMatrix {
+  OpsinInverseMatrix();
+  static const char* Name() { return "OpsinInverseMatrix"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    if (visitor->AllDefault(*this, &all_default)) return true;
+    for (int i = 0; i < 9; ++i) {
+      visitor->F16(DefaultInverseOpsinAbsorbanceMatrix()[i],
+                   &inverse_matrix[i]);
+    }
+    for (int i = 0; i < 3; ++i) {
+      visitor->F16(kNegOpsinAbsorbanceBiasRGB[i], &opsin_biases[i]);
+    }
+    for (int i = 0; i < 4; ++i) {
+      visitor->F16(kDefaultQuantBias[i], &quant_biases[i]);
+    }
+    return true;
+  }
+
+  OpsinParams ToOpsinParams() const {
+    OpsinParams opsin_params;
+    InitSIMDInverseMatrix(inverse_matrix, opsin_params.inverse_opsin_matrix);
+    std::copy(std::begin(opsin_biases), std::end(opsin_biases),
+              opsin_params.opsin_biases);
+    for (int i = 0; i < 3; ++i) {
+      opsin_params.opsin_biases_cbrt[i] =
+          std::cbrt(opsin_params.opsin_biases[i]);
+    }
+    opsin_params.opsin_biases_cbrt[3] = opsin_params.opsin_biases[3] = 255;
+    std::copy(std::begin(quant_biases), std::end(quant_biases),
+              opsin_params.quant_biases);
+    return opsin_params;
+  }
+
+  mutable bool all_default;
+
+  float inverse_matrix[9];
+  float opsin_biases[3];
+  float quant_biases[4];
+};
+
+// Less frequently changed fields, grouped into a separate bundle so they do not
+// need to be signaled when some ImageMetadata fields are non-default.
+struct ImageMetadata2 {
+  ImageMetadata2();
+  static const char* Name() { return "ImageMetadata2"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    if (visitor->AllDefault(*this, &all_default)) return true;
+
+    JXL_RETURN_IF_ERROR(visitor->VisitNested(&opsin_inverse_matrix));
+
+    visitor->Bool(false, &have_preview);
+    visitor->Bool(false, &have_animation);
+
+    visitor->Bits(3, 0, &orientation_minus_1);
+    // (No need for bounds checking because we read exactly 3 bits)
+
+    visitor->U32(Val(0), Val(8), Val(16), Bits(4), 0, &depth_bits);
+    visitor->U32(Val(0), Val(3), Val(4), BitsOffset(3, 1), 0, &depth_shift);
+    if ((1U << depth_shift) > kGroupDim) {
+      return JXL_FAILURE("depth_shift too large");
+    }
+
+    visitor->U32(Val(0), Bits(4), BitsOffset(8, 16), BitsOffset(12, 1), 0,
+                 &num_extra_channels);
+    if (visitor->Conditional(num_extra_channels != 0)) {
+      visitor->U32(Val(0), Val(8), Val(16), Bits(4), 0, &extra_channel_bits);
+      extra_channel_info.resize(num_extra_channels);
+      for (size_t i = 0; i < num_extra_channels; i++) {
+        visitor->U32(Val(0), Val(1), Val(2), Bits(6), 0,
+                     &extra_channel_info[i].rendered);
+        if (visitor->Conditional(extra_channel_info[i].rendered == 1)) {
+          for (float& c : extra_channel_info[i].color) {
+            visitor->F16(0, &c);
+          }
+        }
+      }
+    }
+
+    visitor->BeginExtensions(&extensions);
+    // Extensions: in chronological order of being added to the format.
+    return visitor->EndExtensions();
+  }
+
+  bool HasDepth() const { return depth_bits != 0; }
+
+  mutable bool all_default;
+
+  OpsinInverseMatrix opsin_inverse_matrix;
+
+  bool have_preview;
+  bool have_animation;
+
+  uint32_t orientation_minus_1;
+  uint32_t depth_bits;  // 0 if no depth channel present
+  uint32_t depth_shift;
+
+  uint32_t num_extra_channels;
+  uint32_t extra_channel_bits;  // 0 if num_extra_channels == 0
+  std::vector<ExtraChannelInfo> extra_channel_info;
+
+  uint64_t extensions;
+};
+
+// Properties of the original image bundle. This enables Encode(Decode()) to
+// re-create an equivalent image without user input.
+struct ImageMetadata {
+  ImageMetadata();
+  static const char* Name() { return "ImageMetadata"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    if (visitor->AllDefault(*this, &all_default)) return true;
+
+    visitor->Bool(false, &have_icc);
+
+    visitor->U32(Val(8), Val(16), Val(32), Bits(5), 8, &bits_per_sample);
+    JXL_RETURN_IF_ERROR(visitor->VisitNested(&color_encoding));
+    visitor->U32(Val(0), Val(8), Val(16), Bits(4), 0, &alpha_bits);
+
+    // 250, 1000, 4000 are common; don't anticipate more than 10,000.
+    visitor->U32(Val(5), Val(20), Val(80), BitsOffset(10, 1),
+                 kDefaultIntensityTarget / 50, &target_nits_div50_);
+
+    JXL_RETURN_IF_ERROR(visitor->VisitNested(&m2));
+
+    return true;
+  }
+
+  bool HasAlpha() const { return alpha_bits != 0; }
+
+  void SetIntensityTarget(float intensity_target) {
+    target_nits_div50_ = static_cast<uint32_t>(intensity_target * 0.02f);
+  }
+  float IntensityTarget() const { return target_nits_div50_ * 50.f; }
+
+  mutable bool all_default;
+  bool have_icc;
+
+  ImageMetadata2 m2;  // often default
+
+  uint32_t bits_per_sample;
+  ColorEncoding color_encoding;
+  uint32_t alpha_bits;  // 0 if no alpha channel present
+
+ private:
+  uint32_t target_nits_div50_;
+};
+
+// Chooses a default intensity target based on the transfer function of the
+// image, if known. For SDR images or images not known to be HDR, returns
+// kDefaultIntensityTarget, for images known to have PQ or HLG transfer function
+// returns a higher value.
+float ChooseDefaultIntensityTarget(const ImageMetadata& metadata);
+
+Status ReadImageMetadata(BitReader* JXL_RESTRICT reader,
+                         ImageMetadata* JXL_RESTRICT metadata);
+
+Status WriteImageMetadata(const ImageMetadata& metadata,
+                          BitWriter* JXL_RESTRICT writer, size_t layer,
+                          AuxOut* aux_out);
+
+// A bundle of color/alpha/depth/plane images.
+class ImageBundle {
+ public:
+  // Caller is responsible for setting metadata before calling Set*.
+  explicit ImageBundle(ImageMetadata* metadata) : metadata_(metadata) {}
+
+  // Move-only (allows storing in std::vector).
+  ImageBundle(ImageBundle&&) = default;
+  ImageBundle& operator=(ImageBundle&&) = default;
+
+  ImageBundle Copy() const {
+    ImageBundle copy(metadata_);
+    copy.color_ = CopyImage(color_);
+    copy.c_current_ = c_current_;
+    copy.alpha_ = CopyImage(alpha_);
+    copy.depth_ = CopyImage(depth_);
+    copy.extra_channels_.reserve(extra_channels_.size());
+    for (const ImageU& plane : extra_channels_) {
+      copy.extra_channels_.emplace_back(CopyImage(plane));
+    }
+    return copy;
+  }
+
+  // -- SIZE
+
+  size_t xsize() const {
+    if (jpeg_xsize != 0) return jpeg_xsize;
+    if (color_.xsize() != 0) return color_.xsize();
+    return extra_channels_.empty() ? 0 : extra_channels_[0].xsize();
+  }
+  size_t ysize() const {
+    if (jpeg_ysize != 0) return jpeg_ysize;
+    if (color_.ysize() != 0) return color_.ysize();
+    return extra_channels_.empty() ? 0 : extra_channels_[0].ysize();
+  }
+  void ShrinkTo(size_t xsize, size_t ysize);
+
+  // -- COLOR
+
+  // Whether color() is valid/usable. Returns true in most cases. Even images
+  // with spot colors (one example of when !planes().empty()) typically have a
+  // part that can be converted to RGB.
+  bool HasColor() const { return color_.xsize() != 0; }
+
+  // For resetting the size when switching from a reference to main frame.
+  void RemoveColor() { color_ = Image3F(); }
+
+  // Do not use if !HasColor().
+  const Image3F& color() const {
+    // If this fails, Set* was not called - perhaps because decoding failed?
+    JXL_DASSERT(HasColor());
+    return color_;
+  }
+
+  // Do not use if !HasColor().
+  Image3F* MutableColor() {
+    JXL_DASSERT(HasColor());
+    return &color_;
+  }
+
+  // If c_current.IsGray(), all planes must be identical. NOTE: c_current is
+  // independent of metadata()->color_encoding, which is the original, whereas
+  // a decoder might return pixels in a different c_current.
+  void SetFromImage(Image3F&& color, const ColorEncoding& c_current);
+
+  // Sets image data from 8-bit sRGB pixel array in bytes.
+  // Amount of input bytes per pixel must be:
+  // (is_gray ? 1 : 3) + (has_alpha ? 1 : 0)
+  Status SetFromSRGB(size_t xsize, size_t ysize, bool is_gray, bool has_alpha,
+                     const uint8_t* pixels, const uint8_t* end,
+                     ThreadPool* pool = nullptr);
+
+  // Sets image data from 16-bit sRGB data.
+  // Amount of input uint16_t's per pixel must be:
+  // (is_gray ? 1 : 3) + (has_alpha ? 1 : 0)
+  Status SetFromSRGB(size_t xsize, size_t ysize, bool is_gray, bool has_alpha,
+                     const uint16_t* pixels, const uint16_t* end,
+                     ThreadPool* pool = nullptr);
+
+  // Sets image data from sRGB pixel array in bytes.
+  // This low-level function supports both 8-bit and 16-bit data in bytes to
+  // provide efficient access to arbitrary byte order.
+  // Amount of input bytes per pixel must be:
+  // ((is_gray ? 1 : 3) + (has_alpha ? 1 : 0)) * (is_16bit ? 2 : 1)
+  // The ordering of the channels is interleaved RGBA or gray+alpha in that
+  // order.
+  // The 16-bit byte order is given by big_endian, and this has no effect when
+  // is_16bit is false.
+  Status SetFromSRGB(size_t xsize, size_t ysize, bool is_gray, bool has_alpha,
+                     bool is_16bit, bool big_endian, const uint8_t* pixels,
+                     const uint8_t* end, ThreadPool* pool = nullptr);
+
+  // -- COLOR ENCODING
+
+  const ColorEncoding& c_current() const { return c_current_; }
+
+  // Returns whether the color image has identical planes. Once established by
+  // Set*, remains unchanged until a subsequent Set* or TransformTo.
+  bool IsGray() const { return c_current_.IsGray(); }
+
+  bool IsSRGB() const { return c_current_.IsSRGB(); }
+  bool IsLinearSRGB() const {
+    return c_current_.white_point == WhitePoint::kD65 &&
+           c_current_.primaries == Primaries::kSRGB && c_current_.tf.IsLinear();
+  }
+
+  // Transforms color to c_desired and sets c_current to c_desired. Alpha and
+  // metadata remains unchanged.
+  Status TransformTo(const ColorEncoding& c_desired,
+                     ThreadPool* pool = nullptr);
+
+  // Copies this:rect, converts to c_desired, and allocates+fills out.
+  Status CopyTo(const Rect& rect, const ColorEncoding& c_desired, Image3B* out,
+                ThreadPool* pool = nullptr) const;
+  Status CopyTo(const Rect& rect, const ColorEncoding& c_desired, Image3U* out,
+                ThreadPool* pool = nullptr) const;
+  Status CopyTo(const Rect& rect, const ColorEncoding& c_desired, Image3F* out,
+                ThreadPool* pool = nullptr) const;
+  Status CopyToSRGB(const Rect& rect, Image3B* out,
+                    ThreadPool* pool = nullptr) const;
+
+  // Detect 'real' bit depth, which can be lower than nominal bit depth
+  // (this is common in PNG), returns 'real' bit depth
+  size_t DetectRealBitdepth() const;
+
+  // -- ALPHA
+
+  void SetAlpha(ImageU&& alpha);
+  bool HasAlpha() const { return alpha_.xsize() != 0; }
+  const ImageU& alpha() const {
+    JXL_ASSERT(HasAlpha());
+    return alpha_;
+  }
+
+  // Reverts SetAlpha AND sets metadata->alpha_bits to 0. Called after noticing
+  // that all alpha values are opaque.
+  void RemoveAlpha();
+
+  // -- DEPTH
+
+  void SetDepth(ImageU&& depth);
+  bool HasDepth() const { return depth_.xsize() != 0; }
+  const ImageU& depth() const {
+    JXL_ASSERT(HasDepth());
+    return depth_;
+  }
+  // Returns the dimensions of the depth image
+  size_t DepthSize(size_t size) const {
+    const size_t mask = (1u << metadata_->m2.depth_shift) - 1;
+    return (size + mask) >> metadata_->m2.depth_shift;
+  }
+
+  // -- EXTRA CHANNELS
+
+  // Extra channels of unknown interpretation (e.g. spot colors).
+  void SetExtraChannels(std::vector<ImageU>&& extra_channels);
+  bool HasExtraChannels() const { return !extra_channels_.empty(); }
+  const std::vector<ImageU>& extra_channels() const {
+    JXL_ASSERT(HasExtraChannels());
+    return extra_channels_;
+  }
+
+  const ImageMetadata* metadata() const { return metadata_; }
+
+  void VerifyMetadata() const;
+
+  void SetDecodedBytes(size_t decoded_bytes) { decoded_bytes_ = decoded_bytes; }
+  size_t decoded_bytes() const { return decoded_bytes_; }
+
+  // if not empty: image data represents quantized DCT-8 coefficients
+  // (used for JPEG transcoding)
+  std::vector<int> jpeg_quant_table;
+  size_t jpeg_xsize = 0;  // image dimensions of input JPEG
+  size_t jpeg_ysize = 0;  // (can be up to 7 smaller than color_ dimensions)
+  // these fields are used to signal the input JPEG color space
+  ColorTransform color_transform = ColorTransform::kNone;
+  YCbCrChromaSubsampling chroma_subsampling = YCbCrChromaSubsampling::k444;
+
+ private:
+  // Called after any Set* to ensure their sizes are compatible.
+  void VerifySizes() const;
+
+  // Required for TransformTo so that an ImageBundle is self-sufficient. Always
+  // points to the same thing, but cannot be const-pointer because that prevents
+  // the compiler from generating a move ctor. Mostly pointer-to-const except as
+  // required by RemoveAlpha.
+  ImageMetadata* metadata_;
+
+  // Initialized by Set*:
+  Image3F color_;  // If empty, planes_ is not; all planes equal if IsGray().
+  ColorEncoding c_current_;  // of color_
+
+  // Initialized by SetAlpha.
+  ImageU alpha_;  // Empty or same size as color_.
+
+  // Initialized by SetDepth.
+  ImageU depth_;  // Empty or size >> depth_shift_.
+
+  // Initialized by SetPlanes; size = ImageMetadata.num_extra_channels
+  std::vector<ImageU> extra_channels_;
+
+  // How many bytes of the input were actually read.
+  size_t decoded_bytes_ = 0;
+};
+
+// Does color transformation from in.c_current() to c_desired if the color
+// encodings are different, or nothing if they are already the same.
+// If color transformation is done, stores the transformed values into store and
+// sets the out pointer to store, else leaves store untouched and sets the out
+// pointer to &in.
+// Returns false if color transform fails.
+Status TransformIfNeeded(const ImageBundle& in, const ColorEncoding& c_desired,
+                         ThreadPool* pool, ImageBundle* store,
+                         const ImageBundle** out);
+
+}  // namespace jxl
+
+#endif  // JXL_IMAGE_BUNDLE_H_
diff --git a/jxl/image_ops.h b/jxl/image_ops.h
new file mode 100644
index 000000000..0cf0445ec
--- /dev/null
+++ b/jxl/image_ops.h
@@ -0,0 +1,702 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_IMAGE_OPS_H_
+#define JXL_IMAGE_OPS_H_
+
+// Operations on images.
+
+#include "jxl/image.h"
+
+namespace jxl {
+
+// Sets "thickness" pixels on each border to "value". This is faster than
+// initializing the entire image and overwriting valid/interior pixels.
+template <typename T>
+void SetBorder(const size_t thickness, const T value, Image3<T>* image) {
+  const size_t xsize = image->xsize();
+  const size_t ysize = image->ysize();
+  JXL_ASSERT(2 * thickness < xsize && 2 * thickness < ysize);
+  // Top
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < thickness; ++y) {
+      T* JXL_RESTRICT row = image->PlaneRow(c, y);
+      std::fill(row, row + xsize, value);
+    }
+
+    // Bottom
+    for (size_t y = ysize - thickness; y < ysize; ++y) {
+      T* JXL_RESTRICT row = image->PlaneRow(c, y);
+      std::fill(row, row + xsize, value);
+    }
+
+    // Left/right
+    for (size_t y = thickness; y < ysize - thickness; ++y) {
+      T* JXL_RESTRICT row = image->PlaneRow(c, y);
+      std::fill(row, row + thickness, value);
+      std::fill(row + xsize - thickness, row + xsize, value);
+    }
+  }
+}
+
+template <class ImageIn, class ImageOut>
+void Subtract(const ImageIn& image1, const ImageIn& image2, ImageOut* out) {
+  using T = typename ImageIn::T;
+  const size_t xsize = image1.xsize();
+  const size_t ysize = image1.ysize();
+  JXL_CHECK(xsize == image2.xsize());
+  JXL_CHECK(ysize == image2.ysize());
+
+  for (size_t y = 0; y < ysize; ++y) {
+    const T* const JXL_RESTRICT row1 = image1.Row(y);
+    const T* const JXL_RESTRICT row2 = image2.Row(y);
+    T* const JXL_RESTRICT row_out = out->Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      row_out[x] = row1[x] - row2[x];
+    }
+  }
+}
+
+// In-place.
+template <typename Tin, typename Tout>
+void SubtractFrom(const Plane<Tin>& what, Plane<Tout>* to) {
+  const size_t xsize = what.xsize();
+  const size_t ysize = what.ysize();
+  for (size_t y = 0; y < ysize; ++y) {
+    const Tin* JXL_RESTRICT row_what = what.ConstRow(y);
+    Tout* JXL_RESTRICT row_to = to->Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      row_to[x] -= row_what[x];
+    }
+  }
+}
+
+// In-place.
+template <typename Tin, typename Tout>
+void AddTo(const Plane<Tin>& what, Plane<Tout>* to) {
+  const size_t xsize = what.xsize();
+  const size_t ysize = what.ysize();
+  for (size_t y = 0; y < ysize; ++y) {
+    const Tin* JXL_RESTRICT row_what = what.ConstRow(y);
+    Tout* JXL_RESTRICT row_to = to->Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      row_to[x] += row_what[x];
+    }
+  }
+}
+
+template <typename Tin, typename Tout>
+void AddTo(Rect rectFrom, const Plane<Tin>& what, Rect rectTo,
+           const Plane<Tout>* to) {
+  JXL_ASSERT(SameSize(rectFrom, rectTo));
+  const size_t xsize = rectTo.xsize();
+  const size_t ysize = rectTo.ysize();
+  for (size_t y = 0; y < ysize; ++y) {
+    const Tin* JXL_RESTRICT row_what = rectFrom.ConstRow(what, y);
+    Tout* JXL_RESTRICT row_to = rectTo.MutableRow(to, y);
+    for (size_t x = 0; x < xsize; ++x) {
+      row_to[x] += row_what[x];
+    }
+  }
+}
+
+// Returns linear combination of two grayscale images.
+template <typename T>
+Plane<T> LinComb(const T lambda1, const Plane<T>& image1, const T lambda2,
+                 const Plane<T>& image2) {
+  const size_t xsize = image1.xsize();
+  const size_t ysize = image1.ysize();
+  JXL_CHECK(xsize == image2.xsize());
+  JXL_CHECK(ysize == image2.ysize());
+  Plane<T> out(xsize, ysize);
+  for (size_t y = 0; y < ysize; ++y) {
+    const T* const JXL_RESTRICT row1 = image1.Row(y);
+    const T* const JXL_RESTRICT row2 = image2.Row(y);
+    T* const JXL_RESTRICT row_out = out.Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      row_out[x] = lambda1 * row1[x] + lambda2 * row2[x];
+    }
+  }
+  return out;
+}
+
+// Returns a pixel-by-pixel multiplication of image by lambda.
+template <typename T>
+Plane<T> ScaleImage(const T lambda, const Plane<T>& image) {
+  Plane<T> out(image.xsize(), image.ysize());
+  for (size_t y = 0; y < image.ysize(); ++y) {
+    const T* const JXL_RESTRICT row = image.Row(y);
+    T* const JXL_RESTRICT row_out = out.Row(y);
+    for (size_t x = 0; x < image.xsize(); ++x) {
+      row_out[x] = lambda * row[x];
+    }
+  }
+  return out;
+}
+
+// Multiplies image by lambda in-place
+template <typename T>
+void ScaleImage(const T lambda, Plane<T>* image) {
+  for (size_t y = 0; y < image->ysize(); ++y) {
+    T* const JXL_RESTRICT row = image->Row(y);
+    for (size_t x = 0; x < image->xsize(); ++x) {
+      row[x] = lambda * row[x];
+    }
+  }
+}
+
+template <typename T>
+Plane<T> Product(const Plane<T>& a, const Plane<T>& b) {
+  Plane<T> c(a.xsize(), a.ysize());
+  for (size_t y = 0; y < a.ysize(); ++y) {
+    const T* const JXL_RESTRICT row_a = a.Row(y);
+    const T* const JXL_RESTRICT row_b = b.Row(y);
+    T* const JXL_RESTRICT row_c = c.Row(y);
+    for (size_t x = 0; x < a.xsize(); ++x) {
+      row_c[x] = row_a[x] * row_b[x];
+    }
+  }
+  return c;
+}
+
+float DotProduct(const ImageF& a, const ImageF& b);
+
+template <typename T>
+void FillImage(const T value, Plane<T>* image) {
+  for (size_t y = 0; y < image->ysize(); ++y) {
+    T* const JXL_RESTRICT row = image->Row(y);
+    for (size_t x = 0; x < image->xsize(); ++x) {
+      row[x] = value;
+    }
+  }
+}
+
+template <typename T>
+void ZeroFillImage(Plane<T>* image) {
+  for (size_t y = 0; y < image->ysize(); ++y) {
+    T* const JXL_RESTRICT row = image->Row(y);
+    memset(row, 0, image->xsize() * sizeof(T));
+  }
+}
+
+// Mirrors out of bounds coordinates and returns valid coordinates unchanged.
+// We assume the radius (distance outside the image) is small compared to the
+// image size, otherwise this might not terminate.
+// The mirror is outside the last column (border pixel is also replicated).
+static inline int64_t Mirror(int64_t x, const int64_t xsize) {
+  // TODO(janwas): replace with branchless version
+  while (x < 0 || x >= xsize) {
+    if (x < 0) {
+      x = -x - 1;
+    } else {
+      x = 2 * xsize - 1 - x;
+    }
+  }
+  return x;
+}
+
+// Wrap modes for ensuring X/Y coordinates are in the valid range [0, size):
+
+// Mirrors (repeating the edge pixel once). Useful for convolutions.
+struct WrapMirror {
+  JXL_INLINE int64_t operator()(const int64_t coord, const int64_t size) const {
+    return Mirror(coord, size);
+  }
+};
+
+// Repeats the edge pixel.
+struct WrapClamp {
+  JXL_INLINE int64_t operator()(const int64_t coord, const int64_t size) const {
+    return std::min(std::max<int64_t>(0, coord), size - 1);
+  }
+};
+
+// Returns the same coordinate: required for TFNode with Border(), or useful
+// when we know "coord" is already valid (e.g. interior of an image).
+struct WrapUnchanged {
+  JXL_INLINE int64_t operator()(const int64_t coord, int64_t /*size*/) const {
+    return coord;
+  }
+};
+
+// Similar to Wrap* but for row pointers (reduces Row() multiplications).
+
+class WrapRowMirror {
+ public:
+  template <class ImageOrView>
+  WrapRowMirror(const ImageOrView& image, size_t ysize)
+      : first_row_(image.ConstRow(0)), last_row_(image.ConstRow(ysize - 1)) {}
+
+  const float* operator()(const float* const JXL_RESTRICT row,
+                          const int64_t stride) const {
+    if (row < first_row_) {
+      const int64_t num_before = first_row_ - row;
+      // Mirrored; one row before => row 0, two before = row 1, ...
+      return first_row_ + num_before - stride;
+    }
+    if (row > last_row_) {
+      const int64_t num_after = row - last_row_;
+      // Mirrored; one row after => last row, two after = last - 1, ...
+      return last_row_ - num_after + stride;
+    }
+    return row;
+  }
+
+ private:
+  const float* const JXL_RESTRICT first_row_;
+  const float* const JXL_RESTRICT last_row_;
+};
+
+struct WrapRowUnchanged {
+  JXL_INLINE const float* operator()(const float* const JXL_RESTRICT row,
+                                     int64_t /*stride*/) const {
+    return row;
+  }
+};
+
+// Sets "thickness" pixels on each border to "value". This is faster than
+// initializing the entire image and overwriting valid/interior pixels.
+template <typename T>
+void SetBorder(const size_t thickness, const T value, Plane<T>* image) {
+  const size_t xsize = image->xsize();
+  const size_t ysize = image->ysize();
+  JXL_ASSERT(2 * thickness < xsize && 2 * thickness < ysize);
+  // Top
+  for (size_t y = 0; y < thickness; ++y) {
+    T* const JXL_RESTRICT row = image->Row(y);
+    std::fill(row, row + xsize, value);
+  }
+
+  // Bottom
+  for (size_t y = ysize - thickness; y < ysize; ++y) {
+    T* const JXL_RESTRICT row = image->Row(y);
+    std::fill(row, row + xsize, value);
+  }
+
+  // Left/right
+  for (size_t y = thickness; y < ysize - thickness; ++y) {
+    T* const JXL_RESTRICT row = image->Row(y);
+    std::fill(row, row + thickness, value);
+    std::fill(row + xsize - thickness, row + xsize, value);
+  }
+}
+
+// Computes the minimum and maximum pixel value.
+template <typename T>
+void ImageMinMax(const Plane<T>& image, T* const JXL_RESTRICT min,
+                 T* const JXL_RESTRICT max) {
+  *min = std::numeric_limits<T>::max();
+  *max = std::numeric_limits<T>::lowest();
+  for (size_t y = 0; y < image.ysize(); ++y) {
+    const T* const JXL_RESTRICT row = image.Row(y);
+    for (size_t x = 0; x < image.xsize(); ++x) {
+      *min = std::min(*min, row[x]);
+      *max = std::max(*max, row[x]);
+    }
+  }
+}
+
+// Computes the average pixel value.
+template <typename T>
+double ImageAverage(const Plane<T>& image) {
+  double result = 0;
+  size_t n = 0;
+  for (size_t y = 0; y < image.ysize(); ++y) {
+    const T* const JXL_RESTRICT row = image.Row(y);
+    for (size_t x = 0; x < image.xsize(); ++x) {
+      // Numerically stable method.
+      double v = row[x];
+      double delta = v - result;
+      n++;
+      result += delta / n;
+    }
+  }
+  return result;
+}
+
+// Copies pixels, scaling their value relative to the "from" min/max by
+// "to_range". Example: U8 [0, 255] := [0.0, 1.0], to_range = 1.0 =>
+// outputs [0.0, 1.0].
+template <typename FromType, typename ToType>
+void ImageConvert(const Plane<FromType>& from, const float to_range,
+                  Plane<ToType>* const JXL_RESTRICT to) {
+  JXL_ASSERT(SameSize(from, *to));
+  FromType min_from, max_from;
+  ImageMinMax(from, &min_from, &max_from);
+  const float scale = to_range / (max_from - min_from);
+  for (size_t y = 0; y < from.ysize(); ++y) {
+    const FromType* const JXL_RESTRICT row_from = from.Row(y);
+    ToType* const JXL_RESTRICT row_to = to->Row(y);
+    for (size_t x = 0; x < from.xsize(); ++x) {
+      row_to[x] = static_cast<ToType>((row_from[x] - min_from) * scale);
+    }
+  }
+}
+
+// FromType and ToType are the pixel types.
+template <typename FromType, typename ToType>
+Plane<ToType> StaticCastImage(const Plane<FromType>& from) {
+  Plane<ToType> to(from.xsize(), from.ysize());
+  for (size_t y = 0; y < from.ysize(); ++y) {
+    const FromType* const JXL_RESTRICT row_from = from.Row(y);
+    ToType* const JXL_RESTRICT row_to = to.Row(y);
+    for (size_t x = 0; x < from.xsize(); ++x) {
+      row_to[x] = static_cast<ToType>(row_from[x]);
+    }
+  }
+  return to;
+}
+
+template <typename T>
+Plane<T> ImageFromPacked(const std::vector<T>& packed, const size_t xsize,
+                         const size_t ysize) {
+  Plane<T> out(xsize, ysize);
+  for (size_t y = 0; y < ysize; ++y) {
+    T* const JXL_RESTRICT row = out.Row(y);
+    const T* const JXL_RESTRICT packed_row = &packed[y * xsize];
+    memcpy(row, packed_row, xsize * sizeof(T));
+  }
+  return out;
+}
+
+// Computes independent minimum and maximum values for each plane.
+template <typename T>
+void Image3MinMax(const Image3<T>& image, const Rect& rect,
+                  std::array<T, 3>* out_min, std::array<T, 3>* out_max) {
+  for (size_t c = 0; c < 3; ++c) {
+    T min = std::numeric_limits<T>::max();
+    T max = std::numeric_limits<T>::min();
+    for (size_t y = 0; y < rect.ysize(); ++y) {
+      const T* JXL_RESTRICT row = rect.ConstPlaneRow(image, c, y);
+      for (size_t x = 0; x < rect.xsize(); ++x) {
+        min = std::min(min, row[x]);
+        max = std::max(max, row[x]);
+      }
+    }
+    (*out_min)[c] = min;
+    (*out_max)[c] = max;
+  }
+}
+
+// Computes independent minimum and maximum values for each plane.
+template <typename T>
+void Image3MinMax(const Image3<T>& image, std::array<T, 3>* out_min,
+                  std::array<T, 3>* out_max) {
+  Image3MinMax(image, Rect(image), out_min, out_max);
+}
+
+template <typename T>
+void Image3Max(const Image3<T>& image, std::array<T, 3>* out_max) {
+  for (size_t c = 0; c < 3; ++c) {
+    T max = std::numeric_limits<T>::min();
+    for (size_t y = 0; y < image.ysize(); ++y) {
+      const T* JXL_RESTRICT row = image.ConstPlaneRow(c, y);
+      for (size_t x = 0; x < image.xsize(); ++x) {
+        max = std::max(max, row[x]);
+      }
+    }
+    (*out_max)[c] = max;
+  }
+}
+
+// Computes the sum of the pixels in `rect`.
+template <typename T>
+T ImageSum(const Plane<T>& image, const Rect& rect) {
+  T result = 0;
+  for (size_t y = 0; y < rect.ysize(); ++y) {
+    const T* JXL_RESTRICT row = rect.ConstRow(image, y);
+    for (size_t x = 0; x < rect.xsize(); ++x) {
+      result += row[x];
+    }
+  }
+  return result;
+}
+
+template <typename T>
+T ImageSum(const Plane<T>& image) {
+  return ImageSum(image, Rect(image));
+}
+
+template <typename T>
+std::array<T, 3> Image3Sum(const Image3<T>& image, const Rect& rect) {
+  std::array<T, 3> out_sum = 0;
+  for (size_t c = 0; c < 3; ++c) {
+    (out_sum)[c] = ImageSum(image.Plane(c), rect);
+  }
+  return out_sum;
+}
+
+template <typename T>
+std::array<T, 3> Image3Sum(const Image3<T>& image) {
+  return Image3Sum(image, Rect(image));
+}
+
+template <typename T>
+std::vector<T> PackedFromImage(const Plane<T>& image, const Rect& rect) {
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+  std::vector<T> packed(xsize * ysize);
+  for (size_t y = 0; y < rect.ysize(); ++y) {
+    memcpy(&packed[y * xsize], rect.ConstRow(image, y), xsize * sizeof(T));
+  }
+  return packed;
+}
+
+template <typename T>
+std::vector<T> PackedFromImage(const Plane<T>& image) {
+  return PackedFromImage(image, Rect(image));
+}
+
+// Computes the median pixel value.
+template <typename T>
+T ImageMedian(const Plane<T>& image, const Rect& rect) {
+  std::vector<T> pixels = PackedFromImage(image, rect);
+  return Median(&pixels);
+}
+
+template <typename T>
+T ImageMedian(const Plane<T>& image) {
+  return ImageMedian(image, Rect(image));
+}
+
+template <typename T>
+std::array<T, 3> Image3Median(const Image3<T>& image, const Rect& rect) {
+  std::array<T, 3> out_median;
+  for (size_t c = 0; c < 3; ++c) {
+    (out_median)[c] = ImageMedian(image.Plane(c), rect);
+  }
+  return out_median;
+}
+
+template <typename T>
+std::array<T, 3> Image3Median(const Image3<T>& image) {
+  return Image3Median(image, Rect(image));
+}
+
+template <typename FromType, typename ToType>
+void Image3Convert(const Image3<FromType>& from, const float to_range,
+                   Image3<ToType>* const JXL_RESTRICT to) {
+  JXL_ASSERT(SameSize(from, *to));
+  std::array<FromType, 3> min_from, max_from;
+  Image3MinMax(from, &min_from, &max_from);
+  float scales[3];
+  for (size_t c = 0; c < 3; ++c) {
+    scales[c] = to_range / (max_from[c] - min_from[c]);
+  }
+  float scale = std::min(scales[0], std::min(scales[1], scales[2]));
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < from.ysize(); ++y) {
+      const FromType* JXL_RESTRICT row_from = from.ConstPlaneRow(c, y);
+      ToType* JXL_RESTRICT row_to = to->PlaneRow(c, y);
+      for (size_t x = 0; x < from.xsize(); ++x) {
+        const float to = (row_from[x] - min_from[c]) * scale;
+        row_to[x] = static_cast<ToType>(to);
+      }
+    }
+  }
+}
+
+// FromType and ToType are the pixel types.
+template <typename ToType, typename FromType>
+Image3<ToType> StaticCastImage3(const Image3<FromType>& from) {
+  Image3<ToType> to(from.xsize(), from.ysize());
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < from.ysize(); ++y) {
+      const FromType* JXL_RESTRICT row_from = from.ConstPlaneRow(c, y);
+      ToType* JXL_RESTRICT row_to = to.PlaneRow(c, y);
+      for (size_t x = 0; x < from.xsize(); ++x) {
+        row_to[x] = static_cast<ToType>(row_from[x]);
+      }
+    }
+  }
+  return to;
+}
+
+template <typename Tin, typename Tout>
+void Subtract(const Image3<Tin>& image1, const Image3<Tin>& image2,
+              Image3<Tout>* out) {
+  const size_t xsize = image1.xsize();
+  const size_t ysize = image1.ysize();
+  JXL_CHECK(xsize == image2.xsize());
+  JXL_CHECK(ysize == image2.ysize());
+
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < ysize; ++y) {
+      const Tin* const JXL_RESTRICT row1 = image1.ConstPlaneRow(c, y);
+      const Tin* const JXL_RESTRICT row2 = image2.ConstPlaneRow(c, y);
+      Tout* const JXL_RESTRICT row_out = out->PlaneRow(c, y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row_out[x] = row1[x] - row2[x];
+      }
+    }
+  }
+}
+
+template <typename Tin, typename Tout>
+void SubtractFrom(const Image3<Tin>& what, Image3<Tout>* to) {
+  const size_t xsize = what.xsize();
+  const size_t ysize = what.ysize();
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < ysize; ++y) {
+      const Tin* JXL_RESTRICT row_what = what.ConstPlaneRow(c, y);
+      Tout* JXL_RESTRICT row_to = to->PlaneRow(c, y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row_to[x] -= row_what[x];
+      }
+    }
+  }
+}
+
+template <typename Tin, typename Tout>
+void AddTo(const Image3<Tin>& what, Image3<Tout>* to) {
+  const size_t xsize = what.xsize();
+  const size_t ysize = what.ysize();
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < ysize; ++y) {
+      const Tin* JXL_RESTRICT row_what = what.ConstPlaneRow(c, y);
+      Tout* JXL_RESTRICT row_to = to->PlaneRow(c, y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row_to[x] += row_what[x];
+      }
+    }
+  }
+}
+
+// Adds `what` of the size of `rect` to `to` in the position of `rect`.
+template <typename Tin, typename Tout>
+void AddTo(const Rect& rect, const Image3<Tin>& what, Image3<Tout>* to) {
+  const size_t xsize = what.xsize();
+  const size_t ysize = what.ysize();
+  JXL_ASSERT(xsize == rect.xsize());
+  JXL_ASSERT(ysize == rect.ysize());
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < ysize; ++y) {
+      const Tin* JXL_RESTRICT row_what = what.ConstPlaneRow(c, y);
+      Tout* JXL_RESTRICT row_to = rect.PlaneRow(to, c, y);
+      for (size_t x = 0; x < xsize; ++x) {
+        row_to[x] += row_what[x];
+      }
+    }
+  }
+}
+
+template <typename T>
+Image3<T> ScaleImage(const T lambda, const Image3<T>& image) {
+  Image3<T> out(image.xsize(), image.ysize());
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < image.ysize(); ++y) {
+      const T* JXL_RESTRICT row = image.ConstPlaneRow(c, y);
+      T* JXL_RESTRICT row_out = out.PlaneRow(c, y);
+      for (size_t x = 0; x < image.xsize(); ++x) {
+        row_out[x] = lambda * row[x];
+      }
+    }
+  }
+  return out;
+}
+
+// Multiplies image by lambda in-place
+template <typename T>
+void ScaleImage(const T lambda, Image3<T>* image) {
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < image->ysize(); ++y) {
+      T* const JXL_RESTRICT row = image->PlaneRow(c, y);
+      for (size_t x = 0; x < image->xsize(); ++x) {
+        row[x] = lambda * row[x];
+      }
+    }
+  }
+}
+
+// Initializes all planes to the same "value".
+template <typename T>
+void FillImage(const T value, Image3<T>* image) {
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < image->ysize(); ++y) {
+      T* JXL_RESTRICT row = image->PlaneRow(c, y);
+      for (size_t x = 0; x < image->xsize(); ++x) {
+        row[x] = value;
+      }
+    }
+  }
+}
+
+template <typename T>
+void FillPlane(const T value, Plane<T>* image) {
+  for (size_t y = 0; y < image->ysize(); ++y) {
+    T* JXL_RESTRICT row = image->MutableRow(y);
+    for (size_t x = 0; x < image->xsize(); ++x) {
+      row[x] = value;
+    }
+  }
+}
+
+template <typename T>
+void FillImage(const T value, Image3<T>* image, Rect rect) {
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < rect.ysize(); ++y) {
+      T* JXL_RESTRICT row = rect.Row(image, y);
+      for (size_t x = 0; x < rect.xsize(); ++x) {
+        row[x] = value;
+      }
+    }
+  }
+}
+
+template <typename T>
+void FillPlane(const T value, Plane<T>* image, Rect rect) {
+  for (size_t y = 0; y < rect.ysize(); ++y) {
+    T* JXL_RESTRICT row = rect.Row(image, y);
+    for (size_t x = 0; x < rect.xsize(); ++x) {
+      row[x] = value;
+    }
+  }
+}
+
+template <typename T>
+void ZeroFillImage(Image3<T>* image) {
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < image->ysize(); ++y) {
+      T* JXL_RESTRICT row = image->PlaneRow(c, y);
+      memset(row, 0, image->xsize() * sizeof(T));
+    }
+  }
+}
+
+template <typename T>
+void ZeroFillPlane(Plane<T>* image, Rect rect) {
+  for (size_t y = 0; y < rect.ysize(); ++y) {
+    T* JXL_RESTRICT row = rect.Row(image, y);
+    memset(row, 0, rect.xsize() * sizeof(T));
+  }
+}
+
+// First, image is padded horizontally, with the rightmost value.
+// Next, image is padded vertically, by repeating the last line.
+ImageF PadImage(const ImageF& in, size_t xsize, size_t ysize);
+
+Image3F PadImageSymmetric(const Image3F& in, size_t border);
+
+// First, image is padded horizontally, with the rightmost value.
+// Next, image is padded vertically, by repeating the last line.
+// Prefer PadImageToBlockMultipleInPlace if padding to kBlockDim.
+Image3F PadImageToMultiple(const Image3F& in, size_t N);
+
+// Same as above, but operates in-place. Assumes that the `in` image was
+// allocated large enough.
+void PadImageToBlockMultipleInPlace(Image3F* JXL_RESTRICT in);
+
+}  // namespace jxl
+
+#endif  // JXL_IMAGE_OPS_H_
diff --git a/jxl/image_test.cc b/jxl/image_test.cc
new file mode 100644
index 000000000..bc1e0c364
--- /dev/null
+++ b/jxl/image_test.cc
@@ -0,0 +1,142 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/image.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <random>
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "jxl/image_ops.h"
+#include "jxl/image_test_utils.h"
+
+namespace jxl {
+namespace {
+
+template <typename T>
+void TestPacked(const size_t xsize, const size_t ysize) {
+  Plane<T> image1(xsize, ysize);
+  RandomFillImage(&image1);
+  const std::vector<T>& packed = PackedFromImage(image1);
+  const Plane<T>& image2 = ImageFromPacked(packed, xsize, ysize);
+  EXPECT_TRUE(SamePixels(image1, image2));
+}
+
+TEST(ImageTest, TestPacked) {
+  TestPacked<uint8_t>(1, 1);
+  TestPacked<uint8_t>(7, 1);
+  TestPacked<uint8_t>(1, 7);
+
+  TestPacked<int16_t>(1, 1);
+  TestPacked<int16_t>(7, 1);
+  TestPacked<int16_t>(1, 7);
+
+  TestPacked<uint16_t>(1, 1);
+  TestPacked<uint16_t>(7, 1);
+  TestPacked<uint16_t>(1, 7);
+
+  TestPacked<float>(1, 1);
+  TestPacked<float>(7, 1);
+  TestPacked<float>(1, 7);
+}
+
+// Ensure entire payload is readable/writable for various size/offset combos.
+TEST(ImageTest, TestAllocator) {
+  std::mt19937 rng(129);
+  const size_t k32 = 32;
+  const size_t kAlign = CacheAligned::kAlignment;
+  for (size_t size : {k32 * 1, k32 * 2, k32 * 3, k32 * 4, k32 * 5,
+                      CacheAligned::kAlias, 2 * CacheAligned::kAlias + 4}) {
+    for (size_t offset = 0; offset <= CacheAligned::kAlias; offset += kAlign) {
+      uint8_t* bytes =
+          static_cast<uint8_t*>(CacheAligned::Allocate(size, offset));
+      JXL_CHECK(reinterpret_cast<uintptr_t>(bytes) % kAlign == 0);
+      // Ensure we can write/read the last byte. Use RNG to fool the compiler
+      // into thinking the write is necessary.
+      memset(bytes, 0, size);
+      bytes[size - 1] = 1;  // greatest element
+      std::uniform_int_distribution<uint32_t> dist(0, size - 1);
+      uint32_t pos = dist(rng);  // random but != greatest
+      while (pos == size - 1) {
+        pos = dist(rng);
+      }
+      JXL_CHECK(bytes[pos] < bytes[size - 1]);
+
+      CacheAligned::Free(bytes);
+    }
+  }
+}
+
+template <typename T>
+void TestFillImpl(Image3<T>* img, const char* layout) {
+  FillImage(T(1), img);
+  for (size_t y = 0; y < img->ysize(); ++y) {
+    for (size_t c = 0; c < 3; ++c) {
+      T* JXL_RESTRICT row = img->PlaneRow(c, y);
+      for (size_t x = 0; x < img->xsize(); ++x) {
+        if (row[x] != T(1)) {
+          printf("Not 1 at c=%zu %zu, %zu (%zu x %zu) (%s)\n", c, x, y,
+                 img->xsize(), img->ysize(), layout);
+          abort();
+        }
+        row[x] = T(2);
+      }
+    }
+  }
+
+  // Same for ZeroFillImage and swapped c/y loop ordering.
+  ZeroFillImage(img);
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < img->ysize(); ++y) {
+      T* JXL_RESTRICT row = img->PlaneRow(c, y);
+      for (size_t x = 0; x < img->xsize(); ++x) {
+        if (row[x] != T(0)) {
+          printf("Not 0 at c=%zu %zu, %zu (%zu x %zu) (%s)\n", c, x, y,
+                 img->xsize(), img->ysize(), layout);
+          abort();
+        }
+        row[x] = T(3);
+      }
+    }
+  }
+}
+
+template <typename T>
+void TestFillT() {
+  for (uint32_t xsize : {0, 1, 15, 16, 31, 32}) {
+    for (uint32_t ysize : {0, 1, 15, 16, 31, 32}) {
+      Image3<T> image(xsize, ysize);
+      TestFillImpl(&image, "size ctor");
+
+      Image3<T> planar(Plane<T>(xsize, ysize), Plane<T>(xsize, ysize),
+                       Plane<T>(xsize, ysize));
+      TestFillImpl(&planar, "planar");
+    }
+  }
+}
+
+// Ensure y/c/x and c/y/x loops visit pixels no more than once.
+TEST(ImageTest, TestFill) {
+  TestFillT<uint8_t>();
+  TestFillT<int16_t>();
+  TestFillT<float>();
+  TestFillT<double>();
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/image_test_utils.h b/jxl/image_test_utils.h
new file mode 100644
index 000000000..20c51ff7c
--- /dev/null
+++ b/jxl/image_test_utils.h
@@ -0,0 +1,319 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_IMAGE_TEST_UTILS_H_
+#define JXL_IMAGE_TEST_UTILS_H_
+
+#include <stddef.h>
+
+#include <cmath>
+#include <limits>
+#include <random>
+
+#include "gtest/gtest.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/image.h"
+
+namespace jxl {
+
+template <typename T>
+void VerifyEqual(const Plane<T>& expected, const Plane<T>& actual) {
+  JXL_CHECK(SameSize(expected, actual));
+  for (size_t y = 0; y < expected.ysize(); ++y) {
+    const T* const JXL_RESTRICT row_expected = expected.Row(y);
+    const T* const JXL_RESTRICT row_actual = actual.Row(y);
+    for (size_t x = 0; x < expected.xsize(); ++x) {
+      ASSERT_EQ(row_expected[x], row_actual[x]) << x << " " << y;
+    }
+  }
+}
+
+template <typename T>
+void VerifyEqual(const Image3<T>& expected, const Image3<T>& actual) {
+  for (size_t c = 0; c < 3; ++c) {
+    VerifyEqual(expected.Plane(c), actual.Plane(c));
+  }
+}
+
+template <typename T>
+bool SamePixels(const Plane<T>& image1, const Plane<T>& image2) {
+  const size_t xsize = image1.xsize();
+  const size_t ysize = image1.ysize();
+  JXL_CHECK(xsize == image2.xsize());
+  JXL_CHECK(ysize == image2.ysize());
+  size_t mismatches = 0;
+  for (size_t y = 0; y < ysize; ++y) {
+    const T* const JXL_RESTRICT row1 = image1.Row(y);
+    const T* const JXL_RESTRICT row2 = image2.Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      if (row1[x] != row2[x]) {
+        printf("%zu, %zu: %f != %f\n", x, y, double(row1[x]), double(row2[x]));
+        if (++mismatches > 4) return false;
+      }
+    }
+  }
+  return mismatches == 0;
+}
+
+template <typename T>
+bool SamePixels(const Image3<T>& image1, const Image3<T>& image2) {
+  JXL_CHECK(SameSize(image1, image2));
+  const size_t xsize = image1.xsize();
+  const size_t ysize = image1.ysize();
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < ysize; ++y) {
+      const T* JXL_RESTRICT row1 = image1.PlaneRow(c, y);
+      const T* JXL_RESTRICT row2 = image2.PlaneRow(c, y);
+      for (size_t x = 0; x < xsize; ++x) {
+        if (row1[x] != row2[x]) {
+          // printf("%zu, %zu, %zu: %f != %f\n", x, y, c, double(row1[x]),
+          //       double(row2[x]));
+          return false;
+        }
+      }
+    }
+  }
+  return true;
+}
+
+// Use for floating-point images with fairly large numbers; tolerates small
+// absolute errors and/or small relative errors. Returns max_relative.
+template <typename T>
+void VerifyRelativeError(const Plane<T>& expected, const Plane<T>& actual,
+                           const double threshold_l1,
+                           const double threshold_relative,
+                           const size_t border = 0, const size_t c = 0) {
+  JXL_CHECK(SameSize(expected, actual));
+  // Max over current scanline to give a better idea whether there are
+  // systematic errors or just one outlier. Invalid if negative.
+  double max_l1 = -1;
+  double max_relative = -1;
+  bool any_bad = false;
+  for (size_t y = border; y < expected.ysize() - border; ++y) {
+    const T* const JXL_RESTRICT row_expected = expected.Row(y);
+    const T* const JXL_RESTRICT row_actual = actual.Row(y);
+    for (size_t x = border; x < expected.xsize() - border; ++x) {
+      const double l1 = std::abs(row_expected[x] - row_actual[x]);
+
+      // Cannot compute relative, only check/update L1.
+      if (row_expected[x] < 1E-10) {
+        if (l1 > threshold_l1) {
+          any_bad = true;
+          max_l1 = std::max(max_l1, l1);
+        }
+      } else {
+        const double relative = l1 / std::abs(double(row_expected[x]));
+        if (l1 > threshold_l1 && relative > threshold_relative) {
+          // Fails both tolerances => will exit below, update max_*.
+          any_bad = true;
+          max_l1 = std::max(max_l1, l1);
+          max_relative = std::max(max_relative, relative);
+        }
+      }
+    }
+  }
+  if (any_bad) {
+    // Never had a valid relative value, don't print it.
+    if (max_relative < 0) {
+      fprintf(stderr, "c=%zu: max +/- %E exceeds +/- %.2E\n", c, max_l1,
+              threshold_l1);
+    } else {
+      fprintf(stderr, "c=%zu: max +/- %E, x %E exceeds +/- %.2E, x %.2E\n", c,
+              max_l1, max_relative, threshold_l1, threshold_relative);
+    }
+    // Dump the expected image and actual image if the region is small enough.
+    const size_t kMaxTestDumpSize = 16;
+    if (expected.xsize() <= kMaxTestDumpSize + 2 * border &&
+        expected.ysize() <= kMaxTestDumpSize + 2 * border) {
+      fprintf(stderr, "Expected image:\n");
+      for (size_t y = border; y < expected.ysize() - border; ++y) {
+        const T* const JXL_RESTRICT row_expected = expected.Row(y);
+        for (size_t x = border; x < expected.xsize() - border; ++x) {
+          fprintf(stderr, "%10lf ", static_cast<double>(row_expected[x]));
+        }
+        fprintf(stderr, "\n");
+      }
+
+      fprintf(stderr, "Actual image:\n");
+      for (size_t y = border; y < actual.ysize() - border; ++y) {
+        const T* const JXL_RESTRICT row_expected = expected.Row(y);
+        const T* const JXL_RESTRICT row_actual = actual.Row(y);
+        for (size_t x = border; x < actual.xsize() - border; ++x) {
+          const double l1 = std::abs(row_expected[x] - row_actual[x]);
+
+          bool bad = l1 > threshold_l1;
+          if (row_expected[x] > 1E-10) {
+            const double relative = l1 / std::abs(double(row_expected[x]));
+            bad &= relative > threshold_relative;
+          }
+          if (bad) {
+            fprintf(stderr, "%10lf ", static_cast<double>(row_actual[x]));
+          } else {
+            fprintf(stderr, "%10s ", "==");
+          }
+        }
+        fprintf(stderr, "\n");
+      }
+    }
+
+    // Find first failing x for further debugging.
+    for (size_t y = border; y < expected.ysize() - border; ++y) {
+      const T* const JXL_RESTRICT row_expected = expected.Row(y);
+      const T* const JXL_RESTRICT row_actual = actual.Row(y);
+
+      for (size_t x = border; x < expected.xsize() - border; ++x) {
+        const double l1 = std::abs(row_expected[x] - row_actual[x]);
+
+        bool bad = l1 > threshold_l1;
+        if (row_expected[x] > 1E-10) {
+          const double relative = l1 / std::abs(double(row_expected[x]));
+          bad &= relative > threshold_relative;
+        }
+        if (bad) {
+          FAIL() << x << ", " << y << " (" << expected.xsize() << " x "
+                        << expected.ysize() << ") expected "
+                        << static_cast<double>(row_expected[x]) << " actual "
+                        << static_cast<double>(row_actual[x]);
+        }
+      }
+    }
+    return; // if any_bad, we should have exited.
+  }
+}
+
+template <typename T>
+void VerifyRelativeError(const Image3<T>& expected, const Image3<T>& actual,
+                           const float threshold_l1,
+                           const float threshold_relative,
+                           const size_t border = 0) {
+  for (size_t c = 0; c < 3; ++c) {
+    VerifyRelativeError(expected.Plane(c), actual.Plane(c), threshold_l1,
+                        threshold_relative, border, c);
+  }
+}
+
+// Generator for independent, uniformly distributed integers [0, max].
+template <typename T, typename Random>
+class GeneratorRandom {
+ public:
+  GeneratorRandom(Random* rng, const T max) : rng_(*rng), dist_(0, max) {}
+
+  GeneratorRandom(Random* rng, const T min, const T max)
+      : rng_(*rng), dist_(min, max) {}
+
+  T operator()(const size_t x, const size_t y, const int c) const {
+    return dist_(rng_);
+  }
+
+ private:
+  Random& rng_;
+  mutable std::uniform_int_distribution<> dist_;
+};
+
+template <typename Random>
+class GeneratorRandom<float, Random> {
+ public:
+  GeneratorRandom(Random* rng, const float max)
+      : rng_(*rng), dist_(0.0f, max) {}
+
+  GeneratorRandom(Random* rng, const float min, const float max)
+      : rng_(*rng), dist_(min, max) {}
+
+  float operator()(const size_t x, const size_t y, const int c) const {
+    return dist_(rng_);
+  }
+
+ private:
+  Random& rng_;
+  mutable std::uniform_real_distribution<float> dist_;
+};
+
+template <typename Random>
+class GeneratorRandom<double, Random> {
+ public:
+  GeneratorRandom(Random* rng, const double max)
+      : rng_(*rng), dist_(0.0, max) {}
+
+  GeneratorRandom(Random* rng, const double min, const double max)
+      : rng_(*rng), dist_(min, max) {}
+
+  double operator()(const size_t x, const size_t y, const int c) const {
+    return dist_(rng_);
+  }
+
+ private:
+  Random& rng_;
+  mutable std::uniform_real_distribution<> dist_;
+};
+
+// Assigns generator(x, y, 0) to each pixel (x, y).
+template <class Generator, class Image>
+void GenerateImage(const Generator& generator, Image* image) {
+  using T = typename Image::T;
+  for (size_t y = 0; y < image->ysize(); ++y) {
+    T* const JXL_RESTRICT row = image->Row(y);
+    for (size_t x = 0; x < image->xsize(); ++x) {
+      row[x] = generator(x, y, 0);
+    }
+  }
+}
+
+template <typename T>
+void RandomFillImage(Plane<T>* image,
+                     const T max = std::numeric_limits<T>::max()) {
+  std::mt19937_64 rng(129);
+  const GeneratorRandom<T, std::mt19937_64> generator(&rng, max);
+  GenerateImage(generator, image);
+}
+
+template <typename T>
+void RandomFillImage(Plane<T>* image, const T min, const T max,
+                     const int seed) {
+  std::mt19937_64 rng(seed);
+  const GeneratorRandom<T, std::mt19937_64> generator(&rng, min, max);
+  GenerateImage(generator, image);
+}
+
+// Assigns generator(x, y, c) to each pixel (x, y).
+template <class Generator, typename T>
+void GenerateImage(const Generator& generator, Image3<T>* image) {
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < image->ysize(); ++y) {
+      T* JXL_RESTRICT row = image->PlaneRow(c, y);
+      for (size_t x = 0; x < image->xsize(); ++x) {
+        row[x] = generator(x, y, c);
+      }
+    }
+  }
+}
+
+template <typename T>
+void RandomFillImage(Image3<T>* image,
+                     const T max = std::numeric_limits<T>::max()) {
+  std::mt19937_64 rng(129);
+  const GeneratorRandom<T, std::mt19937_64> generator(&rng, max);
+  GenerateImage(generator, image);
+}
+
+template <typename T>
+void RandomFillImage(Image3<T>* image, const T min, const T max,
+                     const int seed) {
+  std::mt19937_64 rng(seed);
+  const GeneratorRandom<T, std::mt19937_64> generator(&rng, min, max);
+  GenerateImage(generator, image);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_IMAGE_TEST_UTILS_H_
diff --git a/jxl/jpeg_kpasses_test.cc b/jxl/jpeg_kpasses_test.cc
new file mode 100644
index 000000000..679bf8508
--- /dev/null
+++ b/jxl/jpeg_kpasses_test.cc
@@ -0,0 +1,94 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/dec_file.h"
+#include "jxl/enc_butteraugli_comparator.h"
+#include "jxl/enc_file.h"
+#include "jxl/extras/codec.h"
+#include "jxl/extras/codec_jpg.h"
+#include "jxl/image_ops.h"
+#include "jxl/image_test_utils.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+
+TEST(JPEGkPassesTest, RoundtripLarge) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+
+  // encode to JPEG
+  PaddedBytes encoded;
+  io.jpeg_quality = 90;
+  JXL_CHECK(Encode(io, Codec::kJPG, io.metadata.color_encoding,
+                   io.metadata.bits_per_sample, &encoded, pool));
+
+  // decode JPEG to pixels (using libjpeg)
+  CodecInOut io2;
+  JXL_CHECK(jxl::DecodeImageJPG(Span<const uint8_t>(encoded), &io2));
+  const ImageBundle& ib2 = io2.Main();
+
+  // decode JPEG to DCT coeffs
+  CodecInOut io3;
+  JXL_CHECK(SetFromBytes(Span<const uint8_t>(encoded), &io3, pool,
+                         jxl::DecodeTarget::kQuantizedCoeffs));
+
+  CompressParams cparams;
+  DecompressParams dparams;
+
+  PaddedBytes compressed;
+  AuxOut* aux_out = nullptr;
+  PassesEncoderState enc_state;
+
+  // encode DCT coeffs as kPasses JXL
+  EXPECT_TRUE(
+      EncodeFile(cparams, &io3, &enc_state, &compressed, aux_out, pool));
+  // decode JXL to pixels
+  CodecInOut io4;
+  EXPECT_TRUE(DecodeFile(dparams, compressed, &io4, aux_out, pool));
+  const ImageBundle& ib4 = io4.Main();
+
+  // TODO: investigate where the difference between libjpeg and kPasses comes
+  //       from (and see if it can be reduced)
+  EXPECT_LE(ButteraugliDistance(ib2, ib4, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            0.87654f);
+
+  // decode JXL to DCT coefficients
+  CodecInOut io5;
+  dparams.keep_dct = true;
+  EXPECT_TRUE(DecodeFile(dparams, compressed, &io5, aux_out, pool));
+
+  // encode the result to JPEG
+  PaddedBytes encoded2;
+  io5.jpeg_quality = 100;  // ignored but has to be set
+  JXL_CHECK(Encode(io5, Codec::kJPG, io5.metadata.color_encoding,
+                   io5.metadata.bits_per_sample, &encoded2, pool));
+
+  // decode JPEG to pixels (using libjpeg)
+  CodecInOut io6;
+  JXL_CHECK(jxl::DecodeImageJPG(Span<const uint8_t>(encoded2), &io6));
+  const ImageBundle& ib6 = io6.Main();
+
+  EXPECT_TRUE(SamePixels(ib2.color(), ib6.color()));
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/jpegxl.syms b/jxl/jpegxl.syms
new file mode 100644
index 000000000..0f398d715
--- /dev/null
+++ b/jxl/jpegxl.syms
@@ -0,0 +1,5 @@
+{
+  extern "C" {
+    jpegxl_*;
+  };
+};
diff --git a/jxl/jpegxl.version b/jxl/jpegxl.version
new file mode 100644
index 000000000..24aa3a7ef
--- /dev/null
+++ b/jxl/jpegxl.version
@@ -0,0 +1,7 @@
+JPEGXL_0 {
+  global:
+    Jpegxl*;
+
+  local:
+    *;
+};
diff --git a/jxl/jpegxl/decode.cc b/jxl/jpegxl/decode.cc
new file mode 100644
index 000000000..e6ba26898
--- /dev/null
+++ b/jxl/jpegxl/decode.cc
@@ -0,0 +1,68 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jpegxl/decode.h"
+
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/brunsli.h"
+#include "jxl/headers.h"
+#include "jxl/memory_manager_internal.h"
+
+uint32_t JpegxlDecoderVersion(void) {
+  return JPEGXL_MAJOR_VERSION * 1000000 + JPEGXL_MINOR_VERSION * 1000 +
+         JPEGXL_PATCH_VERSION;
+}
+
+enum JpegxlSignature JpegxlSignatureCheck(const uint8_t* buf, size_t len) {
+  enum JpegxlSignature ret = JPEGXL_SIG_INVALID;
+  jxl::BrunsliFileSignature brn =
+      IsBrunsliFile(jxl::Span<const uint8_t>(buf, len));
+  if (brn == jxl::BrunsliFileSignature::kBrunsli) return JPEGXL_SIG_BRUNSLI;
+  if (brn == jxl::BrunsliFileSignature::kNotEnoughData)
+    ret = JPEGXL_SIG_NOT_ENOUGH_BYTES;
+
+  // Check JPEG XL signature.
+  if (len >= 1 && buf[0] == 0xff) {
+    if (len < 2) {
+      ret = JPEGXL_SIG_NOT_ENOUGH_BYTES;
+    } else if (buf[1] == jxl::kMarkerShort || buf[1] == jxl::kMarkerFlexible) {
+      return JPEGXL_SIG_JPEGXL;
+    }
+  }
+  return ret;
+}
+
+struct JpegxlDecoderStruct {
+  JpegxlMemoryManager memory_manager;
+};
+
+JpegxlDecoder* JpegxlDecoderCreate(const JpegxlMemoryManager* memory_manager) {
+  JpegxlMemoryManager local_memory_manager;
+  if (!jxl::MemoryManagerInit(&local_memory_manager, memory_manager))
+    return nullptr;
+
+  JpegxlDecoder* dec = static_cast<JpegxlDecoder*>(
+      jxl::MemoryManagerAlloc(&local_memory_manager, sizeof(JpegxlDecoder)));
+  if (!dec) return nullptr;
+  dec->memory_manager = local_memory_manager;
+
+  return dec;
+}
+
+void JpegxlDecoderDestroy(JpegxlDecoder* dec) {
+  if (dec) {
+    jxl::MemoryManagerFree(&dec->memory_manager, dec);
+  }
+}
diff --git a/jxl/jpegxl/decode_test.cc b/jxl/jpegxl/decode_test.cc
new file mode 100644
index 000000000..5cce97bfd
--- /dev/null
+++ b/jxl/jpegxl/decode_test.cc
@@ -0,0 +1,81 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jpegxl/decode.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+
+TEST(DecodeTest, JpegxlSignatureCheckTest) {
+  std::vector<std::pair<int, std::vector<uint8_t>>> tests = {
+      // No JPEGXL header starts with 'a'.
+      {JPEGXL_SIG_INVALID, {'a'}},
+      {JPEGXL_SIG_INVALID, {'a', 'b', 'c', 'd', 'e', 'f'}},
+
+      // JPEGXL headers.
+      {JPEGXL_SIG_NOT_ENOUGH_BYTES, {0xff}},  // Part of a signature.
+      {JPEGXL_SIG_JPEGXL, {0xff, 0x58}},
+      {JPEGXL_SIG_JPEGXL, {0xff, 0x0a}},
+
+      // A header could start with 0x0a, but it is not a complete signature.
+      {JPEGXL_SIG_NOT_ENOUGH_BYTES, {0x0a}},
+
+      // This is the beginning of a Brunsli file, but not a complete signature.
+      {JPEGXL_SIG_NOT_ENOUGH_BYTES, {0x0a, 0x04, 'B', 0xd2}},
+      {JPEGXL_SIG_BRUNSLI, {0x0a, 0x04, 'B', 0xd2, 0xd5, 'N', 0x12}},
+      {JPEGXL_SIG_INVALID, {0x0a, 0x04, 'B', 0xd2, 0xd5, 'N', 0x13}},
+  };
+  for (const auto& test : tests) {
+    EXPECT_EQ(test.first,
+              JpegxlSignatureCheck(test.second.data(), test.second.size()))
+        << "Where test data is " << ::testing::PrintToString(test.second);
+  }
+}
+
+TEST(DecodeTest, DefaultAllocTest) {
+  JpegxlDecoder* dec = JpegxlDecoderCreate(nullptr);
+  EXPECT_NE(nullptr, dec);
+  JpegxlDecoderDestroy(dec);
+}
+
+TEST(DecodeTest, CustomAllocTest) {
+  struct CalledCounters {
+    int allocs = 0;
+    int frees = 0;
+  } counters;
+
+  JpegxlMemoryManager mm;
+  mm.opaque = &counters;
+  mm.alloc = [](void* opaque, size_t size) {
+    reinterpret_cast<CalledCounters*>(opaque)->allocs++;
+    return malloc(size);
+  };
+  mm.free = [](void* opaque, void* address) {
+    reinterpret_cast<CalledCounters*>(opaque)->frees++;
+    free(address);
+  };
+
+  JpegxlDecoder* dec = JpegxlDecoderCreate(&mm);
+  EXPECT_NE(nullptr, dec);
+  EXPECT_LE(1, counters.allocs);
+  EXPECT_EQ(0, counters.frees);
+  JpegxlDecoderDestroy(dec);
+  EXPECT_LE(1, counters.frees);
+}
diff --git a/jxl/jpegxl_emcc.cc b/jxl/jpegxl_emcc.cc
new file mode 100644
index 000000000..043bd94e1
--- /dev/null
+++ b/jxl/jpegxl_emcc.cc
@@ -0,0 +1,146 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <jxl/base/span.h>
+#include <jxl/brunsli.h>
+#include <jxl/color_management.h>
+#include <jxl/common.h>
+#include <jxl/external_image.h>
+#include <jxl/image_ops.h>
+
+#include <cstdio>
+#include <cstring>
+
+using namespace jxl;
+
+extern "C" {
+
+ExternalImage* decompress(const uint8_t* data, size_t size) {
+  ThreadPool* pool = nullptr;
+  Span<const uint8_t> compressed(data, size);
+  std::unique_ptr<ExternalImage> result;
+
+  CodecInOut io;
+  BrunsliDecoderOptions options;
+  BrunsliDecoderMeta metadata;
+  if (!BrunsliToPixels(compressed, &io, options, &metadata, pool)) {
+    printf("Failed to decompress.\n");
+    return nullptr;
+  }
+
+  if (!metadata.hdr_orig_colorspace.empty()) {
+    printf("Original colorspace: %s\n", metadata.hdr_orig_colorspace.c_str());
+    // Hopefully, that is something like Chrome / rec2020.
+    metadata.hdr_orig_colorspace = "RGB_D65_202_Rel_Lin";
+    printf("Output colorspace: %s\n", metadata.hdr_orig_colorspace.c_str());
+
+    ColorEncoding c;
+    if (!ParseDescription(metadata.hdr_orig_colorspace, &c)) {
+      printf("Failed to parse color profile description.\n");
+      return nullptr;
+    }
+    if (!ColorManagement::CreateProfile(&c)) {
+      printf("Failed to create color profile.\n");
+      return nullptr;
+    }
+    if (!io.Main().TransformTo(c, pool)) {
+      printf("Failed to transform colorspace.\n");
+      return nullptr;
+    }
+    io.metadata.color_encoding = c;
+  }
+
+  const ImageBundle& ib = io.Main();
+  const ColorEncoding& c_desired = io.metadata.color_encoding;
+  const bool has_alpha = true;
+  ImageU alpha(ib.color().xsize(), ib.color().ysize());
+  const size_t alpha_bits = 8;
+  size_t bits_per_sample = 32;
+  const bool big_endian = false;
+  CodecIntervals* temp_intervals = nullptr;
+  Rect rect = Rect(ib);
+
+  result = make_unique<ExternalImage>(pool, ib.color(), rect,
+      ib.c_current(), c_desired, has_alpha, &alpha, alpha_bits,
+      bits_per_sample, big_endian, temp_intervals);
+
+  if (!result->IsHealthy()) {
+    printf("ExternalImage is unhealthy.\n");
+    return nullptr;
+  }
+
+  size_t w = result->xsize();
+  size_t h = result->ysize();
+  float* pixels = const_cast<float*>(reinterpret_cast<const float*>(result->Bytes().data()));
+  for (size_t i = 0; i < w * h; ++i) {
+    pixels[i * 4 + 3] = 1.0f;
+  }
+
+  return result.release();
+}
+
+void freeImage(ExternalImage* img) {
+  delete img;
+}
+
+int getImageWidth(ExternalImage* img) {
+  return img->xsize();
+}
+
+int getImageHeight(ExternalImage* img) {
+  return img->ysize();
+}
+
+const void* getImagePixels(ExternalImage* img) {
+  return img->Bytes().data();
+}
+
+}  // extern "C"
+
+/*
+// Chrome must be launched with "--enable-blink-features=CanvasColorManagement" option.
+
+function showImage(bytes, amp) {
+  if (!amp) amp = 1.0 / 256;
+  console.log("Encoded size: " + bytes.length);
+  var buf = Module._malloc(bytes.length);
+  Module.HEAPU8.set(bytes, buf);
+  var img = Module._decompress(buf, bytes.length);
+  Module._free(buf);
+  if (!img) return;
+  var w = Module._getImageWidth(img);
+  var h = Module._getImageHeight(img);
+  var pixelsPtr = Module._getImagePixels(img);
+  var pixels = new Float32Array(Module.HEAPF32.subarray(pixelsPtr >> 2, (pixelsPtr >> 2) + w * h * 4));
+  for (var i = 0; i < w * h * 4; ++i) if ((i & 3) != 3) pixels[i] *= amp;
+  var canvas = document.getElementById("canvas");
+  canvas.width = w;
+  canvas.height = h;
+  var ctx = canvas.getContext("2d", {"colorSpace": "rec2020", "pixelFormat": "float16"});
+  // Is there a way to create F16 ImageData?
+  var imageData = ctx.getImageData(0, 0, w, h);
+  imageData.dataUnion.set(pixels);
+  ctx.putImageData(imageData, 0, 0);
+  Module._freeImage(img);
+}
+
+function loadAndShow(path, amp) {
+  var xhr = new XMLHttpRequest();
+  xhr.open("GET", path, true);
+  xhr.responseType = "arraybuffer";
+  xhr.onload = (e) => { showImage(new Uint8Array(xhr.response), amp); }
+  xhr.send(null);
+}
+
+*/
diff --git a/jxl/jpegxl_osx.syms b/jxl/jpegxl_osx.syms
new file mode 100644
index 000000000..7b3c0123f
--- /dev/null
+++ b/jxl/jpegxl_osx.syms
@@ -0,0 +1 @@
+_Jpegxl*
diff --git a/jxl/jxl_inspection.h b/jxl/jxl_inspection.h
new file mode 100644
index 000000000..36b6da56e
--- /dev/null
+++ b/jxl/jxl_inspection.h
@@ -0,0 +1,31 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+
+#ifndef JXL_JXL_INSPECTION_H_
+#define JXL_JXL_INSPECTION_H_
+
+#include <functional>
+#include "jxl/image.h"
+
+namespace jxl {
+// Type of the inspection-callback which, if enabled, will be called on various
+// intermediate data during image processing, allowing inspection access.
+//
+// Returns false if processing can be stopped at that point, true otherwise.
+// This is only advisory - it is always OK to just continue processing.
+using InspectorImage3F = std::function<bool(const char*, const Image3F&)>;
+}  // namespace jxl
+
+#endif  // JXL_JXL_INSPECTION_H_
diff --git a/jxl/jxl_test.cc b/jxl/jxl_test.cc
new file mode 100644
index 000000000..397808455
--- /dev/null
+++ b/jxl/jxl_test.cc
@@ -0,0 +1,775 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdio.h>
+
+#include <array>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/override.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+#include "jxl/enc_butteraugli_comparator.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_file.h"
+#include "jxl/enc_params.h"
+#include "jxl/extras/codec.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/image_test_utils.h"
+#include "jxl/modular/encoding/context_predict.h"
+#include "jxl/test_utils.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+using test::Roundtrip;
+
+#define JXL_TEST_NL 0  // Disabled in code
+
+void CreateImage1x1(CodecInOut* io) {
+  Image3F image(1, 1);
+  ZeroFillImage(&image);
+  io->metadata.bits_per_sample = 8;
+  io->metadata.color_encoding = ColorManagement::SRGB();
+  io->SetFromImage(std::move(image), io->metadata.color_encoding);
+}
+
+TEST(JxlTest, HeaderSize) {
+  CodecInOut io;
+  CreateImage1x1(&io);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 1.3;
+  DecompressParams dparams;
+  ThreadPool* pool = nullptr;
+
+  {
+    CodecInOut io2;
+    AuxOut aux_out;
+    Roundtrip(&io, cparams, dparams, pool, &io2, &aux_out);
+    EXPECT_LE(aux_out.layers[kLayerHeader].total_bits, 51);
+  }
+
+  {
+    CodecInOut io2;
+    io.metadata.alpha_bits = 8;
+    ImageU alpha(1, 1);
+    alpha.Row(0)[0] = 1;
+    io.Main().SetAlpha(std::move(alpha));
+    AuxOut aux_out;
+    Roundtrip(&io, cparams, dparams, pool, &io2, &aux_out);
+    EXPECT_LE(aux_out.layers[kLayerHeader].total_bits, 60);
+  }
+}
+
+TEST(JxlTest, RoundtripSinglePixel) {
+  CodecInOut io;
+  CreateImage1x1(&io);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 1.0;
+  DecompressParams dparams;
+  ThreadPool* pool = nullptr;
+  CodecInOut io2;
+  Roundtrip(&io, cparams, dparams, pool, &io2);
+}
+
+// Changing serialized signature causes Decode to fail.
+#ifndef JXL_CRASH_ON_ERROR
+TEST(JxlTest, RoundtripMarker) {
+  CodecInOut io;
+  CreateImage1x1(&io);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 1.0;
+  DecompressParams dparams;
+  AuxOut* aux_out = nullptr;
+  ThreadPool* pool = nullptr;
+
+  PassesEncoderState enc_state;
+  for (size_t i = 0; i < 2; ++i) {
+    PaddedBytes compressed;
+    EXPECT_TRUE(
+        EncodeFile(cparams, &io, &enc_state, &compressed, aux_out, pool));
+    compressed[i] ^= 0xFF;
+    CodecInOut io2;
+    EXPECT_FALSE(DecodeFile(dparams, compressed, &io2, aux_out, pool));
+  }
+}
+#endif
+
+TEST(JxlTest, RoundtripTinyFast) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(32, 32);
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  cparams.butteraugli_distance = 4.0f;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  const size_t enc_bytes = Roundtrip(&io, cparams, dparams, pool, &io2);
+  printf("32x32 image size %zu bytes\n", enc_bytes);
+}
+
+TEST(JxlTest, RoundtripSmallD1) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 1.0;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 1000);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            1.5);
+}
+
+TEST(JxlTest, RoundtripUnalignedD2) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 12, io.ysize() / 7);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 2.0;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 700);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            3.2);
+}
+
+#if JXL_TEST_NL
+
+TEST(JxlTest, RoundtripMultiGroupNL) {
+  ThreadPoolInternal pool(4);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+  io.ShrinkTo(600, 1024);  // partial X, full Y group
+
+  CompressParams cparams;
+  DecompressParams dparams;
+
+  cparams.fast_mode = true;
+  cparams.butteraugli_distance = 1.0f;
+  CodecInOut io2;
+  Roundtrip(&io, cparams, dparams, &pool, &io2);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, &pool),
+            0.9f);
+
+  cparams.butteraugli_distance = 2.0f;
+  CodecInOut io3;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io3), 80000);
+  EXPECT_LE(ButteraugliDistance(io, io3, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, &pool),
+            1.5f);
+}
+
+#endif
+
+TEST(JxlTest, RoundtripMultiGroup) {
+  ThreadPoolInternal pool(4);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+  io.ShrinkTo(600, 1024);  // partial X, full Y group
+
+  CompressParams cparams;
+  DecompressParams dparams;
+
+  cparams.butteraugli_distance = 1.0f;
+  CodecInOut io2;
+  Roundtrip(&io, cparams, dparams, &pool, &io2);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, &pool),
+            1.99f);
+
+  cparams.butteraugli_distance = 2.0f;
+  CodecInOut io3;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io3), 20000);
+  EXPECT_LE(ButteraugliDistance(io, io3, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, &pool),
+            3.0f);
+}
+
+TEST(JxlTest, RoundtripLargeFast) {
+  ThreadPoolInternal pool(8);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 265000);
+}
+
+// Checks for differing size/distance in two consecutive runs of distance 2,
+// which involves additional processing including adaptive reconstruction.
+// Failing this may be a sign of race conditions or invalid memory accesses.
+TEST(JxlTest, RoundtripD2Consistent) {
+  ThreadPoolInternal pool(8);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  cparams.butteraugli_distance = 2.0;
+  DecompressParams dparams;
+
+  // Try each xsize mod kBlockDim to verify right border handling.
+  for (size_t xsize = 48; xsize > 40; --xsize) {
+    io.ShrinkTo(xsize, 15);
+
+    CodecInOut io2;
+    const size_t size2 = Roundtrip(&io, cparams, dparams, &pool, &io2);
+
+    CodecInOut io3;
+    const size_t size3 = Roundtrip(&io, cparams, dparams, &pool, &io3);
+
+    // Exact same compressed size.
+    EXPECT_EQ(size2, size3);
+
+    // Exact same distance.
+    const float dist2 = ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                            /*distmap=*/nullptr, &pool);
+    const float dist3 = ButteraugliDistance(io, io3, cparams.hf_asymmetry,
+                                            /*distmap=*/nullptr, &pool);
+    EXPECT_EQ(dist2, dist3);
+  }
+}
+
+// Same as above, but for full image, testing multiple groups.
+TEST(JxlTest, RoundtripLargeConsistent) {
+  ThreadPoolInternal pool(8);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  cparams.butteraugli_distance = 2.0;
+  DecompressParams dparams;
+
+  // Try each xsize mod kBlockDim to verify right border handling.
+  CodecInOut io2;
+  const size_t size2 = Roundtrip(&io, cparams, dparams, &pool, &io2);
+
+  CodecInOut io3;
+  const size_t size3 = Roundtrip(&io, cparams, dparams, &pool, &io3);
+
+  // Exact same compressed size.
+  EXPECT_EQ(size2, size3);
+
+  // Exact same distance.
+  const float dist2 = ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                          /*distmap=*/nullptr, &pool);
+  const float dist3 = ButteraugliDistance(io, io3, cparams.hf_asymmetry,
+                                          /*distmap=*/nullptr, &pool);
+  EXPECT_EQ(dist2, dist3);
+}
+
+#if JXL_TEST_NL
+
+TEST(JxlTest, RoundtripSmallNL) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 1.0;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 1500);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            1.7);
+}
+
+#endif
+
+TEST(JxlTest, RoundtripSmallNoGaborish) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+
+  CompressParams cparams;
+  cparams.gaborish = Override::kOff;
+  cparams.butteraugli_distance = 1.0;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 900);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            1.7);
+}
+
+TEST(JxlTest, RoundtripSmallPatches) {
+  ThreadPool* pool = nullptr;
+  CodecInOut io;
+  io.metadata.color_encoding = ColorManagement::LinearSRGB();
+  Image3F black_with_small_lines(256, 256);
+  ZeroFillImage(&black_with_small_lines);
+  // This pattern should be picked up by the patch detection heuristics.
+  for (size_t y = 0; y < black_with_small_lines.ysize(); y++) {
+    float* JXL_RESTRICT row = black_with_small_lines.PlaneRow(1, y);
+    for (size_t x = 0; x < black_with_small_lines.xsize(); x++) {
+      if (x % 4 == 0 && (y / 32) % 4 == 0) row[x] = 127.0f;
+    }
+  }
+  io.SetFromImage(std::move(black_with_small_lines),
+                  ColorManagement::LinearSRGB());
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  cparams.butteraugli_distance = 0.1f;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 2000);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            0.5f);
+}
+
+TEST(JxlTest, RoundtripGrayscale) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/cvo9xd_keong_macan_grayscale.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  ASSERT_NE(io.xsize(), 0);
+  io.ShrinkTo(128, 128);
+  EXPECT_TRUE(io.Main().IsGray());
+  EXPECT_EQ(8, io.metadata.bits_per_sample);
+  EXPECT_TRUE(io.metadata.color_encoding.tf.IsSRGB());
+
+  PassesEncoderState enc_state;
+  AuxOut* aux_out = nullptr;
+
+  {
+    CompressParams cparams;
+    cparams.butteraugli_distance = 1.0;
+    DecompressParams dparams;
+
+    PaddedBytes compressed;
+    EXPECT_TRUE(
+        EncodeFile(cparams, &io, &enc_state, &compressed, aux_out, pool));
+    CodecInOut io2;
+    EXPECT_TRUE(DecodeFile(dparams, compressed, &io2, aux_out, pool));
+    EXPECT_TRUE(io2.Main().IsGray());
+
+    EXPECT_LE(compressed.size(), 7000);
+    EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                  /*distmap=*/nullptr, pool),
+              1.7777777);
+  }
+
+  // Test with larger butteraugli distance and other settings enabled so
+  // different jxl codepaths trigger.
+  {
+    CompressParams cparams;
+    cparams.butteraugli_distance = 8.0;
+    DecompressParams dparams;
+
+    PaddedBytes compressed;
+    EXPECT_TRUE(
+        EncodeFile(cparams, &io, &enc_state, &compressed, aux_out, pool));
+    CodecInOut io2;
+    EXPECT_TRUE(DecodeFile(dparams, compressed, &io2, aux_out, pool));
+    EXPECT_TRUE(io2.Main().IsGray());
+
+    EXPECT_LE(compressed.size(), 1300);
+    EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                  /*distmap=*/nullptr, pool),
+              9.0);
+  }
+}
+
+TEST(JxlTest, RoundtripAlpha) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/tmshre_riaphotographs_alpha.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+
+  ASSERT_NE(io.xsize(), 0);
+  ASSERT_TRUE(io.metadata.HasAlpha());
+  ASSERT_TRUE(io.Main().HasAlpha());
+  io.ShrinkTo(128, 128);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 1.0;
+  DecompressParams dparams;
+
+  EXPECT_EQ(8, io.metadata.bits_per_sample);
+  EXPECT_TRUE(io.metadata.color_encoding.tf.IsSRGB());
+  PassesEncoderState enc_state;
+  AuxOut* aux_out = nullptr;
+  PaddedBytes compressed;
+  EXPECT_TRUE(EncodeFile(cparams, &io, &enc_state, &compressed, aux_out, pool));
+  CodecInOut io2;
+  EXPECT_TRUE(DecodeFile(dparams, compressed, &io2, aux_out, pool));
+
+  EXPECT_LE(compressed.size(), 5500);
+
+  // TODO(robryk): Fix the following line in presence of different alpha_bits in
+  // the two contexts.
+  // EXPECT_TRUE(SamePixels(io.Main().alpha(), io2.Main().alpha()));
+  // TODO(robryk): Fix the distance estimate used in the encoder.
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            6.3);
+}
+
+TEST(JxlTest, RoundtripAlphaNonMultipleOf8) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/tmshre_riaphotographs_alpha.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+
+  ASSERT_NE(io.xsize(), 0);
+  ASSERT_TRUE(io.metadata.HasAlpha());
+  ASSERT_TRUE(io.Main().HasAlpha());
+  io.ShrinkTo(12, 12);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 1.0;
+  DecompressParams dparams;
+
+  EXPECT_EQ(8, io.metadata.bits_per_sample);
+  EXPECT_TRUE(io.metadata.color_encoding.tf.IsSRGB());
+  PassesEncoderState enc_state;
+  AuxOut* aux_out = nullptr;
+  PaddedBytes compressed;
+  EXPECT_TRUE(EncodeFile(cparams, &io, &enc_state, &compressed, aux_out, pool));
+  CodecInOut io2;
+  EXPECT_TRUE(DecodeFile(dparams, compressed, &io2, aux_out, pool));
+
+  EXPECT_LE(compressed.size(), 120);
+
+  // TODO(robryk): Fix the following line in presence of different alpha_bits in
+  // the two contexts.
+  // EXPECT_TRUE(SamePixels(io.Main().alpha(), io2.Main().alpha()));
+  // TODO(robryk): Fix the distance estimate used in the encoder.
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            6.3);
+}
+
+TEST(JxlTest, RoundtripAlpha16) {
+  ThreadPoolInternal pool(4);
+
+  size_t xsize = 1200, ysize = 160;
+  std::vector<uint16_t> pixels(xsize * ysize * 4);
+  // Generate 16-bit pattern that uses various colors and alpha values.
+  for (size_t y = 0; y < ysize; y++) {
+    for (size_t x = 0; x < xsize; x++) {
+      size_t i = y * xsize + x;
+      pixels[i * 4 + 0] = y * 65535 / ysize;
+      pixels[i * 4 + 1] = x * 65535 / xsize;
+      pixels[i * 4 + 2] = (y + x) * 65535 / (xsize + ysize);
+      pixels[i * 4 + 3] = 65535 * x / xsize;
+    }
+  }
+  const bool is_gray = false;
+  CodecInOut io;
+  io.metadata.bits_per_sample = 16;
+  io.metadata.alpha_bits = 16;
+  io.metadata.color_encoding = ColorManagement::SRGB(is_gray);
+  ASSERT_TRUE(io.Main().SetFromSRGB(xsize, ysize, is_gray,
+                                    /*has_alpha=*/true, pixels.data(),
+                                    pixels.data() + pixels.size(), &pool));
+
+  // The image is wider than 512 pixels to ensure multiple groups are tested.
+
+  ASSERT_NE(io.xsize(), 0);
+  ASSERT_TRUE(io.metadata.HasAlpha());
+  ASSERT_TRUE(io.Main().HasAlpha());
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 1.0;
+  // Prevent the test to be too slow, does not affect alpha
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  DecompressParams dparams;
+
+  EXPECT_EQ(16, io.metadata.bits_per_sample);
+  EXPECT_TRUE(io.metadata.color_encoding.tf.IsSRGB());
+  PassesEncoderState enc_state;
+  AuxOut* aux_out = nullptr;
+  PaddedBytes compressed;
+  EXPECT_TRUE(
+      EncodeFile(cparams, &io, &enc_state, &compressed, aux_out, &pool));
+  CodecInOut io2;
+  EXPECT_TRUE(DecodeFile(dparams, compressed, &io2, aux_out, &pool));
+
+  EXPECT_TRUE(SamePixels(io.Main().alpha(), io2.Main().alpha()));
+}
+
+namespace {
+CompressParams CParamsForLossless() {
+  CompressParams cparams;
+  cparams.modular_group_mode = true;
+  cparams.color_transform = jxl::ColorTransform::kNone;
+  cparams.quality_pair = {100, 100};
+  cparams.options.predictor = {int(Predictor::Weighted)};
+  return cparams;
+}
+};  // namespace
+
+TEST(JxlTest, RoundtripLossless8) {
+  ThreadPoolInternal pool(8);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  CompressParams cparams = CParamsForLossless();
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, &pool, &io2), 3500000);
+  // If this test fails with a very close to 0.0 but not exactly 0.0 butteraugli
+  // distance, then there is likely a floating point issue, that could be
+  // happening either in io or io2. The values of io are generated by
+  // external_image.cc, and those in io2 by the jxl decoder. If they use
+  // slightly different floating point operations (say, one casts int to float
+  // while other divides the int through 255.0f and later multiplies it by
+  // 255 again) they will get slightly different values. To fix, ensure both
+  // sides do the following formula for converting integer range 0-255 to
+  // floating point range 0.0f-255.0f: static_cast<float>(i)
+  // without any further intermediate operations.
+  // Note that this precision issue is not a problem in practice if the values
+  // are equal when rounded to 8-bit int, but currently full exact precision is
+  // tested.
+  EXPECT_EQ(0.0, ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                     /*distmap=*/nullptr, &pool));
+}
+
+TEST(JxlTest, RoundtripLossless8Alpha) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/tmshre_riaphotographs_alpha.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  EXPECT_EQ(8, io.metadata.alpha_bits);
+  EXPECT_EQ(8, io.metadata.bits_per_sample);
+
+  CompressParams cparams = CParamsForLossless();
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 350000);
+  // If fails, see note about floating point in RoundtripLossless8.
+  EXPECT_EQ(0.0, ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                     /*distmap=*/nullptr, pool));
+  EXPECT_TRUE(SamePixels(io.Main().alpha(), io2.Main().alpha()));
+  EXPECT_EQ(8, io2.metadata.alpha_bits);
+  EXPECT_EQ(8, io2.metadata.bits_per_sample);
+}
+
+TEST(JxlTest, RoundtripLossless16Alpha) {
+  ThreadPool* pool = nullptr;
+
+  size_t xsize = 1200, ysize = 160;
+  std::vector<uint16_t> pixels(xsize * ysize * 4);
+  // Generate 16-bit pattern that uses various colors and alpha values.
+  for (size_t y = 0; y < ysize; y++) {
+    for (size_t x = 0; x < xsize; x++) {
+      size_t i = y * xsize + x;
+      pixels[i * 4 + 0] = y * 65535 / ysize;
+      pixels[i * 4 + 1] = x * 65535 / xsize;
+      pixels[i * 4 + 2] = (y + x) * 65535 / (xsize + ysize);
+      pixels[i * 4 + 3] = 65535 * y / ysize;
+    }
+  }
+  const bool is_gray = false;
+  CodecInOut io;
+  io.metadata.bits_per_sample = 16;
+  io.metadata.alpha_bits = 16;
+  io.metadata.color_encoding = ColorManagement::SRGB(is_gray);
+  ASSERT_TRUE(io.Main().SetFromSRGB(xsize, ysize, is_gray,
+                                    /*has_alpha=*/true, pixels.data(),
+                                    pixels.data() + pixels.size(), pool));
+
+  EXPECT_EQ(16, io.metadata.alpha_bits);
+  EXPECT_EQ(16, io.metadata.bits_per_sample);
+
+  CompressParams cparams = CParamsForLossless();
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 7000);
+  // If this test fails with a very close to 0.0 but not exactly 0.0 butteraugli
+  // distance, then there is likely a floating point issue, that could be
+  // happening either in io or io2. The values of io are generated by
+  // external_image.cc, and those in io2 by the jxl decoder. If they use
+  // slightly different floating point operations (say, one does "i / 257.0f"
+  // while the other does "i * (1.0f / 257)" they will get slightly different
+  // values. To fix, ensure both sides do the following formula for converting
+  // integer range 0-65535 to Image3F floating point range 0.0f-255.0f:
+  // "i * (1.0f / 257)".
+  // Note that this precision issue is not a problem in practice if the values
+  // are equal when rounded to 16-bit int, but currently full exact precision is
+  // tested.
+  EXPECT_EQ(0.0, ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                     /*distmap=*/nullptr, pool));
+  EXPECT_TRUE(SamePixels(io.Main().alpha(), io2.Main().alpha()));
+  EXPECT_EQ(16, io2.metadata.alpha_bits);
+  EXPECT_EQ(16, io2.metadata.bits_per_sample);
+}
+
+TEST(JxlTest, RoundtripLossless16AlphaNotMisdetectedAs8Bit) {
+  ThreadPool* pool = nullptr;
+
+  size_t xsize = 128, ysize = 128;
+  std::vector<uint16_t> pixels(xsize * ysize * 4);
+  // All 16-bit values, both color and alpha, of this image are below 64.
+  // This allows testing if a code path wrongly concludes it's an 8-bit instead
+  // of 16-bit image (or even 6-bit).
+  for (size_t y = 0; y < ysize; y++) {
+    for (size_t x = 0; x < xsize; x++) {
+      size_t i = y * xsize + x;
+      pixels[i * 4 + 0] = y * 64 / ysize;
+      pixels[i * 4 + 1] = x * 64 / xsize;
+      pixels[i * 4 + 2] = (y + x) * 64 / (xsize + ysize);
+      pixels[i * 4 + 3] = 64 * y / ysize;
+    }
+  }
+  const bool is_gray = false;
+  CodecInOut io;
+  io.metadata.bits_per_sample = 16;
+  io.metadata.alpha_bits = 16;
+  io.metadata.color_encoding = ColorManagement::SRGB(is_gray);
+  ASSERT_TRUE(io.Main().SetFromSRGB(xsize, ysize, /*is_gray=*/false,
+                                    /*has_alpha=*/true, pixels.data(),
+                                    pixels.data() + pixels.size(), pool));
+
+  EXPECT_EQ(16, io.metadata.alpha_bits);
+  EXPECT_EQ(16, io.metadata.bits_per_sample);
+
+  CompressParams cparams = CParamsForLossless();
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 2000);
+  EXPECT_EQ(16, io2.metadata.alpha_bits);
+  EXPECT_EQ(16, io2.metadata.bits_per_sample);
+  // If fails, see note about floating point in RoundtripLossless8.
+  EXPECT_EQ(0.0, ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                     /*distmap=*/nullptr, pool));
+  EXPECT_TRUE(SamePixels(io.Main().alpha(), io2.Main().alpha()));
+}
+
+TEST(JxlTest, RoundtripDots) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/cvo9xd_keong_macan_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+
+  ASSERT_NE(io.xsize(), 0);
+
+  CompressParams cparams;
+  cparams.dots = Override::kOn;
+  cparams.butteraugli_distance = 0.04;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  DecompressParams dparams;
+
+  EXPECT_EQ(8, io.metadata.bits_per_sample);
+  EXPECT_TRUE(io.metadata.color_encoding.tf.IsSRGB());
+  PassesEncoderState enc_state;
+  AuxOut* aux_out = nullptr;
+  PaddedBytes compressed;
+  EXPECT_TRUE(EncodeFile(cparams, &io, &enc_state, &compressed, aux_out, pool));
+  CodecInOut io2;
+  EXPECT_TRUE(DecodeFile(dparams, compressed, &io2, aux_out, pool));
+
+  EXPECT_LE(compressed.size(), 400000);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            2.2);
+}
+
+TEST(JxlTest, RoundtripLossless8Gray) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/cvo9xd_keong_macan_grayscale.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+
+  CompressParams cparams = CParamsForLossless();
+  DecompressParams dparams;
+
+  EXPECT_TRUE(io.Main().IsGray());
+  EXPECT_EQ(8, io.metadata.bits_per_sample);
+  CodecInOut io2;
+  EXPECT_LE(Roundtrip(&io, cparams, dparams, pool, &io2), 130000);
+  // If fails, see note about floating point in RoundtripLossless8.
+  EXPECT_EQ(0.0, ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                     /*distmap=*/nullptr, pool));
+  EXPECT_TRUE(io2.Main().IsGray());
+  EXPECT_EQ(8, io2.metadata.bits_per_sample);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/lehmer_code.h b/jxl/lehmer_code.h
new file mode 100644
index 000000000..569c870b1
--- /dev/null
+++ b/jxl/lehmer_code.h
@@ -0,0 +1,111 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_LEHMER_CODE_H_
+#define JXL_LEHMER_CODE_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Permutation <=> factorial base representation (Lehmer code).
+
+using LehmerT = uint32_t;
+
+template <typename T>
+constexpr T ValueOfLowest1Bit(T t) {
+  return t & -t;
+}
+
+// Computes the Lehmer (factorial basis) code of permutation, an array of n
+// unique indices in [0..n), and stores it in code[0..len). N*logN time.
+// temp must have n + 1 elements but need not be initialized.
+template <typename PermutationT>
+void ComputeLehmerCode(const PermutationT* JXL_RESTRICT permutation,
+                       uint32_t* JXL_RESTRICT temp, const size_t n,
+                       LehmerT* JXL_RESTRICT code) {
+  for (size_t idx = 0; idx < n + 1; ++idx) temp[idx] = 0;
+
+  for (size_t idx = 0; idx < n; ++idx) {
+    const PermutationT s = permutation[idx];
+
+    // Compute sum in Fenwick tree
+    uint32_t penalty = 0;
+    uint32_t i = s + 1;
+    while (i != 0) {
+      penalty += temp[i];
+      i &= i - 1;  // clear lowest bit
+    }
+    JXL_DASSERT(s >= penalty);
+    code[idx] = s - penalty;
+    i = s + 1;
+    // Add operation in Fenwick tree
+    while (i < n + 1) {
+      temp[i] += 1;
+      i += ValueOfLowest1Bit(i);
+    }
+  }
+}
+
+// Decodes the Lehmer code in code[0..n) into permutation[0..n).
+// temp must have 1 << CeilLog2(n) elements but need not be initialized.
+template <typename PermutationT>
+void DecodeLehmerCode(const LehmerT* JXL_RESTRICT code,
+                      uint32_t* JXL_RESTRICT temp, size_t n,
+                      PermutationT* JXL_RESTRICT permutation) {
+  JXL_DASSERT(n != 0);
+  const size_t log2n = CeilLog2Nonzero(n);
+  const size_t padded_n = 1ull << log2n;
+
+  for (size_t i = 0; i < padded_n; i++) {
+    const int32_t i1 = static_cast<int32_t>(i + 1);
+    temp[i] = static_cast<uint32_t>(ValueOfLowest1Bit(i1));
+  }
+
+  for (size_t i = 0; i < n; i++) {
+    JXL_DASSERT(code[i] + i < n);
+    uint32_t rank = code[i] + 1;
+
+    // Extract i-th unused element via implicit order-statistics tree.
+    size_t bit = padded_n;
+    size_t next = 0;
+    for (size_t i = 0; i <= log2n; i++) {
+      const size_t cand = next + bit;
+      JXL_DASSERT(cand >= 1);
+      bit >>= 1;
+      if (temp[cand - 1] < rank) {
+        next = cand;
+        rank -= temp[cand - 1];
+      }
+    }
+
+    permutation[i] = next;
+
+    // Mark as used
+    next += 1;
+    while (next <= padded_n) {
+      temp[next - 1] -= 1;
+      next += ValueOfLowest1Bit(next);
+    }
+  }
+}
+
+}  // namespace jxl
+
+#endif  // JXL_LEHMER_CODE_H_
diff --git a/jxl/lehmer_code_test.cc b/jxl/lehmer_code_test.cc
new file mode 100644
index 000000000..c2c3a5722
--- /dev/null
+++ b/jxl/lehmer_code_test.cc
@@ -0,0 +1,107 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/lehmer_code.h"
+
+#include <stdint.h>
+#include <string.h>
+
+#include <algorithm>
+#include <numeric>
+#include <random>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/thread_pool_internal.h"
+
+namespace jxl {
+namespace {
+
+template <typename PermutationT>
+struct WorkingSet {
+  explicit WorkingSet(size_t max_n)
+      : padded_n(1ull << CeilLog2Nonzero(max_n + 1)),
+        permutation(max_n),
+        temp(padded_n),
+        lehmer(max_n),
+        decoded(max_n) {}
+
+  size_t padded_n;
+  std::vector<PermutationT> permutation;
+  std::vector<uint32_t> temp;
+  std::vector<LehmerT> lehmer;
+  std::vector<PermutationT> decoded;
+};
+
+template <typename PermutationT>
+void Roundtrip(size_t n, WorkingSet<PermutationT>* ws) {
+  JXL_ASSERT(n != 0);
+  const size_t padded_n = 1ull << CeilLog2Nonzero(n);
+
+  std::mt19937 rng(n * 65537 + 13);
+
+  // Ensure indices fit into PermutationT
+  EXPECT_LE(n, 1ULL << (sizeof(PermutationT) * 8));
+
+  std::iota(ws->permutation.begin(), ws->permutation.begin() + n, 0);
+
+  // For various random permutations:
+  for (size_t rep = 0; rep < 100; ++rep) {
+    std::shuffle(ws->permutation.begin(), ws->permutation.begin() + n, rng);
+
+    // Must decode to the same permutation
+    ComputeLehmerCode(ws->permutation.data(), ws->temp.data(), n,
+                      ws->lehmer.data());
+    memset(ws->temp.data(), 0, padded_n * 4);
+    DecodeLehmerCode(ws->lehmer.data(), ws->temp.data(), n, ws->decoded.data());
+
+    for (size_t i = 0; i < n; ++i) {
+      EXPECT_EQ(ws->permutation[i], ws->decoded[i]);
+    }
+  }
+}
+
+// Preallocates arrays and tests n = [begin, end).
+template <typename PermutationT>
+void RoundtripSizeRange(ThreadPool* pool, uint32_t begin, uint32_t end) {
+  ASSERT_NE(0, begin);  // n = 0 not allowed.
+  std::vector<WorkingSet<PermutationT>> working_sets;
+
+  RunOnPool(
+      pool, begin, end,
+      [&working_sets, end](size_t num_threads) {
+        for (size_t i = 0; i < num_threads; i++) {
+          working_sets.emplace_back(end - 1);
+        }
+        return true;
+      },
+      [&working_sets](int n, int thread) {
+        Roundtrip(n, &working_sets[thread]);
+      },
+      "lehmer test");
+}
+
+TEST(LehmerCodeTest, TestRoundtrips) {
+  ThreadPoolInternal pool(8);
+
+  RoundtripSizeRange<uint16_t>(&pool, 1, 1026);
+
+  // Ensures PermutationT can fit > 16 bit values.
+  RoundtripSizeRange<uint32_t>(&pool, 65536, 65540);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/libjpegxl.pc.in b/jxl/libjpegxl.pc.in
new file mode 100644
index 000000000..f66682fe8
--- /dev/null
+++ b/jxl/libjpegxl.pc.in
@@ -0,0 +1,12 @@
+prefix=@CMAKE_INSTALL_PREFIX@
+exec_prefix=${prefix}
+libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@
+includedir=${prefix}/@CMAKE_INSTALL_INCLUDEDIR@
+
+Name: libjpegxl
+Description: Loads and saves JPEG XL files
+Version: @JPEGXL_LIBRARY_VERSION@
+Requires.private: @JPEGXL_LIBRARY_REQUIRES@
+Libs: -L${libdir} -ljpegxl
+Libs.private: -lm
+Cflags: -I${includedir}
diff --git a/jxl/linalg.cc b/jxl/linalg.cc
new file mode 100644
index 000000000..df6a585b7
--- /dev/null
+++ b/jxl/linalg.cc
@@ -0,0 +1,244 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/linalg.h"
+
+#include <stdlib.h>
+
+#include <cmath>
+#include <deque>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/image_ops.h"
+
+namespace jxl {
+
+void AssertSymmetric(const ImageD& A) {
+#if JXL_ENABLE_ASSERT
+  JXL_ASSERT(A.xsize() == A.ysize());
+  for (size_t i = 0; i < A.xsize(); ++i) {
+    for (size_t j = i + 1; j < A.xsize(); ++j) {
+      JXL_ASSERT(std::abs(A.Row(i)[j] - A.Row(j)[i]) < 1e-15);
+    }
+  }
+#endif
+}
+
+void Diagonalize2x2(const double a0, const double a1, const double b, double* c,
+                    double* s) {
+  if (std::abs(b) < 1e-15) {
+    *c = 1.0;
+    *s = 0.0;
+    return;
+  }
+  double phi = std::atan2(2 * b, a1 - a0);
+  double theta = b > 0.0 ? 0.5 * phi : 0.5 * phi + Pi(1.0);
+  *c = std::cos(theta);
+  *s = std::sin(theta);
+}
+
+void GivensRotation(const double x, const double y, double* c, double* s) {
+  if (y == 0.0) {
+    *c = x < 0.0 ? -1.0 : 1.0;
+    *s = 0.0;
+  } else {
+    const double h = std::hypot(x, y);
+    const double d = 1.0 / h;
+    *c = x * d;
+    *s = -y * d;
+  }
+}
+
+void RotateMatrixCols(ImageD* const JXL_RESTRICT U, int i, int j, double c,
+                      double s) {
+  JXL_ASSERT(U->xsize() == U->ysize());
+  const size_t N = U->xsize();
+  double* const JXL_RESTRICT u_i = U->Row(i);
+  double* const JXL_RESTRICT u_j = U->Row(j);
+  std::vector<double> rot_i, rot_j;
+  rot_i.reserve(N);
+  rot_j.reserve(N);
+  for (size_t k = 0; k < N; ++k) {
+    rot_i.push_back(u_i[k] * c - u_j[k] * s);
+    rot_j.push_back(u_i[k] * s + u_j[k] * c);
+  }
+  for (size_t k = 0; k < N; ++k) {
+    u_i[k] = rot_i[k];
+    u_j[k] = rot_j[k];
+  }
+}
+void HouseholderReflector(const size_t N, const double* x, double* u) {
+  const double sigma = x[0] <= 0.0 ? 1.0 : -1.0;
+  u[0] = x[0] - sigma * std::sqrt(DotProduct(N, x, x));
+  for (size_t k = 1; k < N; ++k) {
+    u[k] = x[k];
+  }
+  double u_norm = 1.0 / std::sqrt(DotProduct(N, u, u));
+  for (size_t k = 0; k < N; ++k) {
+    u[k] *= u_norm;
+  }
+}
+
+void ConvertToTridiagonal(const ImageD& A, ImageD* const JXL_RESTRICT T,
+                          ImageD* const JXL_RESTRICT U) {
+  AssertSymmetric(A);
+  const size_t N = A.xsize();
+  *U = Identity<double>(A.xsize());
+  *T = CopyImage(A);
+  std::vector<ImageD> u_stack;
+  for (size_t k = 0; k + 2 < N; ++k) {
+    if (DotProduct(N - k - 2, &T->Row(k)[k + 2], &T->Row(k)[k + 2]) > 1e-15) {
+      ImageD u(N, 1);
+      ZeroFillImage(&u);
+      HouseholderReflector(N - k - 1, &T->Row(k)[k + 1], &u.Row(0)[k + 1]);
+      ImageD v = MatMul(*T, u);
+      double scale = DotProduct(u, v);
+      v = LinComb(2.0, v, -2.0 * scale, u);
+      SubtractFrom(MatMul(u, Transpose(v)), T);
+      SubtractFrom(MatMul(v, Transpose(u)), T);
+      u_stack.emplace_back(std::move(u));
+    }
+  }
+  while (!u_stack.empty()) {
+    const ImageD& u = u_stack.back();
+    ImageD v = MatMul(Transpose(*U), u);
+    SubtractFrom(ScaleImage(2.0, MatMul(u, Transpose(v))), U);
+    u_stack.pop_back();
+  }
+}
+
+double WilkinsonShift(const double a0, const double a1, const double b) {
+  const double d = 0.5 * (a0 - a1);
+  if (d == 0.0) {
+    return a1 - std::abs(b);
+  }
+  const double sign_d = d > 0.0 ? 1.0 : -1.0;
+  return a1 - b * b / (d + sign_d * std::hypot(d, b));
+}
+
+void ImplicitQRStep(ImageD* const JXL_RESTRICT U, double* const JXL_RESTRICT a,
+                    double* const JXL_RESTRICT b, int m0, int m1) {
+  JXL_ASSERT(m1 - m0 > 2);
+  double x = a[m0] - WilkinsonShift(a[m1 - 2], a[m1 - 1], b[m1 - 1]);
+  double y = b[m0 + 1];
+  for (int k = m0; k < m1 - 1; ++k) {
+    double c, s;
+    GivensRotation(x, y, &c, &s);
+    const double w = c * x - s * y;
+    const double d = a[k] - a[k + 1];
+    const double z = (2 * c * b[k + 1] + d * s) * s;
+    a[k] -= z;
+    a[k + 1] += z;
+    b[k + 1] = d * c * s + (c * c - s * s) * b[k + 1];
+    x = b[k + 1];
+    if (k > m0) {
+      b[k] = w;
+    }
+    if (k < m1 - 2) {
+      y = -s * b[k + 2];
+      b[k + 2] *= c;
+    }
+    RotateMatrixCols(U, k, k + 1, c, s);
+  }
+}
+
+void ScanInterval(const double* const JXL_RESTRICT a,
+                  const double* const JXL_RESTRICT b, int istart,
+                  const int iend, const double eps,
+                  std::deque<std::pair<int, int> >* intervals) {
+  for (int k = istart; k < iend; ++k) {
+    if ((k + 1 == iend) ||
+        std::abs(b[k + 1]) < eps * (std::abs(a[k]) + std::abs(a[k + 1]))) {
+      if (k > istart) {
+        intervals->push_back(std::make_pair(istart, k + 1));
+      }
+      istart = k + 1;
+    }
+  }
+}
+
+void ConvertToDiagonal(const ImageD& A, ImageD* const JXL_RESTRICT diag,
+                       ImageD* const JXL_RESTRICT U) {
+  AssertSymmetric(A);
+  const size_t N = A.xsize();
+  ImageD T;
+  ConvertToTridiagonal(A, &T, U);
+  // From now on, the algorithm keeps the transformed matrix tri-diagonal,
+  // so we only need to keep track of the diagonal and the off-diagonal entries.
+  std::vector<double> a(N);
+  std::vector<double> b(N);
+  for (size_t k = 0; k < N; ++k) {
+    a[k] = T.Row(k)[k];
+    if (k > 0) b[k] = T.Row(k)[k - 1];
+  }
+  // Run the symmetric tri-diagonal QR algorithm with implicit Wilkinson shift.
+  const double kEpsilon = 1e-14;
+  std::deque<std::pair<int, int> > intervals;
+  ScanInterval(&a[0], &b[0], 0, N, kEpsilon, &intervals);
+  while (!intervals.empty()) {
+    const int istart = intervals[0].first;
+    const int iend = intervals[0].second;
+    intervals.pop_front();
+    if (iend == istart + 2) {
+      double& a0 = a[istart];
+      double& a1 = a[istart + 1];
+      double& b1 = b[istart + 1];
+      double c, s;
+      Diagonalize2x2(a0, a1, b1, &c, &s);
+      const double d = a0 - a1;
+      const double z = (2 * c * b1 + d * s) * s;
+      a0 -= z;
+      a1 += z;
+      b1 = 0.0;
+      RotateMatrixCols(U, istart, istart + 1, c, s);
+    } else {
+      ImplicitQRStep(U, &a[0], &b[0], istart, iend);
+      ScanInterval(&a[0], &b[0], istart, iend, kEpsilon, &intervals);
+    }
+  }
+  *diag = ImageD(N, 1);
+  double* const JXL_RESTRICT diag_row = diag->Row(0);
+  for (size_t k = 0; k < N; ++k) {
+    diag_row[k] = a[k];
+  }
+}
+
+void ComputeQRFactorization(const ImageD& A, ImageD* const JXL_RESTRICT Q,
+                            ImageD* const JXL_RESTRICT R) {
+  JXL_ASSERT(A.xsize() == A.ysize());
+  const size_t N = A.xsize();
+  *Q = Identity<double>(N);
+  *R = CopyImage(A);
+  std::vector<ImageD> u_stack;
+  for (size_t k = 0; k + 1 < N; ++k) {
+    if (DotProduct(N - k - 1, &R->Row(k)[k + 1], &R->Row(k)[k + 1]) > 1e-15) {
+      ImageD u(N, 1);
+      FillImage(0.0, &u);
+      HouseholderReflector(N - k, &R->Row(k)[k], &u.Row(0)[k]);
+      ImageD v = MatMul(Transpose(u), *R);
+      SubtractFrom(ScaleImage(2.0, MatMul(u, v)), R);
+      u_stack.emplace_back(std::move(u));
+    }
+  }
+  while (!u_stack.empty()) {
+    const ImageD& u = u_stack.back();
+    ImageD v = MatMul(Transpose(u), *Q);
+    SubtractFrom(ScaleImage(2.0, MatMul(u, v)), Q);
+    u_stack.pop_back();
+  }
+}
+}  // namespace jxl
diff --git a/jxl/linalg.h b/jxl/linalg.h
new file mode 100644
index 000000000..282b7ab58
--- /dev/null
+++ b/jxl/linalg.h
@@ -0,0 +1,299 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_LINALG_H_
+#define JXL_LINALG_H_
+
+// Linear algebra.
+
+#include <stddef.h>
+
+#include <algorithm>
+#include <cmath>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/image.h"
+#include "jxl/image_ops.h"
+
+namespace jxl {
+
+using ImageD = Plane<double>;
+
+template <typename T>
+inline T DotProduct(const size_t N, const T* const JXL_RESTRICT a,
+                         const T* const JXL_RESTRICT b) {
+  T sum = 0.0;
+  for (size_t k = 0; k < N; ++k) {
+    sum += a[k] * b[k];
+  }
+  return sum;
+}
+
+template <typename T>
+inline T L2NormSquared(const size_t N, const T* const JXL_RESTRICT a) {
+  return DotProduct(N, a, a);
+}
+
+template <typename T>
+inline T L1Norm(const size_t N, const T* const JXL_RESTRICT a) {
+  T sum = 0;
+  for (size_t k = 0; k < N; ++k) {
+    sum += a[k] >= 0 ? a[k] : -a[k];
+  }
+  return sum;
+}
+
+inline double DotProduct(const ImageD& a, const ImageD& b) {
+  JXL_ASSERT(a.ysize() == 1);
+  JXL_ASSERT(b.ysize() == 1);
+  JXL_ASSERT(a.xsize() == b.xsize());
+  const double* const JXL_RESTRICT row_a = a.Row(0);
+  const double* const JXL_RESTRICT row_b = b.Row(0);
+  return DotProduct(a.xsize(), row_a, row_b);
+}
+
+inline ImageD Transpose(const ImageD& A) {
+  ImageD out(A.ysize(), A.xsize());
+  for (size_t x = 0; x < A.xsize(); ++x) {
+    double* const JXL_RESTRICT row_out = out.Row(x);
+    for (size_t y = 0; y < A.ysize(); ++y) {
+      row_out[y] = A.Row(y)[x];
+    }
+  }
+  return out;
+}
+
+template <typename Tout, typename Tin1, typename Tin2>
+Plane<Tout> MatMul(const Plane<Tin1>& A, const Plane<Tin2>& B) {
+  JXL_ASSERT(A.ysize() == B.xsize());
+  Plane<Tout> out(A.xsize(), B.ysize());
+  for (size_t y = 0; y < B.ysize(); ++y) {
+    const Tin2* const JXL_RESTRICT row_b = B.Row(y);
+    Tout* const JXL_RESTRICT row_out = out.Row(y);
+    for (size_t x = 0; x < A.xsize(); ++x) {
+      row_out[x] = 0.0;
+      for (size_t k = 0; k < B.xsize(); ++k) {
+        row_out[x] += A.Row(k)[x] * row_b[k];
+      }
+    }
+  }
+  return out;
+}
+
+template <typename T1, typename T2>
+ImageD MatMul(const Plane<T1>& A, const Plane<T2>& B) {
+  return MatMul<double, T1, T2>(A, B);
+}
+
+template <typename T1, typename T2>
+ImageI MatMulI(const Plane<T1>& A, const Plane<T2>& B) {
+  return MatMul<int, T1, T2>(A, B);
+}
+
+// Computes A = B * C, with sizes rows*cols: A=ha*wa, B=wa*wb, C=ha*wb
+template <typename T>
+void MatMul(const T* a, const T* b, int ha, int wa, int wb, T* c) {
+  std::vector<T> temp(wa);  // Make better use of cache lines
+  for (int x = 0; x < wb; x++) {
+    for (int z = 0; z < wa; z++) {
+      temp[z] = b[z * wb + x];
+    }
+    for (int y = 0; y < ha; y++) {
+      double e = 0;
+      for (int z = 0; z < wa; z++) {
+        e += a[y * wa + z] * temp[z];
+      }
+      c[y * wb + x] = e;
+    }
+  }
+}
+
+// Computes C = A + factor * B
+template <typename T, typename F>
+void MatAdd(const T* a, const T* b, F factor, int h, int w, T* c) {
+  for (int i = 0; i < w * h; i++) {
+    c[i] = a[i] + b[i] * factor;
+  }
+}
+
+template <typename T>
+inline Plane<T> Identity(const size_t N) {
+  Plane<T> out(N, N);
+  for (size_t i = 0; i < N; ++i) {
+    T* JXL_RESTRICT row = out.Row(i);
+    std::fill(row, row + N, 0);
+    row[i] = static_cast<T>(1.0);
+  }
+  return out;
+}
+
+inline ImageD Diagonal(const ImageD& d) {
+  JXL_ASSERT(d.ysize() == 1);
+  ImageD out(d.xsize(), d.xsize());
+  const double* JXL_RESTRICT row_diag = d.Row(0);
+  for (size_t k = 0; k < d.xsize(); ++k) {
+    double* JXL_RESTRICT row_out = out.Row(k);
+    std::fill(row_out, row_out + d.xsize(), 0.0);
+    row_out[k] = row_diag[k];
+  }
+  return out;
+}
+
+// Computes c, s such that c^2 + s^2 = 1 and
+//   [c -s] [x] = [ * ]
+//   [s  c] [y]   [ 0 ]
+void GivensRotation(double x, double y, double* c, double* s);
+
+// U = U * Givens(i, j, c, s)
+void RotateMatrixCols(ImageD* JXL_RESTRICT U, int i, int j, double c, double s);
+
+// A is symmetric, U is orthogonal, T is tri-diagonal and
+// A = U * T * Transpose(U).
+void ConvertToTridiagonal(const ImageD& A, ImageD* JXL_RESTRICT T,
+                          ImageD* JXL_RESTRICT U);
+
+// A is symmetric, U is orthogonal, and A = U * Diagonal(diag) * Transpose(U).
+void ConvertToDiagonal(const ImageD& A, ImageD* JXL_RESTRICT diag,
+                       ImageD* JXL_RESTRICT U);
+
+// A is square matrix, Q is orthogonal, R is upper triangular and A = Q * R;
+void ComputeQRFactorization(const ImageD& A, ImageD* JXL_RESTRICT Q,
+                            ImageD* JXL_RESTRICT R);
+
+// Inverts a 3x3 matrix in place
+template <typename T>
+void Inv3x3Matrix(T* matrix) {
+  // Intermediate computation is done in double precision.
+  double temp[9];
+  temp[0] = static_cast<double>(matrix[4]) * matrix[8] -
+            static_cast<double>(matrix[5]) * matrix[7];
+  temp[1] = static_cast<double>(matrix[2]) * matrix[7] -
+            static_cast<double>(matrix[1]) * matrix[8];
+  temp[2] = static_cast<double>(matrix[1]) * matrix[5] -
+            static_cast<double>(matrix[2]) * matrix[4];
+  temp[3] = static_cast<double>(matrix[5]) * matrix[6] -
+            static_cast<double>(matrix[3]) * matrix[8];
+  temp[4] = static_cast<double>(matrix[0]) * matrix[8] -
+            static_cast<double>(matrix[2]) * matrix[6];
+  temp[5] = static_cast<double>(matrix[2]) * matrix[3] -
+            static_cast<double>(matrix[0]) * matrix[5];
+  temp[6] = static_cast<double>(matrix[3]) * matrix[7] -
+            static_cast<double>(matrix[4]) * matrix[6];
+  temp[7] = static_cast<double>(matrix[1]) * matrix[6] -
+            static_cast<double>(matrix[0]) * matrix[7];
+  temp[8] = static_cast<double>(matrix[0]) * matrix[4] -
+            static_cast<double>(matrix[1]) * matrix[3];
+  double idet =
+      1.0 / (matrix[0] * temp[0] + matrix[1] * temp[3] + matrix[2] * temp[6]);
+  for (int i = 0; i < 9; i++) {
+    matrix[i] = temp[i] * idet;
+  }
+}
+
+// Solves system of linear equations A * X = B using the conjugate gradient
+// method. Matrix a must be a n*n, symmetric and positive definite.
+// Vectors b and x must have n elements
+template <typename T>
+void ConjugateGradient(const T* a, int n, const T* b, T* x) {
+  std::vector<T> r(n);
+  MatMul(a, x, n, n, 1, r.data());
+  MatAdd(b, r.data(), -1, n, 1, r.data());
+  std::vector<T> p = r;
+  T rr;
+  MatMul(r.data(), r.data(), 1, n, 1, &rr);  // inner product
+
+  if (rr == 0) return;  // The initial values were already optimal
+
+  for (int i = 0; i < n; i++) {
+    std::vector<T> ap(n);
+    MatMul(a, p.data(), n, n, 1, ap.data());
+    T alpha;
+    MatMul(r.data(), ap.data(), 1, n, 1, &alpha);
+    // Normally alpha couldn't be zero here but if numerical issues caused it,
+    // return assuming the solution is close.
+    if (alpha == 0) return;
+    alpha = rr / alpha;
+    MatAdd(x, p.data(), alpha, n, 1, x);
+    MatAdd(r.data(), ap.data(), -alpha, n, 1, r.data());
+
+    T rr2;
+    MatMul(r.data(), r.data(), 1, n, 1, &rr2);  // inner product
+    if (rr2 < 1e-20) break;
+
+    T beta = rr2 / rr;
+    MatAdd(r.data(), p.data(), beta, 1, n, p.data());
+    rr = rr2;
+  }
+}
+
+// Computes optimal coefficients r to approximate points p with linear
+// combination of functions f. The matrix f has h rows and w columns, r has h
+// values, p has w values. h is the amount of functions, w the amount of points.
+// Uses the finite element method and minimizes mean square error.
+template <typename T>
+void FEM(const T* f, int h, int w, const T* p, T* r) {
+  // Compute "Gramian" matrix G = F * F^T
+  // Speed up multiplication by using non-zero intervals in sparse F.
+  std::vector<int> start(h);
+  std::vector<int> end(h);
+  for (int y = 0; y < h; y++) {
+    start[y] = end[y] = 0;
+    for (int x = 0; x < w; x++) {
+      if (f[y * w + x] != 0) {
+        start[y] = x;
+        break;
+      }
+    }
+    for (int x = w - 1; x >= 0; x--) {
+      if (f[y * w + x] != 0) {
+        end[y] = x + 1;
+        break;
+      }
+    }
+  }
+
+  std::vector<T> g(h * h);
+  for (int y = 0; y < h; y++) {
+    for (int x = 0; x <= y; x++) {
+      T v = 0;
+      // Intersection of the two sparse intervals.
+      int s = std::max(start[x], start[y]);
+      int e = std::min(end[x], end[y]);
+      for (int z = s; z < e; z++) {
+        v += f[x * w + z] * f[y * w + z];
+      }
+      // Symmetric, so two values output at once
+      g[y * h + x] = v;
+      g[x * h + y] = v;
+    }
+  }
+
+  // B vector: sum of each column of F multiplied by corresponding p
+  std::vector<T> b(h, 0);
+  for (int y = 0; y < h; y++) {
+    T v = 0;
+    for (int x = 0; x < w; x++) {
+      v += f[y * w + x] * p[x];
+    }
+    b[y] = v;
+  }
+
+  ConjugateGradient(g.data(), h, b.data(), r);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_LINALG_H_
diff --git a/jxl/linalg_test.cc b/jxl/linalg_test.cc
new file mode 100644
index 000000000..4d9d47611
--- /dev/null
+++ b/jxl/linalg_test.cc
@@ -0,0 +1,158 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/linalg.h"
+
+#include <random>
+
+#include "gtest/gtest.h"
+#include "jxl/image_test_utils.h"
+
+namespace jxl {
+namespace {
+
+template <typename T, typename Random>
+Plane<T> RandomMatrix(const size_t xsize, const size_t ysize, Random& rng,
+                      const T vmin, const T vmax) {
+  Plane<T> A(xsize, ysize);
+  GeneratorRandom<T, Random> gen(&rng, vmin, vmax);
+  GenerateImage(gen, &A);
+  return A;
+}
+
+template <typename T, typename Random>
+Plane<T> RandomSymmetricMatrix(const size_t N, Random& rng, const T vmin,
+                               const T vmax) {
+  Plane<T> A = RandomMatrix<T>(N, N, rng, vmin, vmax);
+  for (size_t i = 0; i < N; ++i) {
+    for (size_t j = 0; j < i; ++j) {
+      A.Row(j)[i] = A.Row(i)[j];
+    }
+  }
+  return A;
+}
+void VerifyMatrixEqual(const ImageD& A, const ImageD& B, const double eps) {
+  ASSERT_EQ(A.xsize(), B.xsize());
+  ASSERT_EQ(A.ysize(), B.ysize());
+  for (size_t y = 0; y < A.ysize(); ++y) {
+    for (size_t x = 0; x < A.xsize(); ++x) {
+      ASSERT_NEAR(A.Row(y)[x], B.Row(y)[x], eps);
+    }
+  }
+}
+
+void VerifyOrthogonal(const ImageD& A, const double eps) {
+  VerifyMatrixEqual(Identity<double>(A.xsize()), MatMul(Transpose(A), A), eps);
+}
+
+void VerifyTridiagonal(const ImageD& T, const double eps) {
+  ASSERT_EQ(T.xsize(), T.ysize());
+  for (size_t i = 0; i < T.xsize(); ++i) {
+    for (size_t j = i + 2; j < T.xsize(); ++j) {
+      ASSERT_NEAR(T.Row(i)[j], 0.0, eps);
+      ASSERT_NEAR(T.Row(j)[i], 0.0, eps);
+    }
+  }
+}
+
+void VerifyUpperTriangular(const ImageD& R, const double eps) {
+  ASSERT_EQ(R.xsize(), R.ysize());
+  for (size_t i = 0; i < R.xsize(); ++i) {
+    for (size_t j = i + 1; j < R.xsize(); ++j) {
+      ASSERT_NEAR(R.Row(i)[j], 0.0, eps);
+    }
+  }
+}
+
+TEST(LinAlgTest, ConvertToTridiagonal) {
+  {
+    ImageD I = Identity<double>(5);
+    ImageD T, U;
+    ConvertToTridiagonal(I, &T, &U);
+    VerifyMatrixEqual(I, T, 1e-15);
+    VerifyMatrixEqual(I, U, 1e-15);
+  }
+  {
+    ImageD A = Identity<double>(5);
+    A.Row(0)[1] = A.Row(1)[0] = 2.0;
+    A.Row(0)[4] = A.Row(4)[0] = 3.0;
+    A.Row(2)[3] = A.Row(3)[2] = 2.0;
+    A.Row(3)[4] = A.Row(4)[3] = 2.0;
+    ImageD U, d;
+    ConvertToDiagonal(A, &d, &U);
+    VerifyOrthogonal(U, 1e-12);
+    VerifyMatrixEqual(A, MatMul(U, MatMul(Diagonal(d), Transpose(U))), 1e-12);
+  }
+  std::mt19937_64 rng;
+  for (int N = 2; N < 100; ++N) {
+    ImageD A = RandomSymmetricMatrix(N, rng, -1.0, 1.0);
+    ImageD T, U;
+    ConvertToTridiagonal(A, &T, &U);
+    VerifyOrthogonal(U, 1e-12);
+    VerifyTridiagonal(T, 1e-12);
+    VerifyMatrixEqual(A, MatMul(U, MatMul(T, Transpose(U))), 1e-12);
+  }
+}
+
+TEST(LinAlgTest, ConvertToDiagonal) {
+  {
+    ImageD I = Identity<double>(5);
+    ImageD U, d;
+    ConvertToDiagonal(I, &d, &U);
+    VerifyMatrixEqual(I, U, 1e-15);
+    for (int k = 0; k < 5; ++k) {
+      ASSERT_NEAR(d.Row(0)[k], 1.0, 1e-15);
+    }
+  }
+  {
+    ImageD A = Identity<double>(5);
+    A.Row(0)[1] = A.Row(1)[0] = 2.0;
+    A.Row(2)[3] = A.Row(3)[2] = 2.0;
+    A.Row(3)[4] = A.Row(4)[3] = 2.0;
+    ImageD U, d;
+    ConvertToDiagonal(A, &d, &U);
+    VerifyOrthogonal(U, 1e-12);
+    VerifyMatrixEqual(A, MatMul(U, MatMul(Diagonal(d), Transpose(U))), 1e-12);
+  }
+  std::mt19937_64 rng;
+  for (int N = 2; N < 100; ++N) {
+    ImageD A = RandomSymmetricMatrix(N, rng, -1.0, 1.0);
+    ImageD U, d;
+    ConvertToDiagonal(A, &d, &U);
+    VerifyOrthogonal(U, 1e-12);
+    VerifyMatrixEqual(A, MatMul(U, MatMul(Diagonal(d), Transpose(U))), 1e-12);
+  }
+}
+
+TEST(LinAlgTest, ComputeQRFactorization) {
+  {
+    ImageD I = Identity<double>(5);
+    ImageD Q, R;
+    ComputeQRFactorization(I, &Q, &R);
+    VerifyMatrixEqual(I, Q, 1e-15);
+    VerifyMatrixEqual(I, R, 1e-15);
+  }
+  std::mt19937_64 rng;
+  for (int N = 2; N < 100; ++N) {
+    ImageD A = RandomMatrix(N, N, rng, -1.0, 1.0);
+    ImageD Q, R;
+    ComputeQRFactorization(A, &Q, &R);
+    VerifyOrthogonal(Q, 1e-12);
+    VerifyUpperTriangular(R, 1e-12);
+    VerifyMatrixEqual(A, MatMul(Q, R), 1e-12);
+  }
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/loop_filter.cc b/jxl/loop_filter.cc
new file mode 100644
index 000000000..91bcc8d62
--- /dev/null
+++ b/jxl/loop_filter.cc
@@ -0,0 +1,34 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/loop_filter.h"
+
+#include "jxl/aux_out.h"
+#include "jxl/fields.h"
+
+namespace jxl {
+
+LoopFilter::LoopFilter() { Bundle::Init(this); }
+
+Status ReadLoopFilter(BitReader* JXL_RESTRICT reader,
+                      LoopFilter* JXL_RESTRICT loop_filter) {
+  return Bundle::Read(reader, loop_filter);
+}
+
+Status WriteLoopFilter(const LoopFilter& loop_filter,
+                       BitWriter* JXL_RESTRICT writer, AuxOut* aux_out) {
+  return Bundle::Write(loop_filter, writer, kLayerHeader, aux_out);
+}
+
+}  // namespace jxl
diff --git a/jxl/loop_filter.h b/jxl/loop_filter.h
new file mode 100644
index 000000000..12194c881
--- /dev/null
+++ b/jxl/loop_filter.h
@@ -0,0 +1,131 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_LOOP_FILTER_H_
+#define JXL_LOOP_FILTER_H_
+
+// Parameters for loop filter(s), stored in each frame.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/field_encodings.h"
+
+namespace jxl {
+
+struct LoopFilter {
+  LoopFilter();
+  static const char* Name() { return "LoopFilter"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    // Must come before AllDefault.
+
+    if (visitor->AllDefault(*this, &all_default)) return true;
+
+    visitor->Bool(true, &gab);
+    if (visitor->Conditional(gab)) {
+      visitor->Bool(false, &gab_custom);
+      if (visitor->Conditional(gab_custom)) {
+        visitor->F16(1.1 * 0.104699568f, &gab_x_weight1);
+        visitor->F16(1.1 * 0.055680538f, &gab_x_weight2);
+        visitor->F16(1.1 * 0.104699568f, &gab_y_weight1);
+        visitor->F16(1.1 * 0.055680538f, &gab_y_weight2);
+        visitor->F16(1.1 * 0.104699568f, &gab_b_weight1);
+        visitor->F16(1.1 * 0.055680538f, &gab_b_weight2);
+      }
+    }
+
+    visitor->Bool(true, &epf);
+    if (visitor->Conditional(epf)) {
+      visitor->Bool(false, &epf_sharp_custom);
+      if (visitor->Conditional(epf_sharp_custom)) {
+        for (size_t i = 0; i < kEpfSharpEntries; ++i) {
+          visitor->F16(float(i) / float(kEpfSharpEntries - 1),
+                       &epf_sharp_lut[i]);
+        }
+      }
+
+      visitor->Bool(false, &epf_weight_custom);
+      if (visitor->Conditional(epf_weight_custom)) {
+        visitor->F16(40.0f, &epf_channel_scale[0]);
+        visitor->F16(5.0f, &epf_channel_scale[1]);
+        visitor->F16(3.5f, &epf_channel_scale[2]);
+        visitor->F16(0.45f, &epf_pass1_zeroflush);
+        visitor->F16(0.6f, &epf_pass2_zeroflush);
+      }
+
+      visitor->Bool(false, &epf_sigma_custom);
+      if (visitor->Conditional(epf_sigma_custom)) {
+        visitor->F16(0.315f, &epf_quant_mul);
+        visitor->F16(4.0f, &epf_dc_range_mul);
+        visitor->F16(6.5f, &epf_pass2_sigma_scale);
+        visitor->F16(0.6666666666666666f, &epf_border_sad_mul);
+      }
+    }
+
+    visitor->BeginExtensions(&extensions);
+    // Extensions: in chronological order of being added to the format.
+    return visitor->EndExtensions();
+  }
+
+  mutable bool all_default;
+
+  // --- Gaborish convolution
+  bool gab;
+
+  bool gab_custom;
+  float gab_x_weight1;
+  float gab_x_weight2;
+  float gab_y_weight1;
+  float gab_y_weight2;
+  float gab_b_weight1;
+  float gab_b_weight2;
+
+  // --- Edge-preserving filter
+  bool epf;
+
+  bool epf_sharp_custom;
+  enum { kEpfSharpEntries = 8 };
+  float epf_sharp_lut[kEpfSharpEntries];
+
+  bool epf_weight_custom;      // Custom weight params
+  float epf_channel_scale[3];  // Relative weight of each channel
+  float epf_pass1_zeroflush;   // Minimum weight for first pass
+  float epf_pass2_zeroflush;   // Minimum weight for second pass
+
+  bool epf_sigma_custom;        // Custom sigma parameters
+  float epf_quant_mul;          // Sigma is ~ this * quant
+  float epf_dc_range_mul;       // How much to increase sigma with DC range
+  float epf_pass2_sigma_scale;  // Multiplier for sigma in the second pass
+  float epf_border_sad_mul;     // (inverse) multiplier for sigma on borders
+
+  uint64_t extensions;
+};
+
+Status ReadLoopFilter(BitReader* JXL_RESTRICT reader,
+                      LoopFilter* JXL_RESTRICT loop_filter);
+
+Status WriteLoopFilter(const LoopFilter& loop_filter,
+                       BitWriter* JXL_RESTRICT writer, AuxOut* aux_out);
+
+}  // namespace jxl
+
+#endif  // JXL_LOOP_FILTER_H_
diff --git a/jxl/memory_manager_internal.cc b/jxl/memory_manager_internal.cc
new file mode 100644
index 000000000..5a52070c1
--- /dev/null
+++ b/jxl/memory_manager_internal.cc
@@ -0,0 +1,27 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/memory_manager_internal.h"
+
+#include <stdlib.h>
+
+namespace jxl {
+
+void* MemoryManagerDefaultAlloc(void* opaque, size_t size) {
+  return malloc(size);
+}
+
+void MemoryManagerDefaultFree(void* opaque, void* address) { free(address); }
+
+}  // namespace jxl
diff --git a/jxl/memory_manager_internal.h b/jxl/memory_manager_internal.h
new file mode 100644
index 000000000..391dfd743
--- /dev/null
+++ b/jxl/memory_manager_internal.h
@@ -0,0 +1,71 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MEMORY_MANAGER_INTERNAL_H_
+#define JXL_MEMORY_MANAGER_INTERNAL_H_
+
+// Memory allocator with support for alignment + misalignment.
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>  // memcpy
+
+#include <atomic>
+#include <memory>
+
+#include "include/jpegxl/memory_manager.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Default alloc and free functions.
+void* MemoryManagerDefaultAlloc(void* opaque, size_t size);
+void MemoryManagerDefaultFree(void* opaque, void* address);
+
+// Initializes the memory manager instance with the passed one. The
+// MemoryManager passed in |memory_manager| may be NULL or contain NULL
+// functions which will be initialized with the default ones. If either alloc
+// or free are NULL, then both must be NULL, otherwise this function returns an
+// error.
+static JXL_INLINE Status MemoryManagerInit(
+    JpegxlMemoryManager* self, const JpegxlMemoryManager* memory_manager) {
+  if (memory_manager) {
+    *self = *memory_manager;
+  } else {
+    memset(self, 0, sizeof(*self));
+  }
+  if (!self->alloc != !self->free) {
+    return false;
+  }
+  if (!self->alloc) self->alloc = jxl::MemoryManagerDefaultAlloc;
+  if (!self->free) self->free = jxl::MemoryManagerDefaultFree;
+
+  return true;
+}
+
+static JXL_INLINE void* MemoryManagerAlloc(
+    const JpegxlMemoryManager* memory_manager, size_t size) {
+  return memory_manager->alloc(memory_manager->opaque, size);
+}
+
+static JXL_INLINE void MemoryManagerFree(
+    const JpegxlMemoryManager* memory_manager, void* address) {
+  return memory_manager->free(memory_manager->opaque, address);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MEMORY_MANAGER_INTERNAL_H_
diff --git a/jxl/modular/config.h b/jxl/modular/config.h
new file mode 100644
index 000000000..47edbf909
--- /dev/null
+++ b/jxl/modular/config.h
@@ -0,0 +1,76 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_CONFIG_H_
+#define JXL_MODULAR_CONFIG_H_
+
+#include "jxl/modular/ma/chance.h"
+
+namespace jxl {
+
+#ifndef DECODER_ONLY
+#define HAS_ENCODER
+#endif
+
+// MAX_BIT_DEPTH is the maximum bit depth of the absolute values of the numbers
+// that actually get encoded Squeeze residuals plus YCoCg can result in 17-bit
+// absolute values on 16-bit input, so 17 is needed to encode 16-bit input with
+// default options Higher bit depth is needed when DCT is used on 16-bit input.
+
+#define HDR
+
+#ifndef HDR
+#define MAX_BIT_DEPTH 14
+#else
+#define MAX_BIT_DEPTH 30
+#endif
+
+// The above compile-time constant only determines the size of the chance tables
+// in the MA trees, and in any case the maximum bit depth is limited by the
+// integer type used in the channel buffers
+
+// this is one bit; this number gets multiplied by options.ctx_threshold
+// and adjusted for the options.nb_repeats value and the choice of entropy coder
+// decrease this number to make the learning during encode more eager to create
+// new contexts; increase it to reduce the number of contexts
+#define CONTEXT_TREE_SPLIT_THRESHOLD (5461)
+
+#define CONTEXT_TREE_MIN_SUBTREE_SIZE 0
+
+/**************************************************/
+/* DANGER ZONE: OPTIONS THAT CHANGE THE BITSTREAM */
+/* If you modify these, the bitstream format      */
+/* changes, so it is no longer compatible!        */
+/**************************************************/
+
+// Default squeeze will ensure that the first 'scan' fits in a 8x8 rectangle
+#define MAX_FIRST_PREVIEW_SIZE 8
+// Round truncation offsets to a multiples of 1 byte (using less precise offsets
+// requires a more careful implementation of partial decode)
+#define TRUNCATION_OFFSET_RESOLUTION 1
+
+#ifdef _MSC_VER
+#define ATTRIBUTE_HOT
+#else
+#define ATTRIBUTE_HOT __attribute__((hot))
+#endif
+
+typedef SimpleBitChance ModularBitChanceMeta;
+typedef SimpleBitChance ModularBitChancePass1;
+typedef SimpleBitChance ModularBitChancePass2;
+typedef SimpleBitChance ModularBitChanceTree;
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_CONFIG_H_
diff --git a/jxl/modular/encoding/context_predict.h b/jxl/modular/encoding/context_predict.h
new file mode 100644
index 000000000..1f9bac95c
--- /dev/null
+++ b/jxl/modular/encoding/context_predict.h
@@ -0,0 +1,392 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_ENCODING_CONTEXT_PREDICT_H_
+#define JXL_MODULAR_ENCODING_CONTEXT_PREDICT_H_
+
+#include "jxl/modular/encoding/encoding.h"
+#include "jxl/modular/image/image.h"
+#include "jxl/modular/ma/compound.h"
+#include "jxl/modular/ma/util.h"
+
+namespace jxl {
+
+enum class Predictor : uint32_t {
+  Zero = 0,
+  Left = 1,
+  Top = 2,
+  Average = 3,
+  Select = 4,
+  Gradient = 5,
+  Variable = 6,
+  Weighted = 7,
+  Best = 8  // Best of Gradient and Weighted (encoder only)
+};
+
+// Something that looks like absolute value. Just returning the absolute value
+// works. Wrapped into a function to make it easier to experiment with
+// alternatives.
+inline pixel_type UNSIGNED_VAL(pixel_type x) { return abs(x); }
+
+// Something that preserves the sign but potentially reduces the range
+// Just returning the value works.
+// Wrapped into a function to make it easier to experiment with alternatives
+inline pixel_type SIGNED_VAL(pixel_type x) { return x; }
+
+inline void init_properties(Ranges &pr, const Image &image, int beginc,
+                            int endc, modular_options &options) {
+  int offset = 0;
+  for (int j = beginc - 1; j >= 0 && offset < options.max_properties; j--) {
+    if (image.channel[j].minval == image.channel[j].maxval) continue;
+    if (image.channel[j].w == 0 || image.channel[j].h == 0) continue;
+    if (image.channel[j].hshift < 0) continue;
+    int minval = image.channel[j].minval;
+    if (minval > 0) minval = 0;
+    int maxval = image.channel[j].maxval;
+    if (maxval < 0) maxval = 0;
+    pr.push_back(std::pair<PropertyVal, PropertyVal>(
+        0, UNSIGNED_VAL((maxval > -minval ? maxval : minval))));
+    offset++;
+    pr.push_back(std::pair<PropertyVal, PropertyVal>(SIGNED_VAL(minval),
+                                                     SIGNED_VAL(maxval)));
+    offset++;
+    pr.push_back(std::pair<PropertyVal, PropertyVal>(
+        0, MAX(UNSIGNED_VAL(minval - maxval), UNSIGNED_VAL(maxval - minval))));
+    offset++;
+    pr.push_back(std::pair<PropertyVal, PropertyVal>(
+        SIGNED_VAL(minval - maxval), SIGNED_VAL(maxval - minval)));
+    offset++;
+  }
+
+  pixel_type minval = LARGEST_VAL;
+  pixel_type maxval = SMALLEST_VAL;
+  size_t maxh = 0;
+  size_t maxw = 0;
+  for (int j = beginc; j <= endc; j++) {
+    if (image.channel[j].minval < minval) minval = image.channel[j].minval;
+    if (image.channel[j].maxval > maxval) maxval = image.channel[j].maxval;
+    if (image.channel[j].h > maxh) maxh = image.channel[j].h;
+    if (image.channel[j].w > maxw) maxw = image.channel[j].w;
+  }
+  if (minval > 0) minval = 0;
+  if (maxval < 0) maxval = 0;
+
+  // neighbors
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(
+      0, MAX(UNSIGNED_VAL(minval), UNSIGNED_VAL(maxval))));
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(
+      0, MAX(UNSIGNED_VAL(minval), UNSIGNED_VAL(maxval))));
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(SIGNED_VAL(minval),
+                                                   SIGNED_VAL(maxval)));
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(SIGNED_VAL(minval),
+                                                   SIGNED_VAL(maxval)));
+
+  // location
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(0, maxh - 1));
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(0, maxw - 1));
+
+  // local gradient
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(minval + minval - maxval,
+                                                   maxval + maxval - minval));
+
+  // FFV1
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(
+      SIGNED_VAL(minval - maxval), SIGNED_VAL(maxval - minval)));
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(
+      SIGNED_VAL(minval - maxval), SIGNED_VAL(maxval - minval)));
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(
+      SIGNED_VAL(minval - maxval), SIGNED_VAL(maxval - minval)));
+  pr.push_back(std::pair<PropertyVal, PropertyVal>(
+      SIGNED_VAL(minval - maxval), SIGNED_VAL(maxval - minval)));
+}
+
+inline pixel_type select(pixel_type a, pixel_type b, pixel_type c) {
+  pixel_type p = a + b - c;
+  pixel_type pa = abs(p - a);
+  pixel_type pb = abs(p - b);
+  if (pa < pb)
+    return a;
+  else
+    return b;
+}
+
+inline pixel_type predict_and_compute_properties(
+    Properties &p, const Channel &ch, const pixel_type *JXL_RESTRICT pp,
+    const intptr_t onerow, const size_t x, const size_t y, Predictor predictor,
+    int offset) {
+  pixel_type left = (x ? pp[-1] : ch.zero);
+  pixel_type top = (y ? pp[-onerow] : left);
+  pixel_type topleft = (x && y ? pp[-1 - onerow] : left);
+  pixel_type topright = (x + 1 < ch.w && y ? pp[1 - onerow] : top);
+  pixel_type leftleft = (x > 1 ? pp[-2] : left);
+  //  pixel_type toptop = (y > 1 ? pp[-onerow - onerow] : top);
+
+  // neighbors
+  p[offset++] = UNSIGNED_VAL(top);
+  p[offset++] = UNSIGNED_VAL(left);
+  p[offset++] = SIGNED_VAL(top);
+  p[offset++] = SIGNED_VAL(left);
+
+  // location
+  p[offset++] = y;
+  p[offset++] = x;
+
+  // local gradient
+  p[offset++] = left + top - topleft;
+  //  p[offset++] = topleft + topright - top;
+
+  // FFV1 context properties
+  p[offset++] = SIGNED_VAL(left - topleft);
+  p[offset++] = SIGNED_VAL(topleft - top);
+  p[offset++] = SIGNED_VAL(top - topright);
+  //  p[offset++] = SIGNED_VAL(top - toptop);
+  p[offset++] = SIGNED_VAL(left - leftleft);
+
+  switch (predictor) {
+    case Predictor::Zero:
+      return ch.zero;
+    case Predictor::Left:
+      return left;
+    case Predictor::Top:
+      return top;
+    case Predictor::Average:
+      return (left + top) / 2;
+    case Predictor::Select:
+      return select(left, top, topleft);
+    case Predictor::Gradient:
+    default:
+      return median3((pixel_type)(left + top - topleft), left, top);
+  }
+}
+
+inline pixel_type predict(const Channel &ch, const pixel_type *JXL_RESTRICT pp,
+                          const intptr_t onerow, const int x, const int y,
+                          int predictor) ATTRIBUTE_HOT;
+inline pixel_type predict(const Channel &ch, const pixel_type *JXL_RESTRICT pp,
+                          const intptr_t onerow, const int x, const int y,
+                          Predictor predictor) {
+  pixel_type left = (x ? pp[-1] : ch.zero);
+  pixel_type top = (y ? pp[-onerow] : left);
+  pixel_type topleft = (x && y ? pp[-1 - onerow] : left);
+  switch (predictor) {
+    case Predictor::Zero:
+      return ch.zero;
+    case Predictor::Left:
+      return left;
+    case Predictor::Top:
+      return top;
+    case Predictor::Average:
+      return (left + top) / 2;
+    case Predictor::Select:
+      return select(left, top, topleft);
+    case Predictor::Gradient:
+    default:
+      return median3((pixel_type)(left + top - topleft), left, top);
+  }
+}
+
+inline pixel_type predict_and_compute_properties_no_edge_case(
+    Properties &p, const pixel_type *JXL_RESTRICT pp, const intptr_t onerow,
+    const size_t x, const size_t y, int offset = 0) ATTRIBUTE_HOT;
+inline pixel_type predict_and_compute_properties_no_edge_case(
+    Properties &p, const pixel_type *JXL_RESTRICT pp, const intptr_t onerow,
+    const size_t x, const size_t y, int offset) {
+  JXL_DASSERT(x > 1);
+  JXL_DASSERT(y > 1);
+
+  pixel_type left = pp[-1];
+  pixel_type top = pp[-onerow];
+  pixel_type topleft = pp[-1 - onerow];
+  pixel_type topright = pp[1 - onerow];
+
+  pixel_type leftleft = pp[-2];
+  //  pixel_type toptop = pp[-onerow - onerow];
+
+  // neighbors
+  p[offset++] = UNSIGNED_VAL(top);
+  p[offset++] = UNSIGNED_VAL(left);
+  p[offset++] = SIGNED_VAL(top);
+  p[offset++] = SIGNED_VAL(left);
+
+  // location
+  p[offset++] = y;
+  p[offset++] = x;
+
+  // local gradient
+  p[offset++] = left + top - topleft;
+  //  p[offset++] = topleft + topright - top;
+
+  // FFV1 context properties
+  p[offset++] = SIGNED_VAL(left - topleft);
+  p[offset++] = SIGNED_VAL(topleft - top);
+  p[offset++] = SIGNED_VAL(top - topright);
+  //  p[offset++] = SIGNED_VAL(top - toptop);
+  p[offset++] = SIGNED_VAL(left - leftleft);
+
+  return 0;
+}
+
+#define NB_NONREF_PROPERTIES 11
+
+inline void precompute_references(const Channel &ch, size_t y,
+                                  const Image &image, int i,
+                                  const modular_options &options,
+                                  Channel &references) {
+  int offset = 0;
+  size_t oy = y << ch.vshift;
+  intptr_t onerow = references.plane.PixelsPerRow();
+  pixel_type *lastrow = references.Row(0) + references.h * onerow;
+  for (int j = i - 1; j >= 0 && offset < options.max_properties; j--) {
+    if (image.channel[j].minval == image.channel[j].maxval) continue;
+    if (image.channel[j].w == 0 || image.channel[j].h == 0) continue;
+    if (image.channel[j].hshift < 0) continue;
+    size_t ry = oy >> image.channel[j].vshift;
+    if (ry >= image.channel[j].h) ry = image.channel[j].h - 1;
+    pixel_type *JXL_RESTRICT rp = references.Row(0) + offset;
+    const pixel_type *JXL_RESTRICT rpp = image.channel[j].Row(ry);
+    const pixel_type *JXL_RESTRICT rpprev =
+        image.channel[j].Row(ry ? ry - 1 : 0);
+
+    if (ch.hshift == image.channel[j].hshift && ch.w <= image.channel[j].w) {
+      for (size_t x = 0; x < ch.w; x++, rp += onerow) {
+        size_t rx = x;
+        pixel_type v = rpp[rx];
+        rp[0] = UNSIGNED_VAL(v);
+        rp[1] = SIGNED_VAL(v);
+        pixel_type vleft = (rx ? rpp[rx - 1] : image.channel[j].zero);
+        pixel_type vtop = (ry ? rpprev[rx] : vleft);
+        pixel_type vtopleft = (rx && ry ? rpprev[rx - 1] : vleft);
+        pixel_type vpredicted =
+            median3((pixel_type)(vleft + vtop - vtopleft), vleft, vtop);
+        rp[2] = UNSIGNED_VAL(v - vpredicted);
+        rp[3] = SIGNED_VAL(v - vpredicted);
+      }
+    } else if (ch.hshift < image.channel[j].hshift) {
+      size_t stepsize = (1 << image.channel[j].hshift) >> ch.hshift;
+      size_t rx = 0;
+      pixel_type v;
+      if (stepsize == 2)
+        for (; rx < image.channel[j].w - 1; rx++) {
+          v = rpp[rx];
+          pixel_type vleft = (rx ? rpp[rx - 1] : image.channel[j].zero);
+          pixel_type vtop = (ry ? rpprev[rx] : vleft);
+          pixel_type vtopleft = (rx && ry ? rpprev[rx - 1] : vleft);
+          pixel_type vpredicted =
+              median3((pixel_type)(vleft + vtop - vtopleft), vleft, vtop);
+          rp[0] = UNSIGNED_VAL(v);
+          rp[1] = SIGNED_VAL(v);
+          rp[2] = UNSIGNED_VAL(v - vpredicted);
+          rp[3] = SIGNED_VAL(v - vpredicted);
+          rp += onerow;
+          rp[0] = UNSIGNED_VAL(v);
+          rp[1] = SIGNED_VAL(v);
+          rp[2] = UNSIGNED_VAL(v - vpredicted);
+          rp[3] = SIGNED_VAL(v - vpredicted);
+          rp += onerow;
+        }
+      else
+        for (; rx < image.channel[j].w - 1; rx++) {
+          v = rpp[rx];
+          pixel_type vleft = (rx ? rpp[rx - 1] : image.channel[j].zero);
+          pixel_type vtop = (ry ? rpprev[rx] : vleft);
+          pixel_type vtopleft = (rx && ry ? rpprev[rx - 1] : vleft);
+          pixel_type vpredicted =
+              median3((pixel_type)(vleft + vtop - vtopleft), vleft, vtop);
+          for (size_t s = 0; s < stepsize; s++, rp += onerow) {
+            rp[0] = UNSIGNED_VAL(v);
+            rp[1] = SIGNED_VAL(v);
+            rp[2] = UNSIGNED_VAL(v - vpredicted);
+            rp[3] = SIGNED_VAL(v - vpredicted);
+          }
+        }
+      // assert (x-1 < ch.w);
+      v = rpp[rx];
+      pixel_type vleft = (rx ? rpp[rx - 1] : image.channel[j].zero);
+      pixel_type vtop = (ry ? rpprev[rx] : vleft);
+      pixel_type vtopleft = (rx && ry ? rpprev[rx - 1] : vleft);
+      pixel_type vpredicted =
+          median3((pixel_type)(vleft + vtop - vtopleft), vleft, vtop);
+      while (rp < lastrow) {
+        rp[0] = UNSIGNED_VAL(v);
+        rp[1] = SIGNED_VAL(v);
+        rp[2] = UNSIGNED_VAL(v - vpredicted);
+        rp[3] = SIGNED_VAL(v - vpredicted);
+        rp += onerow;
+      }
+    } else
+      // all the above are just some special cases of this:
+      for (size_t x = 0; x < ch.w; x++, rp += onerow) {
+        size_t ox = x << ch.hshift;
+        size_t rx = ox >> image.channel[j].hshift;
+        if (rx >= image.channel[j].w) rx = image.channel[j].w - 1;
+        pixel_type v = rpp[rx];
+        rp[0] = UNSIGNED_VAL(v);
+        rp[1] = SIGNED_VAL(v);
+        pixel_type vleft = (rx ? rpp[rx - 1] : image.channel[j].zero);
+        pixel_type vtop = (ry ? rpprev[rx] : vleft);
+        pixel_type vtopleft = (rx && ry ? rpprev[rx - 1] : vleft);
+        pixel_type vpredicted =
+            median3((pixel_type)(vleft + vtop - vtopleft), vleft, vtop);
+        rp[2] = UNSIGNED_VAL(v - vpredicted);
+        rp[3] = SIGNED_VAL(v - vpredicted);
+      }
+
+    offset += 4;
+  }
+}
+
+inline pixel_type predict_and_compute_properties_with_precomputed_reference(
+    Properties &p, const Channel &ch, const pixel_type *JXL_RESTRICT pp,
+    const intptr_t onerow, const int x, const int y, Predictor predictor,
+    const Image &image, int i, modular_options &options,
+    const Channel &references) {
+  size_t offset;
+  const pixel_type *JXL_RESTRICT rp = references.Row(x);
+  for (offset = 0; offset < references.w;) {
+    p[offset] = rp[offset];
+    offset++;
+    p[offset] = rp[offset];
+    offset++;
+    p[offset] = rp[offset];
+    offset++;
+    p[offset] = rp[offset];
+    offset++;
+  }
+  return predict_and_compute_properties(p, ch, pp, onerow, x, y, predictor,
+                                        offset);
+}
+
+inline pixel_type
+predict_and_compute_properties_with_precomputed_reference_no_edge_case(
+    Properties &p, const pixel_type *JXL_RESTRICT pp, const intptr_t onerow,
+    const int x, const int y, const Channel &references) {
+  size_t offset;
+  const pixel_type *JXL_RESTRICT rp = references.Row(x);
+  for (offset = 0; offset < references.w;) {
+    p[offset] = rp[offset];
+    offset++;
+    p[offset] = rp[offset];
+    offset++;
+    p[offset] = rp[offset];
+    offset++;
+    p[offset] = rp[offset];
+    offset++;
+  }
+  return predict_and_compute_properties_no_edge_case(p, pp, onerow, x, y,
+                                                     offset);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_ENCODING_CONTEXT_PREDICT_H_
diff --git a/jxl/modular/encoding/encoding.cpp b/jxl/modular/encoding/encoding.cpp
new file mode 100644
index 000000000..f58c2e0bd
--- /dev/null
+++ b/jxl/modular/encoding/encoding.cpp
@@ -0,0 +1,1307 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/modular/encoding/encoding.h"
+
+#include <inttypes.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <random>
+
+#include "jxl/base/status.h"
+#include "jxl/brotli.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/modular/encoding/context_predict.h"
+#include "jxl/modular/encoding/weighted_predict.h"
+#include "jxl/modular/ma/compound.h"
+#include "jxl/modular/ma/rac.h"
+#include "jxl/modular/ma/rac_enc.h"
+#include "jxl/modular/ma/util.h"
+#include "jxl/modular/memio.h"
+#include "jxl/modular/transform/transform.h"
+#include "jxl/modular/util.h"
+#include "jxl/toc.h"
+
+namespace jxl {
+
+void set_default_modular_options(struct modular_options &o) {
+  o.identify = false;
+
+  o.entropy_coder = 0;  // MABEGABRAC
+  o.nb_channels = 1;
+  o.skipchannels = 0;
+  o.max_chan_size = 0xFFFFFF;
+  o.nb_repeats =
+      0.5;  // learn MA tree by looking at 50% of the rows, in random order
+  o.ctx_threshold = 16;  // 2 byte improvement needed to add another context
+  o.max_properties = 0;  // no previous channels
+  o.brotli_effort = 11;
+  o.nb_wp_modes = 1;
+  o.debug = false;
+  o.heatmap = nullptr;
+};
+
+template <typename IO>
+void write_big_endian_varint(IO &io, size_t number, bool done = true) {
+  if (number < 128) {
+    if (done)
+      io.fputc(number);
+    else
+      io.fputc(number + 128);
+  } else {
+    size_t lsb = (number & 127);
+    number >>= 7;
+    write_big_endian_varint(io, number, false);
+    write_big_endian_varint(io, lsb, done);
+  }
+}
+
+template <typename IO>
+int read_big_endian_varint(IO &io) {
+  int result = 0;
+  int bytes_read = 0;
+  while (bytes_read++ < 10) {
+    int number = io.get_c();
+    if (number < 0) break;
+    if (number < 128) return result + number;
+    number -= 128;
+    result += number;
+    result <<= 7;
+  }
+  if (!io.isEOF()) JXL_FAILURE("Invalid number encountered!");
+  return -1;
+}
+
+bool check_bit_depth(pixel_type minv, pixel_type maxv) {
+  uint32_t maxav = abs(maxv - minv);
+  if (ilog2(maxav) + 1 > MAX_BIT_DEPTH) {
+    return JXL_FAILURE(
+        "Error: compiled for a maximum bit depth of %i, while %i "
+        "bits are needed to encode this channel (range=%i..%i)",
+        MAX_BIT_DEPTH, ilog2(maxav) + 1, minv, maxv);
+  }
+  return true;
+}
+
+#define ENTROPY_CODER_NAME(e) \
+  ((e == 0 ? "BEGABRAC" : (e == 1 ? "Brotli" : "ANS")))
+
+#ifdef HAS_ENCODER
+Predictor find_best(const Channel &channel, const pixel_type *JXL_RESTRICT p,
+                    intptr_t onerow, size_t y, Predictor prev_predictor) {
+  uint64_t best = 0;
+  uint64_t prev = -1;
+  int best_predictor = 0;
+  for (uint32_t i = 0; i < 6; i++) {
+    uint64_t sum_of_abs_residuals = 0;
+    for (size_t x = 0; x < channel.w; x++) {
+      pixel_type guess = predict(channel, p + x, onerow, x, y, (Predictor)i);
+      sum_of_abs_residuals += abs(p[x] - guess);
+    }
+    if (i == 0 || sum_of_abs_residuals < best) {
+      best = sum_of_abs_residuals;
+      best_predictor = i;
+    }
+    if ((Predictor)i == prev_predictor) prev = sum_of_abs_residuals;
+  }
+  // only change predictor if residuals are 10% smaller
+  if (prev < best * 1.1) return prev_predictor;
+  return (Predictor)best_predictor;
+}
+
+template <typename IO, typename Rac, typename Coder, bool learn>
+bool modular_encode_channels(IO &io, Tree &tree, modular_options &options,
+                             Predictor predictor, int beginc, int endc,
+                             const Image &image, size_t &header_pos,
+                             int entropy_coder) {
+  const Channel &channel = image.channel[beginc];
+  if (channel.w == 0 || channel.h == 0) return true;
+  // BEGABRAC with left predictor works best for palette data
+  if (channel.hshift < 0) {
+    entropy_coder = 0;
+    predictor = Predictor::Left;
+  }
+  int predictability = 2048;  // 50%
+  {
+    if (predictor == Predictor::Zero && entropy_coder == 0 && learn) {
+      uint64_t zeroes = 0;
+      uint64_t pixels = channel.h * channel.w;
+      for (size_t y = 0; y < channel.h; y++) {
+        const pixel_type *JXL_RESTRICT p = channel.Row(y);
+        for (size_t x = 0; x < channel.w; x++)
+          if (p[x] == 0) zeroes++;
+      }
+      predictability = CLAMP(zeroes * 4096 / pixels, 1, 4095);
+      JXL_DEBUG_V(6,
+                  "Found %" PRIu64 " zeroes in %" PRIu64
+                  " pixels (zero chance=%i/4096)",
+                  zeroes, pixels, predictability);
+    }
+  }
+
+  JXL_ASSERT(endc == beginc);
+
+  pixel_type minv = channel.minval;
+  pixel_type maxv = channel.maxval;
+  if (minv == 0 && maxv == 0) {
+    write_big_endian_varint(io, 0);
+  } else {
+    if (minv <= 0) {
+      write_big_endian_varint(io, 2 - minv);
+    } else {
+      write_big_endian_varint(io, 1);
+      write_big_endian_varint(io, minv - 1);
+    }
+    if (channel.w > 1 || channel.h > 1)
+      write_big_endian_varint(io, maxv - minv);
+  }
+  if (entropy_coder == 0 && !check_bit_depth(minv, maxv)) return false;
+
+  if (!learn) {
+    JXL_DEBUG_V(6,
+                "Encoding %zux%zu channel %i with range %i..%i (predictor %i), "
+                "(shift=%i,%i, cshift=%i,%i)",
+                channel.w, channel.h, beginc, minv, maxv, predictor,
+                channel.hshift, channel.vshift, channel.hcshift,
+                channel.vcshift);
+  } else {
+    JXL_DEBUG_V(7,
+                "Learning %zux%zu channel %i with range %i..%i (predictor %i)",
+                channel.w, channel.h, beginc, minv, maxv, predictor);
+  }
+  channel.setzero();
+
+  header_pos = io.ftell();
+  if (minv == maxv) return true;
+
+  write_big_endian_varint(io, ((int)predictor * 3) + entropy_coder);
+
+  if (predictor == Predictor::Weighted) {
+    if (learn) return true;
+    const Channel &channel = image.channel[beginc];
+    if (channel.minval == channel.maxval) return true;
+    PaddedBytes bytes;
+    if (!wp_compress(image.channel[beginc], &bytes, options.nb_wp_modes))
+      return false;
+    io.append(bytes);
+    return true;
+  }
+
+  Predictor subpredictor = predictor;
+  if (entropy_coder == 1) {
+    if (learn) return true;
+    size_t bytesperpixel = 1;
+    pixel_type maxdiff = channel.maxval - channel.minval;
+    pixel_type maxval = maxdiff;
+    bool sign2lsb = true;
+    if (predictor == Predictor::Zero &&
+        (channel.maxval <= 0 || channel.minval >= 0))
+      sign2lsb = false;
+
+    if (sign2lsb) {
+      maxval = 2 * maxdiff;
+    }
+
+    if (maxval > 0xff) bytesperpixel++;
+    if (maxval > 0xffff) bytesperpixel++;
+    size_t pixels = channel.w * channel.h;
+    PaddedBytes buffer(bytesperpixel * pixels);
+    PaddedBytes pbuffer;
+    const intptr_t onerow = channel.plane.PixelsPerRow();
+    size_t pos = 0;
+    for (size_t y = 0; y < channel.h; y++) {
+      const pixel_type *JXL_RESTRICT p = channel.Row(y);
+      if (predictor == Predictor::Variable) {
+        subpredictor = find_best(channel, p, onerow, y, subpredictor);
+        pbuffer.push_back((uint8_t)subpredictor);
+      }
+      for (size_t x = 0; x < channel.w; x++) {
+        pixel_type guess = predict(channel, p + x, onerow, x, y, subpredictor);
+        pixel_type v;
+        if (sign2lsb) {
+          v = PackSigned(p[x] - guess);
+        } else
+          v = p[x] - channel.minval;
+        buffer[pos] = (v & 0xff);
+        if (bytesperpixel > 1) buffer[pos + pixels] = ((v >> 8) & 0xff);
+        if (bytesperpixel > 2) buffer[pos + 2 * pixels] = ((v >> 16) & 0xff);
+        pos++;
+      }
+    }
+    pos = 0;
+    pbuffer.append(buffer);
+    PaddedBytes cbuffer;
+    JXL_RETURN_IF_ERROR(
+        BrotliCompress(options.brotli_effort, pbuffer, &cbuffer));
+    pos = cbuffer.size();
+    JXL_DEBUG_V(4,
+                "Encoded %zu Brotli bytes from %zu uncompressed "
+                "bytes (%f bpp)",
+                pos, pixels * bytesperpixel, pos * 8.0 / pixels);
+    io.append(std::move(cbuffer));
+    return true;
+  }
+
+  Ranges propRanges;
+  init_properties(propRanges, image, beginc, endc, options);
+  size_t treebegin = io.ftell();
+  Rac rac(io);
+  if (!learn && entropy_coder != 1) {
+    // encode tree here
+    MetaPropertySymbolCoder<ModularBitChanceTree, Rac> metacoder(rac,
+                                                                 propRanges);
+    metacoder.write_tree(tree, entropy_coder == 0);
+  }
+  Properties properties(propRanges.size());
+
+  if (!learn && entropy_coder == 2) {
+    rac.flush();
+    BlobReader tmp(io.ptr() + treebegin, io.ftell() - treebegin);
+    RacIn<BlobReader> drac(tmp);
+    // decode tree we just encoded, so it has the same node ordering as what the
+    // decoder gets. Doesn't matter for MABEGABRAC, but does matter for MARANS
+    // and MABrotli
+    MetaPropertySymbolCoder<ModularBitChanceTree, RacIn<BlobReader>>
+        metadecoder(drac, propRanges);
+    metadecoder.read_tree(tree, false);
+    Coder coder(rac, propRanges, tree);
+    JXL_DEBUG_V(3, "Encoding using a MA tree with %zu nodes (%d contexts)",
+                tree.size(), coder.nb_contexts());
+    bool sign2lsb = true;
+    if (predictor == Predictor::Zero &&
+        (channel.maxval <= 0 || channel.minval >= 0))
+      sign2lsb = false;
+
+    int nbctx = coder.nb_contexts();
+    if (predictor == Predictor::Variable) nbctx++;
+    BitWriter writer;
+    std::vector<std::vector<Token>> tokens(1);
+
+    const intptr_t onerow = channel.plane.PixelsPerRow();
+    Channel references(properties.size() - NB_NONREF_PROPERTIES, channel.w, 0,
+                       0);
+    for (size_t y = 0; y < channel.h; y++) {
+      const pixel_type *JXL_RESTRICT p = channel.Row(y);
+      precompute_references(channel, y, image, beginc, options, references);
+      if (predictor == Predictor::Variable) {
+        subpredictor = find_best(channel, p, onerow, y, subpredictor);
+        TokenizeHybridUint(nbctx - 1, (uint32_t)subpredictor, &tokens[0]);
+      }
+      for (size_t x = 0; x < channel.w; x++) {
+        pixel_type guess =
+            predict_and_compute_properties_with_precomputed_reference(
+                properties, channel, p + x, onerow, x, y, subpredictor, image,
+                beginc, options, references);
+        int ctx = coder.context_id(properties);
+        if (sign2lsb) {
+          TokenizeHybridUint(ctx, PackSigned(p[x] - guess), &tokens[0]);
+        } else {
+          TokenizeHybridUint(ctx, p[x] - channel.minval, &tokens[0]);
+        }
+      }
+    }
+    EntropyEncodingData codes;
+    std::vector<uint8_t> context_map;
+    size_t pixels = channel.h * channel.w;
+    BuildAndEncodeHistograms(HistogramParams(), nbctx, tokens, &codes,
+                             &context_map, &writer, 0, nullptr);
+    WriteTokens(tokens[0], codes, context_map, &writer, 0, nullptr);
+    writer.ZeroPadToByte();
+    size_t pos = writer.BitsWritten() / 8;
+    JXL_DEBUG_V(9, "   Encoded %zu ANS bytes for %zu pixels (%f bpp)", pos,
+                pixels, pos * 8.0 / pixels);
+    io.append(std::move(writer).TakeBytes());
+  } else {  // entropy_coder == 0  (or learning and pretending it is)
+    Coder coder(rac, propRanges, tree, predictability,
+                CONTEXT_TREE_SPLIT_THRESHOLD * options.nb_repeats * 2 *
+                    options.ctx_threshold * (entropy_coder > 0 ? 2 : 1));
+    SimpleSymbolCoder<SimpleBitChance, Rac, 3> pcoder(rac);
+    int rowslearned = 0;
+    std::mt19937_64 gen(1);  // deterministic learning (also between threads)
+    std::uniform_int_distribution<> random_row(0, channel.h - 1);
+    Channel references(properties.size() - NB_NONREF_PROPERTIES, channel.w, 0,
+                       0);
+    const intptr_t onerow = channel.plane.PixelsPerRow();
+    for (size_t y = 0; y < channel.h; y++) {
+      if (learn) {
+        if (++rowslearned > options.nb_repeats * channel.h) break;
+      }
+      // try random rows, to avoid giving priority to the top of the image
+      // (because then the y property cannot be learned)
+      if (learn && options.nb_repeats < 2) y = random_row(gen);
+      const pixel_type *JXL_RESTRICT p = channel.Row(y);
+      pixel_type *JXL_RESTRICT hp;
+      if (!learn && options.debug) hp = options.heatmap->channel[beginc].Row(y);
+      precompute_references(channel, y, image, beginc, options, references);
+      if (predictor == Predictor::Variable) {
+        subpredictor = find_best(channel, p, onerow, y, subpredictor);
+        pcoder.write_int(0, 5, (int)subpredictor);
+      }
+      for (size_t x = 0; x < channel.w; x++) {
+        pixel_type guess =
+            predict_and_compute_properties_with_precomputed_reference(
+                properties, channel, p + x, onerow, x, y, subpredictor, image,
+                beginc, options, references);
+        pixel_type diff = p[x] - guess;
+        if (!learn && options.debug) {
+          int estimate =
+              coder.estimate_int(properties, minv - guess, maxv - guess, diff);
+          hp[x] = estimate;
+        }
+        coder.write_int(properties, minv - guess, maxv - guess, diff);
+      }
+      if (learn && y == channel.h - 1) {
+        y = 0;
+      }  // set to zero in case random row was channel.h-1
+    }
+
+    // collect stats for chance initialization
+    if (learn) {
+      Channel references(properties.size() - NB_NONREF_PROPERTIES, channel.w, 0,
+                         0);
+      const intptr_t onerow = channel.plane.PixelsPerRow();
+      for (size_t y = 0; y < channel.h; y++) {
+        const pixel_type *JXL_RESTRICT p = channel.Row(y);
+        precompute_references(channel, y, image, beginc, options, references);
+        for (size_t x = 0; x < channel.w; x++) {
+          pixel_type guess =
+              predict_and_compute_properties_with_precomputed_reference(
+                  properties, channel, p + x, onerow, x, y, predictor, image,
+                  beginc, options, references);
+          pixel_type diff = p[x] - guess;
+          coder.stats_write_int(properties, minv - guess, maxv - guess, diff);
+        }
+      }
+    }
+
+    coder.simplify();
+    rac.flush();
+  }
+  return true;
+}
+#endif
+
+template <typename IO>
+bool corrupt_or_truncated(IO &io, Channel &channel, size_t bytes_to_load) {
+  if (io.isEOF() ||
+      (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load)) {
+    JXL_DEBUG_V(3, "Premature end-of-file detected.");
+    channel.resize();
+    return true;
+  } else {
+    JXL_DEBUG_V(3, "Corruption detected.");
+    return false;
+  }
+}
+
+template <typename IO, typename Coder>
+HWY_ATTR bool modular_decode_channel(IO &io, modular_options &options,
+                                     size_t &beginc, Image &image,
+                                     size_t bytes_to_load) {
+  Channel &channel = image.channel[beginc];
+
+  // zero pixel channel? could happen
+  if (channel.w == 0 || channel.h == 0) return true;
+  size_t filepos = io.ftell();
+  if (io.isEOF() ||
+      (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load))
+    return true;
+  pixel_type minv = 2 - read_big_endian_varint(io);
+  if (io.isEOF() ||
+      (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load))
+    return true;
+  pixel_type maxv;
+  if (minv == 2) {
+    minv = 0;
+    maxv = 0;
+  } else {
+    if (minv == 1) minv = 1 + read_big_endian_varint(io);
+
+    if (io.isEOF() ||
+        (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load))
+      return true;
+    if (channel.w == 1 && channel.h == 1) {
+      maxv = minv;
+    } else {
+      maxv = minv + read_big_endian_varint(io);
+    }
+  }
+  if (io.isEOF() ||
+      (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load))
+    return true;
+
+  channel.minval = minv;
+  channel.maxval = maxv;
+  if (channel.minval == channel.maxval) {
+    channel.setzero();
+    channel.plane = Plane<pixel_type>();
+    channel.resize();  // fill it with the constant value
+    JXL_DEBUG_V(
+        4, "[File position %zu] Decoding channel %zu: %zux%zu, constant %i",
+        filepos, beginc, channel.w, channel.h, channel.minval);
+  }
+
+  if (io.isEOF() ||
+      (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load))
+    return corrupt_or_truncated(io, channel, bytes_to_load);
+
+  if (minv == maxv) return true;
+
+  int firstbyte = read_big_endian_varint(io);
+  if (io.isEOF() ||
+      (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load))
+    return true;
+  int entropy_coder = (firstbyte % 3);
+  Predictor predictor = (Predictor)(firstbyte / 3);
+  JXL_DEBUG_V(4,
+              "[File position %zu] %s-decoding channel %zu: %zux%zu with range "
+              "%i..%i, predictor %i",
+              filepos, ENTROPY_CODER_NAME(entropy_coder), beginc, channel.w,
+              channel.h, channel.minval, channel.maxval, predictor);
+  JXL_DEBUG_V(6, " (shift=%i,%i, cshift=%i,%i)", channel.hshift, channel.vshift,
+              channel.hcshift, channel.vcshift);
+
+  if (entropy_coder == 0 && !check_bit_depth(channel.minval, channel.maxval))
+    return false;
+
+  channel.setzero();
+  channel.resize(channel.w, channel.h);
+
+  if (predictor == Predictor::Weighted) {
+    Channel &channel = image.channel[beginc];
+    channel.setzero();
+    channel.resize(channel.w, channel.h);
+    if (channel.minval == channel.maxval) return true;
+    size_t iopos = io.ftell();
+    const Span<const uint8_t> bytes(io.ptr(), io.size());
+    if (!wp_decompress(bytes, &iopos, channel)) return false;
+    io.fseek(iopos, SEEK_SET);
+    return true;
+  }
+  Predictor subpredictor = predictor;
+  // Brotli decode
+  if (entropy_coder == 1) {
+    size_t bytesperpixel = 1;
+    pixel_type maxdiff = channel.maxval - channel.minval;
+    pixel_type maxval = maxdiff;
+    bool sign2lsb = true;
+    if (predictor == Predictor::Zero &&
+        (channel.maxval <= 0 || channel.minval >= 0))
+      sign2lsb = false;
+
+    if (sign2lsb) maxval = 2 * maxdiff;
+    if (maxval > 0xff) bytesperpixel++;
+    if (maxval > 0xffff) bytesperpixel++;
+    size_t pixels = channel.w * channel.h;
+    PaddedBytes obuffer;
+    size_t pos = 0;
+    size_t iopos = io.ftell();
+    bool decodestatus = BrotliDecompress(
+        Span<const uint8_t>(io.ptr() + iopos, io.size() - iopos),
+        pixels * bytesperpixel +
+            (predictor == Predictor::Variable ? channel.h : 0),
+        &pos, &obuffer);
+    io.fseek(iopos + pos, SEEK_SET);
+    JXL_DEBUG_V(4, "   Decoded %zu bytes for %zu pixels", pos, pixels);
+    pos = (predictor == Predictor::Variable ? channel.h : 0);
+    if (!decodestatus) {
+      if (io.isEOF() ||
+          (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load)) {
+        JXL_DEBUG_V(3, "Premature end-of-file at channel %zu.", beginc);
+        return true;
+      }
+      return JXL_FAILURE("Problem during Brotli decode");
+    }
+
+    if (predictor == Predictor::Zero && bytesperpixel <= 2) {
+      // special optimized case: no predictor
+      JXL_DEBUG_V(8, "Fast track.");
+      for (size_t y = 0; y < channel.h; y++) {
+        pixel_type *JXL_RESTRICT r = channel.Row(y);
+        if (sign2lsb) {
+          if (bytesperpixel == 1)
+            for (size_t x = 0; x < channel.w; x++) {
+              r[x] = UnpackSigned(obuffer[pos++]);
+            }
+          else
+            for (size_t x = 0; x < channel.w; x++) {
+              pixel_type v = obuffer[pos];
+              v += obuffer[pixels + pos] << 8;
+              pos++;
+              r[x] = UnpackSigned(v);
+            }
+        } else {
+          if (bytesperpixel == 1)
+            for (size_t x = 0; x < channel.w; x++) {
+              pixel_type v = obuffer[pos++];
+              v += channel.minval;
+              r[x] = v;
+            }
+          else
+            for (size_t x = 0; x < channel.w; x++) {
+              pixel_type v = obuffer[pos];
+              v += obuffer[pixels + pos] << 8;
+              pos++;
+              v += channel.minval;
+              r[x] = v;
+            }
+        }
+      }
+    } else {
+      const intptr_t onerow = channel.plane.PixelsPerRow();
+      for (size_t y = 0; y < channel.h; y++) {
+        pixel_type *JXL_RESTRICT r = channel.Row(y);
+        if (predictor == Predictor::Variable)
+          subpredictor = (Predictor)obuffer[y];
+        if (bytesperpixel == 1)
+          for (size_t x = 0; x < channel.w; x++) {
+            pixel_type g = predict(channel, r + x, onerow, x, y, subpredictor);
+            pixel_type v = obuffer[pos++];
+            r[x] = UnpackSigned(v) + g;
+          }
+        else if (bytesperpixel == 2)
+          for (size_t x = 0; x < channel.w; x++) {
+            pixel_type g = predict(channel, r + x, onerow, x, y, subpredictor);
+            pixel_type v = obuffer[pos];
+            v += obuffer[pixels + pos] << 8;
+            pos++;
+            r[x] = UnpackSigned(v) + g;
+          }
+        else if (bytesperpixel == 3)
+          for (size_t x = 0; x < channel.w; x++) {
+            pixel_type g = predict(channel, r + x, onerow, x, y, subpredictor);
+            pixel_type v = obuffer[pos];
+            v += obuffer[pixels + pos] << 8;
+            v += obuffer[2 * pixels + pos] << 16;
+            pos++;
+            r[x] = UnpackSigned(v) + g;
+          }
+      }
+    }
+
+    return true;
+  }
+  Ranges propRanges;
+  init_properties(propRanges, image, beginc, beginc, options);
+
+  int predictability = 2048;
+
+  RacIn<IO> rac(io);
+
+  // decode trees
+  Tree tree;
+  MetaPropertySymbolCoder<ModularBitChanceTree, RacIn<IO>> metacoder(
+      rac, propRanges);
+  if (!metacoder.read_tree(tree, entropy_coder == 0))
+    return corrupt_or_truncated(io, image.channel[beginc], bytes_to_load);
+
+  Coder coder(rac, propRanges, tree, predictability);
+  JXL_DEBUG_V(3, "Decoded MA tree with %zu nodes (%d contexts)", tree.size(),
+              coder.nb_contexts());
+  Properties properties(propRanges.size());
+
+  // MAANS decode
+  if (entropy_coder == 2) {
+    bool sign2lsb = true;
+    if (predictor == Predictor::Zero &&
+        (channel.maxval <= 0 || channel.minval >= 0))
+      sign2lsb = false;
+
+    int nbctx = coder.nb_contexts();
+    if (predictor == Predictor::Variable) nbctx++;
+
+    size_t iopos = io.ftell();
+    BitReader br(Span<const uint8_t>(io.ptr() + iopos, io.size() - iopos));
+    std::vector<uint8_t> context_map;
+    ANSCode code;
+    JXL_RETURN_IF_ERROR(
+        DecodeHistograms(&br, nbctx, ANS_MAX_ALPHA_SIZE, &code, &context_map));
+    ANSSymbolReader reader(&code, &br);
+    if (tree.size() == 1 && predictor == Predictor::Zero) {
+      // special optimized case: no meta-adaptation, no predictor, so no need
+      // to compute properties
+      JXL_DEBUG_V(8, "Fast track.");
+      for (size_t y = 0; y < channel.h; y++) {
+        pixel_type *JXL_RESTRICT r = channel.Row(y);
+        if (sign2lsb) {
+          for (size_t x = 0; x < channel.w; x++) {
+            uint32_t v = ReadHybridUint(0, &br, &reader, context_map);
+            r[x] = UnpackSigned(v);
+          }
+        } else {
+          for (size_t x = 0; x < channel.w; x++) {
+            pixel_type v = ReadHybridUint(0, &br, &reader, context_map);
+            v += channel.minval;
+            r[x] = v;
+          }
+        }
+      }
+    } else if (tree.size() == 1) {
+      // special optimized case: no meta-adaptation, so no need to compute
+      // properties
+      JXL_DEBUG_V(8, "Quite fast track.");
+      const intptr_t onerow = channel.plane.PixelsPerRow();
+      for (size_t y = 0; y < channel.h; y++) {
+        pixel_type *JXL_RESTRICT r = channel.Row(y);
+        if (predictor == Predictor::Variable)
+          subpredictor =
+              (Predictor)ReadHybridUint(nbctx - 1, &br, &reader, context_map);
+        for (size_t x = 0; x < channel.w; x++) {
+          pixel_type g = predict(channel, r + x, onerow, x, y, subpredictor);
+          uint32_t v = ReadHybridUint(0, &br, &reader, context_map);
+          r[x] = UnpackSigned(v) + g;
+        }
+      }
+    } else {
+      JXL_DEBUG_V(8, "Slow track.");
+      const intptr_t onerow = channel.plane.PixelsPerRow();
+      Channel references(properties.size() - NB_NONREF_PROPERTIES, channel.w, 0,
+                         0);
+      for (size_t y = 0; y < channel.h; y++) {
+        pixel_type *JXL_RESTRICT p = channel.Row(y);
+        precompute_references(channel, y, image, beginc, options, references);
+        if (predictor == Predictor::Variable)
+          subpredictor =
+              (Predictor)ReadHybridUint(nbctx - 1, &br, &reader, context_map);
+        for (size_t x = 0; x < channel.w; x++) {
+          pixel_type guess =
+              predict_and_compute_properties_with_precomputed_reference(
+                  properties, channel, p + x, onerow, x, y, subpredictor, image,
+                  beginc, options, references);
+          int ctx = coder.context_id(properties);
+          uint32_t v = ReadHybridUint(ctx, &br, &reader, context_map);
+          if (sign2lsb) {
+            p[x] = UnpackSigned(v) + guess;
+          } else {
+            p[x] = channel.minval + v;
+          }
+        }
+      }
+    }
+    if (!reader.CheckANSFinalState()) {
+      return JXL_FAILURE("ANS decode final state failed");
+    }
+    JXL_RETURN_IF_ERROR(br.JumpToByteBoundary());
+    io.fseek(iopos + br.TotalBitsConsumed() / 8, SEEK_SET);
+    JXL_RETURN_IF_ERROR(br.Close());
+    return true;
+  }
+
+  // MABEGABRAC decode
+
+  SimpleSymbolCoder<SimpleBitChance, RacIn<IO>, 3> pcoder(rac);
+  if (tree.size() == 1 && predictor == Predictor::Zero && channel.zero == 0) {
+    // special optimized case: no meta-adaptation, no predictor, so no need to
+    // compute properties
+    JXL_DEBUG_V(8, "Fast track.");
+    for (size_t y = 0; y < channel.h; y++) {
+      if (io.isEOF() ||
+          (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load)) {
+        JXL_DEBUG_V(3, "Premature end-of-file at row %zu of channel %zu.", y,
+                    beginc);
+        break;
+      }
+      pixel_type *JXL_RESTRICT r = channel.Row(y);
+      for (size_t x = 0; x < channel.w; x++) {
+        r[x] = coder.read_int(properties, channel.minval, channel.maxval);
+      }
+    }
+  } else if (tree.size() == 1) {
+    // special optimized case: no meta-adaptation, no predictor, so no need to
+    // compute properties
+    JXL_DEBUG_V(8, "Quite fast track.");
+    const intptr_t onerow = channel.plane.PixelsPerRow();
+    for (size_t y = 0; y < channel.h; y++) {
+      if (io.isEOF() ||
+          (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load)) {
+        JXL_DEBUG_V(3, "Premature end-of-file at row %zu of channel %zu.", y,
+                    beginc);
+        break;
+      }
+      pixel_type *JXL_RESTRICT r = channel.Row(y);
+      if (predictor == Predictor::Variable)
+        subpredictor = (Predictor)pcoder.read_int(0, 5);
+      for (size_t x = 0; x < channel.w; x++) {
+        pixel_type g = predict(channel, r + x, onerow, x, y, subpredictor);
+        r[x] = g + coder.read_int(properties, channel.minval - g,
+                                  channel.maxval - g);
+      }
+    }
+  } else {
+    JXL_DEBUG_V(8, "Slow track.");
+
+    Channel references(properties.size() - NB_NONREF_PROPERTIES, channel.w, 0,
+                       0);
+    const intptr_t onerow = channel.plane.PixelsPerRow();
+    for (size_t y = 0; y < channel.h; y++) {
+      if (io.isEOF() ||
+          (bytes_to_load && static_cast<size_t>(io.ftell()) >= bytes_to_load)) {
+        JXL_DEBUG_V(3, "Premature end-of-file at row %zu of channel %zu.", y,
+                    beginc);
+        break;
+      }
+      precompute_references(channel, y, image, beginc, options, references);
+      pixel_type *JXL_RESTRICT p = channel.Row(y);
+      if (predictor == Predictor::Variable)
+        subpredictor = (Predictor)pcoder.read_int(0, 5);
+      if (y <= 1 || subpredictor != Predictor::Zero || channel.zero != 0) {
+        for (size_t x = 0; x < channel.w; x++) {
+          pixel_type guess;
+          guess = predict_and_compute_properties_with_precomputed_reference(
+              properties, channel, p + x, onerow, x, y, subpredictor, image,
+              beginc, options, references);
+          pixel_type diff = coder.read_int(properties, channel.minval - guess,
+                                           channel.maxval - guess);
+          p[x] = diff + guess;
+        }
+      } else {
+        size_t x = 0;
+        pixel_type guess, diff;
+        for (; x < channel.w && x < 2; x++) {
+          guess = predict_and_compute_properties_with_precomputed_reference(
+              properties, channel, p + x, onerow, x, y, subpredictor, image,
+              beginc, options, references);
+          diff = coder.read_int(properties, channel.minval - guess,
+                                channel.maxval - guess);
+          p[x] = diff + guess;
+        }
+        for (; x < channel.w - 1; x++) {
+          guess =
+              predict_and_compute_properties_with_precomputed_reference_no_edge_case(
+                  properties, p + x, onerow, x, y, references);
+          diff = coder.read_int(properties, channel.minval - guess,
+                                channel.maxval - guess);
+          p[x] = diff + guess;
+        }
+        for (; x < channel.w; x++) {
+          guess = predict_and_compute_properties_with_precomputed_reference(
+              properties, channel, p + x, onerow, x, y, subpredictor, image,
+              beginc, options, references);
+          diff = coder.read_int(properties, channel.minval - guess,
+                                channel.maxval - guess);
+          p[x] = diff + guess;
+        }
+      }
+    }
+  }
+
+  return true;
+}
+
+#ifdef HAS_ENCODER
+template <typename IO>
+bool modular_encode(IO &realio, const Image &image, modular_options &options) {
+  if (image.error) return false;
+  size_t nb_channels = image.real_nb_channels;
+  int bit_depth = 1, maxval = 1;
+  while (maxval < image.maxval) {
+    bit_depth++;
+    maxval = maxval * 2 + 1;
+  }
+  JXL_DEBUG_V(2, "Encoding %zu-channel, %i-bit, %zux%zu image.", nb_channels,
+              bit_depth, image.w, image.h);
+
+  BlobWriter io;
+
+  if (nb_channels < 1)
+    return true;  // is there any use for a zero-channel image?
+
+  int max_properties_nb_transforms = (options.max_properties << 2);
+
+  // encode transforms
+  int nb_transforms = image.transform.size();
+  max_properties_nb_transforms += (nb_transforms < 3 ? nb_transforms : 3);
+  write_big_endian_varint(io, max_properties_nb_transforms);
+  if (nb_transforms >= 3) write_big_endian_varint(io, nb_transforms - 3);
+  JXL_DEBUG_V(5, "Image data underwent %i transformations: ", nb_transforms);
+  for (int i = 0; i < nb_transforms; i++) {
+    int id = image.transform[i].ID;
+    int nb_params = image.transform[i].parameters.size();
+    write_big_endian_varint(io, (nb_params << 4) + id);
+    for (int j = 0; j < nb_params; j++)
+      write_big_endian_varint(io, image.transform[i].parameters[j]);
+  }
+
+  nb_channels = image.channel.size();
+
+  std::vector<int> responsive_offsets(6, -1);  //   LQIP, 1/16 1/8 1/4 1/2 full
+
+  // encode channel data
+  for (size_t i = options.skipchannels; i < nb_channels; i++) {
+    if (!image.channel[i].w || !image.channel[i].h)
+      continue;  // skip empty channels
+    if (i >= image.nb_meta_channels &&
+        (image.channel[i].w > options.max_chan_size ||
+         image.channel[i].h > options.max_chan_size)) {
+      break;
+    }
+    Predictor predictor = Predictor::Zero;
+    if (options.predictor.size() > i) {
+      predictor = (Predictor)options.predictor[i];
+    } else {
+      if (options.predictor.size() > 0) {
+        predictor = (Predictor)options.predictor.back();
+      } else {
+        // if nothing at all is specified, use Gradient,
+        // seems to be the best general-purpose predictor
+        predictor = Predictor::Gradient;
+      }
+    }
+    size_t j = i;
+    Tree tree;
+    size_t header_pos = 0;
+    size_t before = io.ftell();
+    Predictor learn_predictor = predictor;
+    if (predictor == Predictor::Best) learn_predictor = Predictor::Gradient;
+    DummyWriter dummyio;
+    if (!modular_encode_channels<
+            DummyWriter, RacDummy<DummyWriter>,
+            PropertySymbolCoder<ModularBitChancePass1, RacDummy<DummyWriter>,
+                                MAX_BIT_DEPTH>,
+            true>(dummyio, tree, options, learn_predictor, i, j, image,
+                  header_pos, options.entropy_coder))
+      return false;
+    if (!modular_encode_channels<
+            BlobWriter, RacOut<BlobWriter>,
+            FinalPropertySymbolCoder<ModularBitChancePass2, RacOut<BlobWriter>,
+                                     MAX_BIT_DEPTH>,
+            false>(io, tree, options, learn_predictor, i, j, image, header_pos,
+                   options.entropy_coder))
+      return false;
+    size_t after = io.ftell();
+
+    if (predictor == Predictor::Best) {
+      BlobWriter try_io;
+      size_t dummy_header_pos;
+      if (!modular_encode_channels<
+              BlobWriter, RacOut<BlobWriter>,
+              FinalPropertySymbolCoder<ModularBitChancePass2,
+                                       RacOut<BlobWriter>, MAX_BIT_DEPTH>,
+              false>(try_io, tree, options, Predictor::Weighted, i, j, image,
+                     dummy_header_pos, 2))
+        return false;
+      if (after - before > try_io.ftell()) {
+        io.fseek(before, SEEK_SET);
+        io.append(try_io);
+        after = io.ftell();
+      }
+    }
+    float bits = (after - header_pos) * 8.0;
+    float pixels = 0.0;
+    float ubits = 0.0;
+    for (size_t k = i; k <= j; k++) {
+      float chpixels = image.channel[k].w * image.channel[k].h;
+      float uncompressed_bpp =
+          ilog2(image.channel[k].maxval - image.channel[k].minval) + 1;
+      if (uncompressed_bpp < 8)
+        uncompressed_bpp = 8;
+      else if (uncompressed_bpp < 16)
+        uncompressed_bpp = 16;
+      else
+        uncompressed_bpp = 24;
+
+      pixels += chpixels;
+      if (image.channel[k].maxval > image.channel[k].minval)
+        ubits += chpixels * uncompressed_bpp;
+      if (ubits > 0.0) ubits += 32;
+    }
+    float bpp = bits / pixels;
+    float ubpp = ubits / pixels;
+
+    JXL_DEBUG_V(
+        4,
+        "Encoded channel %zu (%zux%zu, range %i..%i), %zu+%zu bytes "
+        "[%zu-%zu] (%f bpp; uncompressed estimate: %f bpp; %.2f%% reduction)",
+        i, image.channel[i].w, image.channel[i].h, image.channel[i].minval,
+        image.channel[i].maxval, after - header_pos, header_pos - before,
+        before, after, bpp, ubpp, (ubpp ? 100.0 - 100.0 * bpp / ubpp : 0));
+
+    if (bits > ubits && ubits > 0.0) {
+      io.fseek(before, SEEK_SET);
+      Tree notree;
+      if (!modular_encode_channels<
+              BlobWriter, RacOut<BlobWriter>,
+              FinalPropertySymbolCoder<ModularBitChancePass2,
+                                       RacOut<BlobWriter>, MAX_BIT_DEPTH>,
+              false>(io, notree, options, Predictor::Zero, i, j, image,
+                     header_pos, 1))
+        return false;
+      after = io.ftell();
+      float bpp = (after - header_pos) * 8.0 / pixels;
+      JXL_DEBUG_V(4,
+                  "Rolled back. Encoded channel %zu uncompressed (%zux%zu, "
+                  "range %i..%i), %zu+%zu bytes [%zu-%zu] (%f bpp)",
+                  i, image.channel[i].w, image.channel[i].h,
+                  image.channel[i].minval, image.channel[i].maxval,
+                  after - header_pos, header_pos - before, before, after, bpp);
+    }
+    i = j;
+  }
+
+  if (options.debug) options.heatmap->recompute_minmax();
+
+  realio.append(io);
+  return true;
+}
+#endif
+
+template <typename IO>
+bool modular_decode(IO &io, Image &image, modular_options &options,
+                    size_t bytes_to_load) {
+  if (image.nb_channels < 1) return true;
+
+  // decode transforms
+  int max_properties_nb_transforms = read_big_endian_varint(io);
+  options.max_properties = (max_properties_nb_transforms >> 2);
+  JXL_DEBUG_V(4, "Global option: up to %i back-referencing MA properties.",
+              options.max_properties);
+  int nb_transforms = (max_properties_nb_transforms & 3);
+  if (nb_transforms == 3) nb_transforms += read_big_endian_varint(io);
+  JXL_DEBUG_V(3, "Image data underwent %i transformations: ", nb_transforms);
+  for (int i = 0; i < nb_transforms; i++) {
+    int id_and_nb_params = read_big_endian_varint(io);
+    Transform t(id_and_nb_params & 0xf);
+    int nb_params = (id_and_nb_params >> 4);
+    for (int j = 0; j < nb_params; j++)
+      t.parameters.push_back(read_big_endian_varint(io));
+    if (!options.identify) {
+      t.meta_apply(image);
+      image.transform.push_back(t);
+    }
+    JXL_DEBUG_V(2, "%s", t.name());
+    if (t.ID == TRANSFORM_PALETTE) {
+      if (t.parameters[0] == t.parameters[1])
+        JXL_DEBUG_V(3, "[Compact channel %i to ", t.parameters[0]);
+      else
+        JXL_DEBUG_V(3, "[channels %i-%i with ", t.parameters[0],
+                    t.parameters[1]);
+      JXL_DEBUG_V(3, "%i colors]", t.parameters[2]);
+    }
+  }
+  JXL_DEBUG_V(5, "Header decoded. Read %ld bytes so far.", io.ftell());
+  if (options.identify) return true;
+  if (image.error) {
+    return JXL_FAILURE("Corrupt file. Aborting.");
+  }
+
+  size_t nb_channels = image.channel.size();
+
+  // decode channel data
+  for (size_t i = options.skipchannels; i < nb_channels; i++) {
+    if (i >= image.nb_meta_channels &&
+        (image.channel[i].w > options.max_chan_size ||
+         image.channel[i].h > options.max_chan_size)) {
+      break;
+    }
+    if ((bytes_to_load == 0 ||
+         static_cast<size_t>(io.ftell()) < bytes_to_load) &&
+        !io.isEOF()) {
+      if (!modular_decode_channel<
+              IO, FinalPropertySymbolCoder<ModularBitChancePass2, RacIn<IO>,
+                                           MAX_BIT_DEPTH>>(
+              io, options, i, image, bytes_to_load))
+        return false;
+    } else {
+      JXL_DEBUG_V(3, "Skipping decode of channels %zu-%zu.", i,
+                  nb_channels - 1);
+      break;
+    }
+  }
+  JXL_DEBUG_V(3, "Done decoding. Read %lu bytes.", io.ftell());
+  return true;
+}
+
+template bool modular_decode(BlobReader &io, Image &image,
+                             modular_options &options, size_t bytes_to_load);
+
+#ifdef HAS_ENCODER
+
+template bool modular_encode(BlobWriter &io, const Image &image,
+                             modular_options &options);
+
+void modular_prepare_encode(Image &image, modular_options &options) {
+  // ensure that the ranges are correct and tight
+  image.recompute_minmax();
+  if (options.debug) {
+    if (!options.heatmap) options.heatmap = new Image();
+    options.heatmap->w = image.w;
+    options.heatmap->h = image.h;
+    for (size_t i = 0; i < image.channel.size(); i++)
+      options.heatmap->channel.emplace_back(
+          Channel(image.channel[i].w, image.channel[i].h, 0, 255));
+  }
+}
+#endif
+
+#ifdef HAS_ENCODER
+bool modular_generic_compress(Image &image, PaddedBytes *bytes,
+                              modular_options *opts, int loss,
+                              bool try_transforms) {
+  if (image.w == 0 || image.h == 0) return true;
+  modular_options options;
+  if (opts == nullptr)
+    set_default_modular_options(options);
+  else
+    options = *opts;
+
+  if (!options.predictor.size())
+    options.predictor.push_back(5);  // use predictor 5 by default
+
+  if (try_transforms) {
+    if (loss > 1) {
+      // lossy DC
+      image.do_transform(Transform(TRANSFORM_SQUEEZE));
+      Transform quantize(TRANSFORM_QUANTIZE);
+      for (size_t i = image.nb_meta_channels; i < image.channel.size(); i++) {
+        Channel &ch = image.channel[i];
+        int shift = ch.hcshift + ch.vcshift;  // number of pixel halvings
+        int q;
+        q = (loss >> shift);
+        if (q < 1) q = 1;
+        quantize.parameters.push_back(q);
+      }
+      image.do_transform(quantize);
+    } else if (!options.skipchannels) {
+      // simple heuristic: if less than 90 percent of the values in the range
+      // actually occur, it is probably worth it to do a compaction
+      // unless there are few pixels to encode, then the overhead is not worth
+      // it
+      image.recompute_minmax();
+
+      for (size_t c = 0; c < image.nb_channels; c++) {
+        Transform maybe_palette_1(TRANSFORM_PALETTE);
+        maybe_palette_1.parameters.push_back(c + image.nb_meta_channels);
+        maybe_palette_1.parameters.push_back(c + image.nb_meta_channels);
+        int colors = image.channel[c + image.nb_meta_channels].maxval -
+                     image.channel[c + image.nb_meta_channels].minval + 1;
+        float factor = 1.0;
+        if (image.w * image.h < 1000) {
+          factor =
+              static_cast<float>(image.w) * static_cast<float>(image.h) / 1000;
+        }
+        maybe_palette_1.parameters.push_back((int)(0.9 * factor * colors));
+        if (image.do_transform(maybe_palette_1)) {
+          options.predictor.insert(options.predictor.begin(),
+                                   1);  // left predictor for palette encoding
+        }
+      }
+    }
+    image.recompute_minmax();
+  }
+
+  modular_prepare_encode(image, options);
+  BlobWriter io;
+  bool status = modular_encode(io, image, options);
+  if (!status) return false;
+  JXL_DEBUG_V(
+      4, "Modular-encoded a %zux%zu maxval=%i nbchans=%zu image in %zu bytes",
+      image.w, image.h, image.maxval, image.real_nb_channels, io.size());
+  bytes->append(io.blob);
+  return true;
+}
+template <typename T>
+bool modular_rect_compress_1(const Plane<T> &img, const Rect &rect,
+                             PaddedBytes *bytes, modular_options *opts,
+                             int loss) {
+  Image image(rect.xsize(), rect.ysize(), 255, 1);
+  for (size_t y = 0; y < image.h; y++) {
+    pixel_type *JXL_RESTRICT to = image.channel[0].Row(y);
+    const T *JXL_RESTRICT from = rect.ConstRow(img, y);
+    for (size_t x = 0; x < image.w; x++) to[x] = from[x];
+  }
+  return modular_generic_compress(image, bytes, opts, loss);
+}
+
+// Change RGB to GRB, or XYB to YXB  (luma-like channel first)
+int swap01(int i) {
+  if (i == 0) return 1;
+  if (i == 1) return 0;
+  return i;
+}
+template <typename T>
+bool modular_rect_compress_3(const Image3<T> &img, const Rect &rect,
+                             PaddedBytes *bytes, modular_options *opts,
+                             int loss) {
+  Image image(rect.xsize(), rect.ysize(), 255, 3);
+  for (int c = 0; c < 3; c++)
+    for (size_t y = 0; y < image.h; y++) {
+      pixel_type *JXL_RESTRICT to = image.channel[c].Row(y);
+      const T *JXL_RESTRICT from = rect.ConstPlaneRow(img, swap01(c), y);
+      for (size_t x = 0; x < image.w; x++) to[x] = from[x];
+    }
+
+  return modular_generic_compress(image, bytes, opts, loss);
+}
+
+template <typename T>
+bool modular_rect_compress_2(const Plane<T> &img1, const Plane<T> &img2,
+                             const Rect &rect, PaddedBytes *bytes,
+                             modular_options *opts, pixel_type offset1,
+                             pixel_type offset2) {
+  Image image(rect.xsize(), rect.ysize(), 255, 2);
+  for (size_t y = 0; y < image.h; y++) {
+    const T *JXL_RESTRICT from1 = img1.ConstRow(y);
+    pixel_type *JXL_RESTRICT to1 = image.channel[0].Row(y);
+    for (size_t x = 0; x < image.w; x++)
+      to1[x] = (pixel_type)from1[x] - offset1;
+    const T *JXL_RESTRICT from2 = img2.ConstRow(y);
+    pixel_type *JXL_RESTRICT to2 = image.channel[1].Row(y);
+    for (size_t x = 0; x < image.w; x++)
+      to2[x] = (pixel_type)from2[x] - offset2;
+  }
+  return modular_generic_compress(image, bytes, opts);
+}
+
+template <typename T>
+bool modular_compress_2(const Plane<T> &img1, const Plane<T> &img2,
+                        PaddedBytes *bytes, modular_options *opts) {
+  JXL_DASSERT(img1.xsize() > 0);
+  Image image(img1.xsize(), img1.ysize(), 255, 2);
+  JXL_DASSERT(img1.xsize() == img2.xsize());
+  JXL_DASSERT(img1.ysize() == 1);
+  JXL_DASSERT(img2.ysize() == 1);
+  for (size_t y = 0; y < image.h; y++) {
+    const T *JXL_RESTRICT from1 = img1.ConstRow(y);
+    pixel_type *JXL_RESTRICT to1 = image.channel[0].Row(y);
+    for (size_t x = 0; x < image.w; x++) to1[x] = from1[x];
+    const T *JXL_RESTRICT from2 = img2.ConstRow(y);
+    pixel_type *JXL_RESTRICT to2 = image.channel[1].Row(y);
+    for (size_t x = 0; x < image.w; x++) to2[x] = from2[x];
+  }
+  image.w = -image.w;  // flip the sign, so the width gets signalled
+  return modular_generic_compress(image, bytes, opts);
+}
+
+template bool modular_rect_compress_1(const Plane<uint8_t> &img,
+                                      const Rect &rect, PaddedBytes *bytes,
+                                      modular_options *opts, int loss);
+template bool modular_rect_compress_1(const Plane<int16_t> &img,
+                                      const Rect &rect, PaddedBytes *bytes,
+                                      modular_options *opts, int loss);
+template bool modular_rect_compress_1(const Plane<uint16_t> &img,
+                                      const Rect &rect, PaddedBytes *bytes,
+                                      modular_options *opts, int loss);
+template bool modular_rect_compress_3(const Image3<int16_t> &img,
+                                      const Rect &rect, PaddedBytes *bytes,
+                                      modular_options *opts, int loss);
+template bool modular_rect_compress_3(const Image3<int32_t> &img,
+                                      const Rect &rect, PaddedBytes *bytes,
+                                      modular_options *opts, int loss);
+template bool modular_rect_compress_2(const Plane<uint8_t> &img1,
+                                      const Plane<uint8_t> &img2,
+                                      const Rect &rect, PaddedBytes *bytes,
+                                      modular_options *opts, pixel_type offset1,
+                                      pixel_type offset2);
+template bool modular_compress_2(const ImageI &img, const ImageI &img2,
+                                 PaddedBytes *bytes, modular_options *opts);
+#endif
+
+bool modular_generic_decompress(const Span<const uint8_t> bytes, size_t *pos,
+                                Image &image, modular_options &options,
+                                size_t bytes_to_load, int undo_transforms) {
+  BlobReader io(bytes.data(), bytes.size());
+  io.fseek(*pos, SEEK_SET);
+  bool status = modular_decode(io, image, options, bytes_to_load);
+  if (!status) return false;
+  image.undo_transforms(undo_transforms);
+  JXL_DEBUG_V(4, "Modular-decoded a %zux%zu nbchans=%zu image from %lu bytes",
+              image.w, image.h, image.real_nb_channels, io.ftell() - *pos);
+  *pos = io.ftell();
+  return true;
+}
+template <typename T>
+bool modular_rect_decompress_1(const Span<const uint8_t> bytes, size_t *pos,
+                               const Plane<T> *JXL_RESTRICT result,
+                               const Rect &rect) {
+  Image image(rect.xsize(), rect.ysize(), 255, 1);
+  if (image.w == 0 || image.h == 0) return true;
+  modular_options options;
+  set_default_modular_options(options);
+  options.nb_channels = 1;
+  if (!modular_generic_decompress(bytes, pos, image, options)) return false;
+  for (size_t y = 0; y < image.h; y++) {
+    const pixel_type *JXL_RESTRICT from = image.channel[0].Row(y);
+    T *JXL_RESTRICT to = rect.MutableRow(result, y);
+    for (size_t x = 0; x < image.w; x++) to[x] = from[x];
+  }
+  return true;
+}
+template <typename T>
+bool modular_rect_decompress_2(const Span<const uint8_t> bytes, size_t *pos,
+                               const Plane<T> *JXL_RESTRICT result1,
+                               const Plane<T> *JXL_RESTRICT result2,
+                               const Rect &rect, pixel_type offset1,
+                               pixel_type offset2) {
+  Image image(rect.xsize(), rect.ysize(), 255, 2);
+  if (image.w == 0 || image.h == 0) return true;
+  modular_options options;
+  set_default_modular_options(options);
+  options.nb_channels = 2;
+  if (!modular_generic_decompress(bytes, pos, image, options)) return false;
+  for (size_t y = 0; y < image.h; y++) {
+    const pixel_type *JXL_RESTRICT from = image.channel[0].Row(y);
+    T *JXL_RESTRICT to = result1->MutableRow(y);
+    for (size_t x = 0; x < image.w; x++) to[x] = from[x] + offset1;
+  }
+  for (size_t y = 0; y < image.h; y++) {
+    const pixel_type *JXL_RESTRICT from = image.channel[1].Row(y);
+    T *JXL_RESTRICT to = result2->MutableRow(y);
+    for (size_t x = 0; x < image.w; x++) to[x] = from[x] + offset2;
+  }
+  return true;
+}
+template <typename T>
+bool modular_rect_decompress_3(const Span<const uint8_t> bytes, size_t *pos,
+                               const Image3<T> *JXL_RESTRICT result,
+                               const Rect &rect) {
+  Image image(rect.xsize(), rect.ysize(), 255, 3);
+  if (image.w == 0 || image.h == 0) return true;
+  modular_options options;
+  set_default_modular_options(options);
+  options.nb_channels = 3;
+  if (!modular_generic_decompress(bytes, pos, image, options)) return false;
+  for (int c = 0; c < 3; c++)
+    for (size_t y = 0; y < image.h; y++) {
+      const pixel_type *JXL_RESTRICT from = image.channel[c].Row(y);
+      T *JXL_RESTRICT to = const_cast<T *>(result->PlaneRow(swap01(c), y));
+      for (size_t x = 0; x < image.w; x++) to[x] = from[x];
+    }
+  return true;
+}
+
+template bool modular_rect_decompress_1(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const Plane<uint8_t> *JXL_RESTRICT result, const Rect &rect);
+template bool modular_rect_decompress_1(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const Plane<int16_t> *JXL_RESTRICT result, const Rect &rect);
+template bool modular_rect_decompress_1(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const Plane<uint16_t> *JXL_RESTRICT result, const Rect &rect);
+template bool modular_rect_decompress_2(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const Plane<uint8_t> *JXL_RESTRICT result1,
+    const Plane<uint8_t> *JXL_RESTRICT result2, const Rect &rect,
+    pixel_type offset1, pixel_type offset2);
+template bool modular_rect_decompress_3(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const Image3<int16_t> *JXL_RESTRICT result, const Rect &rect);
+template bool modular_rect_decompress_3(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const Image3<int32_t> *JXL_RESTRICT result, const Rect &rect);
+
+}  // namespace jxl
\ No newline at end of file
diff --git a/jxl/modular/encoding/encoding.h b/jxl/modular/encoding/encoding.h
new file mode 100644
index 000000000..44547e422
--- /dev/null
+++ b/jxl/modular/encoding/encoding.h
@@ -0,0 +1,192 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_ENCODING_ENCODING_H_
+#define JXL_MODULAR_ENCODING_ENCODING_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/image.h"
+#include "jxl/modular/config.h"
+#include "jxl/modular/image/image.h"
+#include "jxl/modular/ma/compound.h"
+#include "jxl/modular/memio.h"
+
+namespace jxl {
+
+struct modular_options {
+  // decoding options
+  bool identify;  // don't decode image data, just decode header
+
+  // used in both encode and decode
+  int nb_channels;   // if full_header==false, need to specify how many channels
+                     // to expect
+  int skipchannels;  // the first <skipchannels> channels will not be
+                     // encoded/decoded
+  size_t max_chan_size;  // stop encoding/decoding when reaching a (non-meta)
+                         // channel that has a dimension bigger than this
+
+  // encoding options (some of which are needed during decoding too)
+  int entropy_coder;  // 0 = MABEGABRAC, 1 = MABrotli, 2 = MARANS
+
+  // MA options
+  float nb_repeats;    // number of iterations to do to learn a MA tree (does
+                       // not have to be an integer; if zero there is no MA
+                       // context model)
+  int max_properties;  // maximum number of (previous channel) properties to use
+                       // in the MA trees
+  float ctx_threshold;  // number of bits to be saved to justify adding
+                        // another node to the MA tree (lower value = bigger
+                        // context model)
+
+  // Brotli options
+  int brotli_effort;  // 0..11
+
+  std::vector<int> predictor;  // predictor to use for each channel. last one
+                               // gets repeated if needed
+
+  int nb_wp_modes;
+
+  // deprecated
+  bool debug;      // produce debug images, including (for MABEGABRAC only) a
+                   // compression heatmap
+  Image *heatmap;  // produced if debug==true
+};
+void set_default_modular_options(struct modular_options &o);
+
+#ifdef HAS_ENCODER
+void modular_prepare_encode(Image &image, modular_options &options);
+
+bool modular_generic_compress(Image &image, PaddedBytes *bytes,
+                              modular_options *opts = nullptr, int loss = 1,
+                              bool try_transforms = true);
+
+// undo_transforms == N > 0: undo all transforms except the first N
+//                           (e.g. to represent YCbCr420 losslessly)
+// undo_transforms == 0: undo all transforms
+// undo_transforms == -1: undo all transforms but don't clamp to range
+// undo_transforms == -2: don't undo any transform
+bool modular_generic_decompress(const Span<const uint8_t> bytes, size_t *pos,
+                                Image &image, modular_options &options,
+                                size_t bytes_to_load = 0,
+                                int undo_transforms = -1);
+
+// encode a rect from an arbitrary plane; dimensions etc are implicit
+template <typename T>
+bool modular_rect_compress_1(const Plane<T> &img, const Rect &rect,
+                             PaddedBytes *bytes, modular_options *opts,
+                             int loss = 1);
+extern template bool modular_rect_compress_1<uint8_t>(const ImageB &img,
+                                                      const Rect &rect,
+                                                      PaddedBytes *bytes,
+                                                      modular_options *opts,
+                                                      int loss = 1);
+extern template bool modular_rect_compress_1<uint16_t>(const ImageU &img,
+                                                       const Rect &rect,
+                                                       PaddedBytes *bytes,
+                                                       modular_options *opts,
+                                                       int loss = 1);
+extern template bool modular_rect_compress_1<int16_t>(const ImageS &img,
+                                                      const Rect &rect,
+                                                      PaddedBytes *bytes,
+                                                      modular_options *opts,
+                                                      int loss = 1);
+
+// these two planes are supposed to be single-row with the same width
+// (used specifically to encode the ac_strategy + quant_field)
+// width is not implicit
+template <typename T>
+bool modular_compress_2(const Plane<T> &img1, const Plane<T> &img2,
+                        PaddedBytes *bytes, modular_options *opts);
+
+template <typename T>
+bool modular_rect_compress_2(const Plane<T> &img1, const Plane<T> &img2,
+                             const Rect &rect, PaddedBytes *bytes,
+                             modular_options *opts, pixel_type offset1 = 0,
+                             pixel_type offset2 = 0);
+extern template bool modular_rect_compress_2<uint8_t>(
+    const ImageB &img1, const ImageB &img2, const Rect &rect,
+    PaddedBytes *bytes, modular_options *opts, pixel_type offset1 = 0,
+    pixel_type offset2 = 0);
+
+// encode a rect from an arbitrary Image3; dimensions etc are implicit
+template <typename T>
+bool modular_rect_compress_3(const Image3<T> &img, const Rect &rect,
+                             PaddedBytes *bytes, modular_options *opts,
+                             int loss = 1);
+extern template bool modular_rect_compress_3<int16_t>(const Image3S &img,
+                                                      const Rect &rect,
+                                                      PaddedBytes *bytes,
+                                                      modular_options *opts,
+                                                      int loss = 1);
+
+extern template bool modular_rect_compress_3<int32_t>(const Image3I &img,
+                                                      const Rect &rect,
+                                                      PaddedBytes *bytes,
+                                                      modular_options *opts,
+                                                      int loss = 1);
+
+#endif
+
+template <typename T>
+bool modular_rect_decompress_1(const Span<const uint8_t> bytes, size_t *pos,
+                               const Plane<T> *JXL_RESTRICT result,
+                               const Rect &rect);
+extern template bool modular_rect_decompress_1<uint8_t>(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const ImageB *JXL_RESTRICT result, const Rect &rect);
+extern template bool modular_rect_decompress_1<uint16_t>(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const ImageU *JXL_RESTRICT result, const Rect &rect);
+extern template bool modular_rect_decompress_1<int16_t>(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const ImageS *JXL_RESTRICT result, const Rect &rect);
+
+template <typename T>
+bool modular_rect_decompress_2(const Span<const uint8_t> bytes, size_t *pos,
+                               const Plane<T> *JXL_RESTRICT result1,
+                               const Plane<T> *JXL_RESTRICT result2,
+                               const Rect &rect, pixel_type offset1 = 0,
+                               pixel_type offset2 = 0);
+extern template bool modular_rect_decompress_2<uint8_t>(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const ImageB *JXL_RESTRICT result1, const ImageB *JXL_RESTRICT result2,
+    const Rect &rect, pixel_type offset1 = 0, pixel_type offset2 = 0);
+
+template <typename T>
+bool modular_rect_decompress_3(const Span<const uint8_t> bytes, size_t *pos,
+                               const Image3<T> *JXL_RESTRICT result,
+                               const Rect &rect);
+extern template bool modular_rect_decompress_3<int16_t>(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const Image3S *JXL_RESTRICT result, const Rect &rect);
+extern template bool modular_rect_decompress_3<int32_t>(
+    const Span<const uint8_t> bytes, size_t *pos,
+    const Image3I *JXL_RESTRICT result, const Rect &rect);
+
+template <typename IO>
+bool modular_decode(IO &io, Image &image, modular_options &options,
+                    size_t bytes_to_load = 0);
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_ENCODING_ENCODING_H_
diff --git a/jxl/modular/encoding/weighted_predict.h b/jxl/modular/encoding/weighted_predict.h
new file mode 100644
index 000000000..745049f3a
--- /dev/null
+++ b/jxl/modular/encoding/weighted_predict.h
@@ -0,0 +1,443 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// @author Alexander Rhatushnyak
+
+#ifndef JXL_MODULAR_ENCODING_WEIGHTED_PREDICT_H_
+#define JXL_MODULAR_ENCODING_WEIGHTED_PREDICT_H_
+
+#include "jxl/base/bits.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+namespace {
+constexpr int kWithSign = 7;
+constexpr int kBitsMax = 13;
+constexpr int kNumContexts = 1 + kWithSign + kBitsMax;
+
+// SET ME TO ZERO FOR A FASTER VERSION WITH NO ROUNDING!
+constexpr int PBits = 3;
+constexpr int toRound = ((1 << PBits) >> 1) - 1;
+constexpr int toRound_m1 = toRound;  // (toRound ? toRound - 1 : 0);
+constexpr pixel_type AddPBits(pixel_type x) { return uint32_t(x) << PBits; }
+}  // namespace
+
+struct State {
+  pixel_type prediction0, prediction1, prediction2, prediction3;
+  pixel_type p1C, p2C, p3Ca, p3Cb, p3Cc, p3Cd, p3Ce;
+  uint32_t w0, w1, w2, w3;
+  const size_t lastX;  // xsize - 1, for prediction borders
+  pixel_type* JXL_RESTRICT rowImg;
+  const pixel_type *JXL_RESTRICT rowPrev, *JXL_RESTRICT rowPP;
+  const pixel_type minTpv, maxTpv;
+  const pixel_type shiftedMinTpv = AddPBits(minTpv);
+  const pixel_type shiftedMaxTpv = AddPBits(maxTpv);
+
+  std::vector<uint32_t> errors0;  // Errors of predictor 0
+  std::vector<uint32_t> errors1;  // Errors of predictor 1
+  std::vector<uint32_t> errors2;  // Errors of predictor 2
+  std::vector<uint32_t> errors3;  // Errors of predictor 3
+  std::vector<uint8_t> nbitErr;
+  std::vector<int32_t> trueErr;
+  uint32_t divlookup[64];
+
+  State(size_t imageSizeX, size_t imageSizeY, pixel_type minval,
+        pixel_type maxval)
+      : lastX(imageSizeX - 1), minTpv(minval), maxTpv(maxval) {
+    errors0.resize(imageSizeX * 2 + 4);
+    errors1.resize(imageSizeX * 2 + 4);
+    errors2.resize(imageSizeX * 2 + 4);
+    errors3.resize(imageSizeX * 2 + 4);
+    nbitErr.resize(imageSizeX * 2);
+    trueErr.resize(imageSizeX * 2);
+    for (int i = 0; i < 64; i++) {
+      divlookup[i] = (1 << 24) / (i + 1);
+    }
+  }
+
+  // approximates (maxweight<<24)/(x+1), avoiding division
+  JXL_INLINE uint32_t errorWeight(uint32_t x, uint32_t maxweight) {
+    // cheapest option:
+    //    return maxweight << (24 - jxl::FloorLog2Nonzero(x+1));
+
+    // better and somewhat more expensive
+    //    int shift=jxl::FloorLog2Nonzero(x+1)+12;
+    //    return maxweight * ((0x1000000000LLU - ((uint64_t)x<<(46-shift))) >>
+    //    shift);
+
+    // almost as good as real division
+    int shift = jxl::FloorLog2Nonzero(x + 1) - 5;
+    if (shift < 0) shift = 0;
+    return 4 + (maxweight * divlookup[x >> shift] >> shift);
+  }
+  JXL_INLINE int numBits(int x) {
+    if (x < 2) return x;
+    return std::min(2 + (int)jxl::FloorLog2Nonzero((unsigned int)x - 1),
+                    kBitsMax);
+  }
+
+  JXL_INLINE pixel_type predict1y0(size_t x, size_t yp, size_t yp1,
+                                   int* maxErr) {
+    *maxErr = (x == 0 ? kNumContexts - 1
+                      : x == 1 ? nbitErr[yp - 1]
+                               : std::max(nbitErr[yp - 1], nbitErr[yp - 2]));
+    prediction0 = prediction1 = prediction2 = prediction3 =
+        (x == 0 ? AddPBits(minTpv + maxTpv) / 2
+                : x == 1 ? AddPBits(rowImg[x - 1])
+                         : AddPBits(rowImg[x - 1]) +
+                               AddPBits(rowImg[x - 1] - rowImg[x - 2]) / 4);
+    return std::max(shiftedMinTpv, std::min(prediction0, shiftedMaxTpv));
+  }
+
+  JXL_INLINE pixel_type predict1x0(size_t x, size_t yp, size_t yp1,
+                                   int* maxErr) {
+    *maxErr = std::max(nbitErr[yp1], nbitErr[yp1 + (x < lastX ? 1 : 0)]);
+    prediction0 = prediction2 = prediction3 = prediction1 =
+        AddPBits(rowPrev[x]);
+    return std::max(shiftedMinTpv, std::min(prediction0, shiftedMaxTpv));
+  }
+
+  JXL_INLINE pixel_type WeightedAverage(pixel_type p1, pixel_type p2,
+                                        pixel_type p3, pixel_type p4,
+                                        uint32_t w1, uint32_t w2, uint32_t w3,
+                                        uint32_t w4) {
+    uint32_t sumWeights = w1 + w2 + w3 + w4;
+    JXL_DASSERT(sumWeights > 15);
+    uint32_t log_weight = jxl::FloorLog2Nonzero(sumWeights);  // at least 4.
+    w1 = w1 >> (log_weight - 4);
+    w2 = w2 >> (log_weight - 4);
+    w3 = w3 >> (log_weight - 4);
+    w4 = w4 >> (log_weight - 4);
+    uint32_t new_sum_weights = w1 + w2 + w3 + w4;
+    int64_t s = new_sum_weights >> 1;
+    s += static_cast<int64_t>(p1) * w1;
+    s += static_cast<int64_t>(p2) * w2;
+    s += static_cast<int64_t>(p3) * w3;
+    s += static_cast<int64_t>(p4) * w4;
+    return (s * divlookup[new_sum_weights - 1]) >> 24;
+  }
+
+  JXL_INLINE pixel_type predict1(size_t x, size_t yp, size_t yp1, int* maxErr) {
+    if (!rowPrev) return predict1y0(x, yp, yp1, maxErr);
+    if (x == 0LL) return predict1x0(x, yp, yp1, maxErr);
+
+    int a1 = (x < lastX ? 1 : 0);
+    uint32_t weight0 = errors0[yp1] + errors0[yp1 - 1] + errors0[yp1 + a1];
+    uint32_t weight1 = errors1[yp1] + errors1[yp1 - 1] + errors1[yp1 + a1];
+    uint32_t weight2 = errors2[yp1] + errors2[yp1 - 1] + errors2[yp1 + a1];
+    uint32_t weight3 = errors3[yp1] + errors3[yp1 - 1] + errors3[yp1 + a1];
+
+    uint8_t mxe = nbitErr[yp - 1];
+    mxe = std::max(mxe, nbitErr[yp1]);
+    mxe = std::max(mxe, nbitErr[yp1 - 1]);
+    mxe = std::max(mxe, nbitErr[yp1 + a1]);
+
+    pixel_type N = AddPBits(rowPrev[x]), W = AddPBits(rowImg[x - 1]),
+               NE = AddPBits(rowPrev[x + a1]), NW = AddPBits(rowPrev[x - 1]),
+               NN = AddPBits(rowPP[x]);
+
+    weight0 = errorWeight(weight0, w0);
+    weight1 = errorWeight(weight1, w1);
+    weight2 = errorWeight(weight2, w2);
+    weight3 = errorWeight(weight3, w3);
+
+    int teW = trueErr[yp - 1];
+    int teN = trueErr[yp1];
+    int teNW = trueErr[yp1 - 1];
+    int sumWN = teN + teW;
+    int teNE = trueErr[yp1 + a1];
+
+    prediction0 = W + NE - N;
+    prediction1 = N - (((sumWN + teNE) * p1C) >> 5);
+    prediction2 = W - (((sumWN + teNW) * p2C) >> 5);
+    prediction3 = N - ((teNW * p3Ca + teN * p3Cb + teN * p3Cc +
+                        (NN - N) * p3Cd + (NW - W) * p3Ce) >>
+                       5);
+
+    pixel_type prediction =
+        WeightedAverage(prediction0, prediction1, prediction2, prediction3,
+                        weight0, weight1, weight2, weight3);
+    if (mxe && mxe <= kWithSign * 2) {
+      if (sumWN * 2 + teNW + teNE < 0) --mxe;  // 2 2 1 1
+    }
+    *maxErr = mxe;
+
+    if (((teN ^ teW) | (teN ^ teNW)) > 0) {  // if all three have the same sign
+      return std::max(shiftedMinTpv, std::min(prediction, shiftedMaxTpv));
+    }
+
+    pixel_type mx = (W > NE ? W : NE), mn = W + NE - mx;
+    if (N > mx) mx = N;
+    if (N < mn) mn = N;
+    prediction = std::max(mn, std::min(mx, prediction));
+    return prediction;
+  }
+
+  JXL_INLINE void UpdateSizeAndErrors(pixel_type err, size_t yp, size_t yp1,
+                                      size_t x, pixel_type prediction0,
+                                      pixel_type prediction1,
+                                      pixel_type prediction2,
+                                      pixel_type prediction3,
+                                      pixel_type truePixelValue) {
+    trueErr[yp + x] = err;
+    err = numBits(err >= 0 ? err : -err);
+    nbitErr[yp + x] = (err <= kWithSign ? err * 2 : err + kWithSign);
+    err = ((prediction0 + toRound) >> PBits) - truePixelValue;
+    if (err < 0) err = -err; /* abs() and min()? worse speed! */
+    errors0[yp + x] = err;
+    errors0[1 + yp1 + x] += err;
+    err = ((prediction1 + toRound) >> PBits) - truePixelValue;
+    if (err < 0) err = -err;
+    errors1[yp + x] = err;
+    errors1[1 + yp1 + x] += err;
+    err = ((prediction2 + toRound) >> PBits) - truePixelValue;
+    if (err < 0) err = -err;
+    errors2[yp + x] = err;
+    errors2[1 + yp1 + x] += err;
+    err = ((prediction3 + toRound) >> PBits) - truePixelValue;
+    if (err < 0) err = -err;
+    errors3[yp + x] = err;
+    errors3[1 + yp1 + x] += err;
+  }
+
+  // Encoder helper function to set the parameters to some presets.
+  // Decoder only uses predictor_mode(0), which is the default setting.
+  void predictor_mode(int i) {
+    switch (i) {
+      case 0:
+        // ~ lossless16 predictor
+        w0 = 0xd;
+        w1 = 0xc;
+        w2 = 0xc;
+        w3 = 0xc;
+        p1C = 16;
+        p2C = 10;
+        p3Ca = 7;
+        p3Cb = 7;
+        p3Cc = 7;
+        p3Cd = 0;
+        p3Ce = 0;
+        break;
+      case 1:
+        // ~ default lossless8 predictor
+        w0 = 0xd;
+        w1 = 0xc;
+        w2 = 0xc;
+        w3 = 0xb;
+        p1C = 8;
+        p2C = 8;
+        p3Ca = 4;
+        p3Cb = 0;
+        p3Cc = 3;
+        p3Cd = 23;
+        p3Ce = 2;
+        break;
+      case 2:
+        // ~ west lossless8 predictor
+        w0 = 0xd;
+        w1 = 0xc;
+        w2 = 0xd;
+        w3 = 0xc;
+        p1C = 10;
+        p2C = 9;
+        p3Ca = 7;
+        p3Cb = 0;
+        p3Cc = 0;
+        p3Cd = 16;
+        p3Ce = 9;
+        break;
+      case 3:
+        // ~ north lossless8 predictor
+        w0 = 0xd;
+        w1 = 0xd;
+        w2 = 0xc;
+        w3 = 0xc;
+        p1C = 16;
+        p2C = 8;
+        p3Ca = 0;
+        p3Cb = 16;
+        p3Cc = 0;
+        p3Cd = 23;
+        p3Ce = 0;
+        break;
+      case 4:
+        // something else, because why not
+        w0 = 0xd;
+        w1 = 0xc;
+        w2 = 0xc;
+        w3 = 0xc;
+        p1C = 10;
+        p2C = 10;
+        p3Ca = 5;
+        p3Cb = 5;
+        p3Cc = 5;
+        p3Cd = 12;
+        p3Ce = 4;
+        break;
+    }
+  }
+
+  bool wp_compress(const Channel& img, jxl::PaddedBytes* bytes, int nb_modes) {
+    size_t xsize = img.w;
+    size_t ysize = img.h;
+    size_t pos = bytes->size();
+    size_t best_size = 0;
+    for (int mode = 0; mode < nb_modes; mode++) {
+      predictor_mode(mode);
+      std::vector<std::vector<jxl::Token>> tokens(1);
+
+      for (size_t y = 0, yp = 0, yp1 = xsize; y < ysize;
+           ++y, yp = xsize - yp, yp1 = xsize - yp) {
+        rowImg = const_cast<pixel_type*>(img.Row(y));
+        rowPrev = (y == 0 ? nullptr : img.Row(y - 1));
+        rowPP = (y <= 1 ? rowPrev : img.Row(y - 2));
+        for (size_t x = 0; x < xsize; ++x) {
+          int maxErr;
+          pixel_type prediction = predict1(x, yp + x, yp1 + x, &maxErr);
+          JXL_DASSERT(0 <= maxErr && maxErr <= kNumContexts - 1);
+          JXL_DASSERT(shiftedMinTpv <= prediction &&
+                      prediction <= shiftedMaxTpv);
+          pixel_type truePixelValue = rowImg[x];
+          jxl::TokenizeHybridUint(
+              maxErr,
+              jxl::PackSigned(truePixelValue -
+                              ((prediction + toRound_m1) >> PBits)),
+              tokens.data());
+          pixel_type err = prediction - AddPBits(truePixelValue);
+          UpdateSizeAndErrors(err, yp, yp1, x, prediction0, prediction1,
+                              prediction2, prediction3, truePixelValue);
+        }  // x
+      }    // y
+
+      jxl::BitWriter writer;
+      jxl::EntropyEncodingData codes;
+      std::vector<uint8_t> context_map;
+      jxl::BitWriter::Allotment allotment(&writer, 52);
+      if (mode != 0) {
+        writer.Write(1, 0);
+        writer.Write(5, p1C);
+        writer.Write(5, p2C);
+        writer.Write(5, p3Ca);
+        writer.Write(5, p3Cb);
+        writer.Write(5, p3Cc);
+        writer.Write(5, p3Cd);
+        writer.Write(5, p3Ce);
+        writer.Write(4, w0);
+        writer.Write(4, w1);
+        writer.Write(4, w2);
+        writer.Write(4, w3);
+      } else {
+        writer.Write(1, 1);
+      }
+      jxl::ReclaimAndCharge(&writer, &allotment, 0, nullptr);
+      jxl::BuildAndEncodeHistograms(jxl::HistogramParams(), kNumContexts,
+                                    tokens, &codes, &context_map, &writer, 0,
+                                    nullptr);
+      jxl::WriteTokens(tokens[0], codes, context_map, &writer, 0, nullptr);
+      writer.ZeroPadToByte();
+      jxl::Span<const uint8_t> span = writer.GetSpan();
+      if (mode == 0 || span.size() < best_size) {
+        bytes->resize(pos + span.size());
+        best_size = span.size();
+        memcpy(bytes->data() + pos, span.data(), span.size());
+      }
+    }
+    return true;
+  }
+
+  HWY_ATTR bool wp_decompress(const jxl::Span<const uint8_t> bytes,
+                              size_t* bytes_pos, Channel& img) {
+    if (*bytes_pos > bytes.size()) return JXL_FAILURE("out of bounds");
+    size_t xsize = img.w;
+    size_t ysize = img.h;
+    size_t compressedSize = bytes.size() - *bytes_pos;
+    const uint8_t* compressedData = bytes.data() + *bytes_pos;
+
+    if (!xsize || !ysize) return JXL_FAILURE("invalid image size");
+
+    predictor_mode(0);
+
+    jxl::BitReader bitreader(
+        jxl::Span<const uint8_t>(compressedData, compressedSize));
+    jxl::ANSCode code;
+    std::vector<uint8_t> context_map;
+    if (!bitreader.ReadBits(1)) {
+      p1C = bitreader.ReadBits(5);
+      p2C = bitreader.ReadBits(5);
+      p3Ca = bitreader.ReadBits(5);
+      p3Cb = bitreader.ReadBits(5);
+      p3Cc = bitreader.ReadBits(5);
+      p3Cd = bitreader.ReadBits(5);
+      p3Ce = bitreader.ReadBits(5);
+      w0 = bitreader.ReadBits(4);
+      w1 = bitreader.ReadBits(4);
+      w2 = bitreader.ReadBits(4);
+      w3 = bitreader.ReadBits(4);
+    }
+
+    JXL_RETURN_IF_ERROR(jxl::DecodeHistograms(
+        &bitreader, kNumContexts, ANS_MAX_ALPHA_SIZE, &code, &context_map));
+    jxl::ANSSymbolReader ansreader(&code, &bitreader);
+
+    for (size_t y = 0, yp = 0, yp1 = xsize; y < ysize;
+         ++y, yp = xsize - yp, yp1 = xsize - yp) {
+      rowImg = img.Row(y);
+      rowPrev = (y == 0 ? nullptr : img.Row(y - 1));
+      rowPP = (y <= 1 ? rowPrev : img.Row(y - 2));
+      for (size_t x = 0; x < xsize; ++x) {
+        int maxErr;
+        pixel_type prediction = predict1(x, yp + x, yp1 + x, &maxErr);
+        JXL_DASSERT(0 <= maxErr && maxErr <= kNumContexts - 1);
+        JXL_DASSERT(shiftedMinTpv <= prediction && prediction <= shiftedMaxTpv);
+
+        size_t q =
+            jxl::ReadHybridUint(maxErr, &bitreader, &ansreader, context_map);
+        pixel_type truePixelValue =
+            ((prediction + toRound_m1) >> PBits) + jxl::UnpackSigned(q);
+        rowImg[x] = truePixelValue;
+        pixel_type err = prediction - AddPBits(truePixelValue);
+        UpdateSizeAndErrors(err, yp, yp1, x, prediction0, prediction1,
+                            prediction2, prediction3, truePixelValue);
+      }  // x
+    }    // y
+    if (!ansreader.CheckANSFinalState()) {
+      return JXL_FAILURE("ANS final state invalid");
+    }
+    JXL_RETURN_IF_ERROR(bitreader.JumpToByteBoundary());
+    *bytes_pos += bitreader.TotalBitsConsumed() / 8;
+    JXL_RETURN_IF_ERROR(bitreader.Close());
+    return true;
+  }
+};
+
+bool wp_compress(const Channel& img, jxl::PaddedBytes* bytes, int nb_modes) {
+  std::unique_ptr<State> state(new State(img.w, img.h, img.minval, img.maxval));
+  return state->wp_compress(img, bytes, nb_modes);
+}
+
+HWY_ATTR bool wp_decompress(const jxl::Span<const uint8_t> bytes, size_t* pos,
+                            Channel& img) {
+  std::unique_ptr<State> state(new State(img.w, img.h, img.minval, img.maxval));
+  return state->wp_decompress(bytes, pos, img);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_ENCODING_WEIGHTED_PREDICT_H_
diff --git a/jxl/modular/image/image.cpp b/jxl/modular/image/image.cpp
new file mode 100644
index 000000000..7a0e3df96
--- /dev/null
+++ b/jxl/modular/image/image.cpp
@@ -0,0 +1,76 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/modular/image/image.h"
+
+#include "jxl/base/status.h"
+#include "jxl/modular/transform/transform.h"
+#include "jxl/modular/util.h"
+
+namespace jxl {
+
+#ifdef HAS_ENCODER
+void Channel::actual_minmax(pixel_type *min, pixel_type *max) const {
+  pixel_type realmin = LARGEST_VAL;
+  pixel_type realmax = SMALLEST_VAL;
+  for (size_t y = 0; y < h; y++) {
+    const pixel_type *JXL_RESTRICT p = plane.Row(y);
+    for (size_t x = 0; x < w; x++) {
+      if (p[x] < realmin) realmin = p[x];
+      if (p[x] > realmax) realmax = p[x];
+    }
+  }
+
+  *min = realmin;
+  *max = realmax;
+}
+#endif
+
+void Image::undo_transforms(int keep, jxl::ThreadPool *pool) {
+  if (keep == -2) return;
+  while ((int)transform.size() > keep && transform.size() > 0) {
+    Transform t = transform.back();
+    JXL_DEBUG_V(4, "Undoing transform %s", t.name());
+    bool result = t.apply(*this, true, pool);
+    if (result == false) {
+      JXL_FAILURE("Error while undoing transform %s.", t.name());
+      error = true;
+      return;
+    }
+    JXL_DEBUG_V(8, "Undoing transform %s: done", t.name());
+    transform.pop_back();
+  }
+  if (!keep) {  // clamp the values to the valid range (lossy compression can
+                // produce values outside the range)
+    for (size_t i = 0; i < channel.size(); i++) {
+      for (size_t y = 0; y < channel[i].h; y++) {
+        pixel_type *JXL_RESTRICT p = channel[i].plane.Row(y);
+        for (size_t x = 0; x < channel[i].w; x++, p++) {
+          *p = CLAMP(*p, minval, maxval);
+        }
+      }
+    }
+  }
+}
+
+#ifdef HAS_ENCODER
+bool Image::do_transform(const Transform &tr) {
+  Transform t = tr;
+  bool did_it = t.apply(*this, false);
+  if (did_it) transform.push_back(t);
+  return did_it;
+}
+#endif
+
+}  // namespace jxl
\ No newline at end of file
diff --git a/jxl/modular/image/image.h b/jxl/modular/image/image.h
new file mode 100644
index 000000000..2345810c9
--- /dev/null
+++ b/jxl/modular/image/image.h
@@ -0,0 +1,237 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_IMAGE_IMAGE_H_
+#define JXL_MODULAR_IMAGE_IMAGE_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <utility>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/image.h"
+#include "jxl/modular/config.h"
+#include "jxl/modular/util.h"
+
+namespace jxl {
+
+#ifdef HDR
+
+typedef int32_t pixel_type;  // can use int16_t if it's only for 8-bit images.
+                             // Need some wiggle room for YCoCg / Squeeze etc
+
+// largest possible pixel value (2147483647)
+#define LARGEST_VAL 0x7FFFFFFF
+
+// smallest possible pixel value (-2147483647)
+#define SMALLEST_VAL 0x80000001
+
+#else
+
+typedef int16_t
+    pixel_type;  // enough for up to 14-bit with YCoCg/Squeeze (I think)
+#define LARGEST_VAL 0x7FFF
+#define SMALLEST_VAL 0x8001
+
+#endif
+
+class Channel {
+ public:
+  jxl::Plane<pixel_type> plane;
+  size_t w, h;
+  pixel_type minval, maxval;  // range
+  mutable pixel_type
+      zero;  // should be zero if zero is a valid value for this channel; the
+             // valid value closest to zero otherwise
+  int hshift, vshift;  // w ~= image.w >> hshift;  h ~= image.h >> vshift
+  int hcshift,
+      vcshift;  // cumulative, i.e. when decoding up to this point, we have data
+                // available with these shifts (for this component)
+  Channel(size_t iw, size_t ih, pixel_type iminval, pixel_type imaxval,
+          int hsh = 0, int vsh = 0, int hcsh = 0, int vcsh = 0)
+      : plane(iw, ih),
+        w(iw),
+        h(ih),
+        minval(iminval),
+        maxval(imaxval),
+        hshift(hsh),
+        vshift(vsh),
+        hcshift(hcsh),
+        vcshift(vcsh) {
+    setzero();
+  }
+  Channel()
+      : plane(0, 0),
+        w(0),
+        h(0),
+        minval(0),
+        maxval(0),
+        zero(0),
+        hshift(0),
+        vshift(0),
+        hcshift(0),
+        vcshift(0) {}
+
+  Channel(const Channel& other) = delete;
+  Channel& operator=(const Channel& other) = delete;
+
+  // Move assignment
+  Channel& operator=(Channel&& other) noexcept {
+    w = other.w;
+    h = other.h;
+    minval = other.minval;
+    maxval = other.maxval;
+    hshift = other.hshift;
+    vshift = other.vshift;
+    hcshift = other.hcshift;
+    vcshift = other.vcshift;
+    plane = std::move(other.plane);
+    return *this;
+  }
+
+  // Move constructor
+  Channel(Channel&& other) noexcept = default;
+
+  void setzero() const {
+    if (minval > 0)
+      zero = minval;
+    else if (maxval < 0)
+      zero = maxval;
+    else
+      zero = 0;
+  }
+
+  void resize() {
+    if (plane.xsize() == w && plane.ysize() == h) return;
+    jxl::Plane<pixel_type> resizedplane(w, h);
+    if (plane.xsize() || plane.ysize() || zero) {
+      // copy pixels over from old plane to new plane
+      size_t y = 0;
+      for (; y < plane.ysize() && y < h; y++) {
+        const pixel_type* JXL_RESTRICT p = plane.Row(y);
+        pixel_type* JXL_RESTRICT rp = resizedplane.Row(y);
+        size_t x = y;
+        for (; x < plane.xsize() && x < w; x++) rp[x] = p[x];
+        for (; x < w; x++) rp[x] = zero;
+      }
+      for (; y < h; y++) {
+        pixel_type* JXL_RESTRICT p = resizedplane.Row(y);
+        for (size_t x = 0; x < w; x++) p[x] = zero;
+      }
+    } else if (w && h) {
+      size_t ppr = resizedplane.bytes_per_row();
+      memset(resizedplane.bytes(), 0, ppr * h);
+    }
+    plane = std::move(resizedplane);
+  }
+  void resize(int nw, int nh) {
+    w = nw;
+    h = nh;
+    resize();
+  }
+  bool is_empty() const { return (plane.ysize() == 0); }
+
+  JXL_INLINE pixel_type* Row(const size_t y) { return plane.Row(y); }
+  JXL_INLINE const pixel_type* Row(const size_t y) const {
+    return plane.Row(y);
+  }
+  void actual_minmax(pixel_type* min, pixel_type* max) const;
+};
+
+class Transform;
+
+class Image {
+ public:
+  std::vector<Channel>
+      channel;  // image data, transforms can dramatically change the number of
+                // channels and their semantics
+  std::vector<Transform>
+      transform;  // keeps track of the transforms that have been applied (and
+                  // that have to be undone when rendering the image)
+
+  size_t w, h;  // actual dimensions of the image (channels may have different
+                // dimensions due to transforms like chroma subsampling and DCT)
+  int minval, maxval;  // actual (largest) range of the channels (actual ranges
+                       // might be different due to transforms; after undoing
+                       // transforms, might still be different due to lossy)
+  size_t nb_channels;  // actual number of distinct channels (after undoing all
+                       // transforms except Palette; can be different from
+                       // channel.size())
+  size_t real_nb_channels;  // real number of channels (after undoing all
+                            // transforms)
+  size_t nb_meta_channels;  // first few channels might contain things like
+                            // palettes or compaction data that are not yet real
+                            // image data
+  bool error;               // true if a fatal error occurred, false otherwise
+
+  Image(size_t iw, size_t ih, int maxval, int nb_chans)
+      : w(iw),
+        h(ih),
+        minval(0),
+        maxval(maxval),
+        nb_channels(nb_chans),
+        real_nb_channels(nb_chans),
+        nb_meta_channels(0),
+        error(false) {
+    for (int i = 0; i < nb_chans; i++)
+      channel.emplace_back(Channel(iw, ih, 0, maxval));
+  }
+
+  Image()
+      : w(0),
+        h(0),
+        minval(0),
+        maxval(255),
+        nb_channels(0),
+        real_nb_channels(0),
+        nb_meta_channels(0),
+        error(true) {}
+
+  Image(const Image& other) = delete;
+  Image& operator=(const Image& other) = delete;
+
+  Image& operator=(Image&& other) noexcept {
+    w = other.w;
+    h = other.h;
+    minval = other.minval;
+    maxval = other.maxval;
+    nb_channels = other.nb_channels;
+    real_nb_channels = other.real_nb_channels;
+    nb_meta_channels = other.nb_meta_channels;
+    error = other.error;
+    channel = std::move(other.channel);
+    transform = std::move(other.transform);
+    return *this;
+  }
+  Image(Image&& other) noexcept = default;
+
+  bool do_transform(const Transform& t);
+  // undo all except the first 'keep' transforms
+  void undo_transforms(int keep = 0, jxl::ThreadPool* pool = nullptr);
+  void recompute_minmax() {
+    for (auto& ch : channel) ch.actual_minmax(&ch.minval, &ch.maxval);
+  }
+};
+
+}  // namespace jxl
+
+#include "jxl/modular/transform/transform.h"
+
+#endif  // JXL_MODULAR_IMAGE_IMAGE_H_
diff --git a/jxl/modular/ma/bit.cpp b/jxl/modular/ma/bit.cpp
new file mode 100644
index 000000000..7454af068
--- /dev/null
+++ b/jxl/modular/ma/bit.cpp
@@ -0,0 +1,24 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#if defined(_MSC_VER) && !defined(__clang_major__)
+
+#include <intrin.h>
+int __builtin_clz(unsigned int value) {
+  unsigned long r;
+  _BitScanReverse(&r, value);
+  return (31 - r);
+}
+
+#endif
diff --git a/jxl/modular/ma/bit.h b/jxl/modular/ma/bit.h
new file mode 100644
index 000000000..bfcd08b04
--- /dev/null
+++ b/jxl/modular/ma/bit.h
@@ -0,0 +1,24 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_MA_BIT_H_
+#define JXL_MODULAR_MA_BIT_H_
+
+#if defined(_MSC_VER) && !defined(__clang_major__)
+
+int __builtin_clz(unsigned int value);
+
+#endif
+
+#endif  // JXL_MODULAR_MA_BIT_H_
diff --git a/jxl/modular/ma/chance.cpp b/jxl/modular/ma/chance.cpp
new file mode 100644
index 000000000..474d05b3c
--- /dev/null
+++ b/jxl/modular/ma/chance.cpp
@@ -0,0 +1,45 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/modular/ma/chance.h"
+
+namespace jxl {
+
+/** Computes an approximation of log(4096 / x) / log(2) * base */
+static uint32_t log4kf(int x, uint32_t base) {
+  int bits = 8 * sizeof(int) - __builtin_clz(x);
+  uint64_t y = ((uint64_t)x) << (32 - bits);
+  uint32_t res = base * (13 - bits);
+  uint32_t add = base;
+  while ((add > 1) && ((y & 0x7FFFFFFF) != 0)) {
+    y = (((uint64_t)y) * y + 0x40000000) >> 31;
+    add >>= 1;
+    if ((y >> 32) != 0) {
+      res -= add;
+      y >>= 1;
+    }
+  }
+  return res;
+}
+
+Log4kTable::Log4kTable() {
+  data[0] = 0;
+  for (int i = 1; i <= 4096; i++) {
+    data[i] = (log4kf(i, (65535UL << 16) / 12) + (1 << 15)) >> 16;
+  }
+}
+
+const Log4kTable log4k;
+
+}  // namespace jxl
\ No newline at end of file
diff --git a/jxl/modular/ma/chance.h b/jxl/modular/ma/chance.h
new file mode 100644
index 000000000..a953536e5
--- /dev/null
+++ b/jxl/modular/ma/chance.h
@@ -0,0 +1,59 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_MA_CHANCE_H_
+#define JXL_MODULAR_MA_CHANCE_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <cmath>
+
+namespace jxl {
+
+struct Log4kTable {
+  uint16_t data[4097];
+  Log4kTable();
+};
+
+extern const Log4kTable log4k;
+
+class SimpleBitChance {
+ protected:
+  uint16_t chance;  // stored as a 12-bit number
+
+ public:
+  SimpleBitChance() { chance = 0x800; }
+
+  uint16_t inline get_12bit() const { return chance; }
+  void set_12bit(uint16_t chance) { this->chance = chance; }
+  // update chances according to this function
+  void inline put(bool bit) {
+    if (bit)
+      chance += ((0x1000 - chance) >> 5);
+    else
+      chance -= ((chance) >> 5);
+  }
+  // can also add 30 before shifting, to make sure that the really extreme
+  // chances can be reached. Doesn't seem to help for compression though, and
+  // it's quite a bit slower
+
+  void estim(bool bit, uint64_t &total) const {
+    total += log4k.data[bit ? chance : 4096 - chance];
+  }
+};
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_MA_CHANCE_H_
diff --git a/jxl/modular/ma/compound.h b/jxl/modular/ma/compound.h
new file mode 100644
index 000000000..4d06f2196
--- /dev/null
+++ b/jxl/modular/ma/compound.h
@@ -0,0 +1,315 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_MA_COMPOUND_H_
+#define JXL_MODULAR_MA_COMPOUND_H_
+
+#include <stdint.h>
+#include <stdio.h>
+
+#include <cmath>
+#include <vector>
+
+#include "jxl/base/status.h"
+#include "jxl/modular/ma/symbol.h"
+
+namespace jxl {
+
+typedef int32_t PropertyVal;
+typedef std::vector<std::pair<PropertyVal, PropertyVal> > Ranges;
+typedef std::vector<PropertyVal> Properties;
+
+static const uint16_t CHANCE_INITZ[11] = {4,    128,  512,  1024, 1536, 2048,
+                                          2560, 3072, 3584, 3968, 4088};
+static const uint16_t CHANCE_INITS[7] = {512,  1024, 1536, 2048,
+                                         2560, 3072, 3584};
+// only signal sign initialization if zero is unlikely
+#define NO_SIGN_SIGNAL 3
+
+// inner nodes
+class PropertyDecisionNode {
+ public:
+  PropertyVal splitval;
+  int16_t property;  // <0 : leaf node, childID points to leaf node
+  // 0..nb_properties-1 : childID refers to left branch  (in inner_node)
+  //                      childID+1 refers to right branch
+  uint16_t childID;
+
+  explicit PropertyDecisionNode(int p = -1, int s = 0, int c = 0)
+      : splitval(s), property(p), childID(c) {}
+};
+
+class Tree : public std::vector<PropertyDecisionNode> {
+ public:
+  Tree() : std::vector<PropertyDecisionNode>(1, PropertyDecisionNode()) {}
+};
+
+typedef std::vector<Tree> Trees;
+
+// leaf nodes when tree is known
+template <typename BitChance, int bits>
+class FinalCompoundSymbolChances {
+ public:
+  SymbolChance<BitChance, bits> realChances;
+
+  explicit FinalCompoundSymbolChances(uint16_t zero_chance)
+      : realChances(zero_chance) {}
+
+  const SymbolChance<BitChance, bits> &chances() const { return realChances; }
+};
+
+template <typename BitChance, typename RAC, int bits>
+class FinalCompoundSymbolBitCoder {
+ private:
+  RAC &rac;
+  FinalCompoundSymbolChances<BitChance, bits> &chances;
+
+  void inline updateChances(const SymbolChanceBitType type, const int i,
+                            bool bit) {
+    BitChance &real = chances.realChances.bit(type, i);
+    real.put(bit);
+  }
+
+ public:
+  FinalCompoundSymbolBitCoder(
+      RAC &racIn, FinalCompoundSymbolChances<BitChance, bits> &chancesIn)
+      : rac(racIn), chances(chancesIn) {}
+
+  bool inline read(const SymbolChanceBitType type, const int i = 0) {
+    BitChance &ch = chances.realChances.bit(type, i);
+    bool bit = rac.read_12bit_chance(ch.get_12bit());
+    updateChances(type, i, bit);
+    return bit;
+  }
+
+#ifdef HAS_ENCODER
+  void inline write(const bool bit, const SymbolChanceBitType type,
+                    const int i = 0);
+  void estimate(const bool bit, const SymbolChanceBitType type, const int i,
+                uint64_t &total);
+#endif
+};
+
+template <typename BitChance, typename RAC, int bits>
+class FinalCompoundSymbolCoder {
+ private:
+  RAC &rac;
+
+ public:
+  explicit FinalCompoundSymbolCoder(RAC &racIn) : rac(racIn) {}
+
+  int read_int(FinalCompoundSymbolChances<BitChance, bits> &chancesIn, int min,
+               int max) {
+    FinalCompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn);
+    int val = reader<bits>(bitCoder, min, max);
+    return val;
+  }
+  int read_int(FinalCompoundSymbolChances<BitChance, bits> &chancesIn,
+               int nbits) {
+    FinalCompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn);
+    int val = reader(bitCoder, nbits);
+    return val;
+  }
+
+#ifdef HAS_ENCODER
+  void write_int(FinalCompoundSymbolChances<BitChance, bits> &chancesIn,
+                 int min, int max, int val);
+  int estimate_int(FinalCompoundSymbolChances<BitChance, bits> &chancesIn,
+                   int min, int max, int val);
+  void write_int(FinalCompoundSymbolChances<BitChance, bits> &chancesIn,
+                 int nbits, int val);
+#endif
+};
+
+template <typename BitChance, typename RAC, int bits>
+class FinalPropertySymbolCoder {
+ private:
+  FinalCompoundSymbolCoder<BitChance, RAC, bits> coder;
+  const unsigned int nb_properties;
+  std::vector<FinalCompoundSymbolChances<BitChance, bits> > leaf_node;
+  const Tree &inner_node;
+
+  FinalCompoundSymbolChances<BitChance, bits> inline &find_leaf(
+      const Properties &properties) ATTRIBUTE_HOT {
+    Tree::size_type pos = 0;
+#if JXL_COMPILER_CLANG
+#pragma clang loop unroll_count(8)
+#endif
+    while (true) {
+      const PropertyDecisionNode &node = inner_node[pos];
+      if (node.property < 0) return leaf_node[node.childID];
+      if (properties[node.property] > node.splitval)
+        pos = node.childID;
+      else
+        pos = node.childID + 1;
+    }
+  }
+
+ public:
+  FinalPropertySymbolCoder(RAC &racIn, Ranges &rangeIn, Tree &treeIn,
+                           uint16_t zero_chance = ZERO_CHANCE,
+                           int ignored_split_threshold = 0)
+      : coder(racIn), nb_properties(rangeIn.size()), inner_node(treeIn) {
+    for (size_t i = 0, leafID = 0; i < inner_node.size(); i++) {
+      if (inner_node[i].property < 0) {
+        leafID++;
+        JXL_DASSERT(inner_node[i].property > -16);
+        int zerochance = CHANCE_INITZ[-1 - inner_node[i].property];
+        leaf_node.push_back(
+            FinalCompoundSymbolChances<BitChance, bits>(zerochance));
+        int signchance = CHANCE_INITS[inner_node[i].splitval];
+        leaf_node.back().realChances.bitSign().set_12bit(signchance);
+      }
+    }
+  }
+  int context_id(const Properties &properties) const ATTRIBUTE_HOT {
+    Tree::size_type pos = 0;
+#if JXL_COMPILER_CLANG
+#pragma clang loop unroll_count(4)
+#endif
+    while (true) {
+      const PropertyDecisionNode &node = inner_node[pos];
+      if (node.property < 0) return node.childID;
+      if (properties[node.property] > node.splitval)
+        pos = node.childID;
+      else
+        pos = node.childID + 1;
+    }
+  }
+  int nb_contexts() const ATTRIBUTE_HOT { return leaf_node.size(); }
+  int read_int(const Properties &properties, int min, int max) ATTRIBUTE_HOT {
+    JXL_DASSERT(properties.size() == nb_properties);
+    FinalCompoundSymbolChances<BitChance, bits> &chances =
+        find_leaf(properties);
+    return coder.read_int(chances, min, max);
+  }
+
+  int read_int(const Properties &properties, int nbits) {
+    JXL_DASSERT(properties.size() == nb_properties);
+    FinalCompoundSymbolChances<BitChance, bits> &chances =
+        find_leaf(properties);
+    return coder.read_int(chances, nbits);
+  }
+
+#ifdef HAS_ENCODER
+  void write_int(const Properties &properties, int min, int max, int val);
+  int estimate_int(const Properties &properties, int min, int max, int val);
+  void stats_write_int(const Properties &properties, int min, int max,
+                       int val) {}
+  void write_int(const Properties &properties, int nbits, int val);
+  static void simplify(int min_size = CONTEXT_TREE_MIN_SUBTREE_SIZE) {}
+  static uint64_t compute_total_size() { return 0; }
+#endif
+};
+
+template <typename BitChance, typename RAC>
+class MetaPropertySymbolCoder {
+ public:
+  typedef SimpleSymbolCoder<BitChance, RAC, MAX_BIT_DEPTH> Coder;
+
+ private:
+  std::vector<Coder> coder;
+  const Ranges range;
+  unsigned int nb_properties;
+
+ public:
+  MetaPropertySymbolCoder(RAC &racIn, const Ranges &rangesIn)
+      : coder(4, Coder(racIn)),
+        range(rangesIn),
+        nb_properties(rangesIn.size()) {
+    for (unsigned int i = 0; i < nb_properties; i++) {
+      JXL_DASSERT(range[i].first <= range[i].second);
+    }
+  }
+
+#ifdef HAS_ENCODER
+  void write_subtree(int pos, Ranges &subrange, const Tree &tree,
+                     bool signal_chances);
+  void write_tree(const Tree &tree, bool signal_chances);
+#endif
+
+  bool read_subtree(int pos, Ranges &subrange, Tree &tree, int &maxdepth,
+                    int depth, bool signal_chances) {
+    PropertyDecisionNode &n = tree[pos];
+    int p = coder[0].read_int(0, nb_properties) - 1;
+    if (signal_chances && p < 0) {
+      p = -(1 + coder[1].read_int(-5, 5) + 5);
+      if (p > -1 - NO_SIGN_SIGNAL)
+        n.splitval = coder[2].read_int(-3, 3) + 3;
+      else
+        n.splitval = 3;
+    }
+    n.property = p;
+    depth++;
+    if (depth > maxdepth) maxdepth = depth;
+    if (p >= 0) {
+      int oldmin = subrange[p].first;
+      int oldmax = subrange[p].second;
+      if (oldmin >= oldmax) {
+        return JXL_FAILURE("Invalid tree. Aborting tree decoding.");
+      }
+      JXL_DASSERT(oldmin < oldmax);
+      int splitval = n.splitval = coder[3].read_int2(oldmin, oldmax - 1);
+      int childID = n.childID = tree.size();
+      // JXL_FAILURE("Pos %i: prop %i splitval %i in [%i..%i]", pos,
+      //             n.property, splitval, oldmin, oldmax-1);
+      tree.push_back(PropertyDecisionNode());
+      tree.push_back(PropertyDecisionNode());
+      // > splitval
+      subrange[p].first = splitval + 1;
+      if (!read_subtree(childID, subrange, tree, maxdepth, depth,
+                        signal_chances))
+        return false;
+
+      // <= splitval
+      subrange[p].first = oldmin;
+      subrange[p].second = splitval;
+      if (!read_subtree(childID + 1, subrange, tree, maxdepth, depth,
+                        signal_chances))
+        return false;
+
+      subrange[p].second = oldmax;
+    }
+    return true;
+  }
+  bool read_tree(Tree &tree, bool signal_chances) {
+    Ranges rootrange(range);
+    tree.clear();
+    tree.push_back(PropertyDecisionNode());
+    int depth = 0;
+    if (read_subtree(0, rootrange, tree, depth, 0, signal_chances)) {
+      int neededdepth = ilog2(tree.size()) + 1;
+      JXL_DEBUG_V(8,
+                  "Read MA tree with %u nodes and depth %i (with better "
+                  "balance, depth %i might have been enough).",
+                  (unsigned int)tree.size(), depth, neededdepth);
+      for (size_t i = 0, leafID = 0; i < tree.size(); i++) {
+        if (tree[i].property < 0) {
+          tree[i].childID = leafID;
+          leafID++;
+        }
+      }
+      return true;
+    } else
+      return false;
+  }
+};
+
+}  // namespace jxl
+
+#ifdef HAS_ENCODER
+#include "jxl/modular/ma/compound_enc.h"
+#endif
+
+#endif  // JXL_MODULAR_MA_COMPOUND_H_
diff --git a/jxl/modular/ma/compound_enc.h b/jxl/modular/ma/compound_enc.h
new file mode 100644
index 000000000..379e2c78e
--- /dev/null
+++ b/jxl/modular/ma/compound_enc.h
@@ -0,0 +1,630 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_MA_COMPOUND_ENC_H_
+#define JXL_MODULAR_MA_COMPOUND_ENC_H_
+
+#include <inttypes.h>
+
+#include <cmath>
+
+#include "jxl/modular/ma/compound.h"
+
+namespace jxl {
+
+// leaf nodes during tree construction phase
+template <typename BitChance, int bits>
+class CompoundSymbolChances final
+    : public FinalCompoundSymbolChances<BitChance, bits> {
+ public:
+  std::vector<
+      std::pair<SymbolChance<BitChance, bits>, SymbolChance<BitChance, bits> > >
+      virtChances;
+  uint64_t realSize;
+  std::vector<uint64_t> virtSize;
+  std::vector<int64_t> virtPropSum;
+  int32_t count;
+  int16_t best_property;
+  // used to collect stats for chance initialization
+  uint64_t real_count;
+  uint64_t zero_count;  // zero chance = zero_count / real_count
+  uint64_t sign_count;  // pos chance = pos_count / sign_count
+  uint64_t pos_count;
+
+  void resetCounters() {
+    best_property = -1;
+    realSize = 0;
+    count = 0;
+    virtPropSum.assign(virtPropSum.size(), 0);
+    virtSize.assign(virtSize.size(), 0);
+  }
+
+  CompoundSymbolChances(int nProp, uint16_t zero_chance)
+      : FinalCompoundSymbolChances<BitChance, bits>(zero_chance),
+        virtChances(nProp,
+                    std::make_pair(SymbolChance<BitChance, bits>(zero_chance),
+                                   SymbolChance<BitChance, bits>(zero_chance))),
+        realSize(0),
+        virtSize(nProp),
+        virtPropSum(nProp),
+        count(0),
+        best_property(-1),
+        real_count(0),
+        zero_count(0),
+        sign_count(0),
+        pos_count(0) {}
+};
+
+template <typename BitChance, typename RAC, int bits>
+void inline FinalCompoundSymbolBitCoder<BitChance, RAC, bits>::write(
+    const bool bit, const SymbolChanceBitType type, const int i) {
+  BitChance &ch = chances.realChances.bit(type, i);
+  rac.write_12bit_chance(ch.get_12bit(), bit);
+  updateChances(type, i, bit);
+}
+
+// This function is currently not used, but it could be used to estimate the
+// current cost of writing a number without actually writing it. (that could be
+// useful for a lossy encoder)
+template <typename BitChance, typename RAC, int bits>
+void inline FinalCompoundSymbolBitCoder<BitChance, RAC, bits>::estimate(
+    const bool bit, const SymbolChanceBitType type, const int i,
+    uint64_t &total) {
+  BitChance &ch = chances.realChances.bit(type, i);
+  ch.estim(bit, total);
+}
+
+template <typename BitChance, typename RAC, int bits>
+class CompoundSymbolBitCoder {
+ private:
+  RAC &rac;
+  CompoundSymbolChances<BitChance, bits> &chances;
+  std::vector<bool> &select;
+
+  void inline updateChances(SymbolChanceBitType type, int i, bool bit) {
+    BitChance &real = chances.realChances.bit(type, i);
+    real.estim(bit, chances.realSize);
+    real.put(bit);
+
+    int16_t best_property = -1;
+    uint64_t best_size = chances.realSize;
+    for (unsigned int j = 0; j < chances.virtChances.size(); j++) {
+      BitChance &virt = (select)[j]
+                            ? chances.virtChances[j].first.bit(type, i)
+                            : chances.virtChances[j].second.bit(type, i);
+      virt.estim(bit, chances.virtSize[j]);
+      virt.put(bit);
+      if (chances.virtSize[j] < best_size) {
+        best_size = chances.virtSize[j];
+        best_property = j;
+      }
+    }
+    chances.best_property = best_property;
+  }
+  BitChance inline &bestChance(SymbolChanceBitType type, int i = 0) {
+    signed short int p = chances.best_property;
+    return (p < 0 ? chances.realChances.bit(type, i)
+                  : ((select)[p] ? chances.virtChances[p].first.bit(type, i)
+                                 : chances.virtChances[p].second.bit(type, i)));
+  }
+
+ public:
+  CompoundSymbolBitCoder(RAC &racIn,
+                         CompoundSymbolChances<BitChance, bits> &chancesIn,
+                         std::vector<bool> &selectIn)
+      : rac(racIn), chances(chancesIn), select(selectIn) {}
+
+  bool read(SymbolChanceBitType type, int i = 0) {
+    BitChance &ch = bestChance(type, i);
+    bool bit = rac.read_12bit_chance(ch.get_12bit());
+    updateChances(type, i, bit);
+    return bit;
+  }
+
+  void write(bool bit, SymbolChanceBitType type, int i = 0) {
+    BitChance &ch = bestChance(type, i);
+    rac.write_12bit_chance(ch.get_12bit(), bit);
+    updateChances(type, i, bit);
+  }
+
+  void estimate(const bool bit, const SymbolChanceBitType type, const int i,
+                uint64_t &total) {
+    BitChance &ch = bestChance(type, i);
+    ch.estim(bit, total);
+  }
+};
+
+template <typename BitChance, typename RAC, int bits>
+void FinalCompoundSymbolCoder<BitChance, RAC, bits>::write_int(
+    FinalCompoundSymbolChances<BitChance, bits> &chancesIn, int min, int max,
+    int val) {
+  FinalCompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn);
+  writer<bits>(bitCoder, min, max, val);
+}
+
+template <typename BitChance, typename RAC, int bits>
+int FinalCompoundSymbolCoder<BitChance, RAC, bits>::estimate_int(
+    FinalCompoundSymbolChances<BitChance, bits> &chancesIn, int min, int max,
+    int val) {
+  FinalCompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn);
+  return estimate_writer<bits>(bitCoder, min, max, val);
+}
+
+template <typename BitChance, typename RAC, int bits>
+void FinalCompoundSymbolCoder<BitChance, RAC, bits>::write_int(
+    FinalCompoundSymbolChances<BitChance, bits> &chancesIn, int nbits,
+    int val) {
+  FinalCompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn);
+  writer(bitCoder, nbits, val);
+}
+
+template <typename BitChance, typename RAC, int bits>
+class CompoundSymbolCoder {
+ private:
+  RAC &rac;
+
+ public:
+  explicit CompoundSymbolCoder(RAC &racIn) : rac(racIn) {}
+
+  int read_int(CompoundSymbolChances<BitChance, bits> &chancesIn,
+               std::vector<bool> &selectIn, int min, int max) {
+    if (min == max) {
+      return min;
+    }
+    CompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn,
+                                                          selectIn);
+    return reader<bits>(bitCoder, min, max);
+  }
+
+  void write_int(CompoundSymbolChances<BitChance, bits> &chancesIn,
+                 std::vector<bool> &selectIn, int min, int max, int val) {
+    if (min == max) {
+      JXL_DASSERT(val == min);
+      return;
+    }
+    CompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn,
+                                                          selectIn);
+    writer<bits>(bitCoder, min, max, val);
+  }
+
+  int estimate_int(CompoundSymbolChances<BitChance, bits> &chancesIn,
+                   std::vector<bool> &selectIn, int min, int max, int val) {
+    if (min == max) {
+      JXL_DASSERT(val == min);
+      return 0;
+    }
+    CompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn,
+                                                          selectIn);
+    return estimate_writer<bits>(bitCoder, min, max, val);
+  }
+
+  int read_int(CompoundSymbolChances<BitChance, bits> &chancesIn,
+               std::vector<bool> &selectIn, int nbits) {
+    CompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn,
+                                                          selectIn);
+    return reader(bitCoder, nbits);
+  }
+
+  void write_int(CompoundSymbolChances<BitChance, bits> &chancesIn,
+                 std::vector<bool> &selectIn, int nbits, int val) {
+    CompoundSymbolBitCoder<BitChance, RAC, bits> bitCoder(rac, chancesIn,
+                                                          selectIn);
+    writer(bitCoder, nbits, val);
+  }
+};
+
+template <typename BitChance, typename RAC, int bits>
+void FinalPropertySymbolCoder<BitChance, RAC, bits>::write_int(
+    const Properties &properties, int min, int max, int val) {
+  if (min == max) {
+    JXL_DASSERT(val == min);
+    return;
+  }
+  JXL_DASSERT(properties.size() == nb_properties);
+  FinalCompoundSymbolChances<BitChance, bits> &chances = find_leaf(properties);
+  coder.write_int(chances, min, max, val);
+}
+
+template <typename BitChance, typename RAC, int bits>
+int FinalPropertySymbolCoder<BitChance, RAC, bits>::estimate_int(
+    const Properties &properties, int min, int max, int val) {
+  if (min == max) {
+    JXL_DASSERT(val == min);
+    return 0;
+  }
+  JXL_DASSERT(properties.size() == nb_properties);
+  FinalCompoundSymbolChances<BitChance, bits> &chances = find_leaf(properties);
+  return coder.estimate_int(chances, min, max, val);
+}
+
+template <typename BitChance, typename RAC, int bits>
+void FinalPropertySymbolCoder<BitChance, RAC, bits>::write_int(
+    const Properties &properties, int nbits, int val) {
+  JXL_DASSERT(properties.size() == nb_properties);
+  FinalCompoundSymbolChances<BitChance, bits> &chances = find_leaf(properties);
+  coder.write_int(chances, nbits, val);
+}
+
+template <typename BitChance, typename RAC, int bits>
+class PropertySymbolCoder {
+ public:
+  typedef CompoundSymbolCoder<BitChance, RAC, bits> Coder;
+
+ private:
+  RAC &rac;
+  Coder coder;
+  const Ranges range;
+  unsigned int nb_properties;
+  std::vector<CompoundSymbolChances<BitChance, bits> > leaf_node;
+  Tree &inner_node;
+  std::vector<bool> selection;
+  int split_threshold;
+
+  inline PropertyVal div_down(int64_t sum, int32_t count) const {
+    JXL_DASSERT(count > 0);
+    if (sum >= 0)
+      return sum / count;
+    else
+      return -((-sum + count - 1) / count);
+  }
+  inline PropertyVal compute_splitval(
+      const CompoundSymbolChances<BitChance, bits> &ch, int16_t p,
+      const Ranges &crange) const {
+    // make first branch on > 0 if range has negative and positive numbers
+    if (crange[p].first < 0 && crange[p].second > 0) {
+      return 0;
+    }
+    PropertyVal splitval = div_down(ch.virtPropSum[p], ch.count);
+    if (splitval >= crange[p].second)
+      splitval = crange[p].second -
+                 1;  // == does happen because of rounding and running average
+    return splitval;
+  }
+
+  CompoundSymbolChances<BitChance, bits> inline &find_leaf_readonly(
+      const Properties &properties) {
+    uint32_t pos = 0;
+    Ranges current_ranges = range;
+    while (inner_node[pos].property >= 0) {
+      if (properties[inner_node[pos].property] > inner_node[pos].splitval) {
+        current_ranges[inner_node[pos].property].first =
+            inner_node[pos].splitval + 1;
+        pos = inner_node[pos].childID;
+      } else {
+        current_ranges[inner_node[pos].property].second =
+            inner_node[pos].splitval;
+        pos = inner_node[pos].childID + 1;
+      }
+    }
+    CompoundSymbolChances<BitChance, bits> &result =
+        leaf_node[inner_node[pos].childID];
+    set_selection(properties, result, current_ranges);
+    return result;
+  }
+
+  CompoundSymbolChances<BitChance, bits> inline &find_leaf(
+      const Properties &properties) {
+    uint32_t pos = 0;
+    Ranges current_ranges = range;
+    while (inner_node[pos].property >= 0) {
+      if (properties[inner_node[pos].property] > inner_node[pos].splitval) {
+        current_ranges[inner_node[pos].property].first =
+            inner_node[pos].splitval + 1;
+        pos = inner_node[pos].childID;
+      } else {
+        current_ranges[inner_node[pos].property].second =
+            inner_node[pos].splitval;
+        pos = inner_node[pos].childID + 1;
+      }
+    }
+    CompoundSymbolChances<BitChance, bits> &result =
+        leaf_node[inner_node[pos].childID];
+    set_selection_and_update_property_sums(properties, result, current_ranges);
+
+    // split leaf node if some virtual context is performing (significantly)
+    // better
+    if (result.best_property != -1 &&
+        result.realSize >
+            result.virtSize[result.best_property] + split_threshold &&
+        leaf_node.size() < 0xFFFF && inner_node.size() < 0xFFFF &&
+        current_ranges[result.best_property].first <
+            current_ranges[result.best_property].second) {
+      int16_t p = result.best_property;
+      PropertyVal splitval = compute_splitval(result, p, current_ranges);
+
+      uint32_t new_inner = inner_node.size();
+      inner_node.push_back(inner_node[pos]);
+      inner_node.push_back(inner_node[pos]);
+      inner_node[pos].splitval = splitval;
+      inner_node[pos].property = p;
+      uint32_t new_leaf = leaf_node.size();
+      result.resetCounters();
+      leaf_node.push_back(CompoundSymbolChances<BitChance, bits>(result));
+      uint32_t old_leaf = inner_node[pos].childID;
+      inner_node[pos].childID = new_inner;
+      inner_node[new_inner].childID = old_leaf;
+      inner_node[new_inner + 1].childID = new_leaf;
+      if (properties[p] > inner_node[pos].splitval) {
+        return leaf_node[old_leaf];
+      } else {
+        return leaf_node[new_leaf];
+      }
+    }
+    return result;
+  }
+
+  void inline set_selection_and_update_property_sums(
+      const Properties &properties,
+      CompoundSymbolChances<BitChance, bits> &chances, const Ranges &crange) {
+    chances.count++;
+    for (unsigned int i = 0; i < nb_properties; i++) {
+      JXL_DASSERT(properties[i] >= range[i].first);
+      JXL_DASSERT(properties[i] <= range[i].second);
+      chances.virtPropSum[i] += properties[i];
+      PropertyVal splitval = compute_splitval(chances, i, crange);
+      selection[i] = (properties[i] > splitval);
+    }
+  }
+  void inline set_selection(
+      const Properties &properties,
+      const CompoundSymbolChances<BitChance, bits> &chances,
+      const Ranges &crange) {
+    if (chances.count == 0) return;
+    for (unsigned int i = 0; i < nb_properties; i++) {
+      JXL_DASSERT(properties[i] >= range[i].first);
+      JXL_DASSERT(properties[i] <= range[i].second);
+      PropertyVal splitval = compute_splitval(chances, i, crange);
+      selection[i] = (properties[i] > splitval);
+    }
+  }
+
+ public:
+  PropertySymbolCoder(RAC &racIn, Ranges &rangeIn, Tree &treeIn,
+                      int zero_chance = ZERO_CHANCE,
+                      int st = CONTEXT_TREE_SPLIT_THRESHOLD)
+      : rac(racIn),
+        coder(racIn),
+        range(rangeIn),
+        nb_properties(range.size()),
+        leaf_node(1, CompoundSymbolChances<BitChance, bits>(nb_properties,
+                                                            zero_chance)),
+        inner_node(treeIn),
+        selection(nb_properties, false),
+        split_threshold(st) {}
+
+  int read_int(Properties &properties, int min, int max) {
+    CompoundSymbolChances<BitChance, bits> &chances2 = find_leaf(properties);
+    return coder.read_int(chances2, selection, min, max);
+  }
+
+  void write_int(Properties &properties, int min, int max, int val) {
+    CompoundSymbolChances<BitChance, bits> &chances2 = find_leaf(properties);
+    coder.write_int(chances2, selection, min, max, val);
+  }
+
+  int estimate_int(Properties &properties, int min, int max, int val) {
+    CompoundSymbolChances<BitChance, bits> &chances =
+        find_leaf_readonly(properties);
+    return coder.estimate_int(chances, selection, min, max, val);
+  }
+  void stats_write_int(const Properties &properties, int min, int max,
+                       int val) {
+    CompoundSymbolChances<BitChance, bits> &chances =
+        find_leaf_readonly(properties);
+    if (chances.real_count < 500) {
+      chances.real_count++;
+      if (!val) chances.zero_count++;
+    }
+    if (chances.sign_count < 500 && val && max > 0 && min < 0) {
+      chances.sign_count++;
+      if (val > 0) chances.pos_count++;
+    }
+  }
+
+  int read_int(Properties &properties, int nbits) {
+    CompoundSymbolChances<BitChance, bits> &chances2 = find_leaf(properties);
+    return coder.read_int(chances2, selection, nbits);
+  }
+
+  void write_int(Properties &properties, int nbits, int val) {
+    CompoundSymbolChances<BitChance, bits> &chances2 = find_leaf(properties);
+    coder.write_int(chances2, selection, nbits, val);
+  }
+
+  void kill_children(int pos) {
+    PropertyDecisionNode &n = inner_node[pos];
+    if (n.property < 0)
+      n.property = 0;
+    else
+      kill_children(n.childID);
+    PropertyDecisionNode &n1 = inner_node[pos + 1];
+    if (n1.property < 0)
+      n1.property = 0;
+    else
+      kill_children(n1.childID);
+  }
+
+  int find_nearest_chance_initz(uint16_t chance) {
+    for (int i = 0; i < 11; i++)
+      if (CHANCE_INITZ[i] >= chance) {
+        if (i > 5)
+          return i - 1;
+        else
+          return i;  // round towards fifty-fifty
+      }
+    return 10;
+  }
+  int find_nearest_chance_inits(uint16_t chance) {
+    for (int i = 0; i < 7; i++)
+      if (CHANCE_INITS[i] >= chance) {
+        if (i > 3)
+          return i - 1;
+        else
+          return i;  // round towards fifty-fifty
+      }
+    return 6;
+  }
+
+  // destructive simplification procedure, prunes subtrees with too low counts
+  // also sets initialization chances, so not optional
+  long long int simplify_subtree(int pos, int min_size, int indent) {
+    PropertyDecisionNode &n = inner_node[pos];
+    if (n.property < 0) {
+      int zchance = 2048;
+      if (leaf_node[n.childID].real_count)
+        zchance = leaf_node[n.childID].zero_count * 4096 /
+                  leaf_node[n.childID].real_count;
+      int qchance = find_nearest_chance_initz(zchance);
+      JXL_DASSERT(qchance >= 0);
+      JXL_DASSERT(qchance <= 11);
+      n.property = -1 - qchance;
+      int pchance = 2048;
+      if (n.property > -1 - NO_SIGN_SIGNAL) {
+        if (leaf_node[n.childID].sign_count)
+          pchance = leaf_node[n.childID].pos_count * 4096 /
+                    leaf_node[n.childID].sign_count;
+        n.splitval = find_nearest_chance_inits(pchance);
+      } else
+        n.splitval = 3;
+      for (int i = 0; i < indent; i++) JXL_DEBUG_V(10, "  ");
+      JXL_DEBUG_V(
+          10,
+          "* leaf: count=%lli, size=%.2f bits, bits per int: %f, "
+          "zero chance: %i/%i = %.2f%% ~= %.2f%% ~= %.2f%% = %" PRIu64
+          "/%" PRIu64
+          ", "
+          "sign chance: %i/%i = %.2f%% ~= %.2f%% ~= %.2f%% = %" PRIu64
+          "/%" PRIu64 "",
+          (long long int)leaf_node[n.childID].count,
+          leaf_node[n.childID].realSize / 5461.0,
+          (leaf_node[n.childID].count > 0
+               ? leaf_node[n.childID].realSize * 1.0 /
+                     leaf_node[n.childID].count / 5461
+               : -1),
+          CHANCE_INITZ[qchance], 4096, CHANCE_INITZ[qchance] * 100.0 / 4096,
+          leaf_node[n.childID].realChances.bit_zero.get_12bit() * 100.0 / 4096,
+          zchance * 100.0 / 4096, leaf_node[n.childID].zero_count,
+          leaf_node[n.childID].real_count, CHANCE_INITS[n.splitval], 4096,
+          CHANCE_INITS[n.splitval] * 100.0 / 4096,
+          leaf_node[n.childID].realChances.bit_sign.get_12bit() * 100.0 / 4096,
+          pchance * 100.0 / 4096, leaf_node[n.childID].pos_count,
+          leaf_node[n.childID].sign_count);
+
+      for (int i = 0; i < indent; i++) JXL_DEBUG_V(11, "  ");
+      JXL_DEBUG_V(11, "  chances: ");
+      JXL_DEBUG_V(11, " zero=%u",
+                  leaf_node[n.childID].realChances.bit_zero.get_12bit());
+      JXL_DEBUG_V(11, " sign=%u",
+                  leaf_node[n.childID].realChances.bit_sign.get_12bit());
+      for (int i = 0; i < 5; i++)
+        JXL_DEBUG_V(10, " exp%i=%u", i,
+                    leaf_node[n.childID].realChances.bit_exp[i].get_12bit());
+      JXL_DEBUG_V(11, " mant=");
+      for (int i = 0; i < 8; i++)
+        JXL_DEBUG_V(11, "%u ",
+                    leaf_node[n.childID].realChances.bit_mant[i].get_12bit());
+
+      return leaf_node[n.childID].count;
+
+    } else {
+      for (int i = 0; i < indent; i++) JXL_DEBUG_V(10, "  ");
+      JXL_DEBUG_V(10, "* test: property %i, value > %i ?", n.property,
+                  n.splitval);
+      long long int subtree_size = 0;
+      subtree_size += simplify_subtree(n.childID, min_size, indent + 1);
+      subtree_size += simplify_subtree(n.childID + 1, min_size, indent + 1);
+      if (subtree_size < min_size) {
+        for (int i = 0; i < indent; i++) JXL_DEBUG_V(11, "  ");
+        JXL_DEBUG_V(11, "[PRUNING THE ABOVE SUBTREE]");
+        n.property = (inner_node[n.childID].property +
+                      inner_node[n.childID + 1].property) /
+                     2;
+        n.splitval = (inner_node[n.childID].splitval +
+                      inner_node[n.childID + 1].splitval) /
+                     2;
+        JXL_DASSERT(n.property < 0);
+        JXL_DASSERT(n.property >= -16);
+        kill_children(n.childID);
+      }
+      return subtree_size;
+    }
+  }
+  void simplify(int min_size = CONTEXT_TREE_MIN_SUBTREE_SIZE) {
+    JXL_DEBUG_V(10, "MA TREE BEFORE SIMPLIFICATION:");
+    simplify_subtree(0, min_size, 0);
+
+    for (size_t i = 0, leafID = 0; i < inner_node.size(); i++) {
+      if (inner_node[i].property < 0) {
+        inner_node[i].childID = leafID;
+        leafID++;
+      }
+    }
+  }
+  uint64_t compute_total_size_subtree(int pos) {
+    PropertyDecisionNode &n = inner_node[pos];
+    uint64_t total = 0;
+    if (n.property < 0) {
+      total += leaf_node[n.childID].realSize / 5461;
+    } else {
+      total += compute_total_size_subtree(n.childID);
+      total += compute_total_size_subtree(n.childID + 1);
+    }
+    return total;
+  }
+  uint64_t compute_total_size() { return compute_total_size_subtree(0); }
+  // never used
+  int nb_contexts() const { return 0; }
+  int context_id(const Properties &p) const { return 0; }
+};
+
+template <typename BitChance, typename RAC>
+void MetaPropertySymbolCoder<BitChance, RAC>::write_subtree(
+    int pos, Ranges &subrange, const Tree &tree, bool signal_chances) {
+  const PropertyDecisionNode &n = tree[pos];
+  int p = n.property;
+  if (p < 0) p = -1;
+  coder[0].write_int(0, nb_properties, p + 1);
+  if (signal_chances && p < 0) {
+    coder[1].write_int(-5, 5, -n.property - 1 - 5);
+    if (n.property > -1 - NO_SIGN_SIGNAL)
+      coder[2].write_int(-3, 3, n.splitval - 3);
+  }
+  if (p >= 0) {
+    int oldmin = subrange[p].first;
+    int oldmax = subrange[p].second;
+    JXL_DASSERT(oldmin < oldmax);
+    coder[3].write_int2(oldmin, oldmax - 1, n.splitval);
+
+    // > splitval
+    subrange[p].first = n.splitval + 1;
+    write_subtree(n.childID, subrange, tree, signal_chances);
+
+    // <= splitval
+    subrange[p].first = oldmin;
+    subrange[p].second = n.splitval;
+    write_subtree(n.childID + 1, subrange, tree, signal_chances);
+
+    subrange[p].second = oldmax;
+  }
+}
+template <typename BitChance, typename RAC>
+void MetaPropertySymbolCoder<BitChance, RAC>::write_tree(const Tree &tree,
+                                                         bool signal_chances) {
+  Ranges rootrange(range);
+  write_subtree(0, rootrange, tree, signal_chances);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_MA_COMPOUND_ENC_H_
diff --git a/jxl/modular/ma/rac.h b/jxl/modular/ma/rac.h
new file mode 100644
index 000000000..eaee5a688
--- /dev/null
+++ b/jxl/modular/ma/rac.h
@@ -0,0 +1,131 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_MA_RAC_H_
+#define JXL_MODULAR_MA_RAC_H_
+
+#include <stdint.h>
+
+#include "jxl/base/status.h"
+#include "jxl/modular/config.h"
+
+namespace jxl {
+
+/* RAC configuration for 24-bit RAC */
+class RacConfig24 {
+ public:
+  typedef uint_fast32_t data_t;
+  //  typedef uint32_t data_t;
+  static const data_t MAX_RANGE_BITS = 24;
+  static const data_t MIN_RANGE_BITS = 16;
+  static const data_t MIN_RANGE = (1UL << MIN_RANGE_BITS);
+  static const data_t BASE_RANGE = (1UL << MAX_RANGE_BITS);
+
+  static inline data_t chance_12bit_chance(uint16_t b12,
+                                           data_t range) ATTRIBUTE_HOT {
+    //        JXL_DASSERT(b12 > 0);
+    JXL_DASSERT((b12 >> 12) == 0);
+    if (sizeof(data_t) > 4)
+      return (range * b12) >> 12;
+    else
+      return ((((range & 0xFFF) * b12) >> 12) + ((range >> 12) * b12));
+    //    if (sizeof(data_t) > 4) return (range * b12 + 0x800) >> 12;
+    //    else return ((((range & 0xFFF) * b12) >> 12) + ((range >> 12) * b12));
+    // We want to compute (range * b12 + 0x800) >> 12. On 64-bit architectures
+    // this is no problem
+    //        if (sizeof(data_t) > 4) return (range * b12 + 0x800) >> 12;
+    // Unfortunately, this can overflow the 32-bit data type on 32-bit
+    // architectures, so split range in its lower and upper 12 bits, and compute
+    // separately.
+    //        else return ((((range & 0xFFF) * b12 + 0x800) >> 12) + ((range >>
+    //        12) * b12));
+    // (no worries, the compiler eliminates this branching)
+  }
+};
+
+template <typename Config, typename IO>
+class RacInput {
+ public:
+  typedef typename Config::data_t rac_t;
+
+ private:
+  rac_t range;
+  rac_t low;
+
+ protected:
+  IO& io;
+
+ private:
+  rac_t read_catch_eof() {
+    rac_t c = io.get_c();
+    // no reason to branch here to catch end-of-stream, just return garbage
+    // (0xFF I guess) if a premature EOS happens
+    // if(c == io.EOS) return 0;
+    return c;
+  }
+  void inline input() {
+    if (JXL_UNLIKELY(range <= Config::MIN_RANGE)) {
+      low <<= 8;
+      range <<= 8;
+      low |= read_catch_eof();
+      if (JXL_UNLIKELY(range <= Config::MIN_RANGE)) {
+        low <<= 8;
+        range <<= 8;
+        low |= read_catch_eof();
+      }
+    }
+  }
+  bool inline get(const rac_t chance) ATTRIBUTE_HOT {
+    JXL_DASSERT(chance >= 0);
+    JXL_DASSERT(chance < range);
+    bool result = (low >= range - chance);
+    if (result) {
+      low -= range - chance;
+      range = chance;
+    } else {
+      range -= chance;
+    }
+    input();
+    return result;
+  }
+
+ public:
+  explicit RacInput(IO& ioin) : range(Config::BASE_RANGE), low(0), io(ioin) {
+    rac_t r = Config::BASE_RANGE;
+    while (r > 1) {
+      low <<= 8;
+      low |= read_catch_eof();
+      r >>= 8;
+    }
+  }
+
+  bool inline read_12bit_chance(uint16_t b12) ATTRIBUTE_HOT {
+    return get(Config::chance_12bit_chance(b12, range));
+  }
+
+  bool inline read_bit() { return get(range >> 1); }
+};
+
+template <typename IO>
+class RacInput24 : public RacInput<RacConfig24, IO> {
+ public:
+  explicit RacInput24(IO& io) : RacInput<RacConfig24, IO>(io) {}
+};
+
+template <typename IO>
+using RacIn = RacInput24<IO>;
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_MA_RAC_H_
diff --git a/jxl/modular/ma/rac_enc.h b/jxl/modular/ma/rac_enc.h
new file mode 100644
index 000000000..f0eb4c2e0
--- /dev/null
+++ b/jxl/modular/ma/rac_enc.h
@@ -0,0 +1,127 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_MA_RAC_ENC_H_
+#define JXL_MODULAR_MA_RAC_ENC_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/modular/ma/rac.h"
+
+namespace jxl {
+
+template <class Config, typename IO>
+class RacOutput {
+ public:
+  typedef typename Config::data_t rac_t;
+
+ protected:
+  IO& io;
+
+ private:
+  rac_t range;
+  rac_t low;
+  int delayed_byte;
+  int delayed_count;
+
+  void inline output() {
+    while (range <= Config::MIN_RANGE) {
+      int byte = low >> Config::MIN_RANGE_BITS;
+      if (delayed_byte < 0) {  // first generated byte
+        delayed_byte = byte;
+      } else if (((low + range) >> 8) <
+                 Config::MIN_RANGE) {  // definitely no overflow
+        io.fputc(delayed_byte);
+        while (delayed_count) {
+          io.fputc(0xFF);
+          delayed_count--;
+        }
+        delayed_byte = byte;
+      } else if ((low >> 8) >= Config::MIN_RANGE) {  // definitely overflow
+        io.fputc(delayed_byte + 1);
+        while (delayed_count) {
+          io.fputc(0);
+          delayed_count--;
+        }
+        delayed_byte = byte & 0xFF;
+      } else {
+        delayed_count++;
+      }
+      low = (low & (Config::MIN_RANGE - 1)) << 8;
+      range <<= 8;
+    }
+  }
+  void inline put(rac_t chance, bool bit) {
+    JXL_DASSERT(chance >= 0);
+    JXL_DASSERT(chance < range);
+    if (bit) {
+      low += range - chance;
+      range = chance;
+    } else {
+      range -= chance;
+    }
+    output();
+  }
+
+ public:
+  explicit RacOutput(IO& ioin)
+      : io(ioin),
+        range(Config::BASE_RANGE),
+        low(0),
+        delayed_byte(-1),
+        delayed_count(0) {}
+
+  void inline write_12bit_chance(uint16_t b12, bool bit) {
+    put(Config::chance_12bit_chance(b12, range), bit);
+  }
+
+  void inline write_bit(bool bit) { put(range >> 1, bit); }
+
+  void inline flush() {
+    low += (Config::MIN_RANGE - 1);
+    // is this the correct way to reliably flush?
+    range = Config::MIN_RANGE - 1;
+    output();
+    range = Config::MIN_RANGE - 1;
+    output();
+    range = Config::MIN_RANGE - 1;
+    output();
+    range = Config::MIN_RANGE - 1;
+    output();
+  }
+};
+
+template <typename IO>
+class RacDummy {
+ public:
+  static void inline write_12bit_chance(uint16_t /*b12*/, bool /*unused*/) {}
+  static void inline write_bit(bool /*unused*/) {}
+  static void inline flush() {}
+
+  explicit RacDummy(IO& io) {}
+};
+
+template <typename IO>
+class RacOutput24 : public RacOutput<RacConfig24, IO> {
+ public:
+  explicit RacOutput24(IO& io) : RacOutput<RacConfig24, IO>(io) {}
+};
+
+template <typename IO>
+using RacOut = RacOutput24<IO>;
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_MA_RAC_ENC_H_
diff --git a/jxl/modular/ma/symbol.h b/jxl/modular/ma/symbol.h
new file mode 100644
index 000000000..2c4f69dd5
--- /dev/null
+++ b/jxl/modular/ma/symbol.h
@@ -0,0 +1,258 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXl_MODULAR_MA_SYMBOL_H_
+#define JXl_MODULAR_MA_SYMBOL_H_
+
+#include <vector>
+
+#include "jxl/base/status.h"
+#include "jxl/modular/config.h"
+#include "jxl/modular/ma/chance.h"
+#include "jxl/modular/ma/util.h"
+
+namespace jxl {
+
+template <typename RAC>
+class UniformSymbolCoder {
+ private:
+  RAC &rac;
+
+ public:
+  explicit UniformSymbolCoder(RAC &racIn) : rac(racIn) {}
+
+#ifdef HAS_ENCODER
+  void write_int(int min, int max, int val);
+  void write_int(int bits, int val) { write_int(0, (1 << bits) - 1, val); }
+#endif
+  int read_int(int min, int len) {
+    JXL_DASSERT(len >= 0);
+    if (len == 0) return min;
+
+    // split in [0..med] [med+1..len]
+    int med = len / 2;
+    bool bit = rac.read_bit();
+    if (bit) {
+      return read_int(min + med + 1, len - (med + 1));
+    } else {
+      return read_int(min, med);
+    }
+  }
+  int read_int(int bits) { return read_int(0, (1 << bits) - 1); }
+};
+
+typedef enum {
+  BIT_ZERO,
+  BIT_SIGN,
+  BIT_EXP,
+  BIT_MANT,
+} SymbolChanceBitType;
+
+static const uint16_t ZERO_CHANCE = 1024;
+static const uint16_t SIGN_CHANCE = 2048;
+static const uint16_t MANT_CHANCE = 1024;
+
+template <typename BitChance, int bits>
+class SymbolChance {
+ public:
+  BitChance bit_zero;
+  BitChance bit_sign;
+  BitChance bit_exp[bits - 1];
+  BitChance bit_mant[bits];
+
+ public:
+  BitChance inline &bitZero() { return bit_zero; }
+
+  BitChance inline &bitSign() { return bit_sign; }
+
+  // all exp bits 0         -> int(log2(val)) == 0  [ val == 1 ]
+  // exp bits up to i are 1 -> int(log2(val)) == i+1
+  BitChance inline &bitExp(unsigned int i) {
+    JXL_DASSERT(i >= 0 && i < (bits - 1));
+    return bit_exp[i];
+  }
+  BitChance inline &bitMant(unsigned int i) {
+    JXL_DASSERT(i >= 0 && i < bits);
+    return bit_mant[i];
+  }
+
+  BitChance inline &bit(SymbolChanceBitType typ, unsigned int i = 0) {
+    switch (typ) {
+      default:
+      case BIT_ZERO:
+        return bitZero();
+      case BIT_SIGN:
+        return bitSign();
+      case BIT_EXP:
+        return bitExp(i);
+      case BIT_MANT:
+        return bitMant(i);
+    }
+  }
+  SymbolChance() = default;  // don't init  (needed for fast copy constructor in
+                             // CompoundSymbolChances?)
+
+  explicit SymbolChance(uint16_t zero_chance) {
+    bitZero().set_12bit(zero_chance);
+    //        bitSign().set_12bit(0x800); // 50%, which is the default anyway
+    // uint64_t p = zero_chance;       // implicit denominator of 0x1000
+    uint64_t rp = 0x1000 - zero_chance;  // 1-p
+
+    // assume geometric distribution: Pr(X=k) = (1-p)^k p
+    // (p = zero_chance)
+    // cumulative distribution function: Pr(X < k) = 1-(1-p)^k = 1-rp^k
+    //                                   Pr(X >= k) = (1-p)^k = rp^k
+    // bitExp(i) == true   iff   X >= 2^i && X < 2^(i+1)
+    // bitExp(i) is only used if the lower bound holds
+    // Pr(X >= 2^i && X < 2^(i+1) | X >= 2^i ) = (rp^2^i - rp^2^(i+1)) / rp^2^i
+    // = 1 - rp^2^i
+
+    for (int i = 0; i < bits - 1; i++) {
+      if (rp < 0x100) rp = 0x100;
+      if (rp > 0xf00) rp = 0xf00;
+      bitExp(i).set_12bit(0x1000 - rp);
+      rp = (rp * rp + 0x800) >> 12;  // square it
+    }
+    for (int i = 0; i < bits; i++) {
+      bitMant(i).set_12bit(MANT_CHANCE);
+    }
+  }
+};
+
+template <typename SymbolCoder>
+int reader(SymbolCoder &coder, int bits) {
+  int pos = 0;
+  int value = 0;
+  int b = 1;
+  while (pos++ < bits) {
+    if (coder.read(BIT_MANT, pos)) value += b;
+    b *= 2;
+  }
+  return value;
+}
+
+template <int bits, typename SymbolCoder>
+int reader(SymbolCoder &coder, int min, int max) ATTRIBUTE_HOT;
+
+template <int bits, typename SymbolCoder>
+int reader(SymbolCoder &coder, int min, int max) {
+  JXL_DASSERT(min < max);  // make sure not to call this if min==max can happen
+  bool sign;
+  JXL_DASSERT(min <= 0 && max >= 0);  // should always be the case, because
+                                      // guess should always be in valid range
+
+  if (coder.read(BIT_ZERO)) return 0;
+  if (min < 0) {
+    if (max > 0) {
+      sign = coder.read(BIT_SIGN);
+    } else {
+      sign = false;
+    }
+  } else {
+    sign = true;
+  }
+
+  const unsigned int amax = (sign ? max : -min);
+
+  const unsigned int emax = sizeof(unsigned int) * 8 - 1 - __builtin_clz(amax);
+
+  unsigned int e = 0;
+  for (; e < emax; e++) {
+    if (coder.read(BIT_EXP, e)) break;
+  }
+
+  int have = (1 << e);
+
+  for (unsigned int pos = e; pos > 0;) {
+    pos--;
+    int minabs1 = have | (1 << pos);
+    if (minabs1 > static_cast<int>(amax)) continue;  // 1-bit is impossible
+    if (coder.read(BIT_MANT, pos)) have = minabs1;
+  }
+  return (sign ? have : -have);
+}
+
+template <typename BitChance, typename RAC, int bits>
+class SimpleSymbolBitCoder {
+ private:
+  SymbolChance<BitChance, bits> &ctx;
+  RAC &rac;
+
+ public:
+  SimpleSymbolBitCoder(SymbolChance<BitChance, bits> &ctxIn, RAC &racIn)
+      : ctx(ctxIn), rac(racIn) {}
+
+#ifdef HAS_ENCODER
+  void write(bool bit, SymbolChanceBitType typ, int i = 0);
+#endif
+
+  bool read(SymbolChanceBitType typ, unsigned int i = 0) {
+    BitChance &bch = ctx.bit(typ, i);
+    const bool bit = rac.read_12bit_chance(bch.get_12bit());
+    bch.put(bit);
+    return bit;
+  }
+};
+
+template <typename BitChance, typename RAC, int bits>
+class SimpleSymbolCoder {
+ private:
+  SymbolChance<BitChance, bits> ctx;
+  RAC &rac;
+
+ public:
+  explicit SimpleSymbolCoder(RAC &racIn) : ctx(ZERO_CHANCE), rac(racIn) {}
+
+#ifdef HAS_ENCODER
+  void write_int(int min, int max, int value);
+  void write_int2(int min, int max, int value) {
+    if (min == max) return;
+    if (min > 0)
+      write_int(0, max - min, value - min);
+    else if (max < 0)
+      write_int(min - max, 0, value - max);
+    else
+      write_int(min, max, value);
+  }
+  void write_int(int nbits, int value);
+#endif
+
+  int read_int(int min, int max) {
+    SimpleSymbolBitCoder<BitChance, RAC, bits> bitCoder(ctx, rac);
+    return reader<bits, SimpleSymbolBitCoder<BitChance, RAC, bits>>(bitCoder,
+                                                                    min, max);
+  }
+  int read_int2(int min, int max) {
+    if (min == max) return min;
+    if (min > 0)
+      return read_int(0, max - min) + min;
+    else if (max < 0)
+      return read_int(min - max, 0) + max;
+    else
+      return read_int(min, max);
+  }
+  int read_int(int nbits) {
+    JXL_DASSERT(nbits <= bits);
+    SimpleSymbolBitCoder<BitChance, RAC, bits> bitCoder(ctx, rac);
+    return reader(bitCoder, nbits);
+  }
+};
+
+}  // namespace jxl
+
+#ifdef HAS_ENCODER
+#include "jxl/modular/ma/symbol_enc.h"
+#endif
+
+#endif  // JXl_MODULAR_MA_SYMBOL_H_
diff --git a/jxl/modular/ma/symbol_enc.h b/jxl/modular/ma/symbol_enc.h
new file mode 100644
index 000000000..dd2914b16
--- /dev/null
+++ b/jxl/modular/ma/symbol_enc.h
@@ -0,0 +1,196 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_MA_SYMBOL_ENC_H_
+#define JXL_MODULAR_MA_SYMBOL_ENC_H_
+
+#include <stdlib.h>
+
+#include "jxl/base/status.h"
+#include "jxl/modular/ma/symbol.h"
+
+namespace jxl {
+
+template <typename RAC>
+void UniformSymbolCoder<RAC>::write_int(int min, int max, int val) {
+  JXL_DASSERT(max >= min);
+  if (min != 0) {
+    max -= min;
+    val -= min;
+  }
+  if (max == 0) return;
+
+  // split in [0..med] [med+1..max]
+  int med = max / 2;
+  if (val > med) {
+    rac.write_bit(true);
+    write_int(med + 1, max, val);
+  } else {
+    rac.write_bit(false);
+    write_int(0, med, val);
+  }
+  return;
+}
+
+template <typename SymbolCoder>
+void writer(SymbolCoder& coder, int bits, int value) {
+  int pos = 0;
+  while (pos++ < bits) {
+    coder.write(value & 1, BIT_MANT, pos);
+    value >>= 1;
+  }
+}
+
+template <int bits, typename SymbolCoder>
+void writer(SymbolCoder& coder, int min, int max, int value) {
+  JXL_DASSERT(min <= max);
+  JXL_DASSERT(value >= min);
+  JXL_DASSERT(value <= max);
+
+  // avoid doing anything if the value is already known
+  if (min == max) return;
+
+  if (value == 0) {  // value is zero
+    coder.write(true, BIT_ZERO);
+    return;
+  }
+
+  JXL_DASSERT(min <= 0 && max >= 0);  // should always be the case, because
+                                      // guess should always be in valid range
+
+  coder.write(false, BIT_ZERO);
+  int sign = (value > 0 ? 1 : 0);
+  if (max > 0 && min < 0) {
+    // only output sign bit if value can be both pos and neg
+    coder.write(sign, BIT_SIGN);
+  }
+  const int a = abs(value);
+  const int e = ilog2(a);
+  int amax = sign ? abs(max) : abs(min);
+
+  int emax = ilog2(amax);
+
+  int i = 0;  // ilog2(amin);
+  while (i < emax) {
+    // if exponent >i is impossible, we are done
+    if ((1 << (i + 1)) > amax) break;
+    // if exponent i is possible, output the exponent bit
+    coder.write(i == e, BIT_EXP, i);
+    if (i == e) break;
+    i++;
+  }
+
+  int have = (1 << e);
+  int left = have - 1;
+  for (int pos = e; pos > 0;) {
+    int bit = 1;
+    left ^= (1 << (--pos));
+    int minabs1 = have | (1 << pos);
+    if (minabs1 > amax) {  // 1-bit is impossible
+      bit = 0;
+    } else {
+      bit = (a >> pos) & 1;
+      coder.write(bit, BIT_MANT, pos);
+    }
+    have |= (bit << pos);
+  }
+}
+
+template <int bits, typename SymbolCoder>
+int estimate_writer(SymbolCoder& coder, int min, int max, int value) {
+  JXL_DASSERT(min <= max);
+  JXL_DASSERT(value >= min);
+  JXL_DASSERT(value <= max);
+
+  uint64_t total = 0;
+
+  // avoid doing anything if the value is already known
+  if (min == max) return total;
+
+  if (value == 0) {  // value is zero
+    coder.estimate(true, BIT_ZERO, 0, total);
+    return total;
+  }
+
+  JXL_DASSERT(min <= 0 && max >= 0);  // should always be the case, because
+                                      // guess should always be in valid range
+
+  coder.estimate(false, BIT_ZERO, 0, total);
+  int sign = (value > 0 ? 1 : 0);
+  if (max > 0 && min < 0) {
+    // only output sign bit if value can be both pos and neg
+    coder.estimate(sign, BIT_SIGN, 0, total);
+  }
+  if (sign) min = 1;
+  if (!sign) max = -1;
+  const int a = abs(value);
+  const int e = ilog2(a);
+  int amin = sign ? abs(min) : abs(max);
+  int amax = sign ? abs(max) : abs(min);
+
+  int emax = ilog2(amax);
+  int i = ilog2(amin);
+
+  while (i < emax) {
+    // if exponent >i is impossible, we are done
+    if ((1 << (i + 1)) > amax) break;
+    // if exponent i is possible, output the exponent bit
+    coder.estimate(i == e, BIT_EXP, i, total);
+    if (i == e) break;
+    i++;
+  }
+
+  int have = (1 << e);
+  int left = have - 1;
+  for (int pos = e; pos > 0;) {
+    int bit = 1;
+    left ^= (1 << (--pos));
+    int minabs1 = have | (1 << pos);
+    if (minabs1 > amax) {  // 1-bit is impossible
+      bit = 0;
+    } else {
+      bit = (a >> pos) & 1;
+      coder.estimate(bit, BIT_MANT, pos, total);
+    }
+    have |= (bit << pos);
+  }
+  return total;
+}
+
+template <typename BitChance, typename RAC, int bits>
+void SimpleSymbolBitCoder<BitChance, RAC, bits>::write(bool bit,
+                                                       SymbolChanceBitType typ,
+                                                       int i) {
+  BitChance& bch = ctx.bit(typ, i);
+  rac.write_12bit_chance(bch.get_12bit(), bit);
+  bch.put(bit);
+}
+
+template <typename BitChance, typename RAC, int bits>
+void SimpleSymbolCoder<BitChance, RAC, bits>::write_int(int min, int max,
+                                                        int value) {
+  SimpleSymbolBitCoder<BitChance, RAC, bits> bitCoder(ctx, rac);
+  writer<bits, SimpleSymbolBitCoder<BitChance, RAC, bits> >(bitCoder, min, max,
+                                                            value);
+}
+template <typename BitChance, typename RAC, int bits>
+void SimpleSymbolCoder<BitChance, RAC, bits>::write_int(int nbits, int value) {
+  JXL_DASSERT(nbits <= bits);
+  SimpleSymbolBitCoder<BitChance, RAC, bits> bitCoder(ctx, rac);
+  writer(bitCoder, nbits, value);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_MA_SYMBOL_ENC_H_
diff --git a/jxl/modular/ma/util.h b/jxl/modular/ma/util.h
new file mode 100644
index 000000000..0b8605a18
--- /dev/null
+++ b/jxl/modular/ma/util.h
@@ -0,0 +1,33 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_MA_UTIL_H_
+#define JXL_MODULAR_MA_UTIL_H_
+
+#include <stdint.h>
+
+#include "jxl/modular/ma/bit.h"
+
+namespace jxl {
+
+static inline int ilog2(uint32_t l) {
+  if (l == 0) {
+    return 0;
+  }
+  return sizeof(unsigned int) * 8 - 1 - __builtin_clz(l);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_MA_UTIL_H_
diff --git a/jxl/modular/memio.h b/jxl/modular/memio.h
new file mode 100644
index 000000000..f60bbb751
--- /dev/null
+++ b/jxl/modular/memio.h
@@ -0,0 +1,152 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_MEMIO_H_
+#define JXL_MODULAR_MEMIO_H_
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/padded_bytes.h"
+
+namespace jxl {
+
+/*!
+ * Read-only IO interface for a constant memory block
+ */
+class BlobReader {
+ private:
+  const uint8_t* const JXL_RESTRICT data;
+  const size_t data_array_size;
+  size_t seek_pos;
+
+ public:
+  const int EOS = -1;
+
+  BlobReader(const uint8_t* _data, size_t _data_array_size)
+      : data(_data), data_array_size(_data_array_size), seek_pos(0) {}
+
+  bool isEOF() const { return seek_pos > data_array_size; }
+  long ftell() const { return seek_pos; }
+  const uint8_t* ptr() const { return data; }
+  size_t size() const { return data_array_size; }
+
+  int get_c() {
+    if (JXL_UNLIKELY(seek_pos >= data_array_size)) {
+      if (seek_pos == data_array_size) seek_pos++;
+      return EOS;
+    }
+    return data[seek_pos++];
+  }
+  char* gets(char* buf, int n) {
+    int i = 0;
+    const int max_write = n - 1;
+    while (seek_pos < data_array_size && i < max_write)
+      buf[i++] = data[seek_pos++];
+    buf[n - 1] = '\0';
+
+    if (i < max_write)
+      return nullptr;
+    else
+      return buf;
+  }
+  void fseek(long offset, int where) {
+    switch (where) {
+      case SEEK_SET:
+        seek_pos = offset;
+        break;
+      case SEEK_CUR:
+        seek_pos += offset;
+        break;
+      case SEEK_END:
+        seek_pos = long(data_array_size) + offset;
+        break;
+    }
+  }
+};
+
+/*!
+ * IO interface for a growable memory block
+ */
+class BlobWriter {
+ private:
+  size_t pos = 0;
+
+ public:
+  jxl::PaddedBytes blob;
+  BlobWriter() = default;
+
+  int ftell() const { return pos; }
+  const uint8_t* ptr() const { return blob.data(); }
+  size_t size() const { return blob.size(); }
+  void append(const BlobWriter& other) {
+    blob.resize(pos);
+    pos = size() + other.size();
+    blob.append(other.blob);
+  }
+  void append(const jxl::PaddedBytes& other) {
+    blob.resize(pos);
+    pos = size() + other.size();
+    blob.append(other);
+  }
+  int fputs(const char* s) {
+    size_t i = 0;
+    // null-terminated string
+    while (s[i]) {
+      fputc(s[i]);
+      i++;
+    }
+    return 0;
+  }
+  int fputc(int c) {
+    if (pos == size())
+      blob.push_back(static_cast<uint8_t>(c));
+    else
+      blob[pos] = static_cast<uint8_t>(c);
+    pos++;
+    return c;
+  }
+  void fseek(long offset, int where) {
+    switch (where) {
+      case SEEK_SET:
+        pos = offset;
+        break;
+      case SEEK_CUR:
+        pos += offset;
+        break;
+      case SEEK_END:
+        pos = size() + offset;
+        break;
+    }
+  }
+};
+
+/*!
+ * Dummy writer
+ */
+class DummyWriter {
+ public:
+  const int EOS = -1;
+
+  DummyWriter() = default;
+
+  int ftell() const { return -1; }
+  const uint8_t* ptr() const { return nullptr; }
+  void append(const jxl::PaddedBytes& other) {}
+  int fputs(const char* s) { return 0; }
+  int fputc(int c) { return c; }
+  void fseek(long offset, int where) {}
+};
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_MEMIO_H_
diff --git a/jxl/modular/transform/near-lossless.h b/jxl/modular/transform/near-lossless.h
new file mode 100644
index 000000000..54f036886
--- /dev/null
+++ b/jxl/modular/transform/near-lossless.h
@@ -0,0 +1,89 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_TRANSFORM_NEAR_LOSSLESS_H_
+#define JXL_MODULAR_TRANSFORM_NEAR_LOSSLESS_H_
+
+// Very simple lossy preprocessing step.
+// Quantizes the prediction residual (so the entropy coder has an easier job)
+// Obviously there's room for encoder improvement here
+// The decoder doesn't need to know about this step
+
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+#ifdef HAS_ENCODER
+void delta_quantize(int max_error, pixel_type &d) {
+  int a = (d < 0 ? -d : d);
+  a = (a + (max_error / 2)) / max_error * max_error;
+  d = (d < 0 ? -a : a);
+}
+
+static bool fwd_near_lossless(Image &input, std::vector<int> &parameters) {
+  JXL_DASSERT(parameters.size() == 3);
+  int begin_c = input.nb_meta_channels + parameters[0];
+  int end_c = input.nb_meta_channels + parameters[1];
+  int max_delta_error = parameters[2];
+
+  input.recompute_minmax();
+  JXL_DEBUG_V(8, "Applying loss on channels %i-%i with max delta=%i.", begin_c,
+              end_c, max_delta_error);
+  uint64_t total_error = 0;
+  for (int c = begin_c; c <= end_c; c++) {
+    if (c >= input.channel.size()) return false;
+    int w = input.channel[c].w;
+    int h = input.channel[c].h;
+    intptr_t onerow = input.channel[c].plane.PixelsPerRow();
+
+    Channel out(w, h, 0, 1);
+    for (int y = 0; y < h; y++) {
+      pixel_type *JXL_RESTRICT p_in = input.channel[c].Row(y);
+      pixel_type *JXL_RESTRICT p_out = out.Row(y);
+      for (int x = 0; x < w; x++) {
+        // assuming the default predictor
+        pixel_type left = (x ? p_out[x - 1] : 0);
+        pixel_type top = (y ? p_out[x - onerow] : left);
+        pixel_type prediction = (left + top) / 2;
+        pixel_type delta = p_in[x] - prediction;
+        delta_quantize(max_delta_error, delta);
+        pixel_type reconstructed =
+            CLAMP(prediction + delta, input.channel[c].minval,
+                  input.channel[c].maxval);
+        int e = p_in[x] - reconstructed;
+        total_error += abs(e);
+        p_out[x] = reconstructed;
+      }
+    }
+    input.channel[c] = std::move(out);
+    JXL_DEBUG_V(9, "  Avg error: %f", total_error * 1.0 / (w * h));
+  }
+  return false;  // don't signal this 'transform' in the bitstream, there is no
+                 // inverse transform to be done
+}
+#endif
+
+static bool near_lossless(Image &input, bool inverse,
+                          std::vector<int> &parameters) {
+  JXL_DASSERT(inverse == false);
+#ifdef HAS_ENCODER
+  return fwd_near_lossless(input, parameters);
+#else
+  return false;
+#endif
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_TRANSFORM_NEAR_LOSSLESS_H_
diff --git a/jxl/modular/transform/palette.h b/jxl/modular/transform/palette.h
new file mode 100644
index 000000000..61a088b7e
--- /dev/null
+++ b/jxl/modular/transform/palette.h
@@ -0,0 +1,235 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_TRANSFORM_PALETTE_H_
+#define JXL_MODULAR_TRANSFORM_PALETTE_H_
+
+#include <set>
+#include "jxl/base/data_parallel.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+static bool inv_palette(Image &input, std::vector<int> parameters,
+                        jxl::ThreadPool *pool) {
+  if (input.nb_meta_channels < 1) {
+    return JXL_FAILURE("Error: Palette transform without palette.");
+  }
+  if (parameters.size() != 3) {
+    return JXL_FAILURE("Error: Palette transform with incorrect parameters.");
+  }
+  int nb = input.channel[0].h;
+  size_t c0 = static_cast<size_t>(parameters[0] + 1);
+  if (c0 >= input.channel.size()) {
+    return JXL_FAILURE("Error: Palette transform with incorrect parameters.");
+  }
+  int w = input.channel[c0].w;
+  int h = input.channel[c0].h;
+  // might be false in case of lossy
+  // JXL_DASSERT(input.channel[c0].minval == 0);
+  // JXL_DASSERT(input.channel[c0].maxval == palette.w-1);
+  for (int i = 1; i < nb; i++) {
+    input.channel.insert(input.channel.begin() + c0 + 1, Channel(w, h, 0, 1));
+  }
+  const Channel &palette = input.channel[0];
+  int zero = 0;
+  if (nb == 1) {
+    input.channel[c0].actual_minmax(&input.channel[c0].minval,
+                                    &input.channel[c0].maxval);
+    zero = -input.channel[c0].minval;
+  }
+  const pixel_type *JXL_RESTRICT p_palette = input.channel[0].Row(0);
+  intptr_t onerow = input.channel[0].plane.PixelsPerRow();
+
+  if (nb == 1) {
+    RunOnPool(
+        pool, 0, h, jxl::ThreadPool::SkipInit(),
+        [&](const int task, const int thread) {
+          const size_t y = task;
+          pixel_type *p = input.channel[c0].Row(y);
+          for (int x = 0; x < w; x++) {
+            int index = CLAMP(p[x] + zero, 0, static_cast<int>(palette.w) - 1);
+            p[x] = p_palette[index];
+          }
+        },
+        "UndoChannelPalette");
+  } else {
+    RunOnPool(
+        pool, 0, h, jxl::ThreadPool::SkipInit(),
+        [&](const int task, const int thread) {
+          const size_t y = task;
+          std::vector<pixel_type *> p_out(nb);
+          const pixel_type *p_index = input.channel[c0].Row(y);
+          for (int c = 0; c < nb; c++) p_out[c] = input.channel[c0 + c].Row(y);
+          for (int x = 0; x < w; x++) {
+            int index =
+                CLAMP(p_index[x] + zero, 0, static_cast<int>(palette.w) - 1);
+            for (int c = 0; c < nb; c++)
+              p_out[c][x] = p_palette[c * onerow + index];
+          }
+        },
+        "UndoPalette");
+  }
+  input.nb_channels += nb - 1;
+  input.nb_meta_channels--;
+  input.channel.erase(input.channel.begin(), input.channel.begin() + 1);
+  return true;
+}
+
+static void meta_palette(Image &input, std::vector<int> parameters) {
+  if (parameters.size() != 3) {
+    JXL_FAILURE("Error: Palette transform with incorrect parameters.");
+    input.error = true;
+    return;
+  }
+  int begin_c = parameters[0];
+  int end_c = parameters[1];
+  if (begin_c > end_c || end_c >= static_cast<int>(input.channel.size())) {
+    JXL_FAILURE("Error: Palette transform with incorrect parameters.");
+    input.error = true;
+    return;
+  }
+  int nb = end_c - begin_c + 1;
+  int &nb_colors = parameters[2];
+  input.nb_meta_channels++;
+  input.nb_channels -= nb - 1;
+  input.channel.erase(input.channel.begin() + begin_c + 1,
+                      input.channel.begin() + end_c + 1);
+  Channel pch(nb_colors, nb, 0, 1);
+  pch.hshift = -1;
+  input.channel.insert(input.channel.begin(), std::move(pch));
+}
+
+#ifdef HAS_ENCODER
+static bool fwd_palette(Image &input, std::vector<int> &parameters) {
+  JXL_DASSERT(parameters.size() == 3);
+  int begin_c = parameters[0];
+  int end_c = parameters[1];
+  int &nb_colors = parameters[2];
+  bool ordered = true;
+  if (nb_colors < 0) {
+    nb_colors = -nb_colors;
+    ordered = false;
+  }
+  int nb = end_c - begin_c + 1;
+
+  size_t w = input.channel[begin_c].w;
+  size_t h = input.channel[begin_c].h;
+  for (int c = begin_c + 1; c <= end_c; c++) {
+    if (input.channel[c].w != w) return false;
+    if (input.channel[c].h != h) return false;
+  }
+  JXL_DEBUG_V(
+      7, "Trying to represent channels %i-%i using at most a %i-color palette.",
+      begin_c, end_c, nb_colors);
+  std::set<std::vector<pixel_type> >
+      candidate_palette;  // ordered lexicographically
+  std::vector<std::vector<pixel_type> > candidate_palette_imageorder;
+  std::vector<pixel_type> color(nb);
+  std::vector<const pixel_type *> p_in(nb);
+  size_t count = 0;
+  for (size_t y = 0; y < h; y++) {
+    for (int c = 0; c < nb; c++) p_in[c] = input.channel[begin_c + c].Row(y);
+    for (size_t x = 0; x < w; x++) {
+      for (int c = 0; c < nb; c++) color[c] = p_in[c][x];
+      candidate_palette.insert(color);
+      if (candidate_palette.size() > count) {
+        count++;
+        candidate_palette_imageorder.push_back(color);
+      }
+      if (static_cast<int>(candidate_palette.size()) > nb_colors)
+        return false;  // too many colors
+    }
+  }
+  nb_colors = candidate_palette.size();
+  if (nb_colors == 1) return false;  // don't need palette for this
+  JXL_DEBUG_V(6, "Channels %i-%i can be represented using a %i-color palette.",
+              begin_c, end_c, nb_colors);
+
+  Channel pch(nb_colors, nb, 0, 1);
+  pch.hshift = -1;
+  int x = 0;
+  pixel_type *JXL_RESTRICT p_palette = pch.Row(0);
+  intptr_t onerow = pch.plane.PixelsPerRow();
+  int zero = 0;
+  std::vector<pixel_type> lookup;
+  if (nb == 1)
+    lookup.resize(input.channel[begin_c].maxval -
+                  input.channel[begin_c].minval + 1);
+  if (ordered) {
+    JXL_DEBUG_V(7, "Palette of %i colors, using lexicographic order",
+                nb_colors);
+    for (auto pcol : candidate_palette) {
+      JXL_DEBUG_V(9, "  Color %i :  ", x);
+      for (int i = 0; i < nb; i++) p_palette[i * onerow + x] = pcol[i];
+      if (nb == 1) lookup[pcol[0] - input.channel[begin_c].minval] = x;
+      if (nb == 1 && pcol[0] <= 0) zero = x;
+      for (int i = 0; i < nb; i++) JXL_DEBUG_V(9, "%i ", pcol[i]);
+      x++;
+    }
+  } else {
+    JXL_DEBUG_V(7, "Palette of %i colors, using image order", nb_colors);
+    for (auto pcol : candidate_palette_imageorder) {
+      JXL_DEBUG_V(9, "  Color %i :  ", x);
+      for (int i = 0; i < nb; i++) p_palette[i * onerow + x] = pcol[i];
+      if (nb == 1) lookup[pcol[0] - input.channel[begin_c].minval] = x;
+      for (int i = 0; i < nb; i++) JXL_DEBUG_V(9, "%i ", pcol[i]);
+      x++;
+    }
+  }
+  for (size_t y = 0; y < h; y++) {
+    for (int c = 0; c < nb; c++) p_in[c] = input.channel[begin_c + c].Row(y);
+    pixel_type *JXL_RESTRICT p = input.channel[begin_c].Row(y);
+    if (nb == 1) {
+      for (size_t x = 0; x < w; x++)
+        p[x] = lookup[p[x] - input.channel[begin_c].minval] - zero;
+    } else
+      for (size_t x = 0; x < w; x++) {
+        for (int c = 0; c < nb; c++) color[c] = p_in[c][x];
+        int index = 0;
+        for (; index < nb_colors; index++) {
+          bool found = true;
+          for (int c = 0; c < nb; c++)
+            if (color[c] != p_palette[c * onerow + index]) {
+              found = false;
+              break;
+            }
+          if (found) break;
+        }
+        p[x] = index - zero;
+      }
+  }
+  input.nb_meta_channels++;
+  input.nb_channels -= nb - 1;
+  input.channel.erase(input.channel.begin() + begin_c + 1,
+                      input.channel.begin() + end_c + 1);
+  input.channel.insert(input.channel.begin(), std::move(pch));
+  return true;
+}
+#endif
+
+static bool palette(Image &input, bool inverse, std::vector<int> &parameters,
+                    jxl::ThreadPool *pool) {
+  if (inverse) return inv_palette(input, parameters, pool);
+#ifdef HAS_ENCODER
+  else
+    return fwd_palette(input, parameters);
+#else
+  return false;
+#endif
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_TRANSFORM_PALETTE_H_
diff --git a/jxl/modular/transform/quantize.h b/jxl/modular/transform/quantize.h
new file mode 100644
index 000000000..47c410645
--- /dev/null
+++ b/jxl/modular/transform/quantize.h
@@ -0,0 +1,110 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_TRANSFORM_QUANTIZE_H_
+#define JXL_MODULAR_TRANSFORM_QUANTIZE_H_
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+pixel_type rdiv(pixel_type x, int q) {
+  if (x < 0)
+    return -((-x + q / 2) / q);
+  else
+    return (x + q / 2) / q;
+}
+
+bool inv_quantize(Image &input, const std::vector<int> &parameters,
+                  jxl::ThreadPool *pool) {
+  size_t c = input.nb_meta_channels;
+  pixel_type *JXL_RESTRICT qs = input.channel[0].Row(0);
+  size_t qx = 0;
+  for (; c < input.channel.size(); c++) {
+    Channel &ch = input.channel[c];
+    if (ch.is_empty()) continue;
+    int q = qs[qx] + 1;
+    qx++;
+    if (q == 1) continue;
+    JXL_DEBUG_V(3, "De-quantizing channel %zu with quantization constant %i", c,
+                q);
+    RunOnPool(
+        pool, 0, ch.h, jxl::ThreadPool::SkipInit(),
+        [&](const int task, const int thread) {
+          const size_t y = task;
+          pixel_type *JXL_RESTRICT p = ch.Row(y);
+          for (size_t x = 0; x < ch.w; x++) {
+            p[x] *= q;
+          }
+        },
+        "ModularDequant");
+  }
+  input.nb_meta_channels--;
+  input.channel.erase(input.channel.begin(), input.channel.begin() + 1);
+  return true;
+}
+
+static void meta_quantize(Image &input) {
+  Channel qs(input.channel.size() - input.nb_meta_channels, 1, 0, 255);
+  qs.hshift = -1;
+  input.channel.insert(input.channel.begin(), std::move(qs));
+  input.nb_meta_channels++;
+}
+
+#ifdef HAS_ENCODER
+bool fwd_quantize(Image &input, std::vector<int> &parameters) {
+  meta_quantize(input);
+  pixel_type *JXL_RESTRICT qs = input.channel[0].Row(0);
+  size_t qx = 0;
+  for (size_t c = input.nb_meta_channels; c < input.channel.size(); c++) {
+    Channel &ch = input.channel[c];
+    if (ch.is_empty()) continue;
+    int q = (c - input.nb_meta_channels < parameters.size()
+                 ? parameters[c - input.nb_meta_channels]
+                 : parameters.back());
+    if (q < 1) q = 1;
+    JXL_DEBUG_V(3, "Quantizing channel %zu with quantization constant %i", c,
+                q);
+    for (size_t y = 0; y < ch.h; y++) {
+      pixel_type *JXL_RESTRICT p = ch.Row(y);
+      for (size_t x = 0; x < ch.w; x++) {
+        //            p[x] /= q;
+        p[x] = rdiv(p[x], q);
+      }
+    }
+    ch.minval /= q;
+    ch.maxval /= q;
+    qs[qx] = q - 1;
+    qx++;
+  }
+  return true;
+}
+#endif
+
+bool quantize(Image &input, bool inverse, std::vector<int> &parameters,
+              jxl::ThreadPool *pool) {
+  if (inverse) return inv_quantize(input, parameters, pool);
+#ifdef HAS_ENCODER
+  else
+    return fwd_quantize(input, parameters);
+#else
+  return false;
+#endif
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_TRANSFORM_QUANTIZE_H_
diff --git a/jxl/modular/transform/squeeze.h b/jxl/modular/transform/squeeze.h
new file mode 100644
index 000000000..701363cf8
--- /dev/null
+++ b/jxl/modular/transform/squeeze.h
@@ -0,0 +1,466 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_TRANSFORM_SQUEEZE_H_
+#define JXL_MODULAR_TRANSFORM_SQUEEZE_H_
+
+// Haar-like transform: halves the resolution in one direction
+// A B   -> (A+B)>>1              in one channel (average)  -> same range as
+// original channel
+//          A-B - tendency        in a new channel ('residual' needed to make
+//          the transform reversible)
+//                                        -> theoretically range could be 2.5
+//                                        times larger (2 times without the
+//                                        'tendency'), but there should be lots
+//                                        of zeroes
+// Repeated application (alternating horizontal and vertical squeezes) results
+// in downscaling
+
+// TODO: Chroma subsampling is kind of a special case of this, where the
+// residual is simply dropped
+//       Add an option to drop a residual? Could also be simulated by quantizing
+//       the residual by maxval so it becomes a trivial all-zeroes channel.
+
+#include <stdlib.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/common.h"
+#include "jxl/modular/config.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+/*
+        int avg=(A+B)>>1;
+        int diff=(A-B);
+        int rA=(diff+(avg<<1)+(diff&1))>>1;
+        int rB=rA-diff;
+
+*/
+//         |A B|C D|E F|
+//           p   a   n             p=avg(A,B), a=avg(C,D), n=avg(E,F)
+//
+// Goal: estimate C-D (avoiding ringing artifacts)
+// (ensuring that in smooth areas, a zero residual corresponds to a smooth
+// gradient)
+
+// best estimate for C: (B + 2*a)/3
+// best estimate for D: (n + 3*a)/4
+// best estimate for C-D:  4*B - 3*n - a /12
+
+// avoid ringing by 1) only doing this if B <= a <= n  or  B >= a >= n
+// (otherwise, this is not a smooth area and we cannot really estimate C-D)
+//                  2) making sure that B <= C <= D <= n  or B >= C >= D >= n
+
+inline pixel_type smooth_tendency(pixel_type B, pixel_type a, pixel_type n) {
+  pixel_type diff = 0;
+  if (B >= a && a >= n) {
+    diff = (4 * B - 3 * n - a + 6) / 12;
+    //      2C = a<<1 + diff - diff&1 <= 2B  so diff - diff&1 <= 2B - 2a
+    //      2D = a<<1 - diff - diff&1 >= 2n  so diff + diff&1 <= 2a - 2n
+    if (diff - (diff & 1) > 2 * (B - a)) diff = 2 * (B - a) + 1;
+    if (diff + (diff & 1) > 2 * (a - n)) diff = 2 * (a - n);
+  } else if (B <= a && a <= n) {
+    diff = (4 * B - 3 * n - a - 6) / 12;
+    //      2C = a<<1 + diff + diff&1 >= 2B  so diff + diff&1 >= 2B - 2a
+    //      2D = a<<1 - diff + diff&1 <= 2n  so diff - diff&1 >= 2a - 2n
+    if (diff + (diff & 1) < 2 * (B - a)) diff = 2 * (B - a) - 1;
+    if (diff - (diff & 1) < 2 * (a - n)) diff = 2 * (a - n);
+  }
+  return diff;
+}
+
+void inv_hsqueeze(Image &input, int c, int rc, jxl::ThreadPool *pool) {
+  const Channel &chin = input.channel[c];
+  const Channel &chin_residual = input.channel[rc];
+  if (chin_residual.w == 0) return;
+  Channel chout(chin.w + chin_residual.w, chin.h, chin.minval, chin.maxval,
+                chin.hshift - 1, chin.vshift, chin.hcshift - 1, chin.vcshift);
+  JXL_DEBUG_V(4,
+              "Undoing horizontal squeeze of channel %i using residuals in "
+              "channel %i (going from width %zu to %zu)",
+              c, rc, chin.w, chout.w);
+  RunOnPool(
+      pool, 0, chin.h, jxl::ThreadPool::SkipInit(),
+      [&](const int task, const int thread) {
+        const size_t y = task;
+        const pixel_type *JXL_RESTRICT p_residual = chin_residual.Row(y);
+        const pixel_type *JXL_RESTRICT p_avg = chin.Row(y);
+        pixel_type *JXL_RESTRICT p_out = chout.Row(y);
+
+        // special case for x=0 so we don't have to check x>0
+        pixel_type avg = p_avg[0];
+        pixel_type next_avg = (1 < chin.w ? next_avg = p_avg[1] : avg);
+        pixel_type tendency = smooth_tendency(avg, avg, next_avg);
+        pixel_type diff = p_residual[0] + tendency;
+        pixel_type A =
+            ((avg * 2) + diff + (diff > 0 ? -(diff & 1) : (diff & 1))) >> 1;
+        pixel_type B = A - diff;
+        p_out[0] = A;
+        p_out[1] = B;
+
+        for (size_t x = 1; x < chin_residual.w; x++) {
+          pixel_type diff_minus_tendency = p_residual[x];
+          pixel_type avg = p_avg[x];
+          pixel_type next_avg =
+              (x + 1 < chin.w ? next_avg = p_avg[x + 1] : avg);
+          pixel_type left = p_out[(x << 1) - 1];
+          pixel_type tendency = smooth_tendency(left, avg, next_avg);
+          pixel_type diff = diff_minus_tendency + tendency;
+          pixel_type A =
+              ((avg * 2) + diff + (diff > 0 ? -(diff & 1) : (diff & 1))) >> 1;
+          p_out[x << 1] = A;
+          pixel_type B = A - diff;
+          p_out[(x << 1) + 1] = B;
+        }
+        if (chout.w & 1) p_out[chout.w - 1] = p_avg[chin.w - 1];
+      },
+      "InvHorizontalSqueeze");
+  input.channel[c] = std::move(chout);
+}
+
+#ifdef HAS_ENCODER
+void fwd_hsqueeze(Image &input, int c, int rc) {
+  const Channel &chin = input.channel[c];
+
+  JXL_DEBUG_V(4, "Doing horizontal squeeze of channel %i to new channel %i", c,
+              rc);
+
+  Channel chout((chin.w + 1) / 2, chin.h, chin.minval, chin.maxval,
+                chin.hshift + 1, chin.vshift, chin.hcshift + 1, chin.vcshift);
+  Channel chout_residual(chin.w - chout.w, chout.h, chout.minval - chout.maxval,
+                         chout.maxval - chout.minval, chin.hshift + 1,
+                         chin.vshift, chin.hcshift, chin.vcshift);
+
+  for (size_t y = 0; y < chout.h; y++) {
+    const pixel_type *JXL_RESTRICT p_in = chin.Row(y);
+    pixel_type *JXL_RESTRICT p_out = chout.Row(y);
+    pixel_type *JXL_RESTRICT p_res = chout_residual.Row(y);
+    for (size_t x = 0; x < chout_residual.w; x++) {
+      pixel_type A = p_in[x * 2];
+      pixel_type B = p_in[x * 2 + 1];
+      pixel_type avg = (A + B + (A > B)) >> 1;
+      p_out[x] = avg;
+
+      pixel_type diff = A - B;
+
+      pixel_type next_avg = avg;
+      if (x + 1 < chout_residual.w)
+        next_avg = (p_in[x * 2 + 2] + p_in[x * 2 + 3] +
+                    (p_in[x * 2 + 2] > p_in[x * 2 + 3])) >>
+                   1;  // which will be chout.value(y,x+1)
+      else if (chin.w & 1)
+        next_avg = p_in[x * 2 + 2];
+      pixel_type left = (x > 0 ? p_in[x * 2 - 1] : avg);
+      pixel_type tendency = smooth_tendency(left, avg, next_avg);
+
+      p_res[x] = diff - tendency;
+    }
+    if (chin.w & 1) {
+      int x = chout.w - 1;
+      p_out[x] = p_in[x * 2];
+    }
+  }
+  input.channel[c] = std::move(chout);
+  input.channel.insert(input.channel.begin() + rc, std::move(chout_residual));
+}
+#endif
+
+void inv_vsqueeze(Image &input, int c, int rc, jxl::ThreadPool *pool) {
+  const Channel &chin = input.channel[c];
+  const Channel &chin_residual = input.channel[rc];
+  if (chin_residual.h == 0) return;
+  Channel chout(chin.w, chin.h + chin_residual.h, chin.minval, chin.maxval,
+                chin.hshift, chin.vshift - 1, chin.hcshift, chin.vcshift - 1);
+  JXL_DEBUG_V(
+      4,
+      "Undoing vertical squeeze of channel %i using residuals in channel "
+      "%i (going from height %zu to %zu)",
+      c, rc, chin.h, chout.h);
+
+  intptr_t onerow_in = chin.plane.PixelsPerRow();
+  intptr_t onerow_out = chout.plane.PixelsPerRow();
+  constexpr int kColsPerThread = 64;
+  RunOnPool(
+      pool, 0, jxl::DivCeil(chin.w, kColsPerThread),
+      jxl::ThreadPool::SkipInit(),
+      [&](const int task, const int thread) {
+        const size_t x0 = task * kColsPerThread;
+        const size_t x1 = std::min((size_t)(task + 1) * kColsPerThread, chin.w);
+        for (size_t y = 0; y < chin_residual.h; y++) {
+          const pixel_type *JXL_RESTRICT p_residual = chin_residual.Row(y);
+          const pixel_type *JXL_RESTRICT p_avg = chin.Row(y);
+          pixel_type *JXL_RESTRICT p_out = chout.Row(y << 1);
+          for (size_t x = x0; x < x1; x++) {
+            pixel_type diff_minus_tendency = p_residual[x];
+            pixel_type avg = p_avg[x];
+
+            pixel_type next_avg = avg;
+            if (y + 1 < chin.h) next_avg = p_avg[x + onerow_in];
+            pixel_type top =
+                (y > 0 ? p_out[static_cast<ssize_t>(x) - onerow_out] : avg);
+            pixel_type tendency = smooth_tendency(top, avg, next_avg);
+            pixel_type diff = diff_minus_tendency + tendency;
+
+            p_out[x] =
+                ((avg * 2) + diff + (diff > 0 ? -(diff & 1) : (diff & 1))) >> 1;
+            p_out[x + onerow_out] = p_out[x] - diff;
+          }
+        }
+      },
+      "InvVertSqueeze");
+
+  if (chout.h & 1) {
+    size_t y = chin.h - 1;
+    const pixel_type *p_avg = chin.Row(y);
+    pixel_type *p_out = chout.Row(y << 1);
+    for (size_t x = 0; x < chin.w; x++) {
+      p_out[x] = p_avg[x];
+    }
+  }
+  input.channel[c] = std::move(chout);
+}
+
+#ifdef HAS_ENCODER
+void fwd_vsqueeze(Image &input, int c, int rc) {
+  const Channel &chin = input.channel[c];
+
+  JXL_DEBUG_V(4, "Doing vertical squeeze of channel %i to new channel %i", c,
+              rc);
+
+  Channel chout(chin.w, (chin.h + 1) / 2, chin.minval, chin.maxval, chin.hshift,
+                chin.vshift + 1, chin.hcshift, chin.vcshift + 1);
+  Channel chout_residual(chin.w, chin.h - chout.h, chout.minval - chout.maxval,
+                         chout.maxval - chout.minval, chin.hshift,
+                         chin.vshift + 1, chin.hcshift, chin.vcshift);
+  intptr_t onerow_in = chin.plane.PixelsPerRow();
+  for (size_t y = 0; y < chout_residual.h; y++) {
+    const pixel_type *JXL_RESTRICT p_in = chin.Row(y * 2);
+    pixel_type *JXL_RESTRICT p_out = chout.Row(y);
+    pixel_type *JXL_RESTRICT p_res = chout_residual.Row(y);
+    for (size_t x = 0; x < chout.w; x++) {
+      pixel_type A = p_in[x];
+      pixel_type B = p_in[x + onerow_in];
+      pixel_type avg = (A + B + (A > B)) >> 1;
+      p_out[x] = avg;
+
+      pixel_type diff = A - B;
+
+      pixel_type next_avg = avg;
+      if (y + 1 < chout_residual.h) {
+        next_avg = (p_in[x + 2 * onerow_in] + p_in[x + 3 * onerow_in] +
+                    (p_in[x + 2 * onerow_in] > p_in[x + 3 * onerow_in])) >>
+                   1;  // which will be chout.value(y+1,x)
+      } else if (chin.h & 1) {
+        next_avg = p_in[x + 2 * onerow_in];
+      }
+      pixel_type top =
+          (y > 0 ? p_in[static_cast<ssize_t>(x) - onerow_in] : avg);
+      pixel_type tendency = smooth_tendency(top, avg, next_avg);
+
+      p_res[x] = diff - tendency;
+    }
+  }
+  if (chin.h & 1) {
+    size_t y = chout.h - 1;
+    const pixel_type *p_in = chin.Row(y * 2);
+    pixel_type *p_out = chout.Row(y);
+    for (size_t x = 0; x < chout.w; x++) {
+      p_out[x] = p_in[x];
+    }
+  }
+  input.channel[c] = std::move(chout);
+  input.channel.insert(input.channel.begin() + rc, std::move(chout_residual));
+}
+#endif
+
+void default_squeeze_parameters(std::vector<int> &parameters,
+                                const Image &image) {
+  int nb_channels = image.nb_channels;
+  // maybe other transforms have been applied before, but let's assume the first
+  // nb_channels channels still contain the 'main' data
+
+  parameters.clear();
+  size_t w = image.channel[image.nb_meta_channels].w;
+  size_t h = image.channel[image.nb_meta_channels].h;
+  JXL_DEBUG_V(7, "Default squeeze parameters for %zux%zu image: ", w, h);
+
+  bool wide =
+      (w >
+       h);  // do horizontal first on wide images; vertical first on tall images
+
+  if (nb_channels > 2 && image.channel[image.nb_meta_channels + 1].w == w &&
+      image.channel[image.nb_meta_channels + 1].h == h) {
+    // assume channels 1 and 2 are chroma, and can be squeezed first for 4:2:0
+    // previews
+    JXL_DEBUG_V(7, "(4:2:0 chroma), %zux%zu image", w, h);
+    //        if (!wide) {
+    //        parameters.push_back(0+2); // vertical chroma squeeze
+    //        parameters.push_back(image.nb_meta_channels+1);
+    //        parameters.push_back(image.nb_meta_channels+2);
+    //        }
+    parameters.push_back(1 + 2);  // horizontal chroma squeeze
+    parameters.push_back(image.nb_meta_channels + 1);
+    parameters.push_back(image.nb_meta_channels + 2);
+    //        if (wide) {
+    parameters.push_back(0 + 2);  // vertical chroma squeeze
+    parameters.push_back(image.nb_meta_channels + 1);
+    parameters.push_back(image.nb_meta_channels + 2);
+    //        }
+  }
+
+  if (!wide) {
+    if (h > MAX_FIRST_PREVIEW_SIZE) {
+      parameters.push_back(0);  // vertical squeeze
+      parameters.push_back(image.nb_meta_channels);
+      parameters.push_back(image.nb_meta_channels + nb_channels - 1);
+      h = (h + 1) / 2;
+      JXL_DEBUG_V(7, "Vertical (%zux%zu), ", w, h);
+    }
+  }
+  while (w > MAX_FIRST_PREVIEW_SIZE || h > MAX_FIRST_PREVIEW_SIZE) {
+    if (w > MAX_FIRST_PREVIEW_SIZE) {
+      parameters.push_back(1);  // horizontal squeeze
+      parameters.push_back(image.nb_meta_channels);
+      parameters.push_back(image.nb_meta_channels + nb_channels - 1);
+      w = (w + 1) / 2;
+      JXL_DEBUG_V(7, "Horizontal (%zux%zu), ", w, h);
+    }
+    if (h > MAX_FIRST_PREVIEW_SIZE) {
+      parameters.push_back(0);  // vertical squeeze
+      parameters.push_back(image.nb_meta_channels);
+      parameters.push_back(image.nb_meta_channels + nb_channels - 1);
+      h = (h + 1) / 2;
+      JXL_DEBUG_V(7, "Vertical (%zux%zu), ", w, h);
+    }
+  }
+  JXL_DEBUG_V(7, "that's it");
+}
+
+void meta_squeeze(Image &image, std::vector<int> &parameters) {
+  if (!parameters.size()) default_squeeze_parameters(parameters, image);
+
+  for (size_t i = 0; i + 2 < parameters.size(); i += 3) {
+    bool horizontal = parameters[i] & 1;  // 0=vertical, 1=horizontal
+    bool in_place = !(parameters[i] & 2);
+    int beginc = parameters[i + 1];
+    int endc = parameters[i + 2];
+    int offset;
+    if (in_place)
+      offset = endc + 1;
+    else
+      offset = image.nb_meta_channels + image.nb_channels;
+    for (int c = beginc; c <= endc; c++) {
+      Channel dummy;
+      dummy.hcshift = image.channel[c].hcshift;
+      dummy.vcshift = image.channel[c].vcshift;
+      if (horizontal) {
+        int w = image.channel[c].w;
+        image.channel[c].w = (w + 1) / 2;
+        image.channel[c].hshift++;
+        image.channel[c].hcshift++;
+        dummy.w = w - (w + 1) / 2;
+        dummy.h = image.channel[c].h;
+      } else {
+        int h = image.channel[c].h;
+        image.channel[c].h = (h + 1) / 2;
+        image.channel[c].vshift++;
+        image.channel[c].vcshift++;
+        dummy.h = h - (h + 1) / 2;
+        dummy.w = image.channel[c].w;
+      }
+      dummy.hshift = image.channel[c].hshift;
+      dummy.vshift = image.channel[c].vshift;
+
+      image.channel.insert(image.channel.begin() + offset + c - beginc,
+                           std::move(dummy));
+    }
+  }
+}
+
+// [squeezetype] [beginc] [endc]
+bool squeeze(Image &input, bool inverse, std::vector<int> &parameters,
+             jxl::ThreadPool *pool) {
+  std::vector<int> adj_params =
+      parameters;  // use a copy so empty (default) parameters remain empty
+  if (!adj_params.size()) default_squeeze_parameters(adj_params, input);
+
+  if (inverse) {
+    for (int i = adj_params.size() - 3; i >= 0; i -= 3) {
+      bool horizontal = adj_params[i] & 1;  // 0=vertical, 1=horizontal
+      bool in_place = !(adj_params[i] & 2);
+      int beginc = adj_params[i + 1];
+      int endc = adj_params[i + 2];
+      if (endc >= static_cast<int>(input.channel.size())) {
+        JXL_WARNING(
+            "Invalid parameters for squeeze transform: channel %i does not "
+            "exist",
+            endc);
+      }
+      int offset;
+      if (in_place)
+        offset = endc + 1;
+      else
+        offset = input.nb_meta_channels + input.nb_channels;
+      for (int c = beginc; c <= endc; c++) {
+        if (input.channel[offset + c - beginc].is_empty()) {
+          // stop unsqueezing luma; keep unsqueezing chroma channels
+          //                if (input.channel[beginc].w == input.channel[c].w &&
+          //                input.channel[beginc].h == input.channel[c].h)
+          //                continue;
+          input.channel[offset + c - beginc].resize();  // assume all zeroes
+        }
+        if (horizontal)
+          inv_hsqueeze(input, c, offset + c - beginc, pool);
+        else
+          inv_vsqueeze(input, c, offset + c - beginc, pool);
+      }
+      input.channel.erase(input.channel.begin() + offset,
+                          input.channel.begin() + offset + (endc - beginc + 1));
+    }
+  } else {
+#ifdef HAS_ENCODER
+    for (size_t i = 0; i + 2 < adj_params.size(); i += 3) {
+      bool horizontal = adj_params[i] & 1;  // 0=vertical, 1=horizontal
+      bool in_place = !(adj_params[i] & 2);
+      int beginc = adj_params[i + 1];
+      int endc = adj_params[i + 2];
+      if (endc >= static_cast<int>(input.channel.size())) {
+        JXL_WARNING(
+            "Invalid parameters for squeeze transform: channel %i does not "
+            "exist",
+            endc);
+      }
+      int offset;
+      if (in_place)
+        offset = endc + 1;
+      else
+        offset = input.nb_meta_channels + input.nb_channels;
+      for (int c = beginc; c <= endc; c++) {
+        if (horizontal)
+          fwd_hsqueeze(input, c, offset + c - beginc);
+        else
+          fwd_vsqueeze(input, c, offset + c - beginc);
+      }
+    }
+#else
+    return false;
+#endif
+  }
+  return true;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_TRANSFORM_SQUEEZE_H_
diff --git a/jxl/modular/transform/subsample.h b/jxl/modular/transform/subsample.h
new file mode 100644
index 000000000..0ad0f3f5b
--- /dev/null
+++ b/jxl/modular/transform/subsample.h
@@ -0,0 +1,191 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_TRANSFORM_SUBSAMPLE_H_
+#define JXL_MODULAR_TRANSFORM_SUBSAMPLE_H_
+
+#include "jxl/base/status.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+// JPEG-style (chroma) subsampling. Parameters are: [begin_channel],
+// [end_channel], [sample_ratio_h], [sample_ratio_v], ... e.g. 1, 2, 2
+// corresponds to 4:2:0
+
+void check_subsample_parameters(std::vector<int>& parameters) {
+  if (parameters.size() == 0) {
+    parameters.push_back(0);
+  }
+  if (parameters.size() == 1) {
+    // special case: abbreviated parameters for some common cases
+    switch (parameters[0]) {
+      case 0:  // 4:2:0
+        parameters[0] = 1;
+        parameters.push_back(2);
+        parameters.push_back(1);
+        parameters.push_back(1);
+        break;
+      case 1:  // 4:2:2
+        parameters[0] = 1;
+        parameters.push_back(2);
+        parameters.push_back(1);
+        parameters.push_back(0);
+        break;
+      case 2:  // 4:4:0
+        parameters[0] = 1;
+        parameters.push_back(2);
+        parameters.push_back(0);
+        parameters.push_back(1);
+        break;
+      case 3:  // 4:1:1
+        parameters[0] = 1;
+        parameters.push_back(2);
+        parameters.push_back(2);
+        parameters.push_back(0);
+        break;
+      default:
+        break;
+    }
+  }
+  if (parameters.size() % 4) {
+    JXL_NOTIFY_ERROR("Error: invalid parameters for subsampling.\n");
+    parameters.clear();
+  }
+}
+
+bool inv_subsample(Image& input, std::vector<int> parameters) {
+  check_subsample_parameters(parameters);
+
+  for (size_t i = 0; i < parameters.size(); i += 4) {
+    int c1 = parameters[i + 0];
+    int c2 = parameters[i + 1];
+    int tsrh = parameters[i + 2];
+    int tsrv = parameters[i + 3];
+    while (tsrh || tsrv) {
+      int srh = 1, srv = 1;
+      if (tsrh > 0) {
+        srh = 2;
+        tsrh--;
+      }
+      if (tsrv > 0) {
+        srv = 2;
+        tsrv--;
+      }
+      for (int c = c1; c <= c2; c++) {
+        size_t ow = input.channel[c].w;
+        size_t oh = input.channel[c].h;
+        if (ow >= input.channel[input.nb_meta_channels].w &&
+            oh >= input.channel[input.nb_meta_channels].h) {
+          // this can happen in case of LQIP and 1:16 scale decodes
+          JXL_DEBUG_V(
+              5,
+              "Skipping upscaling of channel %d because it is already as "
+              "large as channel %zu.",
+              c, input.nb_meta_channels);
+          continue;
+        }
+        Channel channel(ow * srh, oh * srv, input.channel[c].minval,
+                        input.channel[c].maxval);
+
+        // 'fancy' horizontal upscale
+        if (srh == 2) {
+          for (size_t y = 0; y < oh; y++) {
+            const pixel_type* JXL_RESTRICT in_p = input.channel[c].Row(y);
+            pixel_type* JXL_RESTRICT out_p = channel.Row(y * srv);
+            for (size_t x = 0; x < ow; x++) {
+              out_p[x * srh] = (3 * in_p[x] + in_p[x ? x - 1 : 0] + 1) >> 2;
+              out_p[x * srh + 1] =
+                  (3 * in_p[x] + in_p[x + 1 < ow ? x + 1 : x] + 2) >> 2;
+            }
+          }
+        } else {
+          for (size_t y = 0; y < oh; y++) {
+            const pixel_type* JXL_RESTRICT in_p = input.channel[c].Row(y);
+            pixel_type* JXL_RESTRICT out_p = channel.Row(y * srv);
+            for (size_t x = 0; x < ow; x++) {
+              out_p[x] = in_p[x];
+            }
+          }
+        }
+        if (srv == 2) {
+          Channel nchannel(ow * srh, oh * srv, input.channel[c].minval,
+                           input.channel[c].maxval);
+          intptr_t onerow =
+              channel.plane
+                  .PixelsPerRow();  // is equal for channel and nchannel since
+                                    // they have the same width
+          for (size_t y = 0; y < oh; y++) {
+            const pixel_type* JXL_RESTRICT in_p = channel.Row(y * srv);
+            pixel_type* JXL_RESTRICT out_p = nchannel.Row(y * srv);
+            for (size_t x = 0; x < ow * srh; x++) {
+              out_p[x] =
+                  (3 * in_p[x] +
+                   in_p[y ? static_cast<ssize_t>(x) - onerow * srv : x] + 1) >>
+                  2;
+              out_p[x + onerow] =
+                  (3 * in_p[x] + in_p[y + 1 < oh ? x + onerow * srv : x] + 2) >>
+                  2;
+            }
+          }
+          channel = std::move(nchannel);
+        }
+
+        JXL_DEBUG_V(5, "Upscaled channel %i from %zux%zu to %zux%zu", c,
+                    input.channel[c].w, input.channel[c].h, channel.w,
+                    channel.h);
+        input.channel[c] = std::move(channel);
+      }
+    }
+  }
+  return true;
+}
+
+bool fwd_subsample(Image& input, const std::vector<int>& parameters) {
+  return false;  // TODO (not really needed though; subsampling is useful if the
+                 // input data is a JPEG or YUV, but then the transform is
+                 // already done) for non-subsampled input it's probably better
+                 // to just stick to 4:4:4 (and quantize most of the chroma
+                 // details away)
+}
+
+void meta_subsample(Image& input, std::vector<int> parameters) {
+  check_subsample_parameters(parameters);
+  if (parameters.size())
+    for (size_t i = 0; i < parameters.size(); i += 4) {
+      int c1 = parameters[i + 0];
+      int c2 = parameters[i + 1];
+      int srh = parameters[i + 2];
+      int srv = parameters[i + 3];
+      for (int c = c1; c <= c2; c++) {
+        input.channel[c].w += (1 << srh) - 1;
+        input.channel[c].w <<= srh;
+        input.channel[c].h += (1 << srv) - 1;
+        input.channel[c].h <<= srv;
+        input.channel[c].hshift += srh;
+        input.channel[c].vshift += srv;
+      }
+    }
+}
+
+bool subsample(Image& input, bool inverse, const std::vector<int>& parameters) {
+  if (inverse)
+    return inv_subsample(input, parameters);
+  else
+    return fwd_subsample(input, parameters);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_TRANSFORM_SUBSAMPLE_H_
diff --git a/jxl/modular/transform/subtractgreen.h b/jxl/modular/transform/subtractgreen.h
new file mode 100644
index 000000000..546bd1859
--- /dev/null
+++ b/jxl/modular/transform/subtractgreen.h
@@ -0,0 +1,145 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_TRANSFORM_SUBTRACTGREEN_H_
+#define JXL_MODULAR_TRANSFORM_SUBTRACTGREEN_H_
+
+#include "jxl/base/status.h"
+#include "jxl/modular/config.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+bool inv_subtract_green(Image& input, const std::vector<int>& parameters) {
+  size_t m = input.nb_meta_channels;
+  int nb_channels = input.nb_channels;
+  if (nb_channels < 3) {
+    return JXL_FAILURE(
+        "Invalid number of channels to apply inverse subtract_green.");
+  }
+  size_t w = input.channel[m + 0].w;
+  size_t h = input.channel[m + 0].h;
+  if (input.channel[m + 1].w < w || input.channel[m + 1].h < h ||
+      input.channel[m + 2].w < w || input.channel[m + 2].h < h) {
+    return JXL_FAILURE(
+        "Invalid channel dimensions to apply inverse subtract_green (maybe "
+        "chroma is subsampled?).");
+  }
+  // Permutation: 0=RGB, 1=GBR, 2=BRG, 3=RBG, 4=GRB, 5=BGR
+  int permutation = 1;
+  // Second: 0=nop, 1=SubtractFirst, 2=SubtractAvgFirstThird
+  int second = 1;
+  // Third: 0=nop, 1=SubtractFirst
+  int third = 1;
+  if (parameters.size() == 1) {
+    int custom = parameters[0];
+    permutation = custom / 6;
+    second = (custom % 6) >> 1;
+    third = (custom & 1);
+  }
+  for (size_t y = 0; y < h; y++) {
+    const pixel_type* in0 = input.channel[m].Row(y);
+    const pixel_type* in1 = input.channel[m + 1].Row(y);
+    const pixel_type* in2 = input.channel[m + 2].Row(y);
+    pixel_type* out0 = input.channel[m + (permutation % 3)].Row(y);
+    pixel_type* out1 =
+        input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y);
+    pixel_type* out2 =
+        input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y);
+    for (size_t x = 0; x < w; x++) {
+      pixel_type First = in0[x];
+      pixel_type Second = in1[x];
+      pixel_type Third = in2[x];
+      if (third) Third = Third + First;
+      if (second == 1) {
+        Second = Second + First;
+      } else if (second == 2) {
+        Second = Second + ((First + Third) >> 1);
+      }
+      out0[x] = First;
+      out1[x] = Second;
+      out2[x] = Third;
+    }
+  }
+  return true;
+}
+
+#ifdef HAS_ENCODER
+bool fwd_subtract_green(Image& input, const std::vector<int>& parameters) {
+  size_t nb_channels = input.nb_channels;
+  if (nb_channels < 3) {
+    return false;
+  }
+  // Permutation: 0=RGB, 1=GBR, 2=BRG, 3=RBG, 4=GRB, 5=BGR
+  int permutation = 1;
+  // Second: 0=nop, 1=SubtractFirst, 2=SubtractAvgFirstThird
+  int second = 1;
+  // Third: 0=nop, 1=SubtractFirst
+  int third = 1;
+  if (parameters.size() == 1) {
+    int custom = parameters[0];
+    permutation = custom / 6;
+    second = (custom % 6) >> 1;
+    third = (custom & 1);
+  }
+  size_t m = input.nb_meta_channels;
+  size_t w = input.channel[m + 0].w;
+  size_t h = input.channel[m + 0].h;
+  if (input.channel[m + 1].w < w || input.channel[m + 1].h < h ||
+      input.channel[m + 2].w < w || input.channel[m + 2].h < h) {
+    return JXL_FAILURE("Invalid channel dimensions to apply subtract_green.");
+  }
+  for (size_t y = 0; y < h; y++) {
+    const pixel_type* in0 = input.channel[m + (permutation % 3)].Row(y);
+    const pixel_type* in1 =
+        input.channel[m + ((permutation + 1 + permutation / 3) % 3)].Row(y);
+    const pixel_type* in2 =
+        input.channel[m + ((permutation + 2 - permutation / 3) % 3)].Row(y);
+    pixel_type* out0 = input.channel[m].Row(y);
+    pixel_type* out1 = input.channel[m + 1].Row(y);
+    pixel_type* out2 = input.channel[m + 2].Row(y);
+    for (size_t x = 0; x < w; x++) {
+      pixel_type First = in0[x];
+      pixel_type Second = in1[x];
+      pixel_type Third = in2[x];
+      if (second == 1) {
+        Second = Second - First;
+      } else if (second == 2) {
+        Second = Second - ((First + Third) >> 1);
+      }
+      if (third) Third = Third - First;
+      out0[x] = First;
+      out1[x] = Second;
+      out2[x] = Third;
+    }
+  }
+  return true;
+}
+#endif
+
+bool subtract_green(Image& input, bool inverse,
+                    const std::vector<int>& parameters) {
+  if (inverse) return inv_subtract_green(input, parameters);
+#ifdef HAS_ENCODER
+  else
+    return fwd_subtract_green(input, parameters);
+#else
+  else
+    return false;
+#endif
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_TRANSFORM_SUBTRACTGREEN_H_
diff --git a/jxl/modular/transform/transform.cpp b/jxl/modular/transform/transform.cpp
new file mode 100644
index 000000000..37fb7a9a7
--- /dev/null
+++ b/jxl/modular/transform/transform.cpp
@@ -0,0 +1,78 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/modular/transform/transform.h"
+
+#include "jxl/modular/image/image.h"
+
+#include "jxl/modular/transform/near-lossless.h"
+#include "jxl/modular/transform/palette.h"
+#include "jxl/modular/transform/quantize.h"
+#include "jxl/modular/transform/squeeze.h"
+#include "jxl/modular/transform/subsample.h"
+#include "jxl/modular/transform/subtractgreen.h"
+#include "jxl/modular/transform/ycocg.h"
+
+namespace jxl {
+
+const std::vector<std::string> transform_name = {
+    "YCoCg",   "RCT",          "Palette",      "Subsample",
+    "Squeeze", "Quantization", "Near-Lossless"};
+
+bool Transform::apply(Image &input, bool inverse, jxl::ThreadPool *pool) {
+  switch (ID) {
+    case TRANSFORM_ChromaSubsample:
+      return subsample(input, inverse, parameters);
+    case TRANSFORM_QUANTIZE:
+      return quantize(input, inverse, parameters, pool);
+    case TRANSFORM_YCoCg:
+      return YCoCg(input, inverse, pool);
+    case TRANSFORM_RCT:
+      return subtract_green(input, inverse, parameters);
+    case TRANSFORM_SQUEEZE:
+      return squeeze(input, inverse, parameters, pool);
+    case TRANSFORM_PALETTE:
+      return palette(input, inverse, parameters, pool);
+    case TRANSFORM_NEAR_LOSSLESS:
+      return near_lossless(input, inverse, parameters);
+    default:
+      return JXL_FAILURE("Unknown transformation (ID=%i)", ID);
+  }
+}
+
+void Transform::meta_apply(Image &input) {
+  switch (ID) {
+    case TRANSFORM_YCoCg:
+      return;
+    case TRANSFORM_ChromaSubsample:
+      meta_subsample(input, parameters);
+      return;
+    case TRANSFORM_QUANTIZE:
+      meta_quantize(input);
+      return;
+    case TRANSFORM_RCT:
+      return;
+    case TRANSFORM_SQUEEZE:
+      meta_squeeze(input, parameters);
+      return;
+    case TRANSFORM_PALETTE:
+      meta_palette(input, parameters);
+      return;
+    default:
+      JXL_FAILURE("Unknown transformation (ID=%i)", ID);
+      return;
+  }
+}
+
+}  // namespace jxl
\ No newline at end of file
diff --git a/jxl/modular/transform/transform.h b/jxl/modular/transform/transform.h
new file mode 100644
index 000000000..635fbd2be
--- /dev/null
+++ b/jxl/modular/transform/transform.h
@@ -0,0 +1,66 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_TRANSFORM_TRANSFORM_H_
+#define JXL_MODULAR_TRANSFORM_TRANSFORM_H_
+
+#include <string>
+#include <vector>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+// Lossless YCoCg color transformation
+#define TRANSFORM_YCoCg 0
+
+// G, R-G, B-G and variants
+#define TRANSFORM_RCT 1
+
+// Color palette. Parameters are: [begin_c] [end_c] [nb_colors]
+#define TRANSFORM_PALETTE 2
+
+// JPEG-style (chroma) subsampling. Parameters are: [nb_subsampled_channels],
+// [channel], [sample_ratio_h], [sample_ratio_v], ... e.g. 2, 1, 2, 2, 2, 2, 2
+// corresponds to 4:2:0
+#define TRANSFORM_ChromaSubsample 3
+
+// Squeezing (Haar-style)
+#define TRANSFORM_SQUEEZE 4
+
+// JPEG-style quantization. Parameters are quantization factors for each channel
+// (encoded as part of channel metadata).
+#define TRANSFORM_QUANTIZE 5
+
+// this is lossy preprocessing, doesn't have an inverse transform and doesn't
+// exist from the decoder point of view
+#define TRANSFORM_NEAR_LOSSLESS 6
+
+extern const std::vector<std::string> transform_name;
+
+class Transform {
+ public:
+  const int ID;
+  std::vector<int> parameters;
+
+  explicit Transform(int id) : ID(id) {}
+  bool apply(Image &input, bool inverse, jxl::ThreadPool *pool = nullptr);
+  void meta_apply(Image &input);
+  const char *name() const { return transform_name[ID].c_str(); }
+};
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_TRANSFORM_TRANSFORM_H_
diff --git a/jxl/modular/transform/ycocg.h b/jxl/modular/transform/ycocg.h
new file mode 100644
index 000000000..d20eca8b7
--- /dev/null
+++ b/jxl/modular/transform/ycocg.h
@@ -0,0 +1,107 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_TRANSFORM_YCOCG_H_
+#define JXL_MODULAR_TRANSFORM_YCOCG_H_
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/modular/config.h"
+#include "jxl/modular/image/image.h"
+
+namespace jxl {
+
+bool inv_YCoCg(Image& input, jxl::ThreadPool* pool) {
+  size_t m = input.nb_meta_channels;
+  int nb_channels = input.nb_channels;
+  if (nb_channels < 3) {
+    return JXL_FAILURE("Invalid number of channels to apply inverse YCoCg.");
+  }
+  size_t w = input.channel[m + 0].w;
+  size_t h = input.channel[m + 0].h;
+  if (input.channel[m + 1].w < w || input.channel[m + 1].h < h ||
+      input.channel[m + 2].w < w || input.channel[m + 2].h < h) {
+    return JXL_FAILURE(
+        "Invalid channel dimensions to apply inverse YCoCg (maybe chroma is "
+        "subsampled?).\n");
+  }
+  RunOnPool(
+      pool, 0, h, jxl::ThreadPool::SkipInit(),
+      [&](const int task, const int thread) {
+        const size_t y = task;
+        pixel_type* JXL_RESTRICT p0 = input.channel[m].Row(y);
+        pixel_type* JXL_RESTRICT p1 = input.channel[m + 1].Row(y);
+        pixel_type* JXL_RESTRICT p2 = input.channel[m + 2].Row(y);
+        for (size_t x = 0; x < w; x++) {
+          pixel_type Y = p0[x];
+          pixel_type Co = p1[x];
+          pixel_type Cg = p2[x];
+          pixel_type tmp = Y - (Cg >> 1);
+          pixel_type G = Cg + tmp;
+          pixel_type B = tmp - (Co >> 1);
+          pixel_type R = B + Co;
+          p0[x] = R;
+          p1[x] = G;
+          p2[x] = B;
+        }
+      },
+      "InvYCoCg");
+  return true;
+}
+
+#ifdef HAS_ENCODER
+bool fwd_YCoCg(Image& input) {
+  int m = input.nb_meta_channels;
+  int nb_channels = input.nb_channels;
+  if (nb_channels < 3) {
+    // return JXL_FAILURE("Invalid number of channels to apply YCoCg.");
+    return false;
+  }
+  size_t w = input.channel[m + 0].w;
+  size_t h = input.channel[m + 0].h;
+  if (input.channel[m + 1].w < w || input.channel[m + 1].h < h ||
+      input.channel[m + 2].w < w || input.channel[m + 2].h < h) {
+    return JXL_FAILURE("Invalid channel dimensions to apply YCoCg.");
+  }
+  for (size_t y = 0; y < h; y++) {
+    pixel_type* JXL_RESTRICT p0 = input.channel[m].Row(y);
+    pixel_type* JXL_RESTRICT p1 = input.channel[m + 1].Row(y);
+    pixel_type* JXL_RESTRICT p2 = input.channel[m + 2].Row(y);
+    for (size_t x = 0; x < w; x++) {
+      pixel_type R = p0[x];
+      pixel_type G = p1[x];
+      pixel_type B = p2[x];
+      p1[x] = R - B;
+      pixel_type tmp = B + (p1[x] >> 1);
+      p2[x] = G - tmp;
+      p0[x] = tmp + (p2[x] >> 1);
+    }
+  }
+  return true;
+}
+#endif
+
+bool YCoCg(Image& input, bool inverse, jxl::ThreadPool* pool) {
+  if (inverse) return inv_YCoCg(input, pool);
+#ifdef HAS_ENCODER
+  else
+    return fwd_YCoCg(input);
+#else
+  return false;
+#endif
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_TRANSFORM_YCOCG_H_
diff --git a/jxl/modular/util.h b/jxl/modular/util.h
new file mode 100644
index 000000000..708c04859
--- /dev/null
+++ b/jxl/modular/util.h
@@ -0,0 +1,43 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MODULAR_UTIL_H_
+#define JXL_MODULAR_UTIL_H_
+
+namespace jxl {
+
+#define CLAMP(x, l, u) (((x) < (l) ? (l) : ((x) > (u) ? (u) : (x))))
+#define MIN(A, B) ((A) < (B) ? (A) : (B))
+#define MAX(A, B) ((A) > (B) ? (A) : (B))
+
+template <typename I>
+I inline median3(I a, I b, I c) {
+  if (a < b) {
+    if (b < c) {
+      return b;
+    } else {
+      return a < c ? c : a;
+    }
+  } else {
+    if (a < c) {
+      return a;
+    } else {
+      return b < c ? c : b;
+    }
+  }
+}
+
+}  // namespace jxl
+
+#endif  // JXL_MODULAR_UTIL_H_
diff --git a/jxl/multiframe.cc b/jxl/multiframe.cc
new file mode 100644
index 000000000..ec69bcd33
--- /dev/null
+++ b/jxl/multiframe.cc
@@ -0,0 +1,131 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/multiframe.h"
+
+#include <string.h>
+
+#include <algorithm>
+#include <memory>
+
+#include "jxl/common.h"
+#include "jxl/image.h"
+
+namespace jxl {
+
+bool Multiframe::SuperblockIsSalient(size_t row_start, size_t col_start,
+                                     size_t num_rows, size_t num_cols) const {
+  if (saliency_map_ == nullptr || saliency_map_->xsize() == 0 ||
+      saliency_threshold_ == 0.0) {
+    // If we do not have a saliency-map, or the threshold says to include
+    // every block, we straightaway classify the superblock as 'salient'.
+    return true;
+  }
+  const size_t row_end = std::min(saliency_map_->ysize(), row_start + num_rows);
+  const size_t col_end = std::min(saliency_map_->xsize(), col_start + num_cols);
+  for (size_t num_row = row_start; num_row < row_end; num_row++) {
+    const float* JXL_RESTRICT map_row = saliency_map_->ConstRow(num_row);
+    for (size_t num_col = col_start; num_col < col_end; num_col++) {
+      if (map_row[num_col] >= saliency_threshold_) {
+        // One of the blocks covered by this superblock is above the saliency
+        // threshold.
+        return true;
+      }
+    }
+  }
+  // We did not see any block above the saliency threshold.
+  return false;
+}
+
+void Multiframe::SplitACCoefficients(
+    const ac_qcoeff_t* JXL_RESTRICT block, size_t size, const AcStrategy& acs,
+    size_t bx, size_t by, size_t offset,
+    ac_qcoeff_t* JXL_RESTRICT output[kMaxNumPasses][3]) {
+  // Early quit for the simple case of only one pass.
+  if (mode_.num_passes == 1) {
+    for (size_t c = 0; c < 3; c++) {
+      memcpy(output[0][c] + offset, block + c * size,
+             sizeof(ac_qcoeff_t) * size);
+    }
+    return;
+  }
+  size_t ncoeffs_all_done_from_earlier_passes = 1;
+  size_t previous_pass_salient_only = false;
+
+  float previous_pass_quant_step = 1.0f;
+  float previous_pass_inv_quant_step = 1.0f;
+  size_t previous_pass_shift = 0;
+  for (size_t num_pass = 0; num_pass < mode_.num_passes; num_pass++) {  // pass
+    // Zero out output block.
+    for (size_t c = 0; c < 3; c++) {
+      memset(output[num_pass][c] + offset, 0, size * sizeof(ac_qcoeff_t));
+    }
+    const bool current_pass_salient_only = mode_.passes[num_pass].salient_only;
+    const float pass_quant_step = 1 << mode_.passes[num_pass].shift;
+    const float pass_inv_quant_step = 1.0f / pass_quant_step;
+    size_t frame_ncoeffs = mode_.passes[num_pass].num_coefficients;
+    for (size_t c = 0; c < 3; c++) {  // color-channel
+      size_t xsize = acs.covered_blocks_x();
+      size_t ysize = acs.covered_blocks_y();
+      CoefficientLayout(&ysize, &xsize);
+      if (current_pass_salient_only || previous_pass_salient_only) {
+        // Current or previous pass is salient-only.
+        const bool superblock_is_salient =
+            SuperblockIsSalient(by, bx, ysize, xsize);
+        if (current_pass_salient_only != superblock_is_salient) {
+          // Current pass is salient-only, but block is not salient,
+          // OR last pass was salient-only, and block is salient
+          // (hence was already included in last pass).
+          continue;
+        }
+      }
+      for (size_t y = 0; y < ysize * frame_ncoeffs; y++) {    // superblk-y
+        for (size_t x = 0; x < xsize * frame_ncoeffs; x++) {  // superblk-x
+          size_t pos = y * xsize * kBlockDim + x;
+          if (x < xsize * ncoeffs_all_done_from_earlier_passes &&
+              y < ysize * ncoeffs_all_done_from_earlier_passes) {
+            // This coefficient was already included in an earlier pass,
+            // which included a genuinely smaller set of coefficients
+            // (= is not about saliency-splitting).
+            continue;
+          }
+          ac_qcoeff_t v = block[c * size + pos];
+          // Previous pass discarded some bits: do not encode them again.
+          if (previous_pass_shift != 0) {
+            ac_qcoeff_t previous_v =
+                static_cast<int32_t>(v * previous_pass_inv_quant_step) *
+                previous_pass_quant_step;
+            v -= previous_v;
+          }
+          output[num_pass][c][offset + pos] =
+              static_cast<int32_t>(v * pass_inv_quant_step);
+        }  // superblk-x
+      }    // superblk-y
+    }      // color-channel
+    if (!current_pass_salient_only) {
+      // We just finished a non-salient pass.
+      // Hence, we are now guaranteed to have included all coeffs up to
+      // frame_ncoeffs in every block, unless the current pass is shifted.
+      if (mode_.passes[num_pass].shift == 0) {
+        ncoeffs_all_done_from_earlier_passes = frame_ncoeffs;
+      }
+    }
+    previous_pass_salient_only = current_pass_salient_only;
+    previous_pass_inv_quant_step = pass_inv_quant_step;
+    previous_pass_quant_step = pass_quant_step;
+    previous_pass_shift = mode_.passes[num_pass].shift;
+  }  // num_pass
+}
+
+}  // namespace jxl
diff --git a/jxl/multiframe.h b/jxl/multiframe.h
new file mode 100644
index 000000000..2f9573246
--- /dev/null
+++ b/jxl/multiframe.h
@@ -0,0 +1,242 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_MULTIFRAME_H_
+#define JXL_MULTIFRAME_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <limits>
+#include <memory>
+#include <vector>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/common.h"
+#include "jxl/dct_util.h"
+#include "jxl/dot_dictionary.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_ops.h"
+#include "jxl/patch_dictionary.h"
+#include "jxl/splines.h"
+
+// A multiframe handler/manager to encode single images. It will run heuristics
+// for quantization, AC strategy and color correlation map only the first time
+// we want to encode a lossy pass, and will then re-use the existing heuristics
+// for further passes. All the passes of a single image are added together.
+
+namespace jxl {
+
+constexpr size_t kNoDownsamplingFactor = std::numeric_limits<size_t>::max();
+
+struct PassDefinition {
+  // Side of the square of the coefficients that should be kept in each 8x8
+  // block. Must be greater than 1, and at most 8. Should be in non-decreasing
+  // order.
+  size_t num_coefficients;
+
+  // How much to shift the encoded values by, with rounding.
+  size_t shift;
+
+  // Whether or not we should include only salient blocks.
+  // TODO(veluca): ignored for now.
+  bool salient_only;
+
+  // If specified, this indicates that if the requested downsampling factor is
+  // sufficiently high, then it is fine to stop decoding after this pass.
+  // By default, passes are not marked as being suitable for any downsampling.
+  size_t suitable_for_downsampling_of_at_least;
+};
+
+struct ProgressiveMode {
+  size_t num_passes = 1;
+  PassDefinition passes[kMaxNumPasses] = {PassDefinition{
+      /*num_coefficients=*/8, /*shift=*/0, /*salient_only=*/false,
+      /*suitable_for_downsampling_of_at_least=*/1}};
+
+  ProgressiveMode() = default;
+
+  template <size_t nump>
+  explicit ProgressiveMode(const PassDefinition (&p)[nump]) {
+    JXL_ASSERT(nump <= kMaxNumPasses);
+    num_passes = nump;
+    PassDefinition previous_pass{
+        /*num_coefficients=*/1, /*shift=*/0,
+        /*salient_only=*/false,
+        /*suitable_for_downsampling_of_at_least=*/kNoDownsamplingFactor};
+    size_t last_downsampling_factor = kNoDownsamplingFactor;
+    for (size_t i = 0; i < nump; i++) {
+      JXL_ASSERT(p[i].num_coefficients > previous_pass.num_coefficients ||
+                 (p[i].num_coefficients == previous_pass.num_coefficients &&
+                  !p[i].salient_only && previous_pass.salient_only) ||
+                 (p[i].num_coefficients == previous_pass.num_coefficients &&
+                  p[i].shift < previous_pass.shift));
+      JXL_ASSERT(p[i].suitable_for_downsampling_of_at_least ==
+                     kNoDownsamplingFactor ||
+                 p[i].suitable_for_downsampling_of_at_least <=
+                     last_downsampling_factor);
+      if (p[i].suitable_for_downsampling_of_at_least != kNoDownsamplingFactor) {
+        last_downsampling_factor = p[i].suitable_for_downsampling_of_at_least;
+      }
+      previous_pass = passes[i] = p[i];
+    }
+  }
+};
+
+// Multiframe holds information about passes and manages
+// MultiframeHandlers. It is assumed that parallelization goes below the manager
+// level (at group level), so all the methods of Multiframe should be
+// invoked from a single thread.
+class Multiframe {
+ public:
+  // Called at the start of each frame.
+  void StartFrame(const FrameHeader& frame_header) {
+    current_header_ = frame_header;
+  }
+
+  bool NeedsSaving() {
+    return current_header_.dc_level != 0 ||
+           (current_header_.save_as_reference != 0) ||
+           (!current_header_.animation_frame.is_last &&
+            !current_header_.animation_frame.have_crop &&
+            current_header_.animation_frame.new_base() ==
+                AnimationFrame::NewBase::kCurrentFrame);
+  }
+
+  bool NeedsRestoring() {
+    return current_header_.dc_level == 0 &&
+           current_header_.save_as_reference == 0 &&
+           !current_header_.animation_frame.have_crop && has_previous_frame_;
+  }
+
+  Image3F* FrameStorage(size_t xsize, size_t ysize) {
+    if (!NeedsSaving() && !NeedsRestoring()) return nullptr;
+    if (current_header_.dc_level != 0) {
+      dc_storage_[current_header_.dc_level - 1] = Image3F(xsize, ysize);
+      return &dc_storage_[current_header_.dc_level - 1];
+    }
+    if (current_header_.save_as_reference != 0) {
+      reference_frames_[current_header_.save_as_reference - 1] =
+          Image3F(xsize, ysize);
+      return &reference_frames_[current_header_.save_as_reference - 1];
+    }
+    if (!NeedsRestoring()) {
+      frame_storage_ = Image3F(xsize, ysize);
+    }
+    JXL_CHECK(frame_storage_.xsize() == xsize);
+    JXL_CHECK(frame_storage_.ysize() == ysize);
+    return &frame_storage_;
+  }
+
+  const Image3F* SavedDc(size_t level) const { return &dc_storage_[level - 1]; }
+
+  // Called when a frame is done.
+  void SetDecodedFrame() {
+    if (!IsDisplayed()) return;
+    if (!current_header_.animation_frame.is_last &&
+        !current_header_.animation_frame.have_crop) {
+      switch (current_header_.animation_frame.new_base()) {
+        case AnimationFrame::NewBase::kExisting:
+          break;
+        case AnimationFrame::NewBase::kCurrentFrame:
+          has_previous_frame_ = true;
+          break;
+        case AnimationFrame::NewBase::kNone:
+          frame_storage_ = Image3F();
+          has_previous_frame_ = false;
+          break;
+      }
+    } else {
+      if (current_header_.animation_frame.new_base() !=
+          AnimationFrame::NewBase::kExisting) {
+        frame_storage_ = Image3F();
+        has_previous_frame_ = false;
+      }
+    }
+  }
+
+  // Modifies img by subtracting the current reference frame.
+  void DecorrelateOpsin(Image3F* img) {
+    if (NeedsRestoring()) {
+      SubtractFrom(frame_storage_, img);
+    }
+  }
+
+  void SetProgressiveMode(ProgressiveMode mode) { mode_ = mode; }
+
+  void SetSaliencyMap(const ImageF* saliency_map) {
+    saliency_map_ = saliency_map;
+  }
+
+  void SetSaliencyThreshold(float threshold) {
+    saliency_threshold_ = threshold;
+  }
+
+  size_t GetNumPasses() const { return mode_.num_passes; }
+
+  void InitPasses(Passes* JXL_RESTRICT passes) const {
+    passes->num_passes = static_cast<uint32_t>(GetNumPasses());
+    passes->num_downsample = 0;
+    JXL_ASSERT(passes->num_passes != 0);
+    if (passes->num_passes == 1) return;  // Done, arrays are empty
+
+    for (uint32_t i = 0; i < mode_.num_passes - 1; ++i) {
+      const size_t min_downsampling_factor =
+          mode_.passes[i].suitable_for_downsampling_of_at_least;
+      passes->shift[i] = mode_.passes[i].shift;
+      if (1 < min_downsampling_factor &&
+          min_downsampling_factor != kNoDownsamplingFactor) {
+        passes->downsample[passes->num_downsample] = min_downsampling_factor;
+        passes->last_pass[passes->num_downsample] = i;
+        passes->num_downsample += 1;
+      }
+    }
+  }
+
+  void SplitACCoefficients(const ac_qcoeff_t* JXL_RESTRICT block, size_t size,
+                           const AcStrategy& acs, size_t bx, size_t by,
+                           size_t offset,
+                           ac_qcoeff_t* JXL_RESTRICT output[kMaxNumPasses][3]);
+
+  const Image3F* GetReferenceFrames() const { return reference_frames_; }
+
+  bool IsDisplayed() const { return current_header_.IsDisplayed(); }
+
+ private:
+  friend class MultiframeHandler;
+
+  bool SuperblockIsSalient(size_t row_start, size_t col_start, size_t num_rows,
+                           size_t num_cols) const;
+
+  FrameHeader current_header_;
+  ProgressiveMode mode_;
+
+  // Not owned, must remain valid.
+  const ImageF* saliency_map_ = nullptr;
+  float saliency_threshold_ = 0.0;
+
+  Image3F frame_storage_;
+  Image3F dc_storage_[3];
+  Image3F reference_frames_[kMaxNumReferenceFrames];
+  bool has_previous_frame_ = false;
+};
+
+}  // namespace jxl
+
+#endif  // JXL_MULTIFRAME_H_
diff --git a/jxl/noise.cc b/jxl/noise.cc
new file mode 100644
index 000000000..bf9b71760
--- /dev/null
+++ b/jxl/noise.cc
@@ -0,0 +1,627 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/noise.h"
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <hwy/compiler_specific.h>
+#include <hwy/static_targets.h>
+#include <numeric>
+#include <utility>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/robust_statistics.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/convolve.h"
+#include "jxl/image_ops.h"
+#include "jxl/opsin_params.h"
+#include "jxl/optimize.h"
+#include "jxl/xorshift128plus.h"
+
+namespace jxl {
+namespace {
+
+using D = HWY_CAPPED(float, 1);
+
+// Converts one vector's worth of random bits to floats in [0, 1).
+HWY_ATTR void BitsToFloat(const uint32_t* JXL_RESTRICT random_bits,
+                          float* JXL_RESTRICT floats) {
+  const HWY_FULL(float) df;
+  const HWY_FULL(uint32_t) du;
+
+  const auto bits = Load(du, random_bits);
+  // 1.0 + 23 random mantissa bits = [1, 2)
+  const auto rand12 =
+      BitCast(df, hwy::ShiftRight<9>(bits) | Set(du, 0x3F800000));
+  const auto rand01 = rand12 - Set(df, 1.0f);
+  Store(rand01, df, floats);
+}
+
+HWY_ATTR void RandomImage(ImageF* JXL_RESTRICT temp, Xorshift128Plus* rng,
+                          const Rect& rect, const ImageF* JXL_RESTRICT noise) {
+  const size_t xsize = temp->xsize();
+  const size_t ysize = temp->ysize();
+
+  // May exceed the vector size, hence we have two loops over x below.
+  constexpr size_t kFloatsPerBatch =
+      Xorshift128Plus::N * sizeof(uint64_t) / sizeof(float);
+  HWY_ALIGN uint64_t batch[Xorshift128Plus::N];
+
+  const HWY_FULL(float) df;
+
+  for (size_t y = 0; y < ysize; ++y) {
+    float* JXL_RESTRICT row = temp->Row(y);
+
+    size_t x = 0;
+    // Only entire batches (avoids exceeding the image padding).
+    for (; x + kFloatsPerBatch <= xsize; x += kFloatsPerBatch) {
+      rng->Fill(batch);
+      for (size_t i = 0; i < kFloatsPerBatch; i += df.N) {
+        BitsToFloat(reinterpret_cast<const uint32_t*>(batch) + i, row + x + i);
+      }
+    }
+
+    // Any remaining pixels, rounded up to vectors (safe due to padding).
+    rng->Fill(batch);
+    size_t batch_pos = 0;  // < kFloatsPerBatch
+    for (; x < xsize; x += df.N) {
+      BitsToFloat(reinterpret_cast<const uint32_t*>(batch) + batch_pos,
+                  row + x);
+      batch_pos += df.N;
+    }
+  }
+
+  // TODO(veluca): SIMD-fy & avoid intermediate copy.
+  ImageF out(xsize, ysize);
+  slow::Laplacian5<2, WrapMirror>::Run(*temp, Rect(*temp),
+                                       /*unused*/ int(), &out);
+  CopyImageTo(out, rect, noise);
+}
+
+float GetScoreSumsOfAbsoluteDifferences(const Image3F& opsin, const int x,
+                                        const int y, const int block_size) {
+  const int small_bl_size_x = 3;
+  const int small_bl_size_y = 4;
+  const int kNumSAD =
+      (block_size - small_bl_size_x) * (block_size - small_bl_size_y);
+  // block_size x block_size reference pixels
+  int counter = 0;
+  const int offset = 2;
+
+  std::vector<float> sad(kNumSAD, 0);
+  for (int y_bl = 0; y_bl + small_bl_size_y < block_size; ++y_bl) {
+    for (int x_bl = 0; x_bl + small_bl_size_x < block_size; ++x_bl) {
+      float sad_sum = 0;
+      // size of the center patch, we compare all the patches inside window with
+      // the center one
+      for (int cy = 0; cy < small_bl_size_y; ++cy) {
+        for (int cx = 0; cx < small_bl_size_x; ++cx) {
+          float wnd = 0.5f * (opsin.PlaneRow(1, y + y_bl + cy)[x + x_bl + cx] +
+                              opsin.PlaneRow(0, y + y_bl + cy)[x + x_bl + cx]);
+          float center =
+              0.5f * (opsin.PlaneRow(1, y + offset + cy)[x + offset + cx] +
+                      opsin.PlaneRow(0, y + offset + cy)[x + offset + cx]);
+          sad_sum += std::abs(center - wnd);
+        }
+      }
+      sad[counter++] = sad_sum;
+    }
+  }
+  const int kSamples = (kNumSAD) / 2;
+  // As with ROAD (rank order absolute distance), we keep the smallest half of
+  // the values in SAD (we use here the more robust patch SAD instead of
+  // absolute single-pixel differences).
+  std::sort(sad.begin(), sad.end());
+  const float total_sad_sum =
+      std::accumulate(sad.begin(), sad.begin() + kSamples, 0.0f);
+  return total_sad_sum / kSamples;
+}
+
+class NoiseHistogram {
+ public:
+  static constexpr int kBins = 256;
+
+  NoiseHistogram() { std::fill(bins, bins + kBins, 0); }
+
+  void Increment(const float x) { bins[Index(x)] += 1; }
+  int Get(const float x) const { return bins[Index(x)]; }
+  int Bin(const size_t bin) const { return bins[bin]; }
+
+  void Print() const {
+    for (unsigned int bin : bins) {
+      printf("%d\n", bin);
+    }
+  }
+
+  int Mode() const {
+    uint32_t cdf[kBins];
+    std::partial_sum(bins, bins + kBins, cdf);
+    return HalfRangeMode()(cdf, kBins);
+  }
+
+  double Quantile(double q01) const {
+    const int64_t total = std::accumulate(bins, bins + kBins, 1LL);
+    const int64_t target = static_cast<int64_t>(q01 * total);
+    // Until sum >= target:
+    int64_t sum = 0;
+    size_t i = 0;
+    for (; i < kBins; ++i) {
+      sum += bins[i];
+      // Exact match: assume middle of bin i
+      if (sum == target) {
+        return i + 0.5;
+      }
+      if (sum > target) break;
+    }
+
+    // Next non-empty bin (in case histogram is sparsely filled)
+    size_t next = i + 1;
+    while (next < kBins && bins[next] == 0) {
+      ++next;
+    }
+
+    // Linear interpolation according to how far into next we went
+    const double excess = target - sum;
+    const double weight_next = bins[Index(next)] / excess;
+    return ClampX(next * weight_next + i * (1.0 - weight_next));
+  }
+
+  // Inter-quartile range
+  double IQR() const { return Quantile(0.75) - Quantile(0.25); }
+
+ private:
+  template <typename T>
+  T ClampX(const T x) const {
+    return std::min(std::max(T(0), x), T(kBins - 1));
+  }
+  size_t Index(const float x) const { return ClampX(static_cast<int>(x)); }
+
+  uint32_t bins[kBins];
+};
+
+std::vector<float> GetSADScoresForPatches(const Image3F& opsin,
+                                          const size_t block_s,
+                                          const size_t num_bin,
+                                          NoiseHistogram* sad_histogram) {
+  std::vector<float> sad_scores(
+      (opsin.ysize() / block_s) * (opsin.xsize() / block_s), 0.0f);
+
+  int block_index = 0;
+
+  for (size_t y = 0; y + block_s <= opsin.ysize(); y += block_s) {
+    for (size_t x = 0; x + block_s <= opsin.xsize(); x += block_s) {
+      float sad_sc = GetScoreSumsOfAbsoluteDifferences(opsin, x, y, block_s);
+      sad_scores[block_index++] = sad_sc;
+      sad_histogram->Increment(sad_sc * num_bin);
+    }
+  }
+  return sad_scores;
+}
+
+float GetSADThreshold(const NoiseHistogram& histogram, const int num_bin) {
+  // Here we assume that the most patches with similar SAD value is a "flat"
+  // patches. However, some images might contain regular texture part and
+  // generate second strong peak at the histogram
+  // TODO(user) handle bimodal and heavy-tailed case
+  const int mode = histogram.Mode();
+  return static_cast<float>(mode) / NoiseHistogram::kBins;
+}
+
+// [0, max_value]
+template <class D, class V>
+static HWY_ATTR HWY_INLINE V Clamp0ToMax(D d, const V x, const V max_value) {
+  const auto clamped = Min(x, max_value);
+  return ZeroIfNegative(clamped);
+}
+
+// x is in [0+delta, 1+delta], delta ~= 0.06
+template <class StrengthEval>
+HWY_ATTR typename StrengthEval::V NoiseStrength(
+    const StrengthEval& eval, const typename StrengthEval::V x) {
+  return Clamp0ToMax(D(), eval(x), Set(D(), 1.0f));
+}
+
+std::pair<int, float> IndexAndFrac(float x) {
+  // TODO: instead of 1, this should be a proper Y range.
+  constexpr float kScale = (NoiseParams::kNumNoisePoints - 2) / 1;
+  float scaled_x = std::max(0.f, x * kScale);
+  size_t floor_x = static_cast<size_t>(scaled_x);
+  if (JXL_UNLIKELY(floor_x > NoiseParams::kNumNoisePoints - 2)) {
+    floor_x = NoiseParams::kNumNoisePoints - 2;
+  }
+  return std::make_pair(floor_x, scaled_x - floor_x);
+}
+
+// TODO(veluca): SIMD-fy.
+class StrengthEvalLut {
+ public:
+  using V = hwy::VT<D>;
+
+  explicit StrengthEvalLut(const NoiseParams& noise_params)
+      : noise_params_(noise_params) {}
+
+  HWY_ATTR V operator()(const V vx) const {
+    float x;
+    Store(vx, D(), &x);
+    std::pair<int, float> pos = IndexAndFrac(x);
+    JXL_DASSERT(pos.first >= 0 && static_cast<size_t>(pos.first) <
+                                      NoiseParams::kNumNoisePoints - 1);
+    float low = noise_params_.lut[pos.first];
+    float hi = noise_params_.lut[pos.first + 1];
+    return Set(D(), low * (1.0f - pos.second) + hi * pos.second);
+  }
+
+ private:
+  const NoiseParams noise_params_;
+};
+
+template <class D>
+HWY_ATTR void AddNoiseToRGB(const D d, const hwy::VT<D> rnd_noise_r,
+                            const hwy::VT<D> rnd_noise_g,
+                            const hwy::VT<D> rnd_noise_cor,
+                            const hwy::VT<D> noise_strength_g,
+                            const hwy::VT<D> noise_strength_r, float ytox,
+                            float ytob, float* JXL_RESTRICT out_x,
+                            float* JXL_RESTRICT out_y,
+                            float* JXL_RESTRICT out_b) {
+  const auto kRGCorr = Set(d, 0.99f);
+  const auto kRGNCorr = Set(d, 0.01f);
+
+  const auto red_noise = kRGNCorr * rnd_noise_r * noise_strength_r +
+                         kRGCorr * rnd_noise_cor * noise_strength_r;
+  const auto green_noise = kRGNCorr * rnd_noise_g * noise_strength_g +
+                           kRGCorr * rnd_noise_cor * noise_strength_g;
+
+  auto vx = Load(d, out_x);
+  auto vy = Load(d, out_y);
+  auto vb = Load(d, out_b);
+
+  vx += red_noise - green_noise + Set(d, ytox) * (red_noise + green_noise);
+  vy += red_noise + green_noise;
+  vb += Set(d, ytob) * (red_noise + green_noise);
+
+  Store(vx, d, out_x);
+  Store(vy, d, out_y);
+  Store(vb, d, out_b);
+}
+
+}  // namespace
+
+HWY_ATTR void AddNoise(const NoiseParams& noise_params, const Rect& noise_rect,
+                       const Image3F& noise, const Rect& opsin_rect,
+                       const ColorCorrelationMap& cmap, Image3F* opsin) {
+  if (!noise_params.HasAny()) return;
+  const StrengthEvalLut noise_model(noise_params);
+  D d;
+  const auto half = Set(d, 0.5f);
+
+  const size_t xsize = opsin_rect.xsize();
+  const size_t ysize = opsin_rect.ysize();
+
+  // With the prior subtract-random Laplacian approximation, rnd_* ranges were
+  // about [-1.5, 1.6]; Laplacian3 about doubles this to [-3.6, 3.6], so the
+  // normalizer is half of what it was before (0.5).
+  const auto norm_const = Set(d, 0.22f);
+
+  float ytox = cmap.YtoXRatio(kColorOffset);
+  float ytob = cmap.YtoBRatio(kColorOffset);
+
+  for (size_t y = 0; y < ysize; ++y) {
+    float* JXL_RESTRICT row_x = opsin_rect.PlaneRow(opsin, 0, y);
+    float* JXL_RESTRICT row_y = opsin_rect.PlaneRow(opsin, 1, y);
+    float* JXL_RESTRICT row_b = opsin_rect.PlaneRow(opsin, 2, y);
+    const float* JXL_RESTRICT row_rnd_r = noise_rect.PlaneRow(opsin, 0, y);
+    const float* JXL_RESTRICT row_rnd_g = noise_rect.PlaneRow(opsin, 1, y);
+    const float* JXL_RESTRICT row_rnd_c = noise_rect.PlaneRow(opsin, 2, y);
+    for (size_t x = 0; x < xsize; x += d.N) {
+      const auto vx = Load(d, row_x + x);
+      const auto vy = Load(d, row_y + x);
+      const auto in_g = vy - vx;
+      const auto in_r = vy + vx;
+      const auto noise_strength_g = NoiseStrength(noise_model, in_g * half);
+      const auto noise_strength_r = NoiseStrength(noise_model, in_r * half);
+      const auto addit_rnd_noise_red = Load(d, row_rnd_r + x) * norm_const;
+      const auto addit_rnd_noise_green = Load(d, row_rnd_g + x) * norm_const;
+      const auto addit_rnd_noise_correlated =
+          Load(d, row_rnd_c + x) * norm_const;
+      AddNoiseToRGB(D(), addit_rnd_noise_red, addit_rnd_noise_green,
+                    addit_rnd_noise_correlated, noise_strength_g,
+                    noise_strength_r, ytox, ytob, row_x + x, row_y + x,
+                    row_b + x);
+    }
+  }
+}
+
+void RandomImage3(const Rect& rect, Image3F* JXL_RESTRICT noise) {
+  const size_t xsize = rect.xsize();
+  const size_t ysize = rect.ysize();
+
+  Xorshift128Plus rng((uint64_t(rect.y0()) << 32) + rect.x0());
+  ImageF temp(xsize, ysize);
+  RandomImage(&temp, &rng, rect, &noise->Plane(0));
+  RandomImage(&temp, &rng, rect, &noise->Plane(1));
+  RandomImage(&temp, &rng, rect, &noise->Plane(2));
+}
+
+// loss = sum asym * (F(x) - nl)^2 + kReg * num_points * sum (w[i] - w[i+1])^2
+// where asym = 1 if F(x) < nl, kAsym if F(x) > nl.
+struct LossFunction {
+  explicit LossFunction(std::vector<NoiseLevel> nl0) : nl(std::move(nl0)) {}
+
+  double Compute(const std::vector<double>& w, std::vector<double>* df,
+                 bool skip_regularization = false) const {
+    constexpr double kReg = 0.005;
+    constexpr double kAsym = 1.1;
+    double loss_function = 0;
+    for (size_t i = 0; i < w.size(); i++) {
+      (*df)[i] = 0;
+    }
+    for (auto ind : nl) {
+      std::pair<int, float> pos = IndexAndFrac(ind.intensity);
+      JXL_DASSERT(pos.first >= 0 && static_cast<size_t>(pos.first) <
+                                        NoiseParams::kNumNoisePoints - 1);
+      double low = w[pos.first];
+      double hi = w[pos.first + 1];
+      double val = low * (1.0f - pos.second) + hi * pos.second;
+      double dist = val - ind.noise_level;
+      if (dist > 0) {
+        loss_function += kAsym * dist * dist;
+        (*df)[pos.first] -= kAsym * (1.0f - pos.second) * dist;
+        (*df)[pos.first + 1] -= kAsym * pos.second * dist;
+      } else {
+        loss_function += dist * dist;
+        (*df)[pos.first] -= (1.0f - pos.second) * dist;
+        (*df)[pos.first + 1] -= pos.second * dist;
+      }
+    }
+    if (skip_regularization) return loss_function;
+    for (size_t i = 0; i + 1 < w.size(); i++) {
+      double diff = w[i] - w[i + 1];
+      loss_function += kReg * nl.size() * diff * diff;
+      (*df)[i] -= kReg * diff * nl.size();
+      (*df)[i + 1] += kReg * diff * nl.size();
+    }
+    return loss_function;
+  }
+
+  std::vector<NoiseLevel> nl;
+};
+
+Status GetNoiseParameter(const Image3F& opsin, NoiseParams* noise_params,
+                         float quality_coef) {
+  // The size of a patch in decoder might be different from encoder's patch
+  // size.
+  // For encoder: the patch size should be big enough to estimate
+  //              noise level, but, at the same time, it should be not too big
+  //              to be able to estimate intensity value of the patch
+  const size_t block_s = 8;
+  const size_t kNumBin = 256;
+  NoiseHistogram sad_histogram;
+  std::vector<float> sad_scores =
+      GetSADScoresForPatches(opsin, block_s, kNumBin, &sad_histogram);
+  float sad_threshold = GetSADThreshold(sad_histogram, kNumBin);
+  // If threshold is too large, the image has a strong pattern. This pattern
+  // fools our model and it will add too much noise. Therefore, we do not add
+  // noise for such images
+  if (sad_threshold > 0.15f || sad_threshold <= 0.0f) {
+    noise_params->Clear();
+    return false;
+  }
+  std::vector<NoiseLevel> nl =
+      GetNoiseLevel(opsin, sad_scores, sad_threshold, block_s);
+
+  OptimizeNoiseParameters(nl, noise_params);
+  for (float& i : noise_params->lut) {
+    i *= quality_coef;
+  }
+  return noise_params->HasAny();
+}
+
+const float kNoisePrecision = 1 << 10;
+
+void EncodeFloatParam(float val, float precision, BitWriter* writer) {
+  JXL_ASSERT(val >= 0);
+  const int absval_quant = static_cast<int>(val * precision + 0.5f);
+  JXL_ASSERT(absval_quant < (1 << 10));
+  writer->Write(10, absval_quant);
+}
+
+HWY_ATTR void DecodeFloatParam(float precision, float* val, BitReader* br) {
+  const int absval_quant = br->ReadFixedBits<10>();
+  *val = absval_quant / precision;
+}
+
+void EncodeNoise(const NoiseParams& noise_params, BitWriter* writer,
+                 size_t layer, AuxOut* aux_out) {
+  JXL_ASSERT(noise_params.HasAny());
+
+  BitWriter::Allotment allotment(writer, NoiseParams::kNumNoisePoints * 16);
+  for (float i : noise_params.lut) {
+    EncodeFloatParam(i, kNoisePrecision, writer);
+  }
+  ReclaimAndCharge(writer, &allotment, layer, aux_out);
+}
+
+Status DecodeNoise(BitReader* br, NoiseParams* noise_params) {
+  for (float& i : noise_params->lut) {
+    DecodeFloatParam(kNoisePrecision, &i, br);
+  }
+  if (!noise_params->HasAny()) {
+    return JXL_FAILURE("DecodedNoise got no noise");
+  }
+  return true;
+}
+
+void OptimizeNoiseParameters(const std::vector<NoiseLevel>& noise_level,
+                             NoiseParams* noise_params) {
+  constexpr double kMaxError = 1e-3;
+  static const double kPrecision = 1e-8;
+  static const int kMaxIter = 40;
+
+  float avg = 0;
+  for (const NoiseLevel& nl : noise_level) {
+    avg += nl.noise_level;
+  }
+  avg /= noise_level.size();
+
+  LossFunction loss_function(noise_level);
+  std::vector<double> parameter_vector(NoiseParams::kNumNoisePoints, avg);
+
+  parameter_vector = optimize::OptimizeWithScaledConjugateGradientMethod(
+      loss_function, parameter_vector, kPrecision, kMaxIter);
+
+  std::vector<double> df = parameter_vector;
+  float loss = loss_function.Compute(parameter_vector, &df,
+                                     /*skip_regularization=*/true) /
+               noise_level.size();
+
+  // Approximation went too badly: escape with no noise at all.
+  if (loss > kMaxError) {
+    noise_params->Clear();
+    return;
+  }
+
+  for (size_t i = 0; i < parameter_vector.size(); i++) {
+    noise_params->lut[i] = std::max(parameter_vector[i], 0.0);
+  }
+}
+
+std::vector<float> GetTextureStrength(const Image3F& opsin,
+                                      const size_t block_s) {
+  std::vector<float> texture_strength_index((opsin.ysize() / block_s) *
+                                            (opsin.xsize() / block_s));
+  size_t block_index = 0;
+
+  for (size_t y = 0; y + block_s <= opsin.ysize(); y += block_s) {
+    for (size_t x = 0; x + block_s <= opsin.xsize(); x += block_s) {
+      float texture_strength = 0;
+      for (size_t y_bl = 0; y_bl < block_s; ++y_bl) {
+        for (size_t x_bl = 0; x_bl + 1 < block_s; ++x_bl) {
+          float diff = opsin.PlaneRow(1, y)[x + x_bl + 1] -
+                       opsin.PlaneRow(1, y)[x + x_bl];
+          texture_strength += diff * diff;
+        }
+      }
+      for (size_t y_bl = 0; y_bl + 1 < block_s; ++y_bl) {
+        for (size_t x_bl = 0; x_bl < block_s; ++x_bl) {
+          float diff = opsin.PlaneRow(1, y + 1)[x + x_bl] -
+                       opsin.PlaneRow(1, y)[x + x_bl];
+          texture_strength += diff * diff;
+        }
+      }
+      texture_strength_index[block_index] = texture_strength;
+      ++block_index;
+    }
+  }
+  return texture_strength_index;
+}
+
+float GetThresholdFlatIndices(const std::vector<float>& texture_strength,
+                              const int n_patches) {
+  std::vector<float> kth_statistic = texture_strength;
+  std::stable_sort(kth_statistic.begin(), kth_statistic.end());
+  return kth_statistic[n_patches];
+}
+
+std::vector<NoiseLevel> GetNoiseLevel(
+    const Image3F& opsin, const std::vector<float>& texture_strength,
+    const float threshold, const size_t block_s) {
+  std::vector<NoiseLevel> noise_level_per_intensity;
+
+  const int filt_size = 1;
+  static const float kLaplFilter[filt_size * 2 + 1][filt_size * 2 + 1] = {
+      {-0.25f, -1.0f, -0.25f},
+      {-1.0f, 5.0f, -1.0f},
+      {-0.25f, -1.0f, -0.25f},
+  };
+
+  // The noise model is built based on channel 0.5 * (X+Y) as we notice that it
+  // is similar to the model 0.5 * (Y-X)
+  size_t patch_index = 0;
+
+  for (size_t y = 0; y + block_s <= opsin.ysize(); y += block_s) {
+    for (size_t x = 0; x + block_s <= opsin.xsize(); x += block_s) {
+      if (texture_strength[patch_index] <= threshold) {
+        // Calculate mean value
+        float mean_int = 0;
+        for (size_t y_bl = 0; y_bl < block_s; ++y_bl) {
+          for (size_t x_bl = 0; x_bl < block_s; ++x_bl) {
+            mean_int += 0.5f * (opsin.PlaneRow(1, y + y_bl)[x + x_bl] +
+                                opsin.PlaneRow(0, y + y_bl)[x + x_bl]);
+          }
+        }
+        mean_int /= block_s * block_s;
+
+        // Calculate Noise level
+        float noise_level = 0;
+        size_t count = 0;
+        for (size_t y_bl = 0; y_bl < block_s; ++y_bl) {
+          for (size_t x_bl = 0; x_bl < block_s; ++x_bl) {
+            float filtered_value = 0;
+            for (int y_f = -1 * filt_size; y_f <= filt_size; ++y_f) {
+              if (((y_bl + y_f) < block_s) && ((y_bl + y_f) >= 0)) {
+                for (int x_f = -1 * filt_size; x_f <= filt_size; ++x_f) {
+                  if ((x_bl + x_f) >= 0 && (x_bl + x_f) < block_s) {
+                    filtered_value +=
+                        0.5f *
+                        (opsin.PlaneRow(1, y + y_bl + y_f)[x + x_bl + x_f] +
+                         opsin.PlaneRow(0, y + y_bl + y_f)[x + x_bl + x_f]) *
+                        kLaplFilter[y_f + filt_size][x_f + filt_size];
+                  } else {
+                    filtered_value +=
+                        0.5f *
+                        (opsin.PlaneRow(1, y + y_bl + y_f)[x + x_bl - x_f] +
+                         opsin.PlaneRow(0, y + y_bl + y_f)[x + x_bl - x_f]) *
+                        kLaplFilter[y_f + filt_size][x_f + filt_size];
+                  }
+                }
+              } else {
+                for (int x_f = -1 * filt_size; x_f <= filt_size; ++x_f) {
+                  if ((x_bl + x_f) >= 0 && (x_bl + x_f) < block_s) {
+                    filtered_value +=
+                        0.5f *
+                        (opsin.PlaneRow(1, y + y_bl - y_f)[x + x_bl + x_f] +
+                         opsin.PlaneRow(0, y + y_bl - y_f)[x + x_bl + x_f]) *
+                        kLaplFilter[y_f + filt_size][x_f + filt_size];
+                  } else {
+                    filtered_value +=
+                        0.5f *
+                        (opsin.PlaneRow(1, y + y_bl - y_f)[x + x_bl - x_f] +
+                         opsin.PlaneRow(0, y + y_bl - y_f)[x + x_bl - x_f]) *
+                        kLaplFilter[y_f + filt_size][x_f + filt_size];
+                  }
+                }
+              }
+            }
+            noise_level += std::abs(filtered_value);
+            ++count;
+          }
+        }
+        noise_level /= count;
+        NoiseLevel nl;
+        nl.intensity = mean_int;
+        nl.noise_level = noise_level;
+        noise_level_per_intensity.push_back(nl);
+      }
+      ++patch_index;
+    }
+  }
+  return noise_level_per_intensity;
+}
+
+}  // namespace jxl
diff --git a/jxl/noise.h b/jxl/noise.h
new file mode 100644
index 000000000..f4744b8c6
--- /dev/null
+++ b/jxl/noise.h
@@ -0,0 +1,89 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_NOISE_H_
+#define JXL_NOISE_H_
+
+// Noise synthesis. Currently disabled.
+
+#include <stddef.h>
+
+#include <vector>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/image.h"
+
+namespace jxl {
+
+struct NoiseParams {
+  // LUT index is an intensity of pixel / mean intensity of patch
+  static constexpr size_t kNumNoisePoints = 8;
+  float lut[kNumNoisePoints];
+
+  void Clear() {
+    for (float& i : lut) i = 0;
+  }
+  bool HasAny() const {
+    for (float i : lut) {
+      if (i != 0) return true;
+    }
+    return false;
+  }
+};
+
+struct NoiseLevel {
+  float noise_level;
+  float intensity;
+};
+
+// Add a noise to Opsin image, loading generated random noise from `noise_rect`
+// in `noise`.
+HWY_ATTR void AddNoise(const NoiseParams& noise_params, const Rect& noise_rect,
+                       const Image3F& noise, const Rect& opsin_rect,
+                       const ColorCorrelationMap& cmap, Image3F* opsin);
+
+void RandomImage3(const Rect& rect, Image3F* JXL_RESTRICT noise);
+
+// Get parameters of the noise for NoiseParams model
+// Returns whether a valid noise model (with HasAny()) is set.
+Status GetNoiseParameter(const Image3F& opsin, NoiseParams* noise_params,
+                         float quality_coef);
+
+// Does not write anything if `noise_params` are empty. Otherwise, caller must
+// set FrameHeader.flags.kNoise.
+void EncodeNoise(const NoiseParams& noise_params, BitWriter* writer,
+                 size_t layer, AuxOut* aux_out);
+
+// Must only call if FrameHeader.flags.kNoise.
+Status DecodeNoise(BitReader* br, NoiseParams* noise_params);
+
+// Texture Strength is defined as tr(A), A = [Gh, Gv]^T[[Gh, Gv]]
+std::vector<float> GetTextureStrength(const Image3F& opsin, size_t block_s);
+
+float GetThresholdFlatIndices(const std::vector<float>& texture_strength,
+                              int n_patches);
+
+std::vector<NoiseLevel> GetNoiseLevel(
+    const Image3F& opsin, const std::vector<float>& texture_strength,
+    float threshold, size_t block_s);
+
+void OptimizeNoiseParameters(const std::vector<NoiseLevel>& noise_level,
+                             NoiseParams* noise_params);
+}  // namespace jxl
+
+#endif  // JXL_NOISE_H_
diff --git a/jxl/noise_distributions.h b/jxl/noise_distributions.h
new file mode 100644
index 000000000..2ee0618ed
--- /dev/null
+++ b/jxl/noise_distributions.h
@@ -0,0 +1,148 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_NOISE_DISTRIBUTIONS_H_
+#define JXL_NOISE_DISTRIBUTIONS_H_
+
+// Noise distributions for testing partial_derivatives and robust_statistics.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <random>  // distributions
+#include <string>
+
+#include "jxl/image.h"
+
+namespace jxl {
+
+// Unmodified input
+struct NoiseNone {
+  std::string Name() const { return "None"; }
+
+  template <class Random>
+  float operator()(const float in, Random* rng) const {
+    return in;
+  }
+};
+
+// Salt+pepper
+class NoiseImpulse {
+ public:
+  explicit NoiseImpulse(const uint32_t threshold) : threshold_(threshold) {}
+  std::string Name() const { return "Impulse" + std::to_string(threshold_); }
+
+  // Sets pixels to 0 if rand < threshold or 1 if rand > ~threshold.
+  template <class Random>
+  float operator()(const float in, Random* rng) const {
+    const uint32_t rand = (*rng)();
+    float out = 0.0f;
+    if (rand > ~threshold_) {
+      out = 1.0f;
+    }
+    if (rand > threshold_) {
+      out = in;
+    }
+    return out;
+  }
+
+ private:
+  const uint32_t threshold_;
+};
+
+class NoiseUniform {
+ public:
+  NoiseUniform(const float min, const float max_exclusive)
+      : dist_(min, max_exclusive) {}
+  std::string Name() const { return "Uniform" + std::to_string(dist_.b()); }
+
+  template <class Random>
+  float operator()(const float in, Random* rng) const {
+    return in + dist_(*rng);
+  }
+
+ private:
+  mutable std::uniform_real_distribution<float> dist_;
+};
+
+// Additive, zero-mean Gaussian.
+class NoiseGaussian {
+ public:
+  explicit NoiseGaussian(const float stddev) : dist_(0.0f, stddev) {}
+  std::string Name() const {
+    return "Gaussian" + std::to_string(dist_.stddev());
+  }
+
+  template <class Random>
+  float operator()(const float in, Random* rng) const {
+    return in + dist_(*rng);
+  }
+
+ private:
+  mutable std::normal_distribution<float> dist_;
+};
+
+// Integer noise is scaled by 1E-3.
+class NoisePoisson {
+ public:
+  explicit NoisePoisson(const double mean) : dist_(mean) {}
+  std::string Name() const { return "Poisson" + std::to_string(dist_.mean()); }
+
+  template <class Random>
+  float operator()(const float in, Random* rng) const {
+    return in + dist_(*rng) * 1E-3f;
+  }
+
+ private:
+  mutable std::poisson_distribution<int> dist_;
+};
+
+// Returns the result of applying the randomized "noise" function to each pixel.
+template <class NoiseType, class Random>
+ImageF AddNoise(const ImageF& in, const NoiseType& noise, Random* rng) {
+  const size_t xsize = in.xsize();
+  const size_t ysize = in.ysize();
+  ImageF out(xsize, ysize);
+  for (size_t y = 0; y < ysize; ++y) {
+    const float* JXL_RESTRICT in_row = in.ConstRow(y);
+    float* JXL_RESTRICT out_row = out.Row(y);
+    for (size_t x = 0; x < xsize; ++x) {
+      out_row[x] = noise(in_row[x], rng);
+    }
+  }
+  return out;
+}
+
+template <class NoiseType, class Random>
+Image3F AddNoise(const Image3F& in, const NoiseType& noise, Random* rng) {
+  const size_t xsize = in.xsize();
+  const size_t ysize = in.ysize();
+  Image3F out(xsize, ysize);
+  // noise_estimator_test requires this loop order.
+  for (size_t c = 0; c < 3; ++c) {
+    for (size_t y = 0; y < ysize; ++y) {
+      const float* JXL_RESTRICT in_row = in.ConstPlaneRow(c, y);
+      float* JXL_RESTRICT out_row = out.PlaneRow(c, y);
+
+      for (size_t x = 0; x < xsize; ++x) {
+        out_row[x] = noise(in_row[x], rng);
+      }
+    }
+  }
+  return out;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_NOISE_DISTRIBUTIONS_H_
diff --git a/jxl/opsin_image_test.cc b/jxl/opsin_image_test.cc
new file mode 100644
index 000000000..438cbb9de
--- /dev/null
+++ b/jxl/opsin_image_test.cc
@@ -0,0 +1,132 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include "gtest/gtest.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/color_management.h"
+#include "jxl/dec_xyb.h"
+#include "jxl/enc_xyb.h"
+#include "jxl/image.h"
+#include "jxl/linalg.h"
+#include "jxl/opsin_params.h"
+
+namespace jxl {
+namespace {
+
+TEST(OpsinImageTest, MaxCubeRootError) { TestCubeRoot(); }
+
+// Convert a single linear sRGB color to xyb, using the exact image conversion
+// procedure that jpeg xl uses.
+void LinearSrgbToOpsin(float rgb_r, float rgb_g, float rgb_b,
+                       float* JXL_RESTRICT xyb_x, float* JXL_RESTRICT xyb_y,
+                       float* JXL_RESTRICT xyb_b) {
+  Image3F linear(1, 1);
+  linear.PlaneRow(0, 0)[0] = rgb_r;
+  linear.PlaneRow(1, 0)[0] = rgb_g;
+  linear.PlaneRow(2, 0)[0] = rgb_b;
+
+  ImageMetadata metadata;
+  metadata.bits_per_sample = 32;
+  metadata.color_encoding = ColorManagement::LinearSRGB();
+  ImageBundle ib(&metadata);
+  ib.SetFromImage(std::move(linear), metadata.color_encoding);
+  Image3F opsin(1, 1);
+  ToXYB(ib, 1.0f, /*pool=*/nullptr, &opsin);
+
+  *xyb_x = opsin.PlaneRow(0, 0)[0];
+  *xyb_y = opsin.PlaneRow(1, 0)[0];
+  *xyb_b = opsin.PlaneRow(2, 0)[0];
+}
+
+// Convert a single XYB color to linear sRGB, using the exact image conversion
+// procedure that jpeg xl uses.
+void OpsinToLinearSrgb(float xyb_x, float xyb_y, float xyb_b,
+                       float* JXL_RESTRICT rgb_r, float* JXL_RESTRICT rgb_g,
+                       float* JXL_RESTRICT rgb_b) {
+  Image3F opsin(1, 1);
+  opsin.PlaneRow(0, 0)[0] = xyb_x;
+  opsin.PlaneRow(1, 0)[0] = xyb_y;
+  opsin.PlaneRow(2, 0)[0] = xyb_b;
+  Image3F linear(1, 1);
+  OpsinParams opsin_params;
+  opsin_params.Init();
+  OpsinToLinear(opsin, Rect(opsin), nullptr, &linear, opsin_params);
+  *rgb_r = linear.PlaneRow(0, 0)[0];
+  *rgb_g = linear.PlaneRow(1, 0)[0];
+  *rgb_b = linear.PlaneRow(2, 0)[0];
+}
+
+void OpsinRoundtripTestRGB(float r, float g, float b) {
+  float xyb_x, xyb_y, xyb_b;
+  LinearSrgbToOpsin(r, g, b, &xyb_x, &xyb_y, &xyb_b);
+  float r2, g2, b2;
+  OpsinToLinearSrgb(xyb_x, xyb_y, xyb_b, &r2, &g2, &b2);
+  EXPECT_NEAR(r, r2, 1e-3);
+  EXPECT_NEAR(g, g2, 1e-3);
+  EXPECT_NEAR(b, b2, 1e-3);
+}
+
+TEST(OpsinImageTest, VerifyOpsinAbsorbanceInverseMatrix) {
+  float matrix[9];  // writable copy
+  for (int i = 0; i < 9; i++) {
+    matrix[i] = GetOpsinAbsorbanceInverseMatrix()[i];
+  }
+  Inv3x3Matrix(matrix);
+  for (int i = 0; i < 9; i++) {
+    EXPECT_NEAR(matrix[i], kOpsinAbsorbanceMatrix[i], 1e-6);
+  }
+}
+
+TEST(OpsinImageTest, OpsinRoundtrip) {
+  OpsinRoundtripTestRGB(0, 0, 0);
+  OpsinRoundtripTestRGB(1, 1, 1);
+  OpsinRoundtripTestRGB(128, 128, 128);
+  OpsinRoundtripTestRGB(255, 255, 255);
+
+  OpsinRoundtripTestRGB(0, 0, 1);
+  OpsinRoundtripTestRGB(0, 0, 128);
+  OpsinRoundtripTestRGB(0, 0, 255);
+
+  OpsinRoundtripTestRGB(0, 1, 0);
+  OpsinRoundtripTestRGB(0, 128, 0);
+  OpsinRoundtripTestRGB(0, 255, 0);
+
+  OpsinRoundtripTestRGB(1, 0, 0);
+  OpsinRoundtripTestRGB(128, 0, 0);
+  OpsinRoundtripTestRGB(255, 0, 0);
+}
+
+TEST(OpsinImageTest, VerifyZero) {
+  // Test that black color (zero energy) is 0,0,0 in xyb.
+  float x, y, b;
+  LinearSrgbToOpsin(0, 0, 0, &x, &y, &b);
+  EXPECT_NEAR(0, x, 1e-9);
+  EXPECT_NEAR(0, y, 1e-7);
+  EXPECT_NEAR(0, b, 1e-7);
+}
+
+TEST(OpsinImageTest, VerifyGray) {
+  // Test that grayscale colors have a fixed y/b ratio and x==0.
+  for (size_t i = 1; i < 255; i++) {
+    float x, y, b;
+    LinearSrgbToOpsin(i, i, i, &x, &y, &b);
+    EXPECT_NEAR(0, x, 1e-6);
+    EXPECT_NEAR(kYToBRatio, b / y, 2e-6);
+  }
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/opsin_inverse_test.cc b/jxl/opsin_inverse_test.cc
new file mode 100644
index 000000000..5ae30260e
--- /dev/null
+++ b/jxl/opsin_inverse_test.cc
@@ -0,0 +1,49 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/dec_xyb.h"
+#include "jxl/enc_xyb.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_test_utils.h"
+
+namespace jxl {
+namespace {
+
+TEST(OpsinInverseTest, LinearInverseInverts) {
+  Image3F linear(128, 128);
+  RandomFillImage(&linear, 255.0f);
+
+  CodecInOut io;
+  io.metadata.bits_per_sample = 32;
+  io.metadata.color_encoding = ColorManagement::LinearSRGB();
+  io.SetFromImage(CopyImage(linear), io.metadata.color_encoding);
+  ThreadPool* null_pool = nullptr;
+  Image3F opsin(io.xsize(), io.ysize());
+  ToXYB(io.Main(), 1.0f, null_pool, &opsin);
+
+  OpsinParams opsin_params;
+  opsin_params.Init();
+  OpsinToLinear(&opsin, /*pool=*/nullptr, opsin_params);
+
+  VerifyRelativeError(linear, opsin, 3E-3, 2E-4);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/opsin_params.cc b/jxl/opsin_params.cc
new file mode 100644
index 000000000..ce0b59af4
--- /dev/null
+++ b/jxl/opsin_params.cc
@@ -0,0 +1,52 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/opsin_params.h"
+
+#include <stdlib.h>
+
+#include "jxl/linalg.h"
+
+namespace jxl {
+
+#define INVERSE_OPSIN_FROM_SPEC 1
+
+const float* GetOpsinAbsorbanceInverseMatrix() {
+#if INVERSE_OPSIN_FROM_SPEC
+  return DefaultInverseOpsinAbsorbanceMatrix();
+#else   // INVERSE_OPSIN_FROM_SPEC
+  // Compute the inverse opsin matrix from the forward matrix. Less precise
+  // than taking the values from the specification, but must be used if the
+  // forward transform is changed and the spec will require updating.
+  static const float* const kInverse = [] {
+    static float inverse[9];
+    for (int i = 0; i < 9; i++) {
+      inverse[i] = kOpsinAbsorbanceMatrix[i];
+    }
+    Inv3x3Matrix(inverse);
+    return inverse;
+  }();
+  return kInverse;
+#endif  // INVERSE_OPSIN_FROM_SPEC
+}
+
+void InitSIMDInverseMatrix(const float* JXL_RESTRICT inverse,
+                           float* JXL_RESTRICT simd_inverse) {
+  for (size_t i = 0; i < 9; ++i) {
+    simd_inverse[4 * i] = simd_inverse[4 * i + 1] = simd_inverse[4 * i + 2] =
+        simd_inverse[4 * i + 3] = inverse[i];
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/opsin_params.h b/jxl/opsin_params.h
new file mode 100644
index 000000000..ef7acd907
--- /dev/null
+++ b/jxl/opsin_params.h
@@ -0,0 +1,90 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_OPSIN_PARAMS_H_
+#define JXL_OPSIN_PARAMS_H_
+
+// Constants that define the XYB color space.
+
+#include <stdlib.h>
+
+#include <cmath>
+
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+
+static const float kScale = 255.0f;
+
+static const float kScaleR = 1.0f;
+static const float kScaleG = 1.0f;
+static const float kInvScaleR = 1.0f;
+static const float kInvScaleG = 1.0f;
+
+// Parameters for opsin absorbance.
+static const float kM02 = 0.078;
+static const float kM00 = 0.30;
+static const float kM01 = 1.0 - kM02 - kM00;
+
+static const float kM12 = 0.078;
+static const float kM10 = 0.23;
+static const float kM11 = 1.0 - kM12 - kM10;
+
+static const float kM20 = 0.24342268924547819;
+static const float kM21 = 0.20476744424496821;
+static const float kM22 = 1.0 - kM20 - kM21;
+
+static const float kBScale = 1.0f;
+static const float kYToBRatio = 1.0f;  // works better with 0.50017729543783418
+static const float kBToYRatio = 1.0f / kYToBRatio;
+
+// Unscaled values for kOpsinAbsorbanceBias
+static const float kB0 = 0.96723368009523958;
+static const float kB1 = kB0;
+static const float kB2 = kB0;
+
+// Opsin absorbance matrix is now frozen.
+static const float kOpsinAbsorbanceMatrix[9] = {
+    kM00 / kScale, kM01 / kScale, kM02 / kScale, kM10 / kScale, kM11 / kScale,
+    kM12 / kScale, kM20 / kScale, kM21 / kScale, kM22 / kScale,
+};
+
+// Must be the inverse matrix of kOpsinAbsorbanceMatrix and match the spec.
+static inline const float* DefaultInverseOpsinAbsorbanceMatrix() {
+  static float kDefaultInverseOpsinAbsorbanceMatrix[9] = {
+      2813.04956,  -2516.07070, -41.9788641, -829.807582, 1126.78645,
+      -41.9788641, -933.007078, 691.795377,  496.211701};
+  return kDefaultInverseOpsinAbsorbanceMatrix;
+}
+
+// Returns 3x3 row-major matrix inverse of kOpsinAbsorbanceMatrix.
+// opsin_image_test verifies this is actually the inverse.
+const float* GetOpsinAbsorbanceInverseMatrix();
+
+void InitSIMDInverseMatrix(const float* JXL_RESTRICT inverse,
+                           float* JXL_RESTRICT simd_inverse);
+
+static const float kOpsinAbsorbanceBias[3] = {
+    kB0 / kScale,
+    kB1 / kScale,
+    kB2 / kScale,
+};
+
+static const float kNegOpsinAbsorbanceBiasRGB[4] = {
+    -kOpsinAbsorbanceBias[0], -kOpsinAbsorbanceBias[1],
+    -kOpsinAbsorbanceBias[2], 255.0f};
+
+}  // namespace jxl
+
+#endif  // JXL_OPSIN_PARAMS_H_
diff --git a/jxl/optimize.cc b/jxl/optimize.cc
new file mode 100644
index 000000000..8d07d6297
--- /dev/null
+++ b/jxl/optimize.cc
@@ -0,0 +1,185 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/optimize.h"
+
+#include <algorithm>
+#include <iostream>
+
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+namespace optimize {
+
+namespace {
+
+void PrintVector(const std::vector<double>& v) {
+  std::cout << v[0] << " [";
+  for (size_t j = 1; j < v.size(); j++) {
+    std::cout << (j == 1 ? "" : ", ") << v[j];
+  }
+  std::cout << "]" << std::endl;
+}
+
+// simplex vector must be sorted by first element of its elements
+std::vector<double> Midpoint(const std::vector<std::vector<double>>& simplex) {
+  JXL_CHECK(!simplex.empty());
+  JXL_CHECK(simplex.size() == simplex[0].size());
+  int dim = simplex.size() - 1;
+  std::vector<double> result(dim + 1, 0);
+  for (int i = 0; i < dim; i++) {
+    for (int k = 0; k < dim; k++) {
+      result[i + 1] += simplex[k][i + 1];
+    }
+    result[i + 1] /= dim;
+  }
+  return result;
+}
+
+// first element ignored
+std::vector<double> Subtract(const std::vector<double>& a,
+                             const std::vector<double>& b) {
+  JXL_CHECK(a.size() == b.size());
+  std::vector<double> result(a.size());
+  result[0] = 0;
+  for (size_t i = 1; i < result.size(); i++) {
+    result[i] = a[i] - b[i];
+  }
+  return result;
+}
+
+// first element ignored
+std::vector<double> Add(const std::vector<double>& a,
+                        const std::vector<double>& b) {
+  JXL_CHECK(a.size() == b.size());
+  std::vector<double> result(a.size());
+  result[0] = 0;
+  for (size_t i = 1; i < result.size(); i++) {
+    result[i] = a[i] + b[i];
+  }
+  return result;
+}
+
+// first element ignored
+std::vector<double> Average(const std::vector<double>& a,
+                            const std::vector<double>& b) {
+  JXL_CHECK(a.size() == b.size());
+  std::vector<double> result(a.size());
+  result[0] = 0;
+  for (size_t i = 1; i < result.size(); i++) {
+    result[i] = 0.5 * (a[i] + b[i]);
+  }
+  return result;
+}
+
+// vec: [0] will contain the objective function, [1:] will
+//   contain the vector position for the objective function.
+// fun: the function evaluates the value.
+void Eval(std::vector<double>* vec,
+          const std::function<double((const std::vector<double>&))>& fun) {
+  std::vector<double> args(vec->begin() + 1, vec->end());
+  (*vec)[0] = fun(args);
+  // std::cout << "eval: ";
+  // PrintVector(*vec);
+}
+
+void Sort(std::vector<std::vector<double>>* simplex) {
+  std::sort(simplex->begin(), simplex->end());
+}
+
+// Main iteration step of Nelder-Mead like optimization.
+void Reflect(std::vector<std::vector<double>>* simplex,
+             const std::function<double((const std::vector<double>&))>& fun) {
+  Sort(simplex);
+  const std::vector<double>& last = simplex->back();
+  std::vector<double> mid = Midpoint(*simplex);
+  std::vector<double> diff = Subtract(mid, last);
+  std::vector<double> mirrored = Add(mid, diff);
+  Eval(&mirrored, fun);
+  if (mirrored[0] > (*simplex)[simplex->size() - 2][0]) {
+    // Still the worst, shrink towards the best.
+    std::vector<double> shrinking = Average(simplex->back(), (*simplex)[0]);
+    Eval(&shrinking, fun);
+    simplex->back() = shrinking;
+  } else if (mirrored[0] < (*simplex)[0][0]) {
+    // new best
+    std::vector<double> even_further = Add(mirrored, diff);
+    Eval(&even_further, fun);
+    if (even_further[0] < mirrored[0]) {
+      mirrored = even_further;
+    }
+    simplex->back() = mirrored;
+  } else {
+    // not a best, not a worst point
+    simplex->back() = mirrored;
+  }
+}
+
+// Initialize the simplex at origin.
+std::vector<std::vector<double>> InitialSimplex(
+    int dim, double amount, const std::vector<double>& init,
+    const std::function<double((const std::vector<double>&))>& fun) {
+  std::vector<double> best(1 + dim, 0);
+  std::copy(init.begin(), init.end(), best.begin() + 1);
+  Eval(&best, fun);
+  std::vector<std::vector<double>> result{best};
+  for (int i = 0; i < dim; i++) {
+    best = result[0];
+    best[i + 1] += amount;
+    Eval(&best, fun);
+    result.push_back(best);
+    Sort(&result);
+  }
+  return result;
+}
+
+// For comparing the same with the python tool
+/*void RunSimplexExternal(
+    int dim, double amount, int max_iterations,
+    const std::function<double((const vector<double>&))>& fun) {
+  vector<double> vars;
+  for (int i = 0; i < dim; i++) {
+    vars.push_back(atof(getenv(StrCat("VAR", i).c_str())));
+  }
+  double result = fun(vars);
+  std::cout << "Result=" << result;
+}*/
+
+}  // namespace
+
+std::vector<double> RunSimplex(
+    int dim, double amount, int max_iterations, const std::vector<double>& init,
+    const std::function<double((const std::vector<double>&))>& fun) {
+  std::vector<std::vector<double>> simplex =
+      InitialSimplex(dim, amount, init, fun);
+  for (int i = 0; i < max_iterations; i++) {
+    Sort(&simplex);
+    // std::cout << "reflect " << i << ": ";
+    // PrintVector(simplex[0]);
+    Reflect(&simplex, fun);
+  }
+  return simplex[0];
+}
+
+std::vector<double> RunSimplex(
+    int dim, double amount, int max_iterations,
+    const std::function<double((const std::vector<double>&))>& fun) {
+  std::vector<double> init(dim, 0.0);
+  return RunSimplex(dim, amount, max_iterations, init, fun);
+}
+
+}  // namespace optimize
+
+}  // namespace jxl
diff --git a/jxl/optimize.h b/jxl/optimize.h
new file mode 100644
index 000000000..0171b41ee
--- /dev/null
+++ b/jxl/optimize.h
@@ -0,0 +1,181 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Utility functions for optimizing multi-dimensional nonlinear functions.
+
+#ifndef JXL_OPTIMIZE_H_
+#define JXL_OPTIMIZE_H_
+
+#include <stdio.h>
+
+#include <cmath>
+#include <cstdio>
+#include <functional>
+#include <vector>
+
+namespace jxl {
+namespace optimize {
+
+// Runs Nelder-Mead like optimization. Runs for max_iterations times,
+// fun gets called with a vector of size dim as argument, and returns the score
+// based on those parameters (lower is better). Returns a vector of dim+1
+// dimensions, where the first value is the optimal value of the function and
+// the rest is the argmin value. Use init to pass an initial guess or where
+// the optimal value is.
+//
+// Usage example:
+//
+// RunSimplex(2, 0.1, 100, [](const vector<float>& v) {
+//   return (v[0] - 5) * (v[0] - 5) + (v[1] - 7) * (v[1] - 7);
+// });
+//
+// Returns (0.0, 5, 7)
+std::vector<double> RunSimplex(
+    int dim, double amount, int max_iterations,
+    const std::function<double((const std::vector<double>&))>& fun);
+std::vector<double> RunSimplex(
+    int dim, double amount, int max_iterations, const std::vector<double>& init,
+    const std::function<double((const std::vector<double>&))>& fun);
+
+template <typename T>
+std::vector<T> operator+(const std::vector<T>& x, const std::vector<T>& y) {
+  std::vector<T> z(x.size());
+  for (size_t i = 0; i < x.size(); ++i) {
+    z[i] = x[i] + y[i];
+  }
+  return z;
+}
+
+template <typename T>
+std::vector<T> operator-(const std::vector<T>& x, const std::vector<T>& y) {
+  std::vector<T> z(x.size());
+  for (size_t i = 0; i < x.size(); ++i) {
+    z[i] = x[i] - y[i];
+  }
+  return z;
+}
+
+template <typename T>
+std::vector<T> operator*(T v, const std::vector<T>& x) {
+  std::vector<T> y(x.size());
+  for (size_t i = 0; i < x.size(); ++i) {
+    y[i] = v * x[i];
+  }
+  return y;
+}
+
+template <typename T>
+T operator*(const std::vector<T>& x, const std::vector<T>& y) {
+  T r = 0.0;
+  for (size_t i = 0; i < x.size(); ++i) {
+    r += x[i] * y[i];
+  }
+  return r;
+}
+
+// Implementation of the Scaled Conjugate Gradient method described in the
+// following paper:
+//   Moller, M. "A Scaled Conjugate Gradient Algorithm for Fast Supervised
+//   Learning", Neural Networks, Vol. 6. pp. 525-533, 1993
+//   http://sci2s.ugr.es/keel/pdf/algorithm/articulo/moller1990.pdf
+//
+// The Function template parameter is a class that has the following method:
+//
+//   // Returns the value of the function at point w and sets *df to be the
+//   // negative gradient vector of the function at point w.
+//   double Compute(const vector<double>& w, vector<double>* df) const;
+//
+// Returns a vector w, such that |df(w)| < grad_norm_threshold.
+template <typename T, typename Function>
+std::vector<T> OptimizeWithScaledConjugateGradientMethod(
+    const Function& f, const std::vector<T>& w0, const T grad_norm_threshold,
+    int max_iters) {
+  const size_t n = w0.size();
+  const T rsq_threshold = grad_norm_threshold * grad_norm_threshold;
+  const T sigma0 = static_cast<T>(0.0001);
+  T lambda = static_cast<T>(0.000001);
+  std::vector<T> w(w0);
+  std::vector<T> p(n);
+  std::vector<T> r(n);
+  T fw = f.Compute(w, &r);
+  T rsq = r * r;
+  T psq = rsq;
+  T mu = rsq;
+  p = r;
+  for (int k = 1; rsq > rsq_threshold; ++k) {
+    if (max_iters > 0 && k > max_iters) break;
+    T sigma = sigma0 / std::sqrt(psq);
+    std::vector<T> r2(n);
+    std::vector<T> w2 = w + (sigma * p);
+    f.Compute(w2, &r2);
+    T delta = (mu - (p * r2)) / sigma;
+    T delta1 = delta + lambda * psq;
+
+    if (delta1 <= 0) {
+      lambda = -2.0 * delta / psq;
+      delta1 = delta + lambda * psq;
+    }
+
+    bool success = true;
+    T alpha;
+    T fw1;
+    T Delta;
+    std::vector<T> w1(n);
+    std::vector<T> r1(n);
+
+    do {
+      alpha = mu / delta1;
+      w1 = w + (alpha * p);
+      fw1 = f.Compute(w1, &r1);
+      const T div = mu * alpha;
+      Delta = div == 0 ? 0 : 2 * (fw - fw1) / div;
+      success = (fw1 <= fw);
+      if (!success) {
+        lambda += delta1 * (1 - Delta) / psq;
+        delta1 = delta + lambda * psq;
+      }
+    } while (!success);
+
+    T r1sq = r1 * r1;
+    T beta = k % n == 0 ? 0.0 : (r1sq - (r1 * r)) / mu;
+
+#if SCG_DEBUG
+    printf(
+        "Step %3d fw=%10.2f |dfw|=%7.3f |p|=%6.2f "
+        "delta=%9.6f lambda=%6.4f mu=%8.4f alpha=%8.4f "
+        "beta=%5.3f Delta=%5.3f\n",
+        k, fw, sqrt(rsq), sqrt(psq), delta, lambda, mu, alpha, beta, Delta);
+#endif
+
+    if (Delta >= 0.75) {
+      lambda *= 0.25;
+    } else if (Delta < 0.25) {
+      lambda += delta1 * (1 - Delta) / psq;
+    }
+
+    w = w1;
+    fw = fw1;
+    r = r1;
+    rsq = r1sq;
+    p = r + (beta * p);
+    psq = p * p;
+    mu = p * r;
+  }
+  return w;
+}
+
+}  // namespace optimize
+}  // namespace jxl
+
+#endif  // JXL_OPTIMIZE_H_
diff --git a/jxl/optimize_test.cc b/jxl/optimize_test.cc
new file mode 100644
index 000000000..9a3add338
--- /dev/null
+++ b/jxl/optimize_test.cc
@@ -0,0 +1,116 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/optimize.h"
+
+#include <stdio.h>
+
+#include "gtest/gtest.h"
+
+namespace jxl {
+namespace optimize {
+namespace {
+
+// The maximum number of iterations for the test.
+static const size_t kMaxTestIter = 100000;
+
+// F(w) = (w - w_min)^2.
+struct SimpleQuadraticFunction {
+  explicit SimpleQuadraticFunction(const std::vector<double>& w0) : w_min(w0) {}
+
+  double Compute(const std::vector<double>& w, std::vector<double>* df) const {
+    std::vector<double> dw = w - w_min;
+    *df = -2.0 * dw;
+    return dw * dw;
+  }
+
+  std::vector<double> w_min;
+};
+
+// F(alpha, beta, gamma| x,y) = \sum_i(y_i - (alpha x_i ^ gamma + beta))^2.
+struct PowerFunction {
+  explicit PowerFunction(const std::vector<double>& x0,
+                         const std::vector<double>& y0)
+      : x(x0), y(y0) {}
+
+  double Compute(const std::vector<double>& w, std::vector<double>* df) const {
+    double loss_function = 0;
+    (*df)[0] = 0;
+    (*df)[1] = 0;
+    (*df)[2] = 0;
+    for (int ind = 0; ind < y.size(); ++ind) {
+      if (x[ind] != 0) {
+        double l_f = y[ind] - (w[0] * pow(x[ind], w[1]) + w[2]);
+        (*df)[0] += 2.0 * l_f * pow(x[ind], w[1]);
+        (*df)[1] += 2.0 * l_f * w[0] * pow(x[ind], w[1]) * log(x[ind]);
+        (*df)[2] += 2.0 * l_f * 1;
+        loss_function += l_f * l_f;
+      }
+    }
+    return loss_function;
+  }
+
+  std::vector<double> x;
+  std::vector<double> y;
+};
+
+TEST(OptimizeTest, SimpleQuadraticFunction) {
+  std::vector<double> w_min(2);
+  w_min[0] = 1.0;
+  w_min[1] = 2.0;
+  SimpleQuadraticFunction f(w_min);
+  std::vector<double> w(2);
+  static const double kPrecision = 1e-8;
+  w = optimize::OptimizeWithScaledConjugateGradientMethod(f, w, kPrecision,
+                                                          kMaxTestIter);
+  EXPECT_NEAR(w[0], 1.0, kPrecision);
+  EXPECT_NEAR(w[1], 2.0, kPrecision);
+}
+
+TEST(OptimizeTest, PowerFunction) {
+  std::vector<double> x(10);
+  std::vector<double> y(10);
+  for (int ind = 0; ind < 10; ++ind) {
+    x[ind] = 1. * ind;
+    y[ind] = 2. * pow(x[ind], 3) + 5.;
+  }
+  PowerFunction f(x, y);
+  std::vector<double> w(3);
+
+  static const double kPrecision = 0.01;
+  w = optimize::OptimizeWithScaledConjugateGradientMethod(f, w, kPrecision,
+                                                          kMaxTestIter);
+  EXPECT_NEAR(w[0], 2.0, kPrecision);
+  EXPECT_NEAR(w[1], 3.0, kPrecision);
+  EXPECT_NEAR(w[2], 5.0, kPrecision);
+}
+
+TEST(OptimizeTest, SimplexOptTest) {
+  auto f = [](const std::vector<double>& x) -> double {
+    double t1 = x[0] - 1.0;
+    double t2 = x[1] + 1.5;
+    return 2.0 + t1 * t1 + t2 * t2;
+  };
+  auto opt = RunSimplex(2, 0.01, 100, f);
+  EXPECT_EQ(opt.size(), 3);
+
+  static const double kPrecision = 0.01;
+  EXPECT_NEAR(opt[0], 2.0, kPrecision);
+  EXPECT_NEAR(opt[1], 1.0, kPrecision);
+  EXPECT_NEAR(opt[2], -1.5, kPrecision);
+}
+
+}  // namespace
+}  // namespace optimize
+}  // namespace jxl
diff --git a/jxl/padded_bytes_test.cc b/jxl/padded_bytes_test.cc
new file mode 100644
index 000000000..415bd5b29
--- /dev/null
+++ b/jxl/padded_bytes_test.cc
@@ -0,0 +1,70 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/padded_bytes.h"
+
+#include "gtest/gtest.h"
+
+namespace jxl {
+namespace {
+
+TEST(PaddedBytesTest, TestNonEmptyFirstByteZero) {
+  PaddedBytes pb(1);
+  EXPECT_EQ(0, pb[0]);
+  // Even after resizing..
+  pb.resize(20);
+  EXPECT_EQ(0, pb[0]);
+  // And reserving.
+  pb.reserve(200);
+  EXPECT_EQ(0, pb[0]);
+}
+
+TEST(PaddedBytesTest, TestEmptyFirstByteZero) {
+  PaddedBytes pb(0);
+  // After resizing - new zero is written despite there being nothing to copy.
+  pb.resize(20);
+  EXPECT_EQ(0, pb[0]);
+}
+
+TEST(PaddedBytesTest, TestFillWithoutReserve) {
+  PaddedBytes pb;
+  for (size_t i = 0; i < 170; ++i) {
+    pb.push_back(i);
+  }
+  EXPECT_EQ(170, pb.size());
+  EXPECT_GE(pb.capacity(), 170);
+}
+
+TEST(PaddedBytesTest, TestFillWithExactReserve) {
+  PaddedBytes pb;
+  pb.reserve(170);
+  for (size_t i = 0; i < 170; ++i) {
+    pb.push_back(i);
+  }
+  EXPECT_EQ(170, pb.size());
+  EXPECT_EQ(pb.capacity(), 170);
+}
+
+TEST(PaddedBytesTest, TestFillWithMoreReserve) {
+  PaddedBytes pb;
+  pb.reserve(171);
+  for (size_t i = 0; i < 170; ++i) {
+    pb.push_back(i);
+  }
+  EXPECT_EQ(170, pb.size());
+  EXPECT_GT(pb.capacity(), 170);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/passes_state.cc b/jxl/passes_state.cc
new file mode 100644
index 000000000..67487ec3a
--- /dev/null
+++ b/jxl/passes_state.cc
@@ -0,0 +1,73 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/passes_state.h"
+
+#include "jxl/chroma_from_luma.h"
+#include "jxl/common.h"
+
+namespace jxl {
+
+Status InitializePassesSharedState(const FrameHeader& frame_header,
+                                   const LoopFilter& loop_filter,
+                                   const ImageMetadata& image_metadata,
+                                   const FrameDimensions& frame_dim,
+                                   Multiframe* JXL_RESTRICT multiframe,
+                                   PassesSharedState* JXL_RESTRICT shared,
+                                   bool encoder) {
+  shared->frame_header = frame_header;
+  shared->metadata = image_metadata;
+  shared->frame_dim = frame_dim;
+  shared->image_features.loop_filter = loop_filter;
+  shared->multiframe = multiframe;
+
+  shared->ac_strategy =
+      AcStrategyImage(frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+  shared->raw_quant_field =
+      ImageI(frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+  shared->epf_sharpness =
+      ImageB(frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+  shared->cmap = ColorCorrelationMap(frame_dim.xsize, frame_dim.ysize);
+
+  shared->opsin_params = image_metadata.m2.opsin_inverse_matrix.ToOpsinParams();
+
+  if (!(frame_header.flags & FrameHeader::kUseDcFrame) || encoder) {
+    shared->dc_storage =
+        Image3F(frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+    shared->dc_quant_field =
+        Image3F(frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+  } else {
+    if (frame_header.dc_level == 3) {
+      return JXL_FAILURE("Invalid DC level for kUseDcFrame: %u",
+                         frame_header.dc_level);
+    }
+    shared->dc = multiframe->SavedDc(frame_header.dc_level + 1);
+    if (shared->dc->xsize() == 0) {
+      return JXL_FAILURE(
+          "kUseDcFrame specified for dc_level %u, but no frame was decoded "
+          "with level %u",
+          frame_header.dc_level, frame_header.dc_level + 1);
+    }
+  }
+  shared->image_features.patches.SetReferenceFrames(
+      multiframe->GetReferenceFrames());
+
+  shared->dc_storage = Image3F(frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+  shared->dc_quant_field =
+      Image3F(frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/passes_state.h b/jxl/passes_state.h
new file mode 100644
index 000000000..1eb9e05b2
--- /dev/null
+++ b/jxl/passes_state.h
@@ -0,0 +1,143 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_PASSES_STATE_H_
+#define JXL_PASSES_STATE_H_
+
+#include "jxl/ac_strategy.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/common.h"
+#include "jxl/dot_dictionary.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/loop_filter.h"
+#include "jxl/multiframe.h"
+#include "jxl/noise.h"
+#include "jxl/patch_dictionary.h"
+#include "jxl/quant_weights.h"
+#include "jxl/quantizer.h"
+#include "jxl/splines.h"
+
+// Structures that hold the (en/de)coder state for a JPEG XL kPasses
+// (en/de)coder.
+
+namespace jxl {
+
+struct ImageFeatures {
+  LoopFilter loop_filter;
+  NoiseParams noise_params;
+  PatchDictionary patches;
+  Splines splines;
+};
+
+// Settings for bitstream-affecting experiments.
+struct Experiments {
+  bool use_new_cmap = false;
+};
+
+// State common to both encoder and decoder.
+// NOLINTNEXTLINE(clang-analyzer-optin.performance.Padding)
+struct PassesSharedState {
+  // Headers and metadata.
+  FrameHeader frame_header;
+  ImageMetadata metadata;
+
+  FrameDimensions frame_dim;
+
+  // Control fields and parameters.
+  AcStrategyImage ac_strategy;
+
+  // Dequant matrices + quantizer.
+  DequantMatrices matrices;
+  Quantizer quantizer{&matrices};
+  ImageI raw_quant_field;
+
+  // Per-block side information for EPF detail preservation.
+  ImageB epf_sharpness;
+
+  ColorCorrelationMap cmap;
+
+  OpsinParams opsin_params;
+
+  ImageFeatures image_features;
+
+  // Memory area for storing coefficient orders.
+  std::vector<coeff_order_t> coeff_orders =
+      std::vector<coeff_order_t>(kMaxNumPasses * kCoeffOrderSize);
+
+  // Decoder-side DC and DC quantization information.
+  Image3F dc_storage;
+  const Image3F* JXL_RESTRICT dc = &dc_storage;
+
+  // Only useful if adaptive DC smoothing is enabled.
+  Image3F dc_quant_field;
+
+  Multiframe* JXL_RESTRICT multiframe = nullptr;
+
+  Experiments experiments;
+
+  // Number of histograms and coefficient orders, per pass (always 1 for
+  // now). Encoded as num_histograms_ - 1.
+  size_t num_histograms = 0;
+
+  bool IsGrayscale() const { return metadata.color_encoding.IsGray(); }
+
+  Rect GroupRect(size_t group_index) const {
+    const size_t gx = group_index % frame_dim.xsize_groups;
+    const size_t gy = group_index / frame_dim.xsize_groups;
+    const Rect rect(gx * kGroupDim, gy * kGroupDim, kGroupDim, kGroupDim,
+                    frame_dim.xsize, frame_dim.ysize);
+    return rect;
+  }
+
+  Rect PaddedGroupRect(size_t group_index) const {
+    const size_t gx = group_index % frame_dim.xsize_groups;
+    const size_t gy = group_index / frame_dim.xsize_groups;
+    const Rect rect(gx * kGroupDim, gy * kGroupDim, kGroupDim, kGroupDim,
+                    frame_dim.xsize_padded, frame_dim.ysize_padded);
+    return rect;
+  }
+
+  Rect BlockGroupRect(size_t group_index) const {
+    const size_t gx = group_index % frame_dim.xsize_groups;
+    const size_t gy = group_index / frame_dim.xsize_groups;
+    const Rect rect(gx * kGroupDimInBlocks, gy * kGroupDimInBlocks,
+                    kGroupDimInBlocks, kGroupDimInBlocks,
+                    frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+    return rect;
+  }
+
+  Rect DCGroupRect(size_t group_index) const {
+    const size_t gx = group_index % frame_dim.xsize_dc_groups;
+    const size_t gy = group_index / frame_dim.xsize_dc_groups;
+    const Rect rect(gx * kDcGroupDimInBlocks, gy * kDcGroupDimInBlocks,
+                    kDcGroupDimInBlocks, kDcGroupDimInBlocks,
+                    frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+    return rect;
+  }
+};
+
+// Initialized the state information that is shared between encoder and decoder.
+Status InitializePassesSharedState(const FrameHeader& frame_header,
+                                   const LoopFilter& loop_filter,
+                                   const ImageMetadata& image_metadata,
+                                   const FrameDimensions& frame_dim,
+                                   Multiframe* JXL_RESTRICT multiframe,
+                                   PassesSharedState* JXL_RESTRICT shared,
+                                   bool encoder = false);
+
+}  // namespace jxl
+
+#endif  // JXL_PASSES_STATE_H_
diff --git a/jxl/passes_test.cc b/jxl/passes_test.cc
new file mode 100644
index 000000000..dd9e7a90d
--- /dev/null
+++ b/jxl/passes_test.cc
@@ -0,0 +1,312 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#include <string>
+#include <utility>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/override.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/color_encoding.h"
+#include "jxl/common.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+#include "jxl/enc_butteraugli_comparator.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_file.h"
+#include "jxl/enc_params.h"
+#include "jxl/extras/codec.h"
+#include "jxl/image_bundle.h"
+#include "jxl/test_utils.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+using test::Roundtrip;
+
+TEST(PassesTest, RoundtripSmallPasses) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 1.0;
+  cparams.progressive_mode = true;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  Roundtrip(&io, cparams, dparams, pool, &io2);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            1.5);
+}
+
+TEST(PassesTest, RoundtripUnalignedPasses) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 12, io.ysize() / 7);
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 2.0;
+  cparams.progressive_mode = true;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  Roundtrip(&io, cparams, dparams, pool, &io2);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            3.2);
+}
+
+TEST(PassesTest, RoundtripMultiGroupPasses) {
+  ThreadPoolInternal pool(4);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+  io.ShrinkTo(600, 1024);  // partial X, full Y group
+
+  CompressParams cparams;
+  DecompressParams dparams;
+
+  cparams.butteraugli_distance = 1.0f;
+  cparams.progressive_mode = true;
+  CodecInOut io2;
+  Roundtrip(&io, cparams, dparams, &pool, &io2);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, &pool),
+            1.99f);
+
+  cparams.butteraugli_distance = 2.0f;
+  CodecInOut io3;
+  Roundtrip(&io, cparams, dparams, &pool, &io3);
+  EXPECT_LE(ButteraugliDistance(io, io3, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, &pool),
+            3.0f);
+}
+
+TEST(PassesTest, RoundtripLargeFastPasses) {
+  ThreadPoolInternal pool(8);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  cparams.progressive_mode = true;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  Roundtrip(&io, cparams, dparams, &pool, &io2);
+}
+
+// Checks for differing size/distance in two consecutive runs of distance 2,
+// which involves additional processing including adaptive reconstruction.
+// Failing this may be a sign of race conditions or invalid memory accesses.
+TEST(PassesTest, RoundtripProgressiveConsistent) {
+  ThreadPoolInternal pool(8);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  cparams.progressive_mode = true;
+  cparams.butteraugli_distance = 2.0;
+  DecompressParams dparams;
+
+  // Try each xsize mod kBlockDim to verify right border handling.
+  for (size_t xsize = 48; xsize > 40; --xsize) {
+    io.ShrinkTo(xsize, 15);
+
+    CodecInOut io2;
+    const size_t size2 = Roundtrip(&io, cparams, dparams, &pool, &io2);
+
+    CodecInOut io3;
+    const size_t size3 = Roundtrip(&io, cparams, dparams, &pool, &io3);
+
+    // Exact same compressed size.
+    EXPECT_EQ(size2, size3);
+
+    // Exact same distance.
+    const float dist2 = ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                            /*distmap=*/nullptr, &pool);
+    const float dist3 = ButteraugliDistance(io, io3, cparams.hf_asymmetry,
+                                            /*distmap=*/nullptr, &pool);
+    EXPECT_EQ(dist2, dist3);
+  }
+}
+
+TEST(PassesTest, AllDownsampleFeasible) {
+  ThreadPoolInternal pool(8);
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  PaddedBytes compressed;
+  AuxOut aux;
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  cparams.progressive_mode = true;
+  cparams.butteraugli_distance = 1.0;
+  PassesEncoderState enc_state;
+  ASSERT_TRUE(EncodeFile(cparams, &io, &enc_state, &compressed, &aux, &pool));
+
+  EXPECT_LE(compressed.size(), 240000);
+  float target_butteraugli[9] = {};
+  target_butteraugli[1] = 2.5f;
+  target_butteraugli[2] = 13.0f;
+  target_butteraugli[4] = 20.0f;
+  target_butteraugli[8] = 60.0f;
+
+  // The default progressive encoding scheme should make all these downsampling
+  // factors achievable.
+  // TODO(veluca): re-enable downsampling 16.
+  std::vector<size_t> downsamplings = {1, 2, 4, 8};  //, 16};
+
+  auto check = [&](uint32_t task, uint32_t /* thread */) -> void {
+    const size_t downsampling = downsamplings[task];
+    DecompressParams dparams;
+    dparams.max_downsampling = downsampling;
+    AuxOut aux_downsampled = aux;
+    CodecInOut output;
+    ASSERT_TRUE(
+        DecodeFile(dparams, compressed, &output, &aux_downsampled, nullptr));
+    EXPECT_EQ(aux_downsampled.downsampling, downsampling);
+    EXPECT_EQ(output.xsize(), io.xsize()) << "downsampling = " << downsampling;
+    EXPECT_EQ(output.ysize(), io.ysize()) << "downsampling = " << downsampling;
+    EXPECT_LE(ButteraugliDistance(io, output, cparams.hf_asymmetry,
+                                  /*distmap=*/nullptr, nullptr),
+              target_butteraugli[downsampling])
+        << "downsampling: " << downsampling;
+  };
+  pool.Run(0, downsamplings.size(), ThreadPool::SkipInit(), check);
+}
+
+TEST(PassesTest, AllDownsampleFeasibleQProgressive) {
+  ThreadPoolInternal pool(8);
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  PaddedBytes compressed;
+  AuxOut aux;
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  cparams.qprogressive_mode = true;
+  cparams.butteraugli_distance = 1.0;
+  PassesEncoderState enc_state;
+  ASSERT_TRUE(EncodeFile(cparams, &io, &enc_state, &compressed, &aux, &pool));
+
+  EXPECT_LE(compressed.size(), 220000);
+
+  float target_butteraugli[9] = {};
+  target_butteraugli[1] = 3.0f;
+  target_butteraugli[2] = 6.0f;
+  target_butteraugli[4] = 10.0f;
+  target_butteraugli[8] = 60.0f;
+
+  // The default progressive encoding scheme should make all these downsampling
+  // factors achievable.
+  std::vector<size_t> downsamplings = {1, 2, 4, 8};
+
+  auto check = [&](uint32_t task, uint32_t /* thread */) -> void {
+    const size_t downsampling = downsamplings[task];
+    DecompressParams dparams;
+    dparams.max_downsampling = downsampling;
+    CodecInOut output;
+    AuxOut aux_downsampled = aux;
+    ASSERT_TRUE(
+        DecodeFile(dparams, compressed, &output, &aux_downsampled, nullptr));
+    EXPECT_EQ(aux_downsampled.downsampling, downsampling);
+    EXPECT_EQ(output.xsize(), io.xsize()) << "downsampling = " << downsampling;
+    EXPECT_EQ(output.ysize(), io.ysize()) << "downsampling = " << downsampling;
+    EXPECT_LE(ButteraugliDistance(io, output, cparams.hf_asymmetry,
+                                  /*distmap=*/nullptr, nullptr),
+              target_butteraugli[downsampling])
+        << "downsampling: " << downsampling;
+  };
+  pool.Run(0, downsamplings.size(), ThreadPool::SkipInit(), check);
+}
+
+TEST(PassesTest, NonProgressiveDCImage) {
+  ThreadPoolInternal pool(8);
+  const std::string pathname =
+      GetTestDataPath("imagecompression.info/flower_foveon.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  PaddedBytes compressed;
+  AuxOut aux;
+
+  CompressParams cparams;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  cparams.progressive_mode = false;
+  cparams.butteraugli_distance = 2.0;
+  PassesEncoderState enc_state;
+  ASSERT_TRUE(EncodeFile(cparams, &io, &enc_state, &compressed, &aux, &pool));
+
+  // Even in non-progressive mode, it should be possible to return a DC-only
+  // image.
+  DecompressParams dparams;
+  dparams.max_downsampling = 100;
+  CodecInOut output;
+  ASSERT_TRUE(DecodeFile(dparams, compressed, &output, &aux, &pool));
+  constexpr decltype(output.xsize()) expected_downscale = 8;
+  EXPECT_EQ(aux.downsampling, expected_downscale);
+  EXPECT_EQ(output.xsize(), io.xsize());
+  EXPECT_EQ(output.ysize(), io.ysize());
+}
+
+TEST(PassesTest, RoundtripSmallNoGaborishPasses) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+
+  CompressParams cparams;
+  cparams.gaborish = Override::kOff;
+  cparams.butteraugli_distance = 1.0;
+  cparams.progressive_mode = true;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  Roundtrip(&io, cparams, dparams, pool, &io2);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            1.7);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/patch_dictionary.cc b/jxl/patch_dictionary.cc
new file mode 100644
index 000000000..37165fd7b
--- /dev/null
+++ b/jxl/patch_dictionary.cc
@@ -0,0 +1,917 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/patch_dictionary.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <sys/types.h>
+
+#include <algorithm>
+#include <hwy/static_targets.h>
+#include <random>
+#include <string>
+#include <tuple>
+#include <utility>
+#include <vector>
+
+#include "jxl/ans_params.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/override.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/color_management.h"
+#include "jxl/common.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_frame.h"
+#include "jxl/detect_dots.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_frame.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "jxl/predictor.h"
+
+namespace jxl {
+
+constexpr int kMaxPatches = 1 << 24;
+
+enum Contexts {
+  kNumRefPatchContext,
+  kReferenceFrameContext,
+  kPatchSizeContext,
+  kPatchReferencePositionContext,
+  kPatchPositionContext,
+  kPatchBlendModeContext,
+  kPatchOffsetContext,
+  kPatchCountContext,
+  kNumPatchDictionaryContexts
+};
+
+void PatchDictionary::Encode(BitWriter* writer, size_t layer,
+                             AuxOut* aux_out) const {
+  JXL_ASSERT(HasAny());
+  std::vector<std::vector<Token>> tokens(1);
+
+  auto add_num = [&](int context, size_t num) {
+    TokenizeHybridUint(context, num, &tokens[0]);
+  };
+  size_t num_ref_patch = 0;
+  for (size_t i = 0; i < positions_.size();) {
+    size_t i_start = i;
+    while (i < positions_.size() &&
+           positions_[i].ref_pos == positions_[i_start].ref_pos) {
+      i++;
+    }
+    num_ref_patch++;
+  }
+  add_num(kNumRefPatchContext, num_ref_patch);
+  for (size_t i = 0; i < positions_.size();) {
+    size_t i_start = i;
+    while (i < positions_.size() &&
+           positions_[i].ref_pos == positions_[i_start].ref_pos) {
+      i++;
+    }
+    size_t num = i - i_start;
+    JXL_ASSERT(num > 0);
+    add_num(kPatchCountContext, num - 1);
+    add_num(kReferenceFrameContext, positions_[i_start].ref_pos.ref);
+    add_num(kPatchReferencePositionContext, positions_[i_start].ref_pos.x0);
+    add_num(kPatchReferencePositionContext, positions_[i_start].ref_pos.y0);
+    add_num(kPatchSizeContext, positions_[i_start].ref_pos.xsize - 1);
+    add_num(kPatchSizeContext, positions_[i_start].ref_pos.ysize - 1);
+    for (size_t j = i_start; j < i; j++) {
+      const PatchPosition& pos = positions_[j];
+      if (j == i_start) {
+        add_num(kPatchPositionContext, pos.x);
+        add_num(kPatchPositionContext, pos.y);
+      } else {
+        add_num(kPatchOffsetContext, PackSigned(pos.x - positions_[j - 1].x));
+        add_num(kPatchOffsetContext, PackSigned(pos.y - positions_[j - 1].y));
+      }
+      add_num(kPatchBlendModeContext, static_cast<uint8_t>(pos.blend_mode));
+    }
+  }
+
+  EntropyEncodingData codes;
+  std::vector<uint8_t> context_map;
+  BuildAndEncodeHistograms(HistogramParams(), kNumPatchDictionaryContexts,
+                           tokens, &codes, &context_map, writer, layer,
+                           aux_out);
+  WriteTokens(tokens[0], codes, context_map, writer, layer, aux_out);
+}
+
+HWY_ATTR Status PatchDictionary::Decode(BitReader* br, size_t xsize,
+                                        size_t ysize,
+                                        size_t save_as_reference) {
+  std::vector<uint8_t> context_map;
+  ANSCode code;
+  JXL_RETURN_IF_ERROR(DecodeHistograms(br, kNumPatchDictionaryContexts,
+                                       ANS_MAX_ALPHA_SIZE, &code,
+                                       &context_map));
+  ANSSymbolReader decoder(&code, br);
+
+  auto read_num = [&](size_t context) HWY_ATTR {
+    size_t r = ReadHybridUint(context, br, &decoder, context_map);
+    return r;
+  };
+
+  size_t num_ref_patch = read_num(kNumRefPatchContext);
+  // TODO(veluca): does this make sense?
+  if (num_ref_patch > kMaxPatches) {
+    return JXL_FAILURE("Too many patches in dictionary");
+  }
+
+  for (size_t id = 0; id < num_ref_patch; id++) {
+    size_t id_count = read_num(kPatchCountContext) + 1;
+    if (id_count > kMaxPatches) {
+      return JXL_FAILURE("Too many patches in dictionary");
+    }
+    positions_.reserve(positions_.size() + id_count);
+    PatchReferencePosition ref_pos;
+    ref_pos.ref = read_num(kReferenceFrameContext);
+    if (ref_pos.ref >= kMaxNumReferenceFrames ||
+        ref_pos.ref + 1 == save_as_reference ||
+        reference_frames_[ref_pos.ref].xsize() == 0) {
+      return JXL_FAILURE("Invalid reference frame ID");
+    }
+    ref_pos.x0 = read_num(kPatchReferencePositionContext);
+    ref_pos.y0 = read_num(kPatchReferencePositionContext);
+    ref_pos.xsize = read_num(kPatchSizeContext) + 1;
+    ref_pos.ysize = read_num(kPatchSizeContext) + 1;
+    if (ref_pos.x0 + ref_pos.xsize > reference_frames_[ref_pos.ref].xsize()) {
+      return JXL_FAILURE("Invalid position specified in reference frame");
+    }
+    if (ref_pos.y0 + ref_pos.ysize > reference_frames_[ref_pos.ref].ysize()) {
+      return JXL_FAILURE("Invalid position specified in reference frame");
+    }
+    for (size_t i = 0; i < id_count; i++) {
+      PatchPosition pos;
+      pos.ref_pos = ref_pos;
+      if (i == 0) {
+        pos.x = read_num(kPatchPositionContext);
+        pos.y = read_num(kPatchPositionContext);
+      } else {
+        pos.x =
+            positions_.back().x + UnpackSigned(read_num(kPatchOffsetContext));
+        pos.y =
+            positions_.back().y + UnpackSigned(read_num(kPatchOffsetContext));
+      }
+      if (pos.x + ref_pos.xsize > xsize) {
+        return JXL_FAILURE("Invalid patch x: at %zu + %zu > %zu", pos.x,
+                           ref_pos.xsize, xsize);
+      }
+      if (pos.y + ref_pos.ysize > ysize) {
+        return JXL_FAILURE("Invalid patch y: at %zu + %zu > %zu", pos.y,
+                           ref_pos.ysize, ysize);
+      }
+      uint8_t blend_mode = read_num(kPatchBlendModeContext);
+      if (blend_mode >= uint8_t(PatchBlendMode::kNumBlendModes)) {
+        return JXL_FAILURE("Invalid patch blend mode: %u", blend_mode);
+      }
+      pos.blend_mode = static_cast<PatchBlendMode>(blend_mode);
+      positions_.push_back(std::move(pos));
+    }
+  }
+
+  if (!decoder.CheckANSFinalState()) {
+    return JXL_FAILURE("ANS checksum failure.");
+  }
+  if (!HasAny()) {
+    return JXL_FAILURE("Decoded patch dictionary but got none");
+  }
+
+  ComputePatchCache();
+  return true;
+}
+
+void PatchDictionary::ComputePatchCache() {
+  if (positions_.empty()) return;
+  std::vector<std::pair<size_t, size_t>> sorted_patches_y;
+  for (size_t i = 0; i < positions_.size(); i++) {
+    const PatchPosition& pos = positions_[i];
+    for (size_t y = pos.y; y < pos.y + pos.ref_pos.ysize; y++) {
+      sorted_patches_y.emplace_back(y, i);
+    }
+  }
+  // The relative order of patches that affect the same pixels is preserved.
+  // This is important for patches that have a blend mode different from kAdd.
+  std::sort(sorted_patches_y.begin(), sorted_patches_y.end());
+  patch_starts_.resize(sorted_patches_y.back().first + 2,
+                       sorted_patches_y.size());
+  sorted_patches_.resize(sorted_patches_y.size());
+  for (size_t i = 0; i < sorted_patches_y.size(); i++) {
+    sorted_patches_[i] = sorted_patches_y[i].second;
+    patch_starts_[sorted_patches_y[i].first] =
+        std::min(patch_starts_[sorted_patches_y[i].first], i);
+  }
+  for (size_t i = patch_starts_.size() - 1; i > 0; i--) {
+    patch_starts_[i - 1] = std::min(patch_starts_[i], patch_starts_[i - 1]);
+  }
+}
+
+template <bool add>
+void PatchDictionary::Apply(Image3F* opsin, const Rect& opsin_rect,
+                            const Rect& image_rect) const {
+  JXL_CHECK(SameSize(opsin_rect, image_rect));
+  size_t num = 0;
+  for (size_t y = image_rect.y0(); y < image_rect.y0() + image_rect.ysize();
+       y++) {
+    if (y + 1 >= patch_starts_.size()) continue;
+    float* JXL_RESTRICT rows[3] = {
+        opsin_rect.PlaneRow(opsin, 0, y - image_rect.y0()),
+        opsin_rect.PlaneRow(opsin, 1, y - image_rect.y0()),
+        opsin_rect.PlaneRow(opsin, 2, y - image_rect.y0()),
+    };
+    for (size_t id = patch_starts_[y]; id < patch_starts_[y + 1]; id++) {
+      num++;
+      const PatchPosition& pos = positions_[sorted_patches_[id]];
+      size_t by = pos.y;
+      size_t bx = pos.x;
+      size_t xsize = pos.ref_pos.xsize;
+      JXL_DASSERT(y >= by);
+      JXL_DASSERT(y < by + pos.ref_pos.ysize);
+      size_t iy = y - by;
+      size_t ref = pos.ref_pos.ref;
+      if (bx >= image_rect.x0() + image_rect.xsize()) continue;
+      if (bx + xsize < image_rect.x0()) continue;
+      const float* JXL_RESTRICT ref_rows[3] = {
+          reference_frames_[ref].ConstPlaneRow(0, pos.ref_pos.y0 + iy) +
+              pos.ref_pos.x0,
+          reference_frames_[ref].ConstPlaneRow(1, pos.ref_pos.y0 + iy) +
+              pos.ref_pos.x0,
+          reference_frames_[ref].ConstPlaneRow(2, pos.ref_pos.y0 + iy) +
+              pos.ref_pos.x0,
+      };
+      for (size_t ix = 0; ix < xsize; ix++) {
+        if (bx + ix < image_rect.x0()) continue;
+        if (bx + ix >= image_rect.x0() + image_rect.xsize()) continue;
+        for (size_t c = 0; c < 3; c++) {
+          if (add) {
+            if (pos.blend_mode == PatchBlendMode::kAdd) {
+              rows[c][bx + ix - image_rect.x0()] += ref_rows[c][ix];
+            } else {
+              rows[c][bx + ix - image_rect.x0()] = ref_rows[c][ix];
+            }
+          } else {
+            if (pos.blend_mode == PatchBlendMode::kAdd) {
+              rows[c][bx + ix - image_rect.x0()] -= ref_rows[c][ix];
+            } else {
+              rows[c][bx + ix - image_rect.x0()] = 0;
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+void PatchDictionary::AddTo(Image3F* opsin, const Rect& opsin_rect,
+                            const Rect& image_rect) const {
+  Apply</*add=*/true>(opsin, opsin_rect, image_rect);
+}
+
+void PatchDictionary::SubtractFrom(Image3F* opsin) const {
+  Apply</*add=*/false>(opsin, Rect(*opsin), Rect(*opsin));
+}
+
+namespace {
+
+struct PatchColorspaceInfo {
+  float kChannelDequant[3];
+  float kChannelWeights[3];
+
+  explicit PatchColorspaceInfo(bool is_xyb) {
+    if (is_xyb) {
+      kChannelDequant[0] = 0.01615;
+      kChannelDequant[1] = 0.08875;
+      kChannelDequant[2] = 0.1922;
+      kChannelWeights[0] = 30.0;
+      kChannelWeights[1] = 3.0;
+      kChannelWeights[2] = 1.0;
+    } else {
+      kChannelDequant[0] = 20;
+      kChannelDequant[1] = 22;
+      kChannelDequant[2] = 20;
+      kChannelWeights[0] = 0.017;
+      kChannelWeights[1] = 0.02;
+      kChannelWeights[2] = 0.017;
+    }
+  }
+
+  float ScaleForQuantization(float val, size_t c) {
+    return val / kChannelDequant[c];
+  }
+
+  int Quantize(float val, size_t c) {
+    return std::trunc(ScaleForQuantization(val, c));
+  }
+
+  bool is_similar_v(const float v1[3], const float v2[3], float threshold) {
+    float distance = 0;
+    for (size_t c = 0; c < 3; c++) {
+      distance += std::fabs(v1[c] - v2[c]) * kChannelWeights[c];
+    }
+    return distance <= threshold;
+  };
+};
+
+std::vector<PatchInfo> FindTextLikePatches(
+    const Image3F& opsin, const PassesEncoderState* JXL_RESTRICT state,
+    ThreadPool* pool, AuxOut* aux_out, bool is_xyb) {
+  if (state->cparams.patches == Override::kOff) return {};
+
+  PatchColorspaceInfo pci(is_xyb);
+  float kSimilarThreshold = 0.8f;
+
+  auto is_similar_impl =
+      [&](std::pair<uint32_t, uint32_t> p1, std::pair<uint32_t, uint32_t> p2,
+          const float* JXL_RESTRICT rows[3], size_t stride, float threshold) {
+        float v1[3], v2[3];
+        for (size_t c = 0; c < 3; c++) {
+          v1[c] = rows[c][p1.second * stride + p1.first];
+          v2[c] = rows[c][p2.second * stride + p2.first];
+        }
+        return pci.is_similar_v(v1, v2, threshold);
+      };
+
+  bool has_screenshot_areas = false;
+  const size_t opsin_stride = opsin.PixelsPerRow();
+  const float* JXL_RESTRICT opsin_rows[3] = {opsin.ConstPlaneRow(0, 0),
+                                             opsin.ConstPlaneRow(1, 0),
+                                             opsin.ConstPlaneRow(2, 0)};
+
+  auto is_same = [&](std::pair<uint32_t, uint32_t> p1,
+                     std::pair<uint32_t, uint32_t> p2) {
+    for (size_t c = 0; c < 3; c++) {
+      float v1 = opsin_rows[c][p1.second * opsin_stride + p1.first];
+      float v2 = opsin_rows[c][p2.second * opsin_stride + p2.first];
+      if (std::fabs(v1 - v2) > 1e-4) {
+        return false;
+      }
+    }
+    return true;
+  };
+
+  auto is_similar = [&](std::pair<uint32_t, uint32_t> p1,
+                        std::pair<uint32_t, uint32_t> p2) {
+    return is_similar_impl(p1, p2, opsin_rows, opsin_stride, kSimilarThreshold);
+  };
+
+  constexpr int64_t kPatchSide = 4;
+  constexpr int64_t kExtraSide = 4;
+
+  // Look for kPatchSide size squares, naturally aligned, that all have the same
+  // pixel values.
+  ImageB is_screenshot_like(DivCeil(opsin.xsize(), kPatchSide),
+                            DivCeil(opsin.ysize(), kPatchSide));
+  ZeroFillImage(&is_screenshot_like);
+  uint8_t* JXL_RESTRICT screenshot_row = is_screenshot_like.Row(0);
+  const size_t screenshot_stride = is_screenshot_like.PixelsPerRow();
+  const auto process_row = [&](uint64_t y, int _) {
+    for (uint64_t x = 0; x < opsin.xsize() / kPatchSide; x++) {
+      bool all_same = true;
+      for (size_t iy = 0; iy < kPatchSide; iy++) {
+        for (size_t ix = 0; ix < kPatchSide; ix++) {
+          size_t cx = x * kPatchSide + ix;
+          size_t cy = y * kPatchSide + iy;
+          if (!is_same({cx, cy}, {x * kPatchSide, y * kPatchSide})) {
+            all_same = false;
+            break;
+          }
+        }
+      }
+      if (!all_same) continue;
+      size_t num = 0;
+      size_t num_same = 0;
+      for (int64_t iy = -kExtraSide; iy < kExtraSide + kPatchSide; iy++) {
+        for (int64_t ix = -kExtraSide; ix < kExtraSide + kPatchSide; ix++) {
+          int64_t cx = x * kPatchSide + ix;
+          int64_t cy = y * kPatchSide + iy;
+          if (cx < 0 || static_cast<uint64_t>(cx) >= opsin.xsize() ||  //
+              cy < 0 || static_cast<uint64_t>(cy) >= opsin.ysize()) {
+            continue;
+          }
+          num++;
+          if (is_same({cx, cy}, {x * kPatchSide, y * kPatchSide})) num_same++;
+        }
+      }
+      // Too few equal pixels nearby.
+      if (num_same * 8 < num * 7) continue;
+      screenshot_row[y * screenshot_stride + x] = 1;
+      has_screenshot_areas = true;
+    }
+  };
+  RunOnPool(pool, 0, opsin.ysize() / kPatchSide, ThreadPool::SkipInit(),
+            process_row, "IsScreenshotLike");
+
+  // TODO(veluca): also parallelize the rest of this function.
+  if (WantDebugOutput(aux_out)) {
+    aux_out->DumpPlaneNormalized("screenshot_like", is_screenshot_like);
+  }
+
+  constexpr int kSearchRadius = 1;
+
+  if (!ApplyOverride(state->cparams.patches, has_screenshot_areas)) {
+    return {};
+  }
+  // Search for "similar enough" pixels near the screenshot-like areas.
+  ImageB is_background(opsin.xsize(), opsin.ysize());
+  ZeroFillImage(&is_background);
+  Image3F background(opsin.xsize(), opsin.ysize());
+  ZeroFillImage(&background);
+  constexpr size_t kDistanceLimit = 50;
+  float* JXL_RESTRICT background_rows[3] = {
+      background.PlaneRow(0, 0),
+      background.PlaneRow(1, 0),
+      background.PlaneRow(2, 0),
+  };
+  const size_t background_stride = background.PixelsPerRow();
+  uint8_t* JXL_RESTRICT is_background_row = is_background.Row(0);
+  const size_t is_background_stride = is_background.PixelsPerRow();
+  std::vector<
+      std::pair<std::pair<uint32_t, uint32_t>, std::pair<uint32_t, uint32_t>>>
+      queue;
+  size_t queue_front = 0;
+  for (size_t y = 0; y < opsin.ysize(); y++) {
+    for (size_t x = 0; x < opsin.xsize(); x++) {
+      if (!screenshot_row[screenshot_stride * (y / kPatchSide) +
+                          (x / kPatchSide)])
+        continue;
+      queue.push_back({{x, y}, {x, y}});
+    }
+  }
+  while (queue.size() != queue_front) {
+    std::pair<uint32_t, uint32_t> cur = queue[queue_front].first;
+    std::pair<uint32_t, uint32_t> src = queue[queue_front].second;
+    queue_front++;
+    if (is_background_row[cur.second * is_background_stride + cur.first])
+      continue;
+    is_background_row[cur.second * is_background_stride + cur.first] = 1;
+    for (size_t c = 0; c < 3; c++) {
+      background_rows[c][cur.second * background_stride + cur.first] =
+          opsin_rows[c][src.second * opsin_stride + src.first];
+    }
+    for (int dx = -kSearchRadius; dx <= kSearchRadius; dx++) {
+      for (int dy = -kSearchRadius; dy <= kSearchRadius; dy++) {
+        if (dx == 0 && dy == 0) continue;
+        int next_first = cur.first + dx;
+        int next_second = cur.second + dy;
+        if (next_first < 0 || next_second < 0 ||
+            static_cast<uint32_t>(next_first) >= opsin.xsize() ||
+            static_cast<uint32_t>(next_second) >= opsin.ysize()) {
+          continue;
+        }
+        if (static_cast<uint32_t>(
+                std::abs(next_first - static_cast<int>(src.first)) +
+                std::abs(next_second - static_cast<int>(src.second))) >
+            kDistanceLimit) {
+          continue;
+        }
+        std::pair<uint32_t, uint32_t> next{next_first, next_second};
+        if (is_similar(src, next)) {
+          if (!screenshot_row[next.second / kPatchSide * screenshot_stride +
+                              next.first / kPatchSide] ||
+              is_same(src, next)) {
+            if (!is_background_row[next.second * is_background_stride +
+                                   next.first])
+              queue.push_back({next, src});
+          }
+        }
+      }
+    }
+  }
+  queue.clear();
+
+  ImageF ccs;
+  std::mt19937 rng;
+  std::uniform_real_distribution<float> dist(0.5, 1.0);
+  bool paint_ccs = false;
+  if (WantDebugOutput(aux_out)) {
+    aux_out->DumpPlaneNormalized("is_background", is_background);
+    aux_out->DumpXybImage("background", background);
+    ccs = ImageF(opsin.xsize(), opsin.ysize());
+    ZeroFillImage(&ccs);
+    paint_ccs = true;
+  }
+
+  constexpr float kVerySimilarThreshold = 0.03f;
+  constexpr float kHasSimilarThreshold = 0.03f;
+
+  const float* JXL_RESTRICT const_background_rows[3] = {
+      background_rows[0], background_rows[1], background_rows[2]};
+  auto is_similar_b = [&](std::pair<int, int> p1, std::pair<int, int> p2) {
+    return is_similar_impl(p1, p2, const_background_rows, background_stride,
+                           kVerySimilarThreshold);
+  };
+
+  constexpr int kMinPeak = 2;
+  constexpr int kHasSimilarRadius = 2;
+
+  std::vector<PatchInfo> info;
+
+  // Find small CC outside the "similar enough" areas, compute bounding boxes,
+  // and run heuristics to exclude some patches.
+  ImageB visited(opsin.xsize(), opsin.ysize());
+  ZeroFillImage(&visited);
+  uint8_t* JXL_RESTRICT visited_row = visited.Row(0);
+  const size_t visited_stride = visited.PixelsPerRow();
+  std::vector<std::pair<uint32_t, uint32_t>> cc;
+  std::vector<std::pair<uint32_t, uint32_t>> stack;
+  for (size_t y = 0; y < opsin.ysize(); y++) {
+    for (size_t x = 0; x < opsin.xsize(); x++) {
+      if (is_background_row[y * is_background_stride + x]) continue;
+      cc.clear();
+      stack.clear();
+      stack.push_back({x, y});
+      size_t min_x = x;
+      size_t max_x = x;
+      size_t min_y = y;
+      size_t max_y = y;
+      std::pair<uint32_t, uint32_t> reference;
+      bool found_border = false;
+      bool all_similar = true;
+      while (!stack.empty()) {
+        std::pair<uint32_t, uint32_t> cur = stack.back();
+        stack.pop_back();
+        if (visited_row[cur.second * visited_stride + cur.first]) continue;
+        visited_row[cur.second * visited_stride + cur.first] = 1;
+        if (cur.first < min_x) min_x = cur.first;
+        if (cur.first > max_x) max_x = cur.first;
+        if (cur.second < min_y) min_y = cur.second;
+        if (cur.second > max_y) max_y = cur.second;
+        if (paint_ccs) {
+          cc.push_back(cur);
+        }
+        for (int dx = -kSearchRadius; dx <= kSearchRadius; dx++) {
+          for (int dy = -kSearchRadius; dy <= kSearchRadius; dy++) {
+            if (dx == 0 && dy == 0) continue;
+            int next_first = static_cast<int32_t>(cur.first) + dx;
+            int next_second = static_cast<int32_t>(cur.second) + dy;
+            if (next_first < 0 || next_second < 0 ||
+                static_cast<uint32_t>(next_first) >= opsin.xsize() ||
+                static_cast<uint32_t>(next_second) >= opsin.ysize()) {
+              continue;
+            }
+            std::pair<uint32_t, uint32_t> next{next_first, next_second};
+            if (!is_background_row[next.second * is_background_stride +
+                                   next.first]) {
+              stack.push_back(next);
+            } else {
+              if (!found_border) {
+                reference = next;
+                found_border = true;
+              } else {
+                if (!is_similar_b(next, reference)) all_similar = false;
+              }
+            }
+          }
+        }
+      }
+      if (!found_border || !all_similar || max_x - min_x >= kMaxPatchSize ||
+          max_y - min_y >= kMaxPatchSize) {
+        continue;
+      }
+      size_t bpos = background_stride * reference.second + reference.first;
+      float ref[3] = {background_rows[0][bpos], background_rows[1][bpos],
+                      background_rows[2][bpos]};
+      bool has_similar = false;
+      for (size_t iy = std::max<int>(
+               static_cast<int32_t>(min_y) - kHasSimilarRadius, 0);
+           iy < std::min(max_y + kHasSimilarRadius + 1, opsin.ysize()); iy++) {
+        for (size_t ix = std::max<int>(
+                 static_cast<int32_t>(min_x) - kHasSimilarRadius, 0);
+             ix < std::min(max_x + kHasSimilarRadius + 1, opsin.xsize());
+             ix++) {
+          size_t opos = opsin_stride * iy + ix;
+          float px[3] = {opsin_rows[0][opos], opsin_rows[1][opos],
+                         opsin_rows[2][opos]};
+          if (pci.is_similar_v(ref, px, kHasSimilarThreshold)) {
+            has_similar = true;
+          }
+        }
+      }
+      if (!has_similar) continue;
+      QuantizedPatch patch;
+      patch.xsize = max_x - min_x + 1;
+      patch.ysize = max_y - min_y + 1;
+      int max_value = 0;
+      for (size_t c : {1, 0, 2}) {
+        for (size_t iy = min_y; iy <= max_y; iy++) {
+          for (size_t ix = min_x; ix <= max_x; ix++) {
+            size_t offset = (iy - min_y) * patch.xsize + ix - min_x;
+            patch.fpixels[c][offset] =
+                opsin_rows[c][iy * opsin_stride + ix] - ref[c];
+            int val = pci.Quantize(patch.fpixels[c][offset], c);
+            patch.pixels[c][offset] = val;
+            if (std::abs(val) > max_value) max_value = std::abs(val);
+          }
+        }
+      }
+      if (max_value < kMinPeak) {
+        continue;
+      }
+      info.push_back({patch, {{min_x, min_y}}});
+      if (paint_ccs) {
+        float cc_color = dist(rng);
+        for (std::pair<uint32_t, uint32_t> p : cc) {
+          ccs.Row(p.second)[p.first] = cc_color;
+        }
+      }
+    }
+  }
+
+  if (paint_ccs) {
+    JXL_ASSERT(WantDebugOutput(aux_out));
+    aux_out->DumpPlaneNormalized("ccs", ccs);
+  }
+  if (info.empty()) {
+    return {};
+  }
+
+  // Remove duplicates.
+  constexpr size_t kMinPatchOccurences = 2;
+  std::sort(info.begin(), info.end());
+  size_t unique = 0;
+  for (size_t i = 1; i < info.size(); i++) {
+    if (info[i].first == info[unique].first) {
+      info[unique].second.insert(info[unique].second.end(),
+                                 info[i].second.begin(), info[i].second.end());
+    } else {
+      if (info[unique].second.size() >= kMinPatchOccurences) {
+        unique++;
+      }
+      info[unique] = info[i];
+    }
+  }
+  if (info[unique].second.size() >= kMinPatchOccurences) {
+    unique++;
+  }
+  info.resize(unique);
+
+  size_t max_patch_size = 0;
+
+  for (size_t i = 0; i < info.size(); i++) {
+    size_t pixels = info[i].first.xsize * info[i].first.ysize;
+    if (pixels > max_patch_size) max_patch_size = pixels;
+  }
+
+  // don't use patches if all patches are smaller than this
+  constexpr size_t kMinMaxPatchSize = 20;
+  if (max_patch_size < kMinMaxPatchSize) return {};
+
+  // Ensure that the specified set of patches doesn't produce out-of-bounds
+  // pixels.
+  // TODO(veluca): figure out why this is still necessary even with RCTs that
+  // don't depend on bit depth.
+  if (state->cparams.modular_group_mode &&
+      state->cparams.quality_pair.first >= 100) {
+    for (size_t i = 0; i < info.size(); i++) {
+      for (size_t c = 0; c < 3; c++) {
+        float min_px[kMaxPatchSize * kMaxPatchSize];
+        float max_px[kMaxPatchSize * kMaxPatchSize] = {};
+        std::fill(std::begin(min_px), std::end(min_px), 255);
+        size_t xsize = info[i].first.xsize;
+        for (size_t j = 0; j < info[i].second.size(); j++) {
+          size_t bx = info[i].second[j].first;
+          size_t by = info[i].second[j].second;
+          for (size_t iy = 0; iy < info[i].first.ysize; iy++) {
+            for (size_t ix = 0; ix < xsize; ix++) {
+              float v = opsin_rows[c][(by + iy) * opsin_stride + bx + ix];
+              if (v < min_px[iy * xsize + ix]) min_px[iy * xsize + ix] = v;
+              if (v > max_px[iy * xsize + ix]) max_px[iy * xsize + ix] = v;
+            }
+          }
+        }
+        for (size_t iy = 0; iy < info[i].first.ysize; iy++) {
+          for (size_t ix = 0; ix < xsize; ix++) {
+            float smallest = min_px[iy * xsize + ix];
+            float biggest = max_px[iy * xsize + ix];
+            JXL_ASSERT(smallest <= biggest);
+            float& out = info[i].first.fpixels[c][iy * xsize + ix];
+            // Clamp fpixels so that subtracting the patch never creates a
+            // negative value, or a value above 255.
+            JXL_ASSERT(biggest - 255 <= smallest);
+            out = std::max(smallest, out);
+            out = std::min(biggest - 255.f, out);
+          }
+        }
+      }
+    }
+  }
+
+  return info;
+}
+
+}  // namespace
+
+void FindBestPatchDictionary(const Image3F& opsin,
+                             PassesEncoderState* JXL_RESTRICT state,
+                             ThreadPool* pool, AuxOut* aux_out, bool is_xyb) {
+  std::vector<PatchInfo> info =
+      FindTextLikePatches(opsin, state, pool, aux_out, is_xyb);
+
+  // TODO(veluca): this doesn't work if both dots and patches are enabled.
+  // For now, since dots and patches are not likely to occur in the same kind of
+  // images, disable dots if some patches were found.
+  if (ApplyOverride(
+          state->cparams.dots,
+          state->cparams.speed_tier <= SpeedTier::kSquirrel &&
+              state->cparams.butteraugli_distance >= kMinButteraugliForDots)) {
+    info = FindDotDictionary(state->cparams, opsin, state->shared.cmap, pool);
+  }
+
+  if (info.empty()) return;
+
+  std::sort(
+      info.begin(), info.end(), [&](const PatchInfo& a, const PatchInfo& b) {
+        return a.first.xsize * a.first.ysize > b.first.xsize * b.first.ysize;
+      });
+
+  size_t max_x_size = 0;
+  size_t max_y_size = 0;
+  size_t total_pixels = 0;
+
+  for (size_t i = 0; i < info.size(); i++) {
+    size_t pixels = info[i].first.xsize * info[i].first.ysize;
+    if (max_x_size < info[i].first.xsize) max_x_size = info[i].first.xsize;
+    if (max_y_size < info[i].first.ysize) max_y_size = info[i].first.ysize;
+    total_pixels += pixels;
+  }
+
+  // Bin-packing & conversion of patches.
+  constexpr float kBinPackingSlackness = 1.05f;
+  size_t ref_xsize = std::max<float>(max_x_size, std::sqrt(total_pixels));
+  size_t ref_ysize = std::max<float>(max_y_size, std::sqrt(total_pixels));
+  std::vector<std::pair<size_t, size_t>> ref_positions(info.size());
+  // TODO(veluca): allow partial overlaps of patches that have the same pixels.
+  size_t max_y = 0;
+  do {
+    max_y = 0;
+    // Increase packed image size.
+    ref_xsize = ref_xsize * kBinPackingSlackness + 1;
+    ref_ysize = ref_ysize * kBinPackingSlackness + 1;
+
+    ImageB occupied(ref_xsize, ref_ysize);
+    ZeroFillImage(&occupied);
+    uint8_t* JXL_RESTRICT occupied_rows = occupied.Row(0);
+    size_t occupied_stride = occupied.PixelsPerRow();
+
+    bool success = true;
+    // For every patch...
+    for (size_t patch = 0; patch < info.size(); patch++) {
+      size_t x0 = 0;
+      size_t y0 = 0;
+      size_t xsize = info[patch].first.xsize;
+      size_t ysize = info[patch].first.ysize;
+      bool found = false;
+      // For every possible start position ...
+      for (; y0 + ysize <= ref_ysize; y0++) {
+        x0 = 0;
+        for (; x0 + xsize <= ref_xsize; x0++) {
+          bool has_occupied_pixel = false;
+          size_t x = x0;
+          // Check if it is possible to place the patch in this position in the
+          // reference frame.
+          for (size_t y = y0; y < y0 + ysize; y++) {
+            x = x0;
+            for (; x < x0 + xsize; x++) {
+              if (occupied_rows[y * occupied_stride + x]) {
+                has_occupied_pixel = true;
+                break;
+              }
+            }
+          }  // end of positioning check
+          if (!has_occupied_pixel) {
+            found = true;
+            break;
+          }
+          x0 = x;  // Jump to next pixel after the occupied one.
+        }
+        if (found) break;
+      }  // end of start position checking
+
+      // We didn't find a possible position: repeat from the beginning with a
+      // larger reference frame size.
+      if (!found) {
+        success = false;
+        break;
+      }
+
+      // We found a position: mark the corresponding positions in the reference
+      // image as used.
+      ref_positions[patch] = {x0, y0};
+      for (size_t y = y0; y < y0 + ysize; y++) {
+        for (size_t x = x0; x < x0 + xsize; x++) {
+          occupied_rows[y * occupied_stride + x] = true;
+        }
+      }
+      max_y = std::max(max_y, y0 + ysize);
+    }
+
+    if (success) break;
+  } while (true);
+
+  JXL_ASSERT(ref_ysize >= max_y);
+
+  ref_ysize = max_y;
+
+  Image3F reference_frame(ref_xsize, ref_ysize);
+  // TODO(veluca): figure out a better way to fill the image.
+  ZeroFillImage(&reference_frame);
+  std::vector<PatchPosition> positions;
+  float* JXL_RESTRICT ref_rows[3] = {
+      reference_frame.PlaneRow(0, 0),
+      reference_frame.PlaneRow(1, 0),
+      reference_frame.PlaneRow(2, 0),
+  };
+  size_t ref_stride = reference_frame.PixelsPerRow();
+
+  for (size_t i = 0; i < info.size(); i++) {
+    PatchReferencePosition ref_pos;
+    ref_pos.xsize = info[i].first.xsize;
+    ref_pos.ysize = info[i].first.ysize;
+    ref_pos.x0 = ref_positions[i].first;
+    ref_pos.y0 = ref_positions[i].second;
+    ref_pos.ref = 0;
+    for (size_t y = 0; y < ref_pos.ysize; y++) {
+      for (size_t x = 0; x < ref_pos.xsize; x++) {
+        for (size_t c = 0; c < 3; c++) {
+          ref_rows[c][(y + ref_pos.y0) * ref_stride + x + ref_pos.x0] =
+              info[i].first.fpixels[c][y * ref_pos.xsize + x];
+        }
+      }
+    }
+    for (const auto pos : info[i].second) {
+      positions.emplace_back(
+          PatchPosition{pos.first, pos.second, PatchBlendMode::kAdd, ref_pos});
+    }
+  }
+
+  if (state->cparams.butteraugli_distance >=
+      kMinButteraugliToSubtractOriginalPatches) {
+    state->reference_frames[0] = CopyImage(reference_frame);
+  }
+
+  CompressParams cparams = state->cparams;
+  FrameDimensions frame_dim = state->shared.frame_dim;
+  AnimationFrame animation_frame;
+  animation_frame.duration = 0;
+  animation_frame.is_last = 0;
+  animation_frame.have_crop = true;
+  animation_frame.x0 = 0;
+  animation_frame.y0 = 0;
+  animation_frame.xsize = ref_xsize;
+  animation_frame.ysize = ref_ysize;
+  cparams.save_as_reference = 1;
+  // Recursive application of patches could create very weird issues.
+  cparams.patches = Override::kOff;
+  cparams.dots = Override::kOff;
+  cparams.modular_group_mode = true;
+  cparams.responsive = 0;
+  // TODO(veluca): possibly change heuristics here.
+  if (!cparams.modular_group_mode) {
+    cparams.quality_pair.first = cparams.quality_pair.second =
+        80 - cparams.butteraugli_distance * 12;
+  } else {
+    cparams.quality_pair.first = (100 + cparams.quality_pair.first) * 0.5f;
+    cparams.quality_pair.second = (100 + cparams.quality_pair.second) * 0.5f;
+  }
+  ImageMetadata metadata;
+  metadata.color_encoding = ColorManagement::LinearSRGB();
+  ImageBundle ib(&metadata);
+  // XYB, not sRGB.
+  ib.SetFromImage(std::move(reference_frame), ColorManagement::LinearSRGB());
+  PassesEncoderState roundtrip_state;
+  state->special_frames.emplace_back();
+  Multiframe multiframe;
+  JXL_CHECK(EncodeFrame(cparams, &animation_frame, ib, &roundtrip_state, pool,
+                        &state->special_frames.back(), nullptr, &multiframe));
+  const Span<const uint8_t> encoded = state->special_frames.back().GetSpan();
+  if (cparams.butteraugli_distance < kMinButteraugliToSubtractOriginalPatches) {
+    BitReader br(encoded);
+    ImageBundle decoded(&metadata);
+    JXL_CHECK(DecodeFrame({}, encoded, nullptr, &frame_dim, &multiframe, pool,
+                          &br, nullptr, &decoded));
+    JXL_CHECK(br.Close());
+    state->reference_frames[0] = CopyImage(multiframe.GetReferenceFrames()[0]);
+  }
+  state->shared.image_features.patches.SetReferenceFrames(
+      state->reference_frames);
+  state->shared.image_features.patches.SetPositions(std::move(positions));
+}
+}  // namespace jxl
diff --git a/jxl/patch_dictionary.h b/jxl/patch_dictionary.h
new file mode 100644
index 000000000..e4027e6fb
--- /dev/null
+++ b/jxl/patch_dictionary.h
@@ -0,0 +1,165 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_PATCH_DICTIONARY_H_
+#define JXL_PATCH_DICTIONARY_H_
+
+// Chooses reference patches, and avoids encoding them once per occurrence.
+
+#include <stddef.h>
+#include <string.h>
+#include <sys/types.h>
+
+#include <tuple>
+#include <vector>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/common.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_params.h"
+#include "jxl/image.h"
+#include "jxl/opsin_params.h"
+
+namespace jxl {
+
+constexpr size_t kMaxPatchSize = 32;
+
+enum class PatchBlendMode : uint8_t {
+  kAdd,
+  kReplace,
+  // TODO(veluca): Add kBlendAbove, kBlendBelow
+  kNumBlendModes,
+};
+
+struct QuantizedPatch {
+  size_t xsize;
+  size_t ysize;
+  int8_t pixels[3][kMaxPatchSize * kMaxPatchSize] = {};
+  // Not compared. Used only to retrieve original pixels to construct the
+  // reference image.
+  float fpixels[3][kMaxPatchSize * kMaxPatchSize] = {};
+  bool operator==(const QuantizedPatch& other) const {
+    if (xsize != other.xsize) return false;
+    if (ysize != other.ysize) return false;
+    for (size_t c = 0; c < 3; c++) {
+      if (memcmp(pixels[c], other.pixels[c], sizeof(int8_t) * xsize * ysize) !=
+          0)
+        return false;
+    }
+    return true;
+  }
+
+  bool operator<(const QuantizedPatch& other) const {
+    if (xsize != other.xsize) return xsize < other.xsize;
+    if (ysize != other.ysize) return ysize < other.ysize;
+    for (size_t c = 0; c < 3; c++) {
+      int cmp =
+          memcmp(pixels[c], other.pixels[c], sizeof(int8_t) * xsize * ysize);
+      if (cmp > 0) return false;
+      if (cmp < 0) return true;
+    }
+    return false;
+  }
+};
+
+// Pair (patch, vector of occurences).
+using PatchInfo =
+    std::pair<QuantizedPatch, std::vector<std::pair<uint32_t, uint32_t>>>;
+
+// Position and size of the patch in the reference frame.
+struct PatchReferencePosition {
+  size_t ref, x0, y0, xsize, ysize;
+  bool operator<(const PatchReferencePosition& oth) const {
+    return std::make_tuple(ref, x0, y0, xsize, ysize) <
+           std::make_tuple(oth.ref, oth.x0, oth.y0, oth.xsize, oth.ysize);
+  }
+  bool operator==(const PatchReferencePosition& oth) const {
+    return !(*this < oth) && !(oth < *this);
+  }
+};
+
+struct PatchPosition {
+  // Position of top-left corner of the patch in the image.
+  size_t x, y;
+  PatchBlendMode blend_mode;
+  PatchReferencePosition ref_pos;
+  bool operator<(const PatchPosition& oth) const {
+    return std::make_tuple(ref_pos, x, y) <
+           std::make_tuple(oth.ref_pos, oth.x, oth.y);
+  }
+};
+
+class PatchDictionary {
+ public:
+  PatchDictionary() = default;
+
+  void SetReferenceFrames(const Image3F* JXL_RESTRICT reference_frames) {
+    reference_frames_ = reference_frames;
+  }
+  void SetPositions(std::vector<PatchPosition> positions) {
+    positions_ = std::move(positions);
+    std::sort(positions_.begin(), positions_.end());
+    ComputePatchCache();
+  }
+
+  bool HasAny() const { return !positions_.empty(); }
+  // Only call if HasAny().
+  void Encode(BitWriter* writer, size_t layer, AuxOut* aux_out) const;
+
+  Status Decode(BitReader* br, size_t xsize, size_t ysize,
+                size_t save_as_reference);
+
+  // Only adds patches that belong to the `image_rect` area of the decoded
+  // image, writing them to the `opsin_rect` area of `opsin`.
+  void AddTo(Image3F* opsin, const Rect& opsin_rect,
+             const Rect& image_rect) const;
+
+  void SubtractFrom(Image3F* opsin) const;
+
+ private:
+  const Image3F* JXL_RESTRICT reference_frames_;
+  std::vector<PatchPosition> positions_;
+
+  // Patch occurences sorted by y.
+  std::vector<size_t> sorted_patches_;
+  // Index of the first patch for each y value.
+  std::vector<size_t> patch_starts_;
+
+  // Patch IDs in position [patch_starts_[y], patch_start_[y+1]) of
+  // sorted_patches_ are all the patches that intersect the horizontal line at
+  // y.
+
+  // Compute sorted_patches_ and patch_start_ after updating positions_.
+  void ComputePatchCache();
+
+  template <bool>
+  void Apply(Image3F* opsin, const Rect& opsin_rect,
+             const Rect& image_rect) const;
+};
+
+// Avoid cyclic header inclusion.
+struct PassesEncoderState;
+
+void FindBestPatchDictionary(const Image3F& opsin,
+                             PassesEncoderState* JXL_RESTRICT state,
+                             ThreadPool* pool, AuxOut* aux_out,
+                             bool is_xyb = true);
+
+}  // namespace jxl
+
+#endif  // JXL_PATCH_DICTIONARY_H_
diff --git a/jxl/predictor.cc b/jxl/predictor.cc
new file mode 100644
index 000000000..72781bbb1
--- /dev/null
+++ b/jxl/predictor.cc
@@ -0,0 +1,17 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/predictor.h"
+
+// This file is left empty for possible future use.
diff --git a/jxl/predictor.h b/jxl/predictor.h
new file mode 100644
index 000000000..cf1d38301
--- /dev/null
+++ b/jxl/predictor.h
@@ -0,0 +1,793 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_PREDICTOR_H_
+#define JXL_PREDICTOR_H_
+
+// DC coefficients serve as an image preview, so they are coded separately.
+// Subtracting predicted values leads to a "residual" distribution with lower
+// entropy and magnitudes than the original values. These can be coded more
+// efficiently, even when context modeling is used.
+//
+// Our predictors use immediately adjacent causal pixels because more distant
+// pixels are only weakly correlated in subsampled DC images.
+//
+// This module decreases final size of DC images by 2-4% vs. the standard
+// MED/MAP predictor from JPEG-LS and processes 330 M coefficients per second.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/compiler_specific.h>
+#include <hwy/scalar.h>
+#include <hwy/static_targets.h>
+#include <limits>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/common.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/image.h"
+#include "jxl/image_ops.h"
+
+namespace jxl {
+
+// TODO(veluca): what happens if all valid predictors have an error of exactly
+// 2**16-1? One option is to ensure that this does not happen.
+
+// Sliding window of "causal" (already decoded) pixels, plus simple functions
+// to predict the next pixel "c" from its neighbors: l n r
+// The single-letter names shorten identifiers.      w c
+//
+// Predictions are more accurate when the preceding w pixel is available, but
+// this interferes with SIMD because subsequent pixels depend on the decoding
+// of their predecessor. The encoder can compute residuals in parallel because
+// it knows all DC values up front, but its speed is less important. A diagonal
+// 'wavefront' order would allow computing multiple predictions efficiently,
+// but scattering those to the corresponding pixel positions would be slow.
+// Interleaving pixels by the lane count (eight pixels with x mod 8 = 0, etc)
+// would work if the two pixels before each prediction are already known, but
+// scattering lanes to multiples of 10 would also be slow.
+//
+// We instead compute the various predictors using SIMD, especially because
+// many of them are similar. Horizontal operations are generally inefficient,
+// but we take advantage of special hardware support for video codecs (minpos).
+//
+// The set of 8 predictors was chosen from a set of 16 as the combination that
+// minimized a simple model of encoding cost. Their order matters because
+// minpos(lanes) returns the lowest i with lanes[i] == min. We again retained
+// the permutation with the lowest encoding cost.
+
+namespace predictor {
+
+constexpr bool kEnableStats = false;
+
+constexpr size_t kNumPredictors = 8;
+
+// Returns the average of two int32_t values without overflowing:
+// floor((a + b) / 2).
+static JXL_INLINE int32_t Average(int64_t a, int64_t b) { return (a + b) / 2; }
+
+// Clamps gradient to the min/max of n, w (and l, implicitly).
+static JXL_INLINE int32_t ClampedGradient(const int32_t n, const int32_t w,
+                                          const int32_t l) {
+  const int32_t m = std::min(n, w);
+  const int32_t M = std::max(n, w);
+  // The end result of this operation doesn't overflow or underflow if the
+  // result is between m and M, but the intermediate value may overflow, so we
+  // do the intermediate operations in uint32_t and check later if we had an
+  // overflow or underflow condition comparing m, M and l directly.
+  // grad = M + m - l = n + w - l
+  const int32_t grad =
+      static_cast<int32_t>(static_cast<uint32_t>(n) + static_cast<uint32_t>(w) -
+                           static_cast<uint32_t>(l));
+  // We use two sets of ternary operators to force the evaluation of them in
+  // any case, allowing the compiler to avoid branches and use cmovl/cmovg in
+  // x86.
+  const int32_t grad_clamp_M = (l < m) ? M : grad;
+  return (l > M) ? m : grad_clamp_M;
+}
+
+struct PackSigned {
+  static HWY_ATTR JXL_INLINE void Compute(const int32_t c,
+                                          const int32_t* JXL_RESTRICT pred,
+                                          uint32_t* JXL_RESTRICT error) {
+    HWY_CAPPED(int32_t, kNumPredictors) di;
+    HWY_CAPPED(uint32_t, kNumPredictors) du;
+    auto cv = Set(di, c);
+    for (size_t i = 0; i < kNumPredictors; i += di.N) {
+      auto res = cv - Load(di, pred + i);
+      auto res2 = hwy::ShiftLeft<1>(BitCast(du, Abs(res)));
+      auto res_sign = BitCast(du, VecFromMask(res < Zero(di)));
+      Store(res2 + res_sign, du, error + i);
+    }
+  }
+
+  static JXL_INLINE uint32_t Residual(int32_t c, int32_t pred) {
+    return jxl::PackSigned(c - pred);
+  }
+
+  static JXL_INLINE int32_t Original(uint32_t res, int32_t pred) {
+    return pred + UnpackSigned(res);
+  }
+};
+
+template <int16_t min, int16_t max>
+struct PackSignedRange {
+  static_assert(min <= max, "min <= max");
+  static HWY_ATTR JXL_INLINE void Compute(const int32_t c,
+                                          const int32_t* JXL_RESTRICT pred,
+                                          uint32_t* JXL_RESTRICT error) {
+    HWY_CAPPED(int32_t, kNumPredictors) di;
+    HWY_CAPPED(uint32_t, kNumPredictors) du;
+    auto cv = Set(di, c);
+    auto minv = Set(di, min);
+    auto maxv = Set(di, max);
+    for (size_t i = 0; i < kNumPredictors; i += di.N) {
+      auto predv = Load(di, pred + i);
+      auto res = cv - predv;
+      auto low = cv - minv;
+      auto high = maxv - cv;
+      auto res2 = hwy::ShiftLeft<1>(Abs(res));
+      auto res_sign = VecFromMask(res < Zero(di));
+      auto reshigh = IfThenElse(maxv == predv, high, res2 + res_sign);
+      Store(BitCast(du, IfThenElse(minv == predv, low, reshigh)), du,
+            error + i);
+    }
+  }
+
+  static JXL_INLINE uint32_t Residual(int32_t c, int32_t pred) {
+    if (pred == min) return c - min;
+    if (pred == max) return max - c;
+    return jxl::PackSigned(c - pred);
+  }
+
+  static JXL_INLINE int32_t Original(uint32_t res, int32_t pred) {
+    if (pred == min) return min + res;
+    if (pred == max) return max - res;
+    return pred + UnpackSigned(res);
+  }
+};
+
+static HWY_ATTR JXL_INLINE void ChoosePredictor(
+    const int32_t prediction[kNumPredictors],
+    const uint32_t expected_error[kNumPredictors], int32_t* JXL_RESTRICT pred,
+    uint32_t* JXL_RESTRICT min_error, uint32_t* JXL_RESTRICT num_correct) {
+#if HWY_BITS >= 256
+  HWY_CAPPED(uint32_t, kNumPredictors) du;
+  HWY_CAPPED(int32_t, kNumPredictors) di;
+  static_assert(kNumPredictors == 8, "");
+  const auto predv = Load(di, prediction);
+  const auto errv = Load(du, expected_error);  // 76543210
+
+  const auto errv1 = Shuffle2301(errv);  // 67452301
+  const auto predv1 = Shuffle2301(predv);
+
+  // errv1 >= errv
+  auto max = Max(errv1, errv);
+  const auto errv2 = IfThenElse(max == errv1, errv, errv1);  // 66442200
+  const auto predv2 =
+      IfThenElse(BitCast(di, max) == BitCast(di, errv1), predv, predv1);
+
+  const auto errv3 = Shuffle1032(errv2);  // 44660022
+  const auto predv3 = Shuffle1032(predv2);
+
+  // errv3 >= errv2
+  max = Max(errv3, errv2);
+  const auto errv4 = IfThenElse(max == errv3, errv2, errv3);  // 44440000
+  const auto predv4 =
+      IfThenElse(BitCast(di, max) == BitCast(di, errv3), predv2, predv3);
+
+  const auto plow = GetLane(LowerHalf(predv4));
+  const auto phi = GetLane(UpperHalf(predv4));
+  const auto elow = GetLane(LowerHalf(errv4));
+  const auto ehi = GetLane(UpperHalf(errv4));
+
+  if (ehi < elow) {
+    *pred = phi;
+    *min_error = ehi;
+  } else {
+    *pred = plow;
+    *min_error = elow;
+  }
+
+  *num_correct = hwy::ext::CountTrue(errv == Zero(du));
+#else
+  // TODO(veluca): 128 bit version.
+  size_t idx_pred = 0;
+  uint32_t min_cost = expected_error[0];
+  for (size_t i = 1; i < kNumPredictors; ++i) {
+    const uint32_t cost = expected_error[i];
+    if (cost < min_cost) {
+      min_cost = cost;
+      idx_pred = i;
+    }
+  }
+  *pred = prediction[idx_pred];
+  *min_error = expected_error[idx_pred];
+
+  HWY_CAPPED(uint32_t, kNumPredictors) du;
+  size_t ret = 0;
+  for (size_t i = 0; i < kNumPredictors; i += du.N) {
+    auto l = Load(du, expected_error + i);
+    ret += hwy::ext::CountTrue(l == Zero(du));
+  }
+  *num_correct = ret;
+#endif
+}
+
+// Ensure that 32-bit systems do not overflow when doing simple arithmetic on
+// the max error.
+static constexpr uint32_t kMaxError = std::numeric_limits<uint32_t>::max() >> 1;
+
+struct YPredictor {
+  static JXL_INLINE void Predict(const int32_t n, const int32_t w,
+                                 const int32_t l, const int32_t r,
+                                 int32_t* JXL_RESTRICT pred) {
+    pred[0] = Average(Average(n, w), r);
+    pred[1] = Average(w, n);
+    pred[2] = Average(n, r);
+    pred[3] = Average(w, l);
+    pred[4] = Average(n, l);
+    pred[5] = w;
+    pred[6] = ClampedGradient(n, w, l);
+    pred[7] = n;
+  }
+
+  // 0 = valid predictor, kMaxError = invalid.
+
+  // All predictors but w are invalid in row 0.
+  static JXL_INLINE const uint32_t* Row0Mask() {
+    HWY_ALIGN static constexpr uint32_t kMask[kNumPredictors] = {
+        kMaxError, kMaxError, kMaxError, kMaxError,
+        kMaxError, 0,         kMaxError, kMaxError};
+    return kMask;
+  }
+
+  // All predictors that use l and w are invalid in col 0.
+  static JXL_INLINE const uint32_t* Col0Mask() {
+    HWY_ALIGN static constexpr uint32_t kMask[kNumPredictors] = {
+        kMaxError, kMaxError, 0, kMaxError, kMaxError, kMaxError, kMaxError, 0};
+    return kMask;
+  }
+
+  // All predictors that use r are invalid in the last column.
+  static JXL_INLINE const uint32_t* LastColMask() {
+    HWY_ALIGN static constexpr uint32_t kMask[kNumPredictors] = {
+        kMaxError, 0, kMaxError, 0, 0, 0, 0, 0};
+    return kMask;
+  }
+};
+
+struct XBPredictor {
+  static HWY_ATTR JXL_INLINE void Predict(const int32_t n, const int32_t w,
+                                          const int32_t l, const int32_t r,
+                                          int32_t* JXL_RESTRICT pred) {
+    pred[0] = ClampedGradient(n, w, l);
+    pred[1] = Average(n, w);
+    pred[2] = n;
+    pred[3] = Average(n, r);
+    pred[4] = w;
+    pred[5] = Average(w, l);
+    pred[6] = r;
+    pred[7] = Average(Average(w, r), n);
+  }
+
+  // All predictors but w are invalid in row 0.
+  static JXL_INLINE const uint32_t* Row0Mask() {
+    HWY_ALIGN static constexpr uint32_t kMask[kNumPredictors] = {
+        kMaxError, kMaxError, kMaxError, kMaxError,
+        0,         kMaxError, kMaxError, kMaxError};
+    return kMask;
+  }
+
+  // All predictors that use l and w are invalid in col 0.
+  static JXL_INLINE const uint32_t* Col0Mask() {
+    HWY_ALIGN static constexpr uint32_t kMask[kNumPredictors] = {
+        kMaxError, kMaxError, 0, 0, kMaxError, kMaxError, 0, kMaxError};
+    return kMask;
+  }
+
+  // All predictors that use r are invalid in the last column.
+  static JXL_INLINE const uint32_t* LastColMask() {
+    HWY_ALIGN static constexpr uint32_t kMask[kNumPredictors] = {
+        0, 0, 0, kMaxError, 0, 0, kMaxError, kMaxError};
+    return kMask;
+  }
+};
+
+static constexpr size_t kMaxLine =
+    kDcGroupDimInBlocks < kGroupDim ? kGroupDim : kDcGroupDimInBlocks;
+
+enum class RowType {
+  kFirstRow,
+  kSecondRow,
+  kRegularRow,
+};
+
+enum ColumnType : uint8_t {
+  kRegularColumn = 0x0,
+  kFirstColumn = 0x1,
+  kSecondColumn = 0x2,
+  kLastColumn = 0x4,
+};
+
+enum class PixelType { kBorderPixel, kInteriorPixel };
+
+namespace error {
+struct Or {
+  static const char* Name() { return "Or"; }
+  static void Apply(uint32_t* error, const uint32_t* mask) {
+    for (size_t i = 0; i < kNumPredictors; i++) error[i] |= mask[i];
+  }
+};
+struct And {
+  static const char* Name() { return "And"; }
+  static void Apply(uint32_t* error, const uint32_t* mask) {
+    for (size_t i = 0; i < kNumPredictors; i++) error[i] &= mask[i];
+  }
+};
+struct AndNot {
+  static const char* Name() { return "AndNot"; }
+  static void Apply(uint32_t* error, const uint32_t* mask) {
+    for (size_t i = 0; i < kNumPredictors; i++) error[i] &= ~mask[i];
+  }
+};
+}  // namespace error
+
+template <typename ErrorMetric, typename Predictor, size_t channel,
+          size_t kNumChannels, PixelType pixel_type>
+struct ApplyPredictor {
+  static_assert(channel < kNumChannels, "Invalid channel!");
+
+  ApplyPredictor(RowType row_type, ColumnType column_type)
+      : row_type(row_type), column_type(column_type) {}
+
+  JXL_INLINE void Mask(uint32_t* JXL_RESTRICT mask) {
+    if (pixel_type != PixelType::kInteriorPixel) {
+      if (row_type == RowType::kFirstRow)
+        error::Or::Apply(mask, Predictor::Row0Mask());
+      if (column_type & kFirstColumn)
+        error::Or::Apply(mask, Predictor::Col0Mask());
+      if (column_type & kLastColumn)
+        error::Or::Apply(mask, Predictor::LastColMask());
+    }
+  }
+
+  template <typename Op>
+  JXL_INLINE void ApplyMask(uint32_t* JXL_RESTRICT errors) {
+    if (pixel_type != PixelType::kInteriorPixel) {
+      uint32_t mask[kNumPredictors] = {};
+      Mask(mask);
+      Op::Apply(errors + channel * kNumPredictors, mask);
+    }
+  }
+
+  HWY_ATTR JXL_INLINE void Prediction(size_t x,
+                                      const int32_t* JXL_RESTRICT row_top,
+                                      const int32_t* JXL_RESTRICT row_cur,
+                                      int32_t* JXL_RESTRICT all_predictions) {
+    const int32_t w =
+        pixel_type != PixelType::kInteriorPixel && (column_type & kFirstColumn)
+            ? 0
+            : row_cur[(x - 1) * kNumChannels + channel];
+    const int32_t l =
+        pixel_type != PixelType::kInteriorPixel &&
+                (row_type == RowType::kFirstRow || (column_type & kFirstColumn))
+            ? 0
+            : row_top[(x - 1) * kNumChannels + channel];
+    // Clang mistakenly detect this as a potentially-undefined assignment, but
+    // the conditions for that to happen cannot occur.
+    // NOLINTNEXTLINE(clang-analyzer-core.uninitialized.Assign)
+    const int32_t n = pixel_type != PixelType::kInteriorPixel &&
+                              row_type == RowType::kFirstRow
+                          ? 0
+                          : row_top[x * kNumChannels + channel];
+    const int32_t r =
+        pixel_type != PixelType::kInteriorPixel &&
+                (row_type == RowType::kFirstRow || (column_type & kLastColumn))
+            ? 0
+            : row_top[(x + 1) * kNumChannels + channel];
+    Predictor::Predict(n, w, l, r, all_predictions + channel * kNumPredictors);
+  }
+
+  HWY_ATTR JXL_INLINE void ComputeError(const int32_t* JXL_RESTRICT pred,
+                                        const int32_t* JXL_RESTRICT actual,
+                                        uint32_t* JXL_RESTRICT error) {
+    ErrorMetric::Compute(actual[channel], pred + channel * kNumPredictors,
+                         error + channel * kNumPredictors);
+    // Invalid predictions should become 0 because they should not affect error
+    // that comes from other pixels for which they are valid. Advance will deal
+    // with setting to max all predictions that have never been valid.
+    ApplyMask<error::AndNot>(error);
+  }
+  RowType row_type;
+  ColumnType column_type;
+};
+
+// Policy structs for 1/2/3 channels.
+template <typename Predictor>
+struct Predictors1 {
+  enum { kNumChannels = 1 };
+  template <typename ErrorMetric, PixelType pixel_type>
+  struct Impl {
+    using ApplyPredictor =
+        ApplyPredictor<ErrorMetric, Predictor, 0, kNumChannels, pixel_type>;
+
+    template <typename Op>
+    HWY_ATTR JXL_INLINE static void ApplyMask(RowType row_type,
+                                              ColumnType column_type,
+                                              uint32_t* JXL_RESTRICT errors) {
+      ApplyPredictor(row_type, column_type).template ApplyMask<Op>(errors);
+    }
+
+    HWY_ATTR JXL_INLINE static void Prediction(
+        RowType row_type, ColumnType column_type, size_t x,
+        const int32_t* JXL_RESTRICT row_top,
+        const int32_t* JXL_RESTRICT row_cur,
+        int32_t* JXL_RESTRICT all_predictions) {
+      ApplyPredictor(row_type, column_type)
+          .Prediction(x, row_top, row_cur, all_predictions);
+    }
+
+    HWY_ATTR JXL_INLINE static void ComputeError(
+        RowType row_type, ColumnType column_type,
+        const int32_t* JXL_RESTRICT pred, const int32_t* JXL_RESTRICT actual,
+        uint32_t* JXL_RESTRICT error) {
+      ApplyPredictor(row_type, column_type).ComputeError(pred, actual, error);
+    }
+  };
+};
+
+template <typename Predictor0, typename Predictor1>
+struct Predictors2 {
+  enum { kNumChannels = 2 };
+  template <typename ErrorMetric, PixelType pixel_type>
+  struct Impl {
+    using ApplyPredictor0 =
+        ApplyPredictor<ErrorMetric, Predictor0, 0, kNumChannels, pixel_type>;
+    using ApplyPredictor1 =
+        ApplyPredictor<ErrorMetric, Predictor1, 1, kNumChannels, pixel_type>;
+
+    template <typename Op>
+    JXL_INLINE static void ApplyMask(RowType row_type, ColumnType column_type,
+                                     uint32_t* JXL_RESTRICT errors) {
+      ApplyPredictor0(row_type, column_type).template ApplyMask<Op>(errors);
+      ApplyPredictor1(row_type, column_type).template ApplyMask<Op>(errors);
+    }
+
+    HWY_ATTR JXL_INLINE static void Prediction(
+        RowType row_type, ColumnType column_type, size_t x,
+        const int32_t* JXL_RESTRICT row_top,
+        const int32_t* JXL_RESTRICT row_cur,
+        int32_t* JXL_RESTRICT all_predictions) {
+      ApplyPredictor0(row_type, column_type)
+          .Prediction(x, row_top, row_cur, all_predictions);
+      ApplyPredictor1(row_type, column_type)
+          .Prediction(x, row_top, row_cur, all_predictions);
+    }
+
+    HWY_ATTR JXL_INLINE static void ComputeError(
+        RowType row_type, ColumnType column_type,
+        const int32_t* JXL_RESTRICT pred, const int32_t* JXL_RESTRICT actual,
+        uint32_t* JXL_RESTRICT error) {
+      ApplyPredictor0(row_type, column_type).ComputeError(pred, actual, error);
+      ApplyPredictor1(row_type, column_type).ComputeError(pred, actual, error);
+    }
+  };
+};
+
+template <typename Predictor0, typename Predictor1, typename Predictor2>
+struct Predictors3 {
+  enum { kNumChannels = 3 };
+  template <typename ErrorMetric, PixelType pixel_type>
+  struct Impl {
+    using ApplyPredictor0 =
+        ApplyPredictor<ErrorMetric, Predictor0, 0, kNumChannels, pixel_type>;
+    using ApplyPredictor1 =
+        ApplyPredictor<ErrorMetric, Predictor1, 1, kNumChannels, pixel_type>;
+    using ApplyPredictor2 =
+        ApplyPredictor<ErrorMetric, Predictor2, 2, kNumChannels, pixel_type>;
+
+    template <typename Op>
+    JXL_INLINE static void ApplyMask(RowType row_type, ColumnType column_type,
+                                     uint32_t* JXL_RESTRICT errors) {
+      ApplyPredictor0(row_type, column_type).template ApplyMask<Op>(errors);
+      ApplyPredictor1(row_type, column_type).template ApplyMask<Op>(errors);
+      ApplyPredictor2(row_type, column_type).template ApplyMask<Op>(errors);
+    }
+
+    HWY_ATTR JXL_INLINE static void Prediction(
+        RowType row_type, ColumnType column_type, size_t x,
+        const int32_t* JXL_RESTRICT row_top,
+        const int32_t* JXL_RESTRICT row_cur,
+        int32_t* JXL_RESTRICT all_predictions) {
+      ApplyPredictor0(row_type, column_type)
+          .Prediction(x, row_top, row_cur, all_predictions);
+      ApplyPredictor1(row_type, column_type)
+          .Prediction(x, row_top, row_cur, all_predictions);
+      ApplyPredictor2(row_type, column_type)
+          .Prediction(x, row_top, row_cur, all_predictions);
+    }
+
+    HWY_ATTR JXL_INLINE static void ComputeError(
+        RowType row_type, ColumnType column_type,
+        const int32_t* JXL_RESTRICT pred, const int32_t* JXL_RESTRICT actual,
+        uint32_t* JXL_RESTRICT error) {
+      ApplyPredictor0(row_type, column_type).ComputeError(pred, actual, error);
+      ApplyPredictor1(row_type, column_type).ComputeError(pred, actual, error);
+      ApplyPredictor2(row_type, column_type).ComputeError(pred, actual, error);
+    }
+  };
+};
+
+template <typename T, typename ErrorMetric, typename Predictors>
+class ComputeResiduals {
+  enum { kNumChannels = Predictors::kNumChannels };
+
+  template <PixelType pixel_type>
+  using ForEachChannel =
+      typename Predictors::template Impl<ErrorMetric, pixel_type>;
+
+ public:
+  HWY_ATTR void Run(size_t xsize, size_t ysize, AuxOut* JXL_RESTRICT aux_out) {
+    ProcessRow<RowType::kFirstRow>(xsize, 0, aux_out);
+    if (ysize == 1) return;
+    ProcessRow<RowType::kSecondRow>(xsize, 1, aux_out);
+    for (size_t y = 2; y < ysize; y++) {
+      ProcessRow<RowType::kRegularRow>(xsize, y, aux_out);
+    }
+  }
+
+ private:
+  // Computes the best prediction for each channel, calls Prediction and
+  // updates error_w_ and error_l_. If kEnableStats is true, also updates
+  // stats in aux_out.
+  template <PixelType pixel_type>
+  HWY_ATTR void Advance(RowType row_type, ColumnType column_type, size_t x,
+                        size_t y, AuxOut* JXL_RESTRICT aux_out) {
+    using DU = HWY_CAPPED(uint32_t, kNumChannels * kNumPredictors);
+    auto max_it = [](uint32_t* JXL_RESTRICT dest,
+                     uint32_t* JXL_RESTRICT mix) HWY_ATTR {
+      for (size_t i = 0; i < kNumChannels * kNumPredictors; i += DU::N) {
+        auto cur = Load(DU(), dest + i);
+        auto m = Load(DU(), mix + i);
+        Store(Max(cur, m), DU(), dest + i);
+      }
+#if defined(__arm__)
+      // Compiler bug in clang-6 on arm can cause successive calls to max_it()
+      // using the same "dest" variable, as it is used here, to  have a
+      // read-after-write in the wrong order. The following statement prevents
+      // that problem
+      asm("" : : "m"(dest[0]));
+#endif  // __arm__
+    };
+    HWY_ALIGN int32_t prediction_cur[kNumChannels * kNumPredictors];
+    HWY_ALIGN uint32_t expected_error[kNumChannels * kNumPredictors];
+    HWY_ALIGN uint32_t error_mask[kNumChannels * kNumPredictors];
+    HWY_ALIGN uint32_t error_n[kNumChannels * kNumPredictors];
+    const int32_t* JXL_RESTRICT row_top =
+        pixel_type != PixelType::kInteriorPixel &&
+                row_type != RowType::kRegularRow
+            ? nullptr
+            : rows_[(y - 2) & 3];
+    const int32_t* JXL_RESTRICT row_mid =
+        pixel_type != PixelType::kInteriorPixel &&
+                row_type == RowType::kFirstRow
+            ? nullptr
+            : rows_[(y - 1) & 3];
+    int32_t* JXL_RESTRICT row_cur = rows_[y & 3];
+
+    // Compute prediction.
+    ForEachChannel<pixel_type>::Prediction(row_type, column_type, x, row_mid,
+                                           row_cur, prediction_cur);
+
+    // Initialize expected error to 0.
+    for (size_t c = 0; c < kNumChannels * kNumPredictors; c++) {
+      expected_error[c] = 0;
+    }
+
+    // Compute previous row and column type. We don't care about correctly
+    // detecting if the preceding row/column is the second, as that does not
+    // affect predictor usability.
+    const RowType previous_row = row_type == RowType::kRegularRow
+                                     ? RowType::kRegularRow
+                                     : RowType::kFirstRow;
+    const ColumnType previous_column =
+        column_type & kSecondColumn ? kFirstColumn : kRegularColumn;
+
+    // Initialize error_mask to all ones. This is only needed in the first two
+    // rows and columns: all predictors will be valid at least once in the 3
+    // preceding pixels otherwise.
+    if (pixel_type != PixelType::kInteriorPixel &&
+        (row_type != RowType::kRegularRow ||
+         column_type & (kFirstColumn | kSecondColumn))) {
+      for (size_t c = 0; c < kNumChannels * kNumPredictors; c++) {
+        error_mask[c] = kMaxError;
+      }
+    }
+
+    // Do not take into account error on pixel n if in the first row.
+    if (pixel_type == PixelType::kInteriorPixel ||
+        row_type != RowType::kFirstRow) {
+      HWY_ALIGN int32_t prediction_n[kNumChannels * kNumPredictors];
+      ForEachChannel<pixel_type>::Prediction(previous_row, column_type, x,
+                                             row_top, row_mid, prediction_n);
+      ForEachChannel<pixel_type>::ComputeError(
+          previous_row, column_type, prediction_n, row_mid + x * kNumChannels,
+          error_n);
+      max_it(expected_error, error_n);
+      // Mark valid predictors as being usable.
+      ForEachChannel<pixel_type>::template ApplyMask<error::And>(
+          previous_row, column_type, error_mask);
+    }
+
+    // Do not take into account error on pixel l in either the first row or
+    // first column.
+    if (pixel_type == PixelType::kInteriorPixel ||
+        (row_type != RowType::kFirstRow && (column_type & kFirstColumn) == 0)) {
+      max_it(expected_error, error_l_);
+      // Mark valid predictors as being usable.
+      ForEachChannel<pixel_type>::template ApplyMask<error::And>(
+          previous_row, previous_column, error_mask);
+    }
+
+    // Do not take into account error on pixel w if in the first column.
+    if (pixel_type == PixelType::kInteriorPixel ||
+        (column_type & kFirstColumn) == 0) {
+      max_it(expected_error, error_w_);
+      // Mark valid predictors as being usable.
+      ForEachChannel<pixel_type>::template ApplyMask<error::And>(
+          row_type, previous_column, error_mask);
+    }
+
+    // Set max error for predictors that did not ever receive error statistics
+    // from neighboring pixels, or that are unavailable for the current pixel.
+    // This only ever happens in the first two rows or columns or the last
+    // column.
+    // If no error information is available for any predictor, use the first
+    // predictor that is available for this pixel.
+    if (pixel_type != PixelType::kInteriorPixel &&
+        (row_type != RowType::kRegularRow ||
+         column_type & (kFirstColumn | kSecondColumn))) {
+      ForEachChannel<pixel_type>::template ApplyMask<error::Or>(
+          row_type, column_type, expected_error);
+
+      for (size_t c = 0; c < kNumChannels; c++) {
+        int32_t pred;
+        uint32_t min_error;
+        uint32_t num_correct;
+        // Re-use ChoosePredictor to compute the minimum error in the error
+        // mask.
+        ChoosePredictor(prediction_cur + c * kNumPredictors,
+                        error_mask + c * kNumPredictors, &pred, &min_error,
+                        &num_correct);
+        if (min_error != kMaxError) {
+          error::Or::Apply(expected_error + c * kNumPredictors,
+                           error_mask + c * kNumPredictors);
+        } else {
+          // Set all errors to kMaxError-1. This ensures that num_correct
+          // will be zero, and that min_error will be high, not to ``poison''
+          // the context model with low errors.
+          for (size_t i = c * kNumPredictors; i < (c + 1) * kNumPredictors;
+               i++) {
+            expected_error[i] |= kMaxError - 1;
+          }
+        }
+      }
+    }
+    if (pixel_type != PixelType::kInteriorPixel && column_type & kLastColumn) {
+      ForEachChannel<pixel_type>::template ApplyMask<error::Or>(
+          row_type, column_type, expected_error);
+    }
+
+    HWY_ALIGN int32_t prediction[kNumChannels < 4 ? 4 : kNumChannels];
+    HWY_ALIGN uint32_t num_correct[kNumChannels];
+    HWY_ALIGN uint32_t min_error[kNumChannels];
+
+#if defined(ADDRESS_SANITIZER) || defined(MEMORY_SANITIZER) || \
+    defined(THREAD_SANITIZER)
+    for (size_t i = kNumChannels; i < 4; i++) {
+      prediction[i] = 0;
+    }
+#endif
+
+    // Choose best predictor, compute stats.
+    for (size_t channel = 0; channel < kNumChannels; channel++) {
+      if (row_type != RowType::kFirstRow) {
+        for (size_t i = 0; i < kNumPredictors; i++) {
+          error_l_[channel * kNumPredictors + i] =
+              error_n[channel * kNumPredictors + i];
+        }
+      }
+      ChoosePredictor(prediction_cur + channel * kNumPredictors,
+                      expected_error + channel * kNumPredictors,
+                      prediction + channel, min_error + channel,
+                      num_correct + channel);
+    }
+
+    int32_t* JXL_RESTRICT decoded = row_cur + x * kNumChannels;
+    Prediction(x, y, prediction, num_correct, min_error, decoded);
+    if (pixel_type == PixelType::kInteriorPixel ||
+        (column_type & kLastColumn) == 0) {
+      // Compute error for current pixel and save it in error_w_.
+      ForEachChannel<pixel_type>::ComputeError(
+          row_type, column_type, prediction_cur, decoded, error_w_);
+    }
+  }
+
+  template <RowType row_type>
+  HWY_ATTR JXL_INLINE void ProcessRow(size_t xsize, size_t y,
+                                      AuxOut* JXL_RESTRICT aux_out) {
+    CallStartRow(y);
+    // First column
+    Advance<PixelType::kBorderPixel>(
+        row_type, ColumnType((xsize == 1 ? kLastColumn : 0) | kFirstColumn), 0,
+        y, aux_out);
+    if (xsize == 1) return;
+    // Second column
+    Advance<PixelType::kBorderPixel>(
+        row_type, ColumnType((xsize == 2 ? kLastColumn : 0) | kSecondColumn), 1,
+        y, aux_out);
+    if (xsize == 2) return;
+    // All other columns
+    for (size_t x = 2; x < xsize - 1; x++) {
+      Advance<row_type == RowType::kRegularRow ? PixelType::kInteriorPixel
+                                               : PixelType::kBorderPixel>(
+          row_type, kRegularColumn, x, y, aux_out);
+    }
+    // Last column
+    Advance<PixelType::kBorderPixel>(row_type, kLastColumn, xsize - 1, y,
+                                     aux_out);
+  }
+
+  // This function is called with the prediction, number of predictors
+  // expected to be correct and minimum prediction error for each channel, and
+  // should write in `decoded` the decoder-side values.
+  HWY_ATTR void Prediction(size_t x, size_t y,
+                           const int32_t* JXL_RESTRICT predictions,
+                           const uint32_t* JXL_RESTRICT num_correct,
+                           const uint32_t* JXL_RESTRICT min_error,
+                           int32_t* JXL_RESTRICT decoded) {
+    static_cast<T*>(this)->Prediction(x, y, predictions, num_correct, min_error,
+                                      decoded);
+  }
+
+  void StartRow(size_t y) {}
+
+  HWY_ATTR void CallStartRow(size_t y) { static_cast<T*>(this)->StartRow(y); }
+
+  // Predictors need the last two rows. We use a ringbuffer of size 4, as it
+  // is faster to compute row numbers modulo 4 than modulo 3.
+  int32_t rows_[4][kMaxLine * kNumChannels];
+
+  // We re-use predictor error for pixel l from pixel n and error w
+  // from the previous pixel.
+  HWY_ALIGN uint32_t error_l_[kNumChannels * kNumPredictors];
+  HWY_ALIGN uint32_t error_w_[kNumChannels * kNumPredictors];
+};
+
+}  // namespace predictor
+
+// TODO(veluca): choose predictors.
+template <typename T>
+using DcPredictor = predictor::ComputeResiduals<
+    T, predictor::PackSigned,
+    predictor::Predictors3<predictor::YPredictor, predictor::YPredictor,
+                           predictor::YPredictor>>;
+
+}  // namespace jxl
+
+#endif  // JXL_PREDICTOR_H_
diff --git a/jxl/predictor_test.cc b/jxl/predictor_test.cc
new file mode 100644
index 000000000..71178ca85
--- /dev/null
+++ b/jxl/predictor_test.cc
@@ -0,0 +1,411 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/predictor.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <algorithm>
+#include <hwy/static_targets.h>
+#include <limits>
+#include <random>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/common.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/image.h"
+
+namespace jxl {
+namespace predictor {
+namespace {
+
+TEST(PredictorTest, AverageTest) {
+  EXPECT_EQ(0, Average(0, 1));
+  EXPECT_EQ(0, Average(-1, 1));
+  EXPECT_EQ(0, Average(-2, 2));
+  EXPECT_EQ(3, Average(2, 4));
+  EXPECT_EQ(3, Average(2, 5));
+  EXPECT_EQ(3, Average(1, 5));
+  EXPECT_EQ(-3, Average(-1, -5));
+  EXPECT_EQ(0x7ffffffe, Average(0x7fffffff, 0x7ffffffe));
+  EXPECT_EQ(-0x7ffffffe, Average(-0x7fffffff, -0x7ffffffe));
+  EXPECT_EQ(0, Average(-0x7fffffff, 0x7ffffffe));  // -0.5
+  EXPECT_EQ(0, Average(-0x7ffffffe, 0x7fffffff));  // 0.5
+}
+
+HWY_ATTR void TestPackSignedRange() {
+  using Pack = PackSignedRange<0, 255>;
+  for (int32_t pred = 0; pred <= 255; pred++) {
+    for (int32_t i = 0; i <= 255; i++) {
+      HWY_ALIGN int32_t predv[kNumPredictors];
+      std::fill(predv, predv + kNumPredictors, pred);
+      HWY_ALIGN uint32_t res_simd[kNumPredictors];
+      Pack::Compute(i, predv, res_simd);
+      uint32_t res = Pack::Residual(i, pred);
+      EXPECT_EQ(i, Pack::Original(res, pred))
+          << "v: " << i << " pred: " << pred;
+      for (size_t p = 0; p < kNumPredictors; p++) {
+        EXPECT_EQ(res, res_simd[p]) << "v: " << i << " pred: " << pred;
+      }
+    }
+  }
+}
+TEST(PredictorTest, TestPackSignedRange) { TestPackSignedRange(); }
+
+HWY_ATTR void TestPackSigned() {
+  for (int32_t i = std::numeric_limits<int16_t>::min();
+       i <= std::numeric_limits<int16_t>::max(); i++) {
+    HWY_ALIGN int32_t zeros[kNumPredictors] = {};
+    HWY_ALIGN uint32_t cost_simd[kNumPredictors];
+    PackSigned::Compute(i, zeros, cost_simd);
+    uint32_t cost = jxl::PackSigned(i);
+    for (size_t i = 0; i < kNumPredictors; i++) {
+      EXPECT_EQ(cost, cost_simd[i]);
+    }
+  }
+}
+TEST(PredictorTest, TestPackSigned) { TestPackSigned(); }
+
+template <typename Pred>
+HWY_ATTR void TestPredictor() {
+  std::mt19937_64 rng;
+  // While the range fits in int32_t, the internal calculations in
+  // uniform_int_distribution overflow when using integer values, which is
+  // undefined. That's why we use int64_t here instead.
+  std::uniform_int_distribution<int64_t> dist(
+      std::numeric_limits<int32_t>::min(), std::numeric_limits<int32_t>::max());
+
+  for (int i = 0; i < 1000000; i++) {
+    int32_t w = dist(rng);
+    int32_t l = dist(rng);
+    int32_t n = dist(rng);
+    int32_t r = dist(rng);
+    int32_t predictions[kNumPredictors];
+    Pred::Predict(n, w, l, r, predictions);
+    const uint32_t* mask_r0 = Pred::Row0Mask();
+    const uint32_t* mask_c0 = Pred::Col0Mask();
+    const uint32_t* mask_lc = Pred::LastColMask();
+    // Set to 0 each pixel value and check that predictions that claim not to
+    // use that pixel are still the same.
+    // n
+    int32_t altered_predictions[kNumPredictors];
+    Pred::Predict(0, w, l, r, altered_predictions);
+    for (size_t p = 0; p < kNumPredictors; p++) {
+      if (mask_r0[p] == 0) EXPECT_EQ(altered_predictions[p], predictions[p]);
+    }
+    // w
+    Pred::Predict(n, 0, l, r, altered_predictions);
+    for (size_t p = 0; p < kNumPredictors; p++) {
+      if (mask_c0[p] == 0) EXPECT_EQ(altered_predictions[p], predictions[p]);
+    }
+    // l
+    Pred::Predict(n, w, 0, r, altered_predictions);
+    for (size_t p = 0; p < kNumPredictors; p++) {
+      if (mask_r0[p] == 0 && mask_c0[p] == 0)
+        EXPECT_EQ(altered_predictions[p], predictions[p]);
+    }
+    // r
+    Pred::Predict(n, w, l, 0, altered_predictions);
+    for (size_t p = 0; p < kNumPredictors; p++) {
+      if (mask_r0[p] == 0 && mask_lc[p] == 0)
+        EXPECT_EQ(altered_predictions[p], predictions[p]);
+    }
+  }
+}
+
+TEST(PredictorTest, TestYPredictor) { TestPredictor<YPredictor>(); }
+
+TEST(PredictorTest, TestXBPredictor) { TestPredictor<XBPredictor>(); }
+
+// Simple set of predictors used only for testing.
+struct PredictorForTesting {
+  static JXL_INLINE void Predict(const int32_t n, const int32_t w,
+                                 const int32_t l, const int32_t r,
+                                 int32_t* JXL_RESTRICT pred) {
+    pred[0] = pred[1] = n;
+    pred[2] = pred[3] = w;
+    pred[4] = pred[5] = l;
+    pred[6] = pred[7] = r;
+  }
+
+  static JXL_INLINE const uint32_t* Row0Mask() {
+    HWY_ALIGN static constexpr uint32_t kMask[kNumPredictors] = {
+        kMaxError, kMaxError, 0, 0, kMaxError, kMaxError, kMaxError, kMaxError};
+    return kMask;
+  }
+
+  static JXL_INLINE const uint32_t* Col0Mask() {
+    HWY_ALIGN static constexpr uint32_t kMask[kNumPredictors] = {
+        0, 0, kMaxError, kMaxError, kMaxError, kMaxError, 0, 0};
+    return kMask;
+  }
+
+  static JXL_INLINE const uint32_t* LastColMask() {
+    HWY_ALIGN static constexpr uint32_t kMask[kNumPredictors] = {
+        0, 0, 0, 0, 0, 0, kMaxError, kMaxError};
+    return kMask;
+  }
+
+  // Returns true if predictor is available.
+  static bool RunPredictor(size_t idx, size_t c, const Image3I& img, size_t x,
+                           size_t y, int32_t* pred) {
+    if (idx == 0 || idx == 1) {
+      if (y == 0) return false;
+      *pred = img.ConstPlaneRow(c, y - 1)[x];
+      return true;
+    }
+    if (idx == 2 || idx == 3) {
+      if (x == 0) return false;
+      *pred = img.ConstPlaneRow(c, y)[x - 1];
+      return true;
+    }
+    if (idx == 4 || idx == 5) {
+      if (y == 0) return false;
+      if (x == 0) return false;
+      *pred = img.ConstPlaneRow(c, y - 1)[x - 1];
+      return true;
+    }
+    if (idx == 6 || idx == 7) {
+      if (y == 0) return false;
+      if (x == img.xsize() - 1) return false;
+      *pred = img.ConstPlaneRow(c, y - 1)[x + 1];
+      return true;
+    }
+    JXL_ASSERT(idx < 8);
+    return false;
+  }
+};
+
+TEST(PredictorTest, TestPredictorForTesting) {
+  TestPredictor<PredictorForTesting>();
+}
+
+class PredictorTester;
+
+using PredictorToTest = ComputeResiduals<
+    PredictorTester, PackSigned,
+    Predictors3<PredictorForTesting, PredictorForTesting, PredictorForTesting>>;
+
+class PredictorTester : public PredictorToTest {
+ public:
+  explicit PredictorTester(const Image3I& test_img) : test_img_(test_img) {}
+
+  void Run() {
+    AuxOut aux_out;
+    PredictorToTest::Run(test_img_.xsize(), test_img_.ysize(), &aux_out);
+    EXPECT_EQ(pred_count_, test_img_.xsize() * test_img_.ysize());
+  }
+
+  JXL_INLINE void Prediction(size_t x, size_t y,
+                             const int32_t* JXL_RESTRICT predictions,
+                             const uint32_t* JXL_RESTRICT num_correct,
+                             const uint32_t* JXL_RESTRICT min_error,
+                             int32_t* JXL_RESTRICT decoded) {
+    pred_count_++;
+    for (size_t c = 0; c < 3; c++) {
+      decoded[c] = test_img_.ConstPlaneRow(c, y)[x];
+
+      // Verify prediction.
+      uint32_t max_error[kNumPredictors] = {};
+      bool has_error_information[kNumPredictors] = {};
+      bool has_any_error_information = false;
+      static constexpr int kErrorPixels[][2] = {{0, -1}, {-1, 0}, {-1, -1}};
+
+      // Compute max error and availability of error information.
+      for (auto px : kErrorPixels) {
+        if (x < -px[0] || y < -px[1]) continue;
+        for (size_t i = 0; i < kNumPredictors; i++) {
+          int32_t pred;
+          if (PredictorForTesting::RunPredictor(i, c, test_img_, x + px[0],
+                                                y + px[1], &pred)) {
+            uint32_t error = PackSigned::Residual(
+                test_img_.ConstPlaneRow(c, y + px[1])[x + px[0]], pred);
+            max_error[i] = std::max(error, max_error[i]);
+            has_error_information[i] = true;
+            has_any_error_information = true;
+          }
+        }
+      }
+
+      // Compute prediction for this pixel.
+      int32_t prediction[kNumPredictors] = {};
+      bool has_prediction[kNumPredictors] = {};
+      bool has_any_prediction = false;
+      for (size_t i = 0; i < kNumPredictors; i++) {
+        has_prediction[i] = PredictorForTesting::RunPredictor(
+            i, c, test_img_, x, y, &prediction[i]);
+        if (has_prediction[i]) {
+          has_any_prediction = true;
+        }
+      }
+
+      // Compute best predictor.
+      uint32_t best_predictor = 0;
+      uint32_t merr = kMaxError;
+      uint32_t corr = 0;
+      bool has_predictor = false;
+      // Implementation detail: if no error information is available, but at
+      // least one predictor is, we give max_error-1 as the minimum error value.
+      if (!has_any_error_information && has_any_prediction)
+        merr = kMaxError - 1;
+      for (size_t i = 0; i < kNumPredictors; i++) {
+        // Count the predictors for which there is error information, the
+        // prediction is valid, and max error is 0.
+        if (has_prediction[i] && has_error_information[i] &&
+            max_error[i] == 0) {
+          corr++;
+        }
+        // Find the prediction with minimum error among the available ones.
+        if (has_error_information[i] && has_prediction[i] &&
+            max_error[i] < merr) {
+          merr = max_error[i];
+          best_predictor = i;
+          has_predictor = true;
+        }
+        // If no error information is available, and this predictor is
+        // available, use it if no other predictor was chosen.
+        if (!has_predictor && !has_any_error_information && has_prediction[i]) {
+          has_predictor = true;
+          best_predictor = i;
+        }
+      }
+
+      // Avoid spamming the output with failure once the test has already
+      // failed.
+      if (!::testing::Test::HasNonfatalFailure()) {
+        EXPECT_EQ(predictions[c], prediction[best_predictor])
+            << "c: " << c << " x: " << x << " y: " << y;
+        EXPECT_EQ(num_correct[c], corr)
+            << "c: " << c << " x: " << x << " y: " << y;
+        EXPECT_EQ(min_error[c], merr)
+            << "c: " << c << " x: " << x << " y: " << y;
+      }
+      if (::testing::Test::HasNonfatalFailure()) {
+        JXL_CHECK(false);
+      }
+    }
+  }
+
+ private:
+  const Image3I& test_img_;
+  size_t pred_count_ = 0;
+};
+
+Image3I Stripes(int32_t low, int32_t high) {
+  Image3I ret(kDcGroupDimInBlocks, kDcGroupDimInBlocks);
+  for (size_t y = 0; y < ret.ysize(); y++) {
+    int32_t* JXL_RESTRICT row = ret.PlaneRow(0, y);
+    for (size_t x = 0; x < ret.xsize(); x++) {
+      row[x] = x & 1 ? low : high;
+    }
+  }
+  for (size_t y = 0; y < ret.ysize(); y++) {
+    int32_t* JXL_RESTRICT row = ret.PlaneRow(1, y);
+    for (size_t x = 0; x < ret.xsize(); x++) {
+      row[x] = y & 1 ? low : high;
+    }
+  }
+  for (size_t y = 0; y < ret.ysize(); y++) {
+    int32_t* JXL_RESTRICT row = ret.PlaneRow(2, y);
+    for (size_t x = 0; x < ret.xsize(); x++) {
+      row[x] = (x + y) & 1 ? low : high;
+    }
+  }
+  return ret;
+}
+
+Image3I Constant(int32_t low, int32_t high) {
+  Image3I ret(kDcGroupDimInBlocks, kDcGroupDimInBlocks);
+  for (size_t y = 0; y < ret.ysize(); y++) {
+    int32_t* JXL_RESTRICT row = ret.PlaneRow(0, y);
+    for (size_t x = 0; x < ret.xsize(); x++) {
+      row[x] = low;
+    }
+  }
+  for (size_t y = 0; y < ret.ysize(); y++) {
+    int32_t* JXL_RESTRICT row = ret.PlaneRow(1, y);
+    for (size_t x = 0; x < ret.xsize(); x++) {
+      row[x] = high;
+    }
+  }
+  for (size_t y = 0; y < ret.ysize(); y++) {
+    int32_t* JXL_RESTRICT row = ret.PlaneRow(2, y);
+    for (size_t x = 0; x < ret.xsize(); x++) {
+      row[x] = (low + high) >> 1;
+    }
+  }
+  return ret;
+}
+
+Image3I Random(int32_t low, int32_t high) {
+  Image3I ret(kDcGroupDimInBlocks, kDcGroupDimInBlocks);
+  std::mt19937_64 rng;
+  std::uniform_int_distribution<int32_t> dist(low, high);
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t y = 0; y < ret.ysize(); y++) {
+      int32_t* JXL_RESTRICT row = ret.PlaneRow(c, y);
+      for (size_t x = 0; x < ret.xsize(); x++) {
+        row[x] = dist(rng);
+      }
+    }
+  }
+  return ret;
+}
+
+TEST(PredictorTest, TestStripes8) {
+  Image3I img = Stripes(0, 255);
+  PredictorTester tester(img);
+  tester.Run();
+}
+
+TEST(PredictorTest, TestConstant8) {
+  Image3I img = Constant(0, 255);
+  PredictorTester tester(img);
+  tester.Run();
+}
+
+TEST(PredictorTest, TestRandom8) {
+  Image3I img = Random(0, 255);
+  PredictorTester tester(img);
+  tester.Run();
+}
+
+TEST(PredictorTest, TestStripes16) {
+  Image3I img = Stripes(std::numeric_limits<int16_t>::min(),
+                        std::numeric_limits<int16_t>::max());
+  PredictorTester tester(img);
+  tester.Run();
+}
+
+TEST(PredictorTest, TestConstant16) {
+  Image3I img = Constant(std::numeric_limits<int16_t>::min(),
+                         std::numeric_limits<int16_t>::max());
+  PredictorTester tester(img);
+  tester.Run();
+}
+
+TEST(PredictorTest, TestRandom16) {
+  Image3I img = Random(std::numeric_limits<int16_t>::min(),
+                       std::numeric_limits<int16_t>::max());
+  PredictorTester tester(img);
+  tester.Run();
+}
+}  // namespace
+}  // namespace predictor
+}  // namespace jxl
diff --git a/jxl/preview_test.cc b/jxl/preview_test.cc
new file mode 100644
index 000000000..7ace687df
--- /dev/null
+++ b/jxl/preview_test.cc
@@ -0,0 +1,86 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+
+#include <string>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/override.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+#include "jxl/enc_butteraugli_comparator.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_file.h"
+#include "jxl/enc_params.h"
+#include "jxl/extras/codec.h"
+#include "jxl/headers.h"
+#include "jxl/image_bundle.h"
+#include "jxl/test_utils.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+using test::Roundtrip;
+
+TEST(PreviewTest, RoundtripGivenPreview) {
+  ThreadPool* pool = nullptr;
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ASSERT_TRUE(SetFromFile(pathname, &io, pool));
+  io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+  // Same as main image
+  io.preview_frame = io.Main().Copy();
+  io.metadata.m2.have_preview = true;
+  ASSERT_TRUE(
+      io.preview.Set(io.preview_frame.xsize(), io.preview_frame.ysize()));
+
+  CompressParams cparams;
+  cparams.butteraugli_distance = 2.0;
+  cparams.speed_tier = SpeedTier::kSquirrel;
+  DecompressParams dparams;
+  dparams.preview = Override::kOff;
+
+  CodecInOut io2;
+  Roundtrip(&io, cparams, dparams, pool, &io2);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            2.5);
+  EXPECT_EQ(0, io2.preview_frame.xsize());
+
+  dparams.preview = Override::kOn;
+
+  CodecInOut io3;
+  Roundtrip(&io, cparams, dparams, pool, &io3);
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, pool),
+            2.5);
+
+  // Preview image also close to original
+  EXPECT_NE(0, io3.preview_frame.xsize());
+  EXPECT_LE(
+      ButteraugliDistance(io.Main(), io3.preview_frame, cparams.hf_asymmetry,
+                          /*distmap=*/nullptr, pool),
+      2.5);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/quant_weights.cc b/jxl/quant_weights.cc
new file mode 100644
index 000000000..8a28e2750
--- /dev/null
+++ b/jxl/quant_weights.cc
@@ -0,0 +1,1225 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "jxl/quant_weights.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <cmath>
+#include <limits>
+#include <utility>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/dct.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/fields.h"
+#include "jxl/image.h"
+#include "jxl/modular/encoding/encoding.h"
+
+namespace jxl {
+
+// kQuantWeights[N * N * c + N * y + x] is the relative weight of the (x, y)
+// coefficient in component c. Higher weights correspond to finer quantization
+// intervals and more bits spent in encoding.
+
+namespace {
+void GetQuantWeightsDCT2(const QuantEncoding::DCT2Weights& dct2weights,
+                         double* weights) {
+  for (size_t c = 0; c < 3; c++) {
+    size_t start = c * 64;
+    weights[start] = 0xBAD;
+    weights[start + 1] = weights[start + 8] = dct2weights[c][0];
+    weights[start + 9] = dct2weights[c][1];
+    for (size_t y = 0; y < 2; y++) {
+      for (size_t x = 0; x < 2; x++) {
+        weights[start + y * 8 + x + 2] = dct2weights[c][2];
+        weights[start + (y + 2) * 8 + x] = dct2weights[c][2];
+      }
+    }
+    for (size_t y = 0; y < 2; y++) {
+      for (size_t x = 0; x < 2; x++) {
+        weights[start + (y + 2) * 8 + x + 2] = dct2weights[c][3];
+      }
+    }
+    for (size_t y = 0; y < 4; y++) {
+      for (size_t x = 0; x < 4; x++) {
+        weights[start + y * 8 + x + 4] = dct2weights[c][4];
+        weights[start + (y + 4) * 8 + x] = dct2weights[c][4];
+      }
+    }
+    for (size_t y = 0; y < 4; y++) {
+      for (size_t x = 0; x < 4; x++) {
+        weights[start + (y + 4) * 8 + x + 4] = dct2weights[c][5];
+      }
+    }
+  }
+}
+
+void GetQuantWeightsIdentity(const QuantEncoding::IdWeights& idweights,
+                             double* weights) {
+  for (size_t c = 0; c < 3; c++) {
+    for (int i = 0; i < 64; i++) {
+      weights[64 * c + i] = idweights[c][0];
+    }
+    weights[64 * c + 1] = idweights[c][1];
+    weights[64 * c + 8] = idweights[c][1];
+    weights[64 * c + 9] = idweights[c][2];
+  }
+}
+
+double Mult(double v) {
+  if (v > 0) return 1 + v;
+  return 1 / (1 - v);
+}
+
+double Interpolate(double pos, double max, const double* array, size_t len) {
+  double scaled_pos = pos * (len - 1) / max;
+  size_t idx = scaled_pos;
+  JXL_ASSERT(idx + 1 < len);
+  double a = array[idx];
+  double b = array[idx + 1];
+  return a * pow(b / a, scaled_pos - idx);
+}
+
+// Computes quant weights for a COLS*ROWS-sized transform, using num_bands
+// eccentricity bands and num_ebands eccentricity bands. If print_mode is 1,
+// prints the resulting matrix; if print_mode is 2, prints the matrix in a
+// format suitable for a 3d plot with gnuplot.
+template <size_t print_mode = 0>
+Status GetQuantWeights(
+    size_t ROWS, size_t COLS,
+    const DctQuantWeightParams::DistanceBandsArray& distance_bands,
+    size_t num_bands, double* out) {
+  for (size_t c = 0; c < 3; c++) {
+    if (print_mode) {
+      fprintf(stderr, "Channel %zu\n", c);
+    }
+    double bands[DctQuantWeightParams::kMaxDistanceBands] = {
+        distance_bands[c][0]};
+    for (size_t i = 1; i < num_bands; i++) {
+      bands[i] = bands[i - 1] * Mult(distance_bands[c][i]);
+      if (bands[i] < 0) return JXL_FAILURE("Invalid distance bands");
+    }
+    for (size_t y = 0; y < ROWS; y++) {
+      for (size_t x = 0; x < COLS; x++) {
+        double dx = 1.0 * x / (COLS - 1);
+        double dy = 1.0 * y / (ROWS - 1);
+        double distance = std::sqrt(dx * dx + dy * dy);
+        double weight =
+            num_bands == 1
+                ? bands[0]
+                : Interpolate(distance, std::sqrt(2) + 1e-6, bands, num_bands);
+
+        if (print_mode == 1) {
+          fprintf(stderr, "%15.12f, ", weight);
+        }
+        if (print_mode == 2) {
+          fprintf(stderr, "%zu %zu %15.12f\n", x, y, weight);
+        }
+        out[c * COLS * ROWS + y * COLS + x] = weight;
+      }
+      if (print_mode) fprintf(stderr, "\n");
+      if (print_mode == 1) fprintf(stderr, "\n");
+    }
+    if (print_mode) fprintf(stderr, "\n");
+  }
+  return true;
+}
+
+Status EncodeDctParams(const DctQuantWeightParams& params, BitWriter* writer) {
+  JXL_ASSERT(params.num_distance_bands >= 1);
+  writer->Write(DctQuantWeightParams::kLog2MaxDistanceBands,
+                params.num_distance_bands - 1);
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t i = 0; i < params.num_distance_bands; i++) {
+      JXL_RETURN_IF_ERROR(F16Coder::Write(
+          params.distance_bands[c][i] * (i == 0 ? (1 / 64.0f) : 1.0f), writer));
+    }
+  }
+  return true;
+}
+
+HWY_ATTR Status DecodeDctParams(BitReader* br, DctQuantWeightParams* params) {
+  params->num_distance_bands =
+      br->ReadFixedBits<DctQuantWeightParams::kLog2MaxDistanceBands>() + 1;
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t i = 0; i < params->num_distance_bands; i++) {
+      JXL_RETURN_IF_ERROR(F16Coder::Read(br, &params->distance_bands[c][i]));
+    }
+    params->distance_bands[c][0] *= 64.0f;
+  }
+  return true;
+}
+
+Status EncodeQuant(const QuantEncoding& encoding, size_t idx, size_t size_x,
+                   size_t size_y, BitWriter* writer) {
+  writer->Write(kLog2NumQuantModes, encoding.mode);
+  size_x *= kBlockDim;
+  size_y *= kBlockDim;
+  switch (encoding.mode) {
+    case QuantEncoding::kQuantModeLibrary: {
+      writer->Write(kCeilLog2NumPredefinedTables, encoding.predefined);
+      break;
+    }
+    case QuantEncoding::kQuantModeID: {
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t i = 0; i < 3; i++) {
+          JXL_RETURN_IF_ERROR(
+              F16Coder::Write(encoding.idweights[c][i] * (1.0f / 64), writer));
+        }
+      }
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT2: {
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t i = 0; i < 6; i++) {
+          JXL_RETURN_IF_ERROR(F16Coder::Write(
+              encoding.dct2weights[c][i] * (1.0f / 64), writer));
+        }
+      }
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT4X8: {
+      for (size_t c = 0; c < 3; c++) {
+        JXL_RETURN_IF_ERROR(
+            F16Coder::Write(encoding.dct4x8multipliers[c], writer));
+      }
+      JXL_RETURN_IF_ERROR(EncodeDctParams(encoding.dct_params, writer));
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT4: {
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t i = 0; i < 2; i++) {
+          JXL_RETURN_IF_ERROR(
+              F16Coder::Write(encoding.dct4multipliers[c][i], writer));
+        }
+      }
+      JXL_RETURN_IF_ERROR(EncodeDctParams(encoding.dct_params, writer));
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT: {
+      JXL_RETURN_IF_ERROR(EncodeDctParams(encoding.dct_params, writer));
+      break;
+    }
+    case QuantEncoding::kQuantModeRAW: {
+      JXL_ASSERT(encoding.qraw.qtable != nullptr);
+      JXL_ASSERT(size_x * size_y * 3 == encoding.qraw.qtable->size());
+      writer->Write(3, encoding.qraw.qtable_den_shift);
+      writer->ZeroPadToByte();
+      Image3I img(size_x, size_y);
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t y = 0; y < size_y; y++) {
+          int* JXL_RESTRICT row = img.PlaneRow(c, y);
+          for (size_t x = 0; x < size_x; x++) {
+            row[x] =
+                (*encoding.qraw.qtable)[c * size_x * size_y + y * size_x + x];
+          }
+        }
+      }
+      modular_options cfopts;
+      set_default_modular_options(cfopts);
+      cfopts.nb_repeats = 0;
+      cfopts.entropy_coder = 2;
+      PaddedBytes enc;
+      modular_rect_compress_3(img, Rect(img), &enc, &cfopts, /*loss=*/0);
+      writer->ZeroPadToByte();
+      *writer += enc;
+      break;
+    }
+    case QuantEncoding::kQuantModeAFV: {
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t i = 0; i < 9; i++) {
+          JXL_RETURN_IF_ERROR(F16Coder::Write(
+              encoding.afv_weights[c][i] * (i < 6 ? 1.0f / 64 : 1.0f), writer));
+        }
+        JXL_RETURN_IF_ERROR(EncodeDctParams(encoding.dct_params, writer));
+        JXL_RETURN_IF_ERROR(
+            EncodeDctParams(encoding.dct_params_afv_4x4, writer));
+      }
+      break;
+    }
+  }
+  return true;
+}
+
+HWY_ATTR Status Decode(BitReader* br, QuantEncoding* encoding,
+                       size_t required_size_x, size_t required_size_y,
+                       size_t idx) {
+  size_t required_size = required_size_x * required_size_y;
+  required_size_x *= kBlockDim;
+  required_size_y *= kBlockDim;
+  int mode = br->ReadFixedBits<kLog2NumQuantModes>();
+  switch (mode) {
+    case QuantEncoding::kQuantModeLibrary: {
+      encoding->predefined = br->ReadFixedBits<kCeilLog2NumPredefinedTables>();
+      if (encoding->predefined >= kNumPredefinedTables) {
+        return JXL_FAILURE("Invalid predefined table");
+      }
+      break;
+    }
+    case QuantEncoding::kQuantModeID: {
+      if (required_size != 1) return JXL_FAILURE("Invalid mode");
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t i = 0; i < 3; i++) {
+          JXL_RETURN_IF_ERROR(F16Coder::Read(br, &encoding->idweights[c][i]));
+          encoding->idweights[c][i] *= 64;
+        }
+      }
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT2: {
+      if (required_size != 1) return JXL_FAILURE("Invalid mode");
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t i = 0; i < 6; i++) {
+          JXL_RETURN_IF_ERROR(F16Coder::Read(br, &encoding->dct2weights[c][i]));
+          encoding->dct2weights[c][i] *= 64;
+        }
+      }
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT4X8: {
+      if (required_size != 1) return JXL_FAILURE("Invalid mode");
+      for (size_t c = 0; c < 3; c++) {
+        JXL_RETURN_IF_ERROR(
+            F16Coder::Read(br, &encoding->dct4x8multipliers[c]));
+      }
+      JXL_RETURN_IF_ERROR(DecodeDctParams(br, &encoding->dct_params));
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT4: {
+      if (required_size != 1) return JXL_FAILURE("Invalid mode");
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t i = 0; i < 2; i++) {
+          JXL_RETURN_IF_ERROR(
+              F16Coder::Read(br, &encoding->dct4multipliers[c][i]));
+        }
+      }
+      JXL_RETURN_IF_ERROR(DecodeDctParams(br, &encoding->dct_params));
+      break;
+    }
+    case QuantEncoding::kQuantModeAFV: {
+      if (required_size != 1) return JXL_FAILURE("Invalid mode");
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t i = 0; i < 9; i++) {
+          JXL_RETURN_IF_ERROR(F16Coder::Read(br, &encoding->afv_weights[c][i]));
+        }
+        for (size_t i = 0; i < 6; i++) {
+          encoding->afv_weights[c][i] *= 64;
+        }
+        JXL_RETURN_IF_ERROR(DecodeDctParams(br, &encoding->dct_params));
+        JXL_RETURN_IF_ERROR(DecodeDctParams(br, &encoding->dct_params_afv_4x4));
+      }
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT: {
+      JXL_RETURN_IF_ERROR(DecodeDctParams(br, &encoding->dct_params));
+      break;
+    }
+    case QuantEncoding::kQuantModeRAW: {
+      encoding->qraw.qtable_den_shift = br->ReadFixedBits<3>();
+      Image3I img(required_size_x, required_size_y);
+      JXL_RETURN_IF_ERROR(br->JumpToByteBoundary());
+      size_t pos = 0;
+      const Span<const uint8_t> compressed = br->GetSpan();
+      if (!modular_rect_decompress_3(compressed, &pos, &img, Rect(img)))
+        return JXL_FAILURE("Failed to decode DC");
+      br->SkipBits(pos * kBitsPerByte);
+      if (!encoding->qraw.qtable) {
+        encoding->qraw.qtable = new std::vector<int>();
+      }
+      encoding->qraw.qtable->resize(required_size_x * required_size_y * 3);
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t y = 0; y < required_size_y; y++) {
+          int* JXL_RESTRICT row = img.PlaneRow(c, y);
+          for (size_t x = 0; x < required_size_x; x++) {
+            (*encoding->qraw.qtable)[c * required_size_x * required_size_y +
+                                     y * required_size_x + x] = row[x];
+            if (row[x] <= 0) {
+              return JXL_FAILURE("Invalid raw quantization table");
+            }
+          }
+        }
+      }
+      JXL_RETURN_IF_ERROR(br->JumpToByteBoundary());
+      break;
+    }
+    default:
+      return JXL_FAILURE("Invalid quantization table encoding");
+  }
+  encoding->mode = QuantEncoding::Mode(mode);
+  return true;
+}
+
+Status ComputeQuantTable(const QuantEncoding& encoding, float* table,
+                         size_t* offsets, size_t table_num,
+                         DequantMatrices::QuantTable kind, size_t* pos) {
+  double weights[3 * kMaxQuantTableSize];
+  double numerators[kMaxQuantTableSize];
+
+  constexpr size_t N = kBlockDim;
+  const float* idct4_scales = IDCTScales<N / 2>();
+  const float* idct_scales = IDCTScales<N>();
+  const float* idct16_scales = IDCTScales<2 * N>();
+  const float* idct32_scales = IDCTScales<4 * N>();
+  size_t wrows = 8, wcols = 8;
+  size_t num = 0;
+  switch (kind) {
+    case DequantMatrices::DCT: {
+      num = kDCTBlockSize;
+      wrows = 8;
+      wcols = 8;
+      for (size_t i = 0; i < num; i++) {
+        const size_t x = i % N;
+        const size_t y = i / N;
+        const float idct_scale = idct_scales[x] * idct_scales[y] * 8;
+        numerators[i] = idct_scale;
+      }
+      break;
+    }
+    case DequantMatrices::DCT16X16: {
+      num = 4 * kDCTBlockSize;
+      wrows = 16;
+      wcols = 16;
+      for (size_t i = 0; i < num; i++) {
+        const size_t x = i % (2 * N);
+        const size_t y = i / (2 * N);
+        const float idct_scale = idct16_scales[x] * idct16_scales[y] * 16;
+        numerators[i] = idct_scale;
+      }
+      break;
+    }
+    case DequantMatrices::DCT32X32: {
+      num = 16 * kDCTBlockSize;
+      wrows = 32;
+      wcols = 32;
+      for (size_t i = 0; i < num; i++) {
+        const size_t x = i % (4 * N);
+        const size_t y = i / (4 * N);
+        const float idct_scale = idct32_scales[x] * idct32_scales[y] * 32;
+        numerators[i] = idct_scale;
+      }
+      break;
+    }
+    case DequantMatrices::DCT8X16: {
+      wrows = 8;
+      wcols = 16;
+      num = 2 * kDCTBlockSize;
+      for (size_t i = 0; i < num; i++) {
+        const size_t x = i % (2 * N);
+        const size_t y = i / (2 * N);
+        const float idct_scale =
+            idct16_scales[x] * idct_scales[y] * std::sqrt(8 * 16);
+        numerators[i] = idct_scale;
+      }
+      break;
+    }
+    case DequantMatrices::DCT8X32: {
+      num = 4 * kDCTBlockSize;
+      wrows = 8;
+      wcols = 32;
+      for (size_t i = 0; i < num; i++) {
+        const size_t x = i % (4 * N);
+        const size_t y = i / (4 * N);
+        const float idct_scale = idct32_scales[x] * idct_scales[y] * 16;
+        numerators[i] = idct_scale;
+      }
+      break;
+    }
+    case DequantMatrices::DCT16X32: {
+      num = 8 * kDCTBlockSize;
+      wrows = 16;
+      wcols = 32;
+      for (size_t i = 0; i < num; i++) {
+        const size_t x = i % (4 * N);
+        const size_t y = i / (4 * N);
+        const float idct_scale =
+            idct32_scales[x] * idct16_scales[y] * std::sqrt(16 * 32);
+        numerators[i] = idct_scale;
+      }
+      break;
+    }
+    case DequantMatrices::DCT4X4: {
+      num = kDCTBlockSize;
+      for (size_t i = 0; i < N * N; i++) {
+        const size_t x = i % N;
+        const size_t y = i / N;
+        float idct_scale = idct4_scales[x / 2] * idct4_scales[y / 2] * 4;
+        numerators[i] = idct_scale;
+      }
+      break;
+    }
+    case DequantMatrices::DCT4X8: {
+      num = kDCTBlockSize;
+      for (size_t i = 0; i < N * N; i++) {
+        const size_t x = i % N;
+        const size_t y = i / N;
+        float idct_scale =
+            idct_scales[x] * idct4_scales[y / 2] * std::sqrt(4 * 8);
+        numerators[i] = idct_scale;
+      }
+      break;
+    }
+    case DequantMatrices::IDENTITY:
+    case DequantMatrices::DCT2X2:
+      num = kDCTBlockSize;
+      std::fill_n(numerators, kDCTBlockSize, 1.0);
+      break;
+    case DequantMatrices::AFV0: {
+      num = kDCTBlockSize;
+      for (size_t i = 0; i < N * N; i++) {
+        const size_t x = i % N;
+        const size_t y = i / N;
+        if (y & 1) {
+          float idct_scale =
+              idct_scales[x] * idct4_scales[y / 2] * std::sqrt(4 * 8);
+          numerators[i] = idct_scale;
+        } else {
+          if (x & 1) {
+            float idct_scale = idct4_scales[x / 2] * idct4_scales[y / 2] * 4;
+            numerators[i] = idct_scale;
+          } else {
+            numerators[i] = 1;
+          }
+        }
+      }
+      break;
+    }
+    default: {
+      JXL_ABORT("Invalid AC strategy value");
+    }
+  }
+
+  switch (encoding.mode) {
+    case QuantEncoding::kQuantModeLibrary: {
+      // Library and copy quant encoding should get replaced by the actual
+      // parameters by the caller.
+      JXL_ASSERT(false);
+      break;
+    }
+    case QuantEncoding::kQuantModeID: {
+      JXL_ASSERT(num == kDCTBlockSize);
+      GetQuantWeightsIdentity(encoding.idweights, weights);
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT2: {
+      JXL_ASSERT(num == kDCTBlockSize);
+      GetQuantWeightsDCT2(encoding.dct2weights, weights);
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT4: {
+      JXL_ASSERT(num == kDCTBlockSize);
+      double weights4x4[3 * 4 * 4];
+      // Always use 4x4 GetQuantWeights for DCT4 quantization tables.
+      JXL_RETURN_IF_ERROR(
+          GetQuantWeights(4, 4, encoding.dct_params.distance_bands,
+                          encoding.dct_params.num_distance_bands, weights4x4));
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t y = 0; y < kBlockDim; y++) {
+          for (size_t x = 0; x < kBlockDim; x++) {
+            weights[c * num + y * kBlockDim + x] =
+                weights4x4[c * 16 + (y / 2) * 4 + (x / 2)];
+          }
+        }
+        weights[c * num + 1] /= encoding.dct4multipliers[c][0];
+        weights[c * num + N] /= encoding.dct4multipliers[c][0];
+        weights[c * num + N + 1] /= encoding.dct4multipliers[c][1];
+      }
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT4X8: {
+      JXL_ASSERT(num == kDCTBlockSize);
+      double weights4x8[3 * 4 * 8];
+      // Always use 4x8 GetQuantWeights for DCT4X8 quantization tables.
+      JXL_RETURN_IF_ERROR(
+          GetQuantWeights(4, 8, encoding.dct_params.distance_bands,
+                          encoding.dct_params.num_distance_bands, weights4x8));
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t y = 0; y < kBlockDim; y++) {
+          for (size_t x = 0; x < kBlockDim; x++) {
+            weights[c * num + y * kBlockDim + x] =
+                weights4x8[c * 32 + (y / 2) * 8 + x];
+          }
+        }
+        weights[c * num + N] /= encoding.dct4x8multipliers[c];
+      }
+      break;
+    }
+    case QuantEncoding::kQuantModeDCT: {
+      JXL_RETURN_IF_ERROR(
+          GetQuantWeights(wrows, wcols, encoding.dct_params.distance_bands,
+                          encoding.dct_params.num_distance_bands, weights));
+      break;
+    }
+    case QuantEncoding::kQuantModeRAW: {
+      float den = 1 << encoding.qraw.qtable_den_shift;
+      if (!encoding.qraw.qtable || encoding.qraw.qtable->size() != 3 * num) {
+        return JXL_FAILURE("Invalid table encoding");
+      }
+      for (size_t i = 0; i < 3 * num; i++) {
+        weights[i] = 8.0 / (den * (*encoding.qraw.qtable)[i]);
+      }
+      break;
+    }
+    case QuantEncoding::kQuantModeAFV: {
+      constexpr float kFreqs[] = {
+          0xBAD,
+          0xBAD,
+          0.8517778890324296,
+          5.37778436506804,
+          0xBAD,
+          0xBAD,
+          4.734747904497923,
+          5.449245381693219,
+          1.6598270267479331,
+          4,
+          7.275749096817861,
+          10.423227632456525,
+          2.662932286148962,
+          7.630657783650829,
+          8.962388608184032,
+          12.97166202570235,
+      };
+
+      double weights4x8[3 * 4 * 8];
+      JXL_RETURN_IF_ERROR((
+          GetQuantWeights(4, 8, encoding.dct_params.distance_bands,
+                          encoding.dct_params.num_distance_bands, weights4x8)));
+      double weights4x4[3 * 4 * 4];
+      JXL_RETURN_IF_ERROR((GetQuantWeights(
+          4, 4, encoding.dct_params_afv_4x4.distance_bands,
+          encoding.dct_params_afv_4x4.num_distance_bands, weights4x4)));
+
+      constexpr float lo = 0.8517778890324296;
+      constexpr float hi = 12.97166202570235 - lo + 1e-6;
+      for (size_t c = 0; c < 3; c++) {
+        double bands[4];
+        bands[0] = encoding.afv_weights[c][5];
+        if (bands[0] < 0) return JXL_FAILURE("Invalid AFV bands");
+        for (size_t i = 1; i < 4; i++) {
+          bands[i] = bands[i - 1] * Mult(encoding.afv_weights[c][i + 5]);
+          if (bands[i] < 0) return JXL_FAILURE("Invalid AFV bands");
+        }
+        size_t start = c * 64;
+        auto set_weight = [&start, &weights](size_t x, size_t y, float val) {
+          weights[start + y * 8 + x] = val;
+        };
+        weights[start] = 1;  // Not used, but causes MSAN error otherwise.
+        // Weights for (0, 1) and (1, 0).
+        set_weight(0, 1, encoding.afv_weights[c][0]);
+        set_weight(1, 0, encoding.afv_weights[c][1]);
+        // AFV special weights for 3-pixel corner.
+        set_weight(0, 2, encoding.afv_weights[c][2]);
+        set_weight(2, 0, encoding.afv_weights[c][3]);
+        set_weight(2, 2, encoding.afv_weights[c][4]);
+
+        // All other AFV weights.
+        for (size_t y = 0; y < 4; y++) {
+          for (size_t x = 0; x < 4; x++) {
+            if (x < 2 && y < 2) continue;
+            float val = Interpolate(kFreqs[y * 4 + x] - lo, hi, bands, 4);
+            set_weight(2 * x, 2 * y, val);
+          }
+        }
+
+        // Put 4x8 weights in odd rows, except (1, 0).
+        for (size_t y = 0; y < kBlockDim / 2; y++) {
+          for (size_t x = 0; x < kBlockDim; x++) {
+            if (x == 0 && y == 0) continue;
+            weights[c * num + (2 * y + 1) * kBlockDim + x] =
+                weights4x8[c * 32 + y * 8 + x];
+          }
+        }
+        // Put 4x4 weights in even rows / odd columns, except (0, 1).
+        for (size_t y = 0; y < kBlockDim / 2; y++) {
+          for (size_t x = 0; x < kBlockDim / 2; x++) {
+            if (x == 0 && y == 0) continue;
+            weights[c * num + (2 * y) * kBlockDim + 2 * x + 1] =
+                weights4x4[c * 16 + y * 4 + x];
+          }
+        }
+      }
+      break;
+    }
+  }
+  for (size_t c = 0; c < 3; c++) {
+    offsets[table_num * 3 + c] = *pos;
+    for (size_t i = 0; i < num; i++) {
+      double val = numerators[i] / weights[c * num + i];
+      if (val > std::numeric_limits<float>::max() || val < 0) {
+        return JXL_FAILURE("Invalid quantization table");
+      }
+      table[(*pos)++] = val;
+    }
+  }
+  return true;
+}
+
+}  // namespace
+
+// These definitions are needed before C++17.
+constexpr size_t DequantMatrices::required_size_[];
+constexpr size_t DequantMatrices::required_size_x_[];
+constexpr size_t DequantMatrices::required_size_y_[];
+constexpr DequantMatrices::QuantTable DequantMatrices::kQuantTable[];
+
+Status DequantMatrices::Encode(BitWriter* writer, size_t layer,
+                               AuxOut* aux_out) const {
+  bool all_default = true;
+  for (size_t i = 0; i < encodings_.size(); i++) {
+    if (encodings_[i].mode != QuantEncoding::kQuantModeLibrary ||
+        encodings_[i].predefined != 0) {
+      all_default = false;
+    }
+  }
+  // TODO(janwas): better bound
+  BitWriter::Allotment allotment(writer, 512 * 1024);
+  writer->Write(1, all_default);
+  if (!all_default) {
+    for (size_t i = 0; i < encodings_.size(); i++) {
+      JXL_RETURN_IF_ERROR(EncodeQuant(encodings_[i], i, required_size_x_[i],
+                                      required_size_y_[i], writer));
+    }
+  }
+  ReclaimAndCharge(writer, &allotment, layer, aux_out);
+  return true;
+}
+
+Status DequantMatrices::EncodeDC(BitWriter* writer, size_t layer,
+                                 AuxOut* aux_out) const {
+  bool all_default = true;
+  for (size_t c = 0; c < 3; c++) {
+    if (dc_quant_[c] != kDCQuant[c]) {
+      all_default = false;
+    }
+  }
+  BitWriter::Allotment allotment(writer, 1 + sizeof(float) * kBitsPerByte * 3);
+  writer->Write(1, all_default);
+  if (!all_default) {
+    for (size_t c = 0; c < 3; c++) {
+      JXL_RETURN_IF_ERROR(F16Coder::Write(dc_quant_[c] * 128.0f, writer));
+    }
+  }
+  ReclaimAndCharge(writer, &allotment, layer, aux_out);
+  return true;
+}
+
+HWY_ATTR Status DequantMatrices::Decode(BitReader* br) {
+  size_t all_default = br->ReadBits(1);
+  size_t num_tables = all_default ? 0 : kNum;
+  encodings_.clear();
+  encodings_.resize(kNum, QuantEncoding::Library(0));
+  for (size_t i = 0; i < num_tables; i++) {
+    JXL_RETURN_IF_ERROR(jxl::Decode(br, &encodings_[i],
+                                    required_size_x_[i % kNum],
+                                    required_size_y_[i % kNum], i));
+  }
+  return DequantMatrices::Compute();
+}
+
+HWY_ATTR Status DequantMatrices::DecodeDC(BitReader* br) {
+  bool all_default = br->ReadBits(1);
+  if (!all_default) {
+    for (size_t c = 0; c < 3; c++) {
+      JXL_RETURN_IF_ERROR(F16Coder::Read(br, &dc_quant_[c]));
+      dc_quant_[c] *= 1.0f / 128.0f;
+      if (dc_quant_[c] == 0.)
+        return JXL_FAILURE("Invalid dc_quant coefficient 0.");
+      inv_dc_quant_[c] = 1.0f / dc_quant_[c];
+    }
+  }
+  return true;
+}
+
+constexpr float V(double v) { return static_cast<float>(v); }
+
+namespace {
+struct DequantMatricesLibraryDef {
+  // DCT8
+  static constexpr const QuantEncodingInternal DCT() {
+    return QuantEncodingInternal::DCT(DctQuantWeightParams({{{
+                                                                 V(3150.0),
+                                                                 V(0.0),
+                                                                 V(-0.4),
+                                                                 V(-0.4),
+                                                                 V(-0.4),
+                                                                 V(-2.0),
+                                                             },
+                                                             {
+                                                                 V(560.0),
+                                                                 V(0.0),
+                                                                 V(-0.3),
+                                                                 V(-0.3),
+                                                                 V(-0.3),
+                                                                 V(-0.3),
+                                                             },
+                                                             {
+                                                                 V(512.0),
+                                                                 V(-2.0),
+                                                                 V(-1.0),
+                                                                 V(0.0),
+                                                                 V(-1.0),
+                                                                 V(-2.0),
+                                                             }}},
+                                                           6));
+  }
+
+  // Identity
+  static constexpr const QuantEncodingInternal IDENTITY() {
+    return QuantEncodingInternal::Identity({{{
+                                                 V(280.0),
+                                                 V(3160.0),
+                                                 V(3160.0),
+                                             },
+                                             {
+                                                 V(60.0),
+                                                 V(864.0),
+                                                 V(864.0),
+                                             },
+                                             {
+                                                 V(18.0),
+                                                 V(200.0),
+                                                 V(200.0),
+                                             }}});
+  }
+
+  // DCT2
+  static constexpr const QuantEncodingInternal DCT2X2() {
+    return QuantEncodingInternal::DCT2({{{
+                                             V(3840.0),
+                                             V(2560.0),
+                                             V(1280.0),
+                                             V(640.0),
+                                             V(480.0),
+                                             V(300.0),
+                                         },
+                                         {
+                                             V(960.0),
+                                             V(640.0),
+                                             V(320.0),
+                                             V(180.0),
+                                             V(140.0),
+                                             V(120.0),
+                                         },
+                                         {
+                                             V(640.0),
+                                             V(320.0),
+                                             V(128.0),
+                                             V(64.0),
+                                             V(32.0),
+                                             V(16.0),
+                                         }}});
+  }
+
+  // DCT4 (quant_kind 3)
+  static constexpr const QuantEncodingInternal DCT4X4() {
+    return QuantEncodingInternal::DCT4(DctQuantWeightParams({{{
+                                                                  V(2200.0),
+                                                                  V(0.0),
+                                                                  V(0.0),
+                                                                  V(0.0),
+                                                              },
+                                                              {
+                                                                  V(392.0),
+                                                                  V(0.0),
+                                                                  V(0.0),
+                                                                  V(0.0),
+                                                              },
+                                                              {
+                                                                  V(112.0),
+                                                                  V(-0.25),
+                                                                  V(-0.25),
+                                                                  V(-0.5),
+                                                              }}},
+                                                            4),
+                                       /* kMul */
+                                       {{{
+                                             V(1.0),
+                                             V(1.0),
+                                         },
+                                         {
+                                             V(1.0),
+                                             V(1.0),
+                                         },
+                                         {
+                                             V(1.0),
+                                             V(1.0),
+                                         }}});
+  }
+
+  // DCT16
+  static constexpr const QuantEncodingInternal DCT16X16() {
+    return QuantEncodingInternal::DCT(
+        DctQuantWeightParams({{{
+                                   V(8996.8725711814115328),
+                                   V(-1.3000777393353804),
+                                   V(-0.49424529824571225),
+                                   V(-0.439093774457103443),
+                                   V(-0.6350101832695744),
+                                   V(-0.90177264050827612),
+                                   V(-1.6162099239887414),
+                               },
+                               {
+                                   V(3191.48366296844234752),
+                                   V(-0.67424582104194355),
+                                   V(-0.80745813428471001),
+                                   V(-0.44925837484843441),
+                                   V(-0.35865440981033403),
+                                   V(-0.31322389111877305),
+                                   V(-0.37615025315725483),
+                               },
+                               {
+                                   V(1157.50408145487200256),
+                                   V(-2.0531423165804414),
+                                   V(-1.4),
+                                   V(-0.50687130033378396),
+                                   V(-0.42708730624733904),
+                                   V(-1.4856834539296244),
+                                   V(-4.9209142884401604),
+                               }}},
+                             7));
+  }
+
+  // DCT32
+  static constexpr const QuantEncodingInternal DCT32X32() {
+    return QuantEncodingInternal::DCT(
+        DctQuantWeightParams({{{
+                                   V(15718.40830982518931456),
+                                   V(-1.025),
+                                   V(-0.98),
+                                   V(-0.9012),
+                                   V(-0.4),
+                                   V(-0.48819395464),
+                                   V(-0.421064),
+                                   V(-0.27),
+                               },
+                               {
+                                   V(7305.7636810695983104),
+                                   V(-0.8041958212306401),
+                                   V(-0.7633036457487539),
+                                   V(-0.55660379990111464),
+                                   V(-0.49785304658857626),
+                                   V(-0.43699592683512467),
+                                   V(-0.40180866526242109),
+                                   V(-0.27321683125358037),
+                               },
+                               {
+                                   V(3803.53173721215041536),
+                                   V(-3.060733579805728),
+                                   V(-2.0413270132490346),
+                                   V(-2.0235650159727417),
+                                   V(-0.5495389509954993),
+                                   V(-0.4),
+                                   V(-0.4),
+                                   V(-0.3),
+                               }}},
+                             8));
+  }
+
+  // DCT16X8
+  static constexpr const QuantEncodingInternal DCT8X16() {
+    return QuantEncodingInternal::DCT(
+        DctQuantWeightParams({{{
+                                   V(7240.7734393502),
+                                   V(-0.7),
+                                   V(-0.7),
+                                   V(-0.2),
+                                   V(-0.2),
+                                   V(-0.2),
+                                   V(-0.5),
+                               },
+                               {
+                                   V(1448.15468787004),
+                                   V(-0.5),
+                                   V(-0.5),
+                                   V(-0.5),
+                                   V(-0.2),
+                                   V(-0.2),
+                                   V(-0.2),
+                               },
+                               {
+                                   V(506.854140754517),
+                                   V(-1.4),
+                                   V(-0.2),
+                                   V(-0.5),
+                                   V(-0.5),
+                                   V(-1.5),
+                                   V(-3.6),
+                               }}},
+                             7));
+  }
+
+  // DCT32X8
+  static constexpr const QuantEncodingInternal DCT8X32() {
+    return QuantEncodingInternal::DCT(
+        DctQuantWeightParams({{{
+                                   V(16283.2494710648897),
+                                   V(-1.7812845336559429),
+                                   V(-1.6309059012653515),
+                                   V(-1.0382179034313539),
+                                   V(-0.85),
+                                   V(-0.7),
+                                   V(-0.9),
+                                   V(-1.2360638576849587),
+                               },
+                               {
+                                   V(5089.15750884921511936),
+                                   V(-0.320049391452786891),
+                                   V(-0.35362849922161446),
+                                   V(-0.30340000000000003),
+                                   V(-0.61),
+                                   V(-0.5),
+                                   V(-0.5),
+                                   V(-0.6),
+                               },
+                               {
+                                   V(3397.77603275308720128),
+                                   V(-0.321327362693153371),
+                                   V(-0.34507619223117997),
+                                   V(-0.70340000000000003),
+                                   V(-0.9),
+                                   V(-1.0),
+                                   V(-1.0),
+                                   V(-1.1754605576265209),
+                               }}},
+                             8));
+  }
+
+  // DCT32X16
+  static constexpr const QuantEncodingInternal DCT16X32() {
+    return QuantEncodingInternal::DCT(
+        DctQuantWeightParams({{{
+                                   V(13844.97076442300573),
+                                   V(-0.97113799999999995),
+                                   V(-0.658),
+                                   V(-0.42026),
+                                   V(-0.22712),
+                                   V(-0.2206),
+                                   V(-0.226),
+                                   V(-0.6),
+                               },
+                               {
+                                   V(4798.964084220744293),
+                                   V(-0.61125308982767057),
+                                   V(-0.83770786552491361),
+                                   V(-0.79014862079498627),
+                                   V(-0.2692727459704829),
+                                   V(-0.38272769465388551),
+                                   V(-0.22924222653091453),
+                                   V(-0.20719098826199578),
+                               },
+                               {
+                                   V(1807.236946760964614),
+                                   V(-1.2),
+                                   V(-1.2),
+                                   V(-0.7),
+                                   V(-0.7),
+                                   V(-0.7),
+                                   V(-0.4),
+                                   V(-0.5),
+                               }}},
+                             8));
+  }
+
+  // DCT4X8 and 8x4
+  static constexpr const QuantEncodingInternal DCT4X8() {
+    return QuantEncodingInternal::DCT4X8(
+        DctQuantWeightParams({{
+                                 {
+                                     V(2198.050556016380522),
+                                     V(-0.96269623020744692),
+                                     V(-0.76194253026666783),
+                                     V(-0.6551140670773547),
+                                 },
+                                 {
+                                     V(764.3655248643528689),
+                                     V(-0.92630200888366945),
+                                     V(-0.9675229603596517),
+                                     V(-0.27845290869168118),
+                                 },
+                                 {
+                                     V(527.107573587542228),
+                                     V(-1.4594385811273854),
+                                     V(-1.450082094097871593),
+                                     V(-1.5843722511996204),
+                                 },
+                             }},
+                             4),
+        /* kMuls */
+        {{
+            V(1.0),
+            V(1.0),
+            V(1.0),
+        }});
+  }
+
+  // AFV
+  static const QuantEncodingInternal AFV0() {
+    return QuantEncodingInternal::AFV(DCT4X8().dct_params, DCT4X4().dct_params,
+                                      {{{
+                                            // 4x4/4x8 DC tendency.
+                                            V(3072.0),
+                                            V(3072.0),
+                                            // AFV corner.
+                                            V(256.0),
+                                            V(256.0),
+                                            V(256.0),
+                                            // AFV high freqs.
+                                            V(414.0),
+                                            V(0.0),
+                                            V(0.0),
+                                            V(0.0),
+                                        },
+                                        {
+                                            // 4x4/4x8 DC tendency.
+                                            V(1024.0),
+                                            V(1024.0),
+                                            // AFV corner.
+                                            V(50),
+                                            V(50),
+                                            V(50),
+                                            // AFV high freqs.
+                                            V(58.0),
+                                            V(0.0),
+                                            V(0.0),
+                                            V(0.0),
+                                        },
+                                        {
+                                            // 4x4/4x8 DC tendency.
+                                            V(384.0),
+                                            V(384.0),
+                                            // AFV corner.
+                                            V(12.0),
+                                            V(12.0),
+                                            V(12.0),
+                                            // AFV high freqs.
+                                            V(22.0),
+                                            V(-0.25),
+                                            V(-0.25),
+                                            V(-0.25),
+                                        }}});
+  }
+};
+}  // namespace
+
+const DequantMatrices::DequantLibraryInternal DequantMatrices::LibraryInit() {
+  static_assert(kNum == 11,
+                "Update this function when adding new quantization kinds.");
+  static_assert(kNumPredefinedTables == 1,
+                "Update this function when adding new quantization matrices to "
+                "the library.");
+
+  // The library and the indices need to be kept in sync manually.
+  static_assert(0 == DCT, "Update the DequantLibrary array below.");
+  static_assert(1 == IDENTITY, "Update the DequantLibrary array below.");
+  static_assert(2 == DCT2X2, "Update the DequantLibrary array below.");
+  static_assert(3 == DCT4X4, "Update the DequantLibrary array below.");
+  static_assert(4 == DCT16X16, "Update the DequantLibrary array below.");
+  static_assert(5 == DCT32X32, "Update the DequantLibrary array below.");
+  static_assert(6 == DCT8X16, "Update the DequantLibrary array below.");
+  static_assert(7 == DCT8X32, "Update the DequantLibrary array below.");
+  static_assert(8 == DCT16X32, "Update the DequantLibrary array below.");
+  static_assert(9 == DCT4X8, "Update the DequantLibrary array below.");
+  static_assert(10 == AFV0, "Update the DequantLibrary array below.");
+  return DequantMatrices::DequantLibraryInternal{
+      DequantMatricesLibraryDef::DCT(),
+      DequantMatricesLibraryDef::IDENTITY(),
+      DequantMatricesLibraryDef::DCT2X2(),
+      DequantMatricesLibraryDef::DCT4X4(),
+      DequantMatricesLibraryDef::DCT16X16(),
+      DequantMatricesLibraryDef::DCT32X32(),
+      DequantMatricesLibraryDef::DCT8X16(),
+      DequantMatricesLibraryDef::DCT8X32(),
+      DequantMatricesLibraryDef::DCT16X32(),
+      DequantMatricesLibraryDef::DCT4X8(),
+      DequantMatricesLibraryDef::AFV0(),
+  };
+}
+
+namespace {
+const DequantMatrices::DequantLibraryInternal kDequantLibrary =
+    DequantMatrices::LibraryInit();
+}  // namespace
+
+const QuantEncoding* DequantMatrices::Library() {
+  // Downcast the result to a const QuantEncoding* from QuantEncodingInternal*
+  // since the subclass (QuantEncoding) doesn't add any new members and users
+  // will need to upcast to QuantEncodingInternal to access the members of that
+  // class. This allows to have kDequantLibrary as a constexpr value while still
+  // allowing to create QuantEncoding::RAW() instances that use std::vector in
+  // C++11.
+  return reinterpret_cast<const QuantEncoding*>(kDequantLibrary.data());
+}
+
+Status DequantMatrices::Compute() {
+  size_t pos = 0;
+
+  const QuantEncoding* library = Library();
+
+  // Avoid changing encodings_.
+  auto encodings = encodings_;
+
+  std::vector<size_t> offsets(kNum * 3);
+
+  for (size_t table = 0; table < encodings.size(); table++) {
+    if (encodings[table].mode == QuantEncoding::kQuantModeLibrary) {
+      encodings[table] = library[encodings[table].predefined *
+                                     AcStrategy::kNumValidStrategies +
+                                 table % kNum];
+    }
+    JXL_RETURN_IF_ERROR(ComputeQuantTable(encodings[table], table_,
+                                          offsets.data(), table,
+                                          QuantTable(table % kNum), &pos));
+  }
+
+  JXL_ASSERT(pos == kTotalTableSize);
+
+  size_ = pos;
+  for (size_t i = 0; i < pos; i++) {
+    inv_table_[i] = 1.0f / table_[i];
+  }
+  for (size_t i = 0; i < AcStrategy::kNumValidStrategies; i++) {
+    for (size_t c = 0; c < 3; c++) {
+      table_offsets_[i * 3 + c] = offsets[kQuantTable[i] * 3 + c];
+    }
+  }
+  return true;
+}
+
+void FindBestDequantMatrices(const CompressParams& cparams,
+                             const Image3F& opsin,
+                             DequantMatrices* dequant_matrices) {
+  // TODO(veluca): heuristics for in-bitstream quant tables.
+  *dequant_matrices = DequantMatrices();
+  if (cparams.max_error_mode) {
+    // Set numerators of all quantization matrices to constant values.
+    float weights[3][1] = {{1.0f / cparams.max_error[0]},
+                           {1.0f / cparams.max_error[1]},
+                           {1.0f / cparams.max_error[2]}};
+    DctQuantWeightParams dct_params(weights);
+    std::vector<QuantEncoding> encodings(DequantMatrices::kNum,
+                                         QuantEncoding::DCT(dct_params));
+    dequant_matrices->SetCustom(encodings);
+    float dc_weights[3] = {1.0f / cparams.max_error[0],
+                           1.0f / cparams.max_error[1],
+                           1.0f / cparams.max_error[2]};
+    dequant_matrices->SetCustomDC(dc_weights);
+  }
+}
+
+}  // namespace jxl
diff --git a/jxl/quant_weights.h b/jxl/quant_weights.h
new file mode 100644
index 000000000..fc9d5c4e0
--- /dev/null
+++ b/jxl/quant_weights.h
@@ -0,0 +1,467 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_QUANT_WEIGHTS_H_
+#define JXL_QUANT_WEIGHTS_H_
+
+#include <stdint.h>
+#include <string.h>
+
+#include <array>
+#include <utility>
+#include <vector>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/cache_aligned.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/enc_params.h"
+#include "jxl/image.h"
+
+namespace jxl {
+
+template <typename T, size_t N>
+constexpr T ArraySum(T (&a)[N], size_t i = N - 1) {
+  static_assert(N > 0, "Trying to compute the sum of an empty array");
+  return i == 0 ? a[0] : a[i] + ArraySum(a, i - 1);
+}
+
+static constexpr size_t kMaxQuantTableSize = kDCTBlockSize * 16;
+static constexpr size_t kNumPredefinedTables = 1;
+static constexpr size_t kCeilLog2NumPredefinedTables = 0;
+static constexpr size_t kLog2NumQuantModes = 3;
+
+struct DctQuantWeightParams {
+  static constexpr size_t kLog2MaxDistanceBands = 4;
+  static constexpr size_t kMaxDistanceBands = 1 + (1 << kLog2MaxDistanceBands);
+  typedef std::array<std::array<float, kMaxDistanceBands>, 3>
+      DistanceBandsArray;
+
+  size_t num_distance_bands = 0;
+  DistanceBandsArray distance_bands = {};
+
+  constexpr DctQuantWeightParams() : num_distance_bands(0) {}
+
+  constexpr DctQuantWeightParams(const DistanceBandsArray& dist_bands,
+                                 size_t num_dist_bands)
+      : num_distance_bands(num_dist_bands), distance_bands(dist_bands) {}
+
+  template <size_t num_dist_bands>
+  explicit DctQuantWeightParams(const float dist_bands[3][num_dist_bands]) {
+    num_distance_bands = num_dist_bands;
+    for (size_t c = 0; c < 3; c++) {
+      memcpy(distance_bands[c].data(), dist_bands[c],
+             sizeof(float) * num_dist_bands);
+    }
+  }
+};
+
+// NOLINTNEXTLINE(clang-analyzer-optin.performance.Padding)
+struct QuantEncodingInternal {
+  enum Mode {
+    kQuantModeLibrary,
+    kQuantModeID,
+    kQuantModeDCT2,
+    kQuantModeDCT4,
+    kQuantModeDCT4X8,
+    kQuantModeAFV,
+    kQuantModeDCT,
+    kQuantModeRAW,
+  };
+
+  template <Mode mode>
+  struct Tag {};
+
+  typedef std::array<std::array<float, 3>, 3> IdWeights;
+  typedef std::array<std::array<float, 6>, 3> DCT2Weights;
+  typedef std::array<std::array<float, 2>, 3> DCT4Multipliers;
+  typedef std::array<std::array<float, 9>, 3> AFVWeights;
+  typedef std::array<float, 3> DCT4x8Multipliers;
+
+  static constexpr QuantEncodingInternal Library(uint8_t predefined) {
+    return ((predefined < kNumPredefinedTables) ||
+            ::jxl::Abort(__FILE__, __LINE__,
+                         "Assert predefined < kNumPredefinedTables")),
+           QuantEncodingInternal(Tag<kQuantModeLibrary>(), predefined);
+  }
+  constexpr QuantEncodingInternal(Tag<kQuantModeLibrary> /* tag */,
+                                  uint8_t predefined)
+      : mode(kQuantModeLibrary), predefined(predefined) {}
+
+  // Identity
+  // xybweights is an array of {xweights, yweights, bweights}.
+  static constexpr QuantEncodingInternal Identity(const IdWeights& xybweights) {
+    return QuantEncodingInternal(Tag<kQuantModeID>(), xybweights);
+  }
+  constexpr QuantEncodingInternal(Tag<kQuantModeID> /* tag */,
+                                  const IdWeights& xybweights)
+      : mode(kQuantModeID), idweights(xybweights) {}
+
+  // DCT2
+  static constexpr QuantEncodingInternal DCT2(const DCT2Weights& xybweights) {
+    return QuantEncodingInternal(Tag<kQuantModeDCT2>(), xybweights);
+  }
+  constexpr QuantEncodingInternal(Tag<kQuantModeDCT2> /* tag */,
+                                  const DCT2Weights& xybweights)
+      : mode(kQuantModeDCT2), dct2weights(xybweights) {}
+
+  // DCT4
+  static constexpr QuantEncodingInternal DCT4(
+      const DctQuantWeightParams& params, const DCT4Multipliers& xybmul) {
+    return QuantEncodingInternal(Tag<kQuantModeDCT4>(), params, xybmul);
+  }
+  constexpr QuantEncodingInternal(Tag<kQuantModeDCT4> /* tag */,
+                                  const DctQuantWeightParams& params,
+                                  const DCT4Multipliers& xybmul)
+      : mode(kQuantModeDCT4), dct_params(params), dct4multipliers(xybmul) {}
+
+  // DCT4x8
+  static constexpr QuantEncodingInternal DCT4X8(
+      const DctQuantWeightParams& params, const DCT4x8Multipliers& xybmul) {
+    return QuantEncodingInternal(Tag<kQuantModeDCT4X8>(), params, xybmul);
+  }
+  constexpr QuantEncodingInternal(Tag<kQuantModeDCT4X8> /* tag */,
+                                  const DctQuantWeightParams& params,
+                                  const DCT4x8Multipliers& xybmul)
+      : mode(kQuantModeDCT4X8), dct_params(params), dct4x8multipliers(xybmul) {}
+
+  // DCT
+  static constexpr QuantEncodingInternal DCT(
+      const DctQuantWeightParams& params) {
+    return QuantEncodingInternal(Tag<kQuantModeDCT>(), params);
+  }
+  constexpr QuantEncodingInternal(Tag<kQuantModeDCT> /* tag */,
+                                  const DctQuantWeightParams& params)
+      : mode(kQuantModeDCT), dct_params(params) {}
+
+  // AFV
+  static constexpr QuantEncodingInternal AFV(
+      const DctQuantWeightParams& params4x8,
+      const DctQuantWeightParams& params4x4, const AFVWeights& weights) {
+    return QuantEncodingInternal(Tag<kQuantModeAFV>(), params4x8, params4x4,
+                                 weights);
+  }
+  constexpr QuantEncodingInternal(Tag<kQuantModeAFV> /* tag */,
+                                  const DctQuantWeightParams& params4x8,
+                                  const DctQuantWeightParams& params4x4,
+                                  const AFVWeights& weights)
+      : mode(kQuantModeAFV),
+        dct_params(params4x8),
+        afv_weights(weights),
+        dct_params_afv_4x4(params4x4) {}
+
+  // This constructor is not constexpr so it can't be used in any of the
+  // constexpr cases above.
+  explicit QuantEncodingInternal(Mode mode) : mode(mode) {}
+
+  Mode mode;
+
+  // Weights for DCT4+ tables.
+  DctQuantWeightParams dct_params;
+
+  union {
+    // Weights for identity.
+    IdWeights idweights;
+
+    // Weights for DCT2.
+    DCT2Weights dct2weights;
+
+    // Extra multipliers for coefficients 01/10 and 11 for DCT4 and AFV.
+    DCT4Multipliers dct4multipliers;
+
+    // Weights for AFV. {0, 1} are used directly for coefficients (0, 1) and (1,
+    // 0);  {2, 3, 4} are used directly corner DC, (1,0) - (0,1) and (0, 1) +
+    // (1, 0) - (0, 0) inside the AFV block. Values from 5 to 8 are interpolated
+    // as in GetQuantWeights for DC and are used for other coefficients.
+    AFVWeights afv_weights = {};
+
+    // Extra multipliers for coefficients 01 or 10 for DCT4X8 and DCT8X4.
+    DCT4x8Multipliers dct4x8multipliers;
+
+    // Only used in kQuantModeRAW mode.
+    struct {
+      // explicit quantization table (like in JPEG)
+      std::vector<int>* qtable = nullptr;
+      int qtable_den_shift = 0;
+    } qraw;
+  };
+
+  // Weights for 4x4 sub-block in AFV.
+  DctQuantWeightParams dct_params_afv_4x4;
+
+  union {
+    // Which predefined table to use. Only used if mode is kQuantModeLibrary.
+    uint8_t predefined = 0;
+
+    // Which other quant table to copy; must copy from a table that comes before
+    // the current one. Only used if mode is kQuantModeCopy.
+    uint8_t source;
+  };
+};
+
+class QuantEncoding final : public QuantEncodingInternal {
+ public:
+  QuantEncoding(const QuantEncoding& other)
+      : QuantEncodingInternal(
+            static_cast<const QuantEncodingInternal&>(other)) {
+    if (mode == kQuantModeRAW && qraw.qtable) {
+      // Need to make a copy of the passed *qtable.
+      qraw.qtable = new std::vector<int>(*other.qraw.qtable);
+    }
+  }
+  QuantEncoding(QuantEncoding&& other)
+      : QuantEncodingInternal(
+            static_cast<const QuantEncodingInternal&>(other)) {
+    // Steal the qtable from the other object if any.
+    if (mode == kQuantModeRAW) {
+      other.qraw.qtable = nullptr;
+    }
+  }
+  QuantEncoding& operator=(const QuantEncoding& other) {
+    if (mode == kQuantModeRAW && qraw.qtable) {
+      delete qraw.qtable;
+    }
+    *static_cast<QuantEncodingInternal*>(this) =
+        QuantEncodingInternal(static_cast<const QuantEncodingInternal&>(other));
+    if (mode == kQuantModeRAW && qraw.qtable) {
+      // Need to make a copy of the passed *qtable.
+      qraw.qtable = new std::vector<int>(*other.qraw.qtable);
+    }
+    return *this;
+  }
+
+  ~QuantEncoding() {
+    if (mode == kQuantModeRAW && qraw.qtable) {
+      delete qraw.qtable;
+    }
+  }
+
+  // Wrappers of the QuantEncodingInternal:: static functions that return a
+  // QuantEncoding instead. This is using the explicit and private cast from
+  // QuantEncodingInternal to QuantEncoding, which would be inlined anyway.
+  // In general, you should use this wrappers. The only reason to directly
+  // create a QuantEncodingInternal instance is if you need a constexpr version
+  // of this class. Note that RAW() is not supported in that case since it uses
+  // a std::vector.
+  static QuantEncoding Library(uint8_t predefined) {
+    return QuantEncoding(QuantEncodingInternal::Library(predefined));
+  }
+  static QuantEncoding Identity(const IdWeights& xybweights) {
+    return QuantEncoding(QuantEncodingInternal::Identity(xybweights));
+  }
+  static QuantEncoding DCT2(const DCT2Weights& xybweights) {
+    return QuantEncoding(QuantEncodingInternal::DCT2(xybweights));
+  }
+  static QuantEncoding DCT4(const DctQuantWeightParams& params,
+                            const DCT4Multipliers& xybmul) {
+    return QuantEncoding(QuantEncodingInternal::DCT4(params, xybmul));
+  }
+  static QuantEncoding DCT4X8(const DctQuantWeightParams& params,
+                              const DCT4x8Multipliers& xybmul) {
+    return QuantEncoding(QuantEncodingInternal::DCT4X8(params, xybmul));
+  }
+  static QuantEncoding DCT(const DctQuantWeightParams& params) {
+    return QuantEncoding(QuantEncodingInternal::DCT(params));
+  }
+  static QuantEncoding AFV(const DctQuantWeightParams& params4x8,
+                           const DctQuantWeightParams& params4x4,
+                           const AFVWeights& weights) {
+    return QuantEncoding(
+        QuantEncodingInternal::AFV(params4x8, params4x4, weights));
+  }
+
+  // RAW, note that this one is not a constexpr one.
+  static QuantEncoding RAW(const std::vector<int>& qtable, int shift = 0) {
+    QuantEncoding encoding(kQuantModeRAW);
+    encoding.qraw.qtable = new std::vector<int>();
+    *encoding.qraw.qtable = qtable;
+    encoding.qraw.qtable_den_shift = shift;
+    return encoding;
+  }
+
+ private:
+  explicit QuantEncoding(const QuantEncodingInternal& other)
+      : QuantEncodingInternal(other) {}
+
+  explicit QuantEncoding(QuantEncodingInternal::Mode mode)
+      : QuantEncodingInternal(mode) {}
+};
+
+// A constexpr QuantEncodingInternal instance is often downcasted to the
+// QuantEncoding subclass even if the instance wasn't an instance of the
+// subclass. This is safe because user will upcast to QuantEncodingInternal to
+// access any of its members.
+static_assert(sizeof(QuantEncoding) == sizeof(QuantEncodingInternal),
+              "Don't add any members to QuantEncoding");
+
+// Let's try to keep these 2**N for possible future simplicity.
+const float kInvDCQuant[3] = {
+    4096.0f,
+    512.0f,
+    256.0f,
+};
+
+const float kDCQuant[3] = {
+    1.0f / kInvDCQuant[0],
+    1.0f / kInvDCQuant[1],
+    1.0f / kInvDCQuant[2],
+};
+
+class DequantMatrices {
+ public:
+  enum QuantTable : size_t {
+    DCT = 0,
+    IDENTITY,
+    DCT2X2,
+    DCT4X4,
+    DCT16X16,
+    DCT32X32,
+    // DCT16X8
+    DCT8X16,
+    // DCT32X8
+    DCT8X32,
+    // DCT32X16
+    DCT16X32,
+    DCT4X8,
+    // DCT8X4
+    AFV0,
+    // AFV1
+    // AFV2
+    // AFV3
+    kNum
+  };
+
+  static_assert(AcStrategy::kNumValidStrategies == 18,
+                "Update this array when adding or removing AC strategies.");
+  static constexpr QuantTable kQuantTable[] = {
+      QuantTable::DCT,     QuantTable::IDENTITY, QuantTable::DCT2X2,
+      QuantTable::DCT4X4,  QuantTable::DCT16X16, QuantTable::DCT32X32,
+      QuantTable::DCT8X16, QuantTable::DCT8X16,  QuantTable::DCT8X32,
+      QuantTable::DCT8X32, QuantTable::DCT16X32, QuantTable::DCT16X32,
+      QuantTable::DCT4X8,  QuantTable::DCT4X8,   QuantTable::AFV0,
+      QuantTable::AFV0,    QuantTable::AFV0,     QuantTable::AFV0,
+  };
+
+  DequantMatrices() {
+    encodings_.resize(size_t(QuantTable::kNum), QuantEncoding::Library(0));
+    for (size_t i = 0; i < size_t(QuantTable::kNum); i++) {
+      encodings_[i] = QuantEncoding::Library(0);
+    }
+    // Default quantization tables need to be valid.
+    JXL_CHECK(Compute());
+  }
+
+  static const QuantEncoding* Library();
+
+  typedef std::array<QuantEncodingInternal, kNumPredefinedTables * kNum>
+      DequantLibraryInternal;
+  // Return the array of library kNumPredefinedTables QuantEncoding entries as
+  // a constexpr array. Use Library() to obtain a pointer to the copy in the
+  // .cc file.
+  static const DequantLibraryInternal LibraryInit();
+
+  JXL_INLINE size_t MatrixOffset(size_t quant_kind, size_t c) const {
+    JXL_DASSERT(quant_kind < AcStrategy::kNumValidStrategies);
+    return table_offsets_[quant_kind * 3 + c];
+  }
+
+  // Returns aligned memory.
+  JXL_INLINE const float* Matrix(size_t quant_kind, size_t c) const {
+    JXL_DASSERT(quant_kind < AcStrategy::kNumValidStrategies);
+    return &table_[MatrixOffset(quant_kind, c)];
+  }
+
+  JXL_INLINE const float* InvMatrix(size_t quant_kind, size_t c) const {
+    JXL_DASSERT(quant_kind < AcStrategy::kNumValidStrategies);
+    return &inv_table_[MatrixOffset(quant_kind, c)];
+  }
+
+  JXL_INLINE float DCQuant(size_t c) const { return dc_quant_[c]; }
+
+  JXL_INLINE float InvDCQuant(size_t c) const { return inv_dc_quant_[c]; }
+
+  size_t Size() const { return size_; }
+
+  void SetCustom(const std::vector<QuantEncoding>& encodings) {
+    JXL_ASSERT(encodings.size() == kNum);
+    encodings_ = encodings;
+    // Roundtrip encode/decode the matrices to ensure same values as decoder.
+    BitWriter writer;
+    JXL_CHECK(Encode(&writer, 0, nullptr));
+    writer.ZeroPadToByte();
+    BitReader br(writer.GetSpan());
+    // Called only in the encoder: should fail only for programmer errors.
+    JXL_CHECK(Decode(&br));
+    JXL_CHECK(br.Close());
+  }
+
+  // For consistency with QuantEncoding, higher values correspond to more
+  // precision.
+  void SetCustomDC(const float* dc) {
+    for (size_t c = 0; c < 3; c++) {
+      dc_quant_[c] = 1.0f / dc[c];
+      inv_dc_quant_[c] = dc[c];
+    }
+    // Roundtrip encode/decode DC to ensure same values as decoder.
+    BitWriter writer;
+    JXL_CHECK(EncodeDC(&writer, 0, nullptr));
+    writer.ZeroPadToByte();
+    BitReader br(writer.GetSpan());
+    // Called only in the encoder: should fail only for programmer errors.
+    JXL_CHECK(DecodeDC(&br));
+    JXL_CHECK(br.Close());
+  }
+
+  Status Encode(BitWriter* writer, size_t layer, AuxOut* aux_out) const;
+  Status EncodeDC(BitWriter* writer, size_t layer, AuxOut* aux_out) const;
+
+  Status Decode(BitReader* br);
+  Status DecodeDC(BitReader* br);
+
+  const std::vector<QuantEncoding>& encodings() const { return encodings_; }
+
+ private:
+  Status Compute();
+  static_assert(kNum == 11,
+                "Update this array when adding or removing quant tables.");
+  static constexpr size_t required_size_x_[kNum] = {1, 1, 1, 1, 2, 4,
+                                                    1, 1, 2, 1, 1};
+  static constexpr size_t required_size_y_[kNum] = {1, 1, 1, 1, 2, 4,
+                                                    2, 4, 4, 1, 1};
+  static constexpr size_t required_size_[kNum] = {1, 1, 1, 1, 4, 16,
+                                                  2, 4, 8, 1, 1};
+  static constexpr size_t kTotalTableSize =
+      ArraySum(required_size_) * kDCTBlockSize * 3;
+
+  HWY_ALIGN float table_[kTotalTableSize];
+  HWY_ALIGN float inv_table_[kTotalTableSize];
+  float dc_quant_[3] = {kDCQuant[0], kDCQuant[1], kDCQuant[2]};
+  float inv_dc_quant_[3] = {kInvDCQuant[0], kInvDCQuant[1], kInvDCQuant[2]};
+  size_t table_offsets_[AcStrategy::kNumValidStrategies * 3];
+  size_t size_;
+  std::vector<QuantEncoding> encodings_;
+};
+
+void FindBestDequantMatrices(const CompressParams& cparams,
+                             const Image3F& opsin,
+                             DequantMatrices* dequant_matrices);
+
+}  // namespace jxl
+
+#endif  // JXL_QUANT_WEIGHTS_H_
diff --git a/jxl/quant_weights_test.cc b/jxl/quant_weights_test.cc
new file mode 100644
index 000000000..948d7d075
--- /dev/null
+++ b/jxl/quant_weights_test.cc
@@ -0,0 +1,235 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "jxl/quant_weights.h"
+
+#include <stdlib.h>
+
+#include <algorithm>
+#include <cmath>
+
+#include "gtest/gtest.h"
+#include "jxl/dct.h"
+
+namespace jxl {
+namespace {
+
+template <typename T>
+void CheckSimilar(T a, T b) {
+  EXPECT_EQ(a, b);
+}
+// minimum exponent = -15.
+template <>
+void CheckSimilar(float a, float b) {
+  float m = std::max(std::abs(a), std::abs(b));
+  // 10 bits of precision are used in the format. Relative error should be
+  // below 2^-10.
+  EXPECT_LE(std::abs(a - b), m / 1024.0f) << "a: " << a << " b: " << b;
+}
+
+TEST(QuantWeightsTest, DC) {
+  DequantMatrices mat;
+  float dc_quant[3] = {1e+5, 1e+3, 1e+1};
+  mat.SetCustomDC(dc_quant);
+  for (size_t c = 0; c < 3; c++) {
+    CheckSimilar(mat.InvDCQuant(c), dc_quant[c]);
+  }
+}
+
+void RoundtripMatrices(const std::vector<QuantEncoding>& encodings) {
+  ASSERT_TRUE(encodings.size() == DequantMatrices::kNum);
+  DequantMatrices mat;
+  mat.SetCustom(encodings);
+  const std::vector<QuantEncoding>& encodings_dec = mat.encodings();
+  for (size_t i = 0; i < encodings.size(); i++) {
+    const QuantEncoding& e = encodings[i];
+    const QuantEncoding& d = encodings_dec[i];
+    // Check values roundtripped correctly.
+    EXPECT_EQ(e.mode, d.mode);
+    EXPECT_EQ(e.predefined, d.predefined);
+    EXPECT_EQ(e.source, d.source);
+
+    EXPECT_EQ(static_cast<uint64_t>(e.dct_params.num_distance_bands),
+              static_cast<uint64_t>(d.dct_params.num_distance_bands));
+    for (size_t c = 0; c < 3; c++) {
+      for (size_t j = 0; j < DctQuantWeightParams::kMaxDistanceBands; j++) {
+        CheckSimilar(e.dct_params.distance_bands[c][j],
+                     d.dct_params.distance_bands[c][j]);
+      }
+    }
+
+    if (e.mode == QuantEncoding::kQuantModeRAW) {
+      EXPECT_FALSE(!e.qraw.qtable);
+      EXPECT_FALSE(!d.qraw.qtable);
+      EXPECT_EQ(e.qraw.qtable->size(), d.qraw.qtable->size());
+      for (size_t j = 0; j < e.qraw.qtable->size(); j++) {
+        EXPECT_EQ((*e.qraw.qtable)[j], (*d.qraw.qtable)[j]);
+      }
+      EXPECT_EQ(e.qraw.qtable_den_shift, d.qraw.qtable_den_shift);
+    } else {
+      // modes different than kQuantModeRAW use one of the other fields used
+      // here, which all happen to be arrays of floats.
+      for (size_t c = 0; c < 3; c++) {
+        for (size_t j = 0; j < 3; j++) {
+          CheckSimilar(e.idweights[c][j], d.idweights[c][j]);
+        }
+        for (size_t j = 0; j < 6; j++) {
+          CheckSimilar(e.dct2weights[c][j], d.dct2weights[c][j]);
+        }
+        for (size_t j = 0; j < 2; j++) {
+          CheckSimilar(e.dct4multipliers[c][j], d.dct4multipliers[c][j]);
+        }
+        CheckSimilar(e.dct4x8multipliers[c], d.dct4x8multipliers[c]);
+        for (size_t j = 0; j < 9; j++) {
+          CheckSimilar(e.afv_weights[c][j], d.afv_weights[c][j]);
+        }
+        for (size_t j = 0; j < DctQuantWeightParams::kMaxDistanceBands; j++) {
+          CheckSimilar(e.dct_params_afv_4x4.distance_bands[c][j],
+                       d.dct_params_afv_4x4.distance_bands[c][j]);
+        }
+      }
+    }
+  }
+}
+
+TEST(QuantWeightsTest, AllDefault) {
+  std::vector<QuantEncoding> encodings(DequantMatrices::kNum,
+                                       QuantEncoding::Library(0));
+  RoundtripMatrices(encodings);
+}
+
+void TestSingleQuantMatrix(DequantMatrices::QuantTable kind) {
+  std::vector<QuantEncoding> encodings(DequantMatrices::kNum,
+                                       QuantEncoding::Library(0));
+  encodings[kind] = DequantMatrices::Library()[kind];
+  RoundtripMatrices(encodings);
+}
+
+// Ensure we can reasonably represent default quant tables.
+TEST(QuantWeightsTest, DCT) { TestSingleQuantMatrix(DequantMatrices::DCT); }
+TEST(QuantWeightsTest, IDENTITY) {
+  TestSingleQuantMatrix(DequantMatrices::IDENTITY);
+}
+TEST(QuantWeightsTest, DCT2X2) {
+  TestSingleQuantMatrix(DequantMatrices::DCT2X2);
+}
+TEST(QuantWeightsTest, DCT4X4) {
+  TestSingleQuantMatrix(DequantMatrices::DCT4X4);
+}
+TEST(QuantWeightsTest, DCT16X16) {
+  TestSingleQuantMatrix(DequantMatrices::DCT16X16);
+}
+TEST(QuantWeightsTest, DCT32X32) {
+  TestSingleQuantMatrix(DequantMatrices::DCT32X32);
+}
+TEST(QuantWeightsTest, DCT8X16) {
+  TestSingleQuantMatrix(DequantMatrices::DCT8X16);
+}
+TEST(QuantWeightsTest, DCT8X32) {
+  TestSingleQuantMatrix(DequantMatrices::DCT8X32);
+}
+TEST(QuantWeightsTest, DCT16X32) {
+  TestSingleQuantMatrix(DequantMatrices::DCT16X32);
+}
+TEST(QuantWeightsTest, DCT4X8) {
+  TestSingleQuantMatrix(DequantMatrices::DCT4X8);
+}
+TEST(QuantWeightsTest, AFV0) { TestSingleQuantMatrix(DequantMatrices::AFV0); }
+TEST(QuantWeightsTest, RAW) {
+  std::vector<QuantEncoding> encodings(DequantMatrices::kNum,
+                                       QuantEncoding::Library(0));
+  std::vector<int> matrix(3 * 32 * 32);
+  std::mt19937 rng;
+  std::uniform_int_distribution<size_t> dist(1, 255);
+  for (size_t i = 0; i < matrix.size(); i++) matrix[i] = dist(rng);
+  encodings[DequantMatrices::kQuantTable[AcStrategy::DCT32X32]] =
+      QuantEncoding::RAW(matrix, 2);
+  RoundtripMatrices(encodings);
+}
+
+TEST(QuantWeightsTest, DCTUniform) {
+  constexpr float kUniformQuant = 4;
+  float weights[3][2] = {{1.0f / kUniformQuant, 0},
+                         {1.0f / kUniformQuant, 0},
+                         {1.0f / kUniformQuant, 0}};
+  DctQuantWeightParams dct_params(weights);
+  std::vector<QuantEncoding> encodings(DequantMatrices::kNum,
+                                       QuantEncoding::DCT(dct_params));
+  DequantMatrices dequant_matrices;
+  dequant_matrices.SetCustom(encodings);
+
+  const float dc_quant[3] = {1.0f / kUniformQuant, 1.0f / kUniformQuant,
+                             1.0f / kUniformQuant};
+  dequant_matrices.SetCustomDC(dc_quant);
+
+  // DCT8
+  {
+    HWY_ALIGN float pixels[64];
+    std::iota(std::begin(pixels), std::end(pixels), 0);
+    HWY_ALIGN float coeffs[64];
+    AcStrategy dct = AcStrategy::FromRawStrategy(AcStrategy::DCT);
+    dct.TransformFromPixels(pixels, 8, coeffs);
+    HWY_ALIGN double slow_coeffs[64];
+    for (size_t i = 0; i < 64; i++) slow_coeffs[i] = pixels[i];
+    slow_dct::DCTSlow<8>(slow_coeffs);
+
+    for (size_t i = 0; i < 64; i++) {
+      // DCTSlow doesn't multiply/divide by 1/N, so we do it manually.
+      slow_coeffs[i] =
+          std::round(slow_coeffs[i] / 8 / kUniformQuant) * 8 * kUniformQuant;
+      coeffs[i] = std::round(coeffs[i] * dequant_matrices.InvMatrix(
+                                             dct.RawStrategy(), 0)[i]) *
+                  dequant_matrices.Matrix(dct.RawStrategy(), 0)[i];
+    }
+    slow_dct::IDCTSlow<8>(slow_coeffs);
+    dct.TransformToPixels(coeffs, pixels, 8);
+    for (size_t i = 0; i < 64; i++) {
+      EXPECT_NEAR(pixels[i], slow_coeffs[i], 1e-4);
+    }
+  }
+
+  // DCT16
+  {
+    HWY_ALIGN float pixels[64 * 4];
+    std::iota(std::begin(pixels), std::end(pixels), 0);
+    HWY_ALIGN float coeffs[64 * 4];
+    AcStrategy dct = AcStrategy::FromRawStrategy(AcStrategy::DCT16X16);
+    dct.TransformFromPixels(pixels, 16, coeffs);
+    HWY_ALIGN double slow_coeffs[64 * 4];
+    for (size_t i = 0; i < 64 * 4; i++) slow_coeffs[i] = pixels[i];
+    slow_dct::DCTSlow<16>(slow_coeffs);
+
+    for (size_t i = 0; i < 64 * 4; i++) {
+      slow_coeffs[i] =
+          std::round(slow_coeffs[i] / 16 / kUniformQuant) * 16 * kUniformQuant;
+      coeffs[i] = std::round(coeffs[i] * dequant_matrices.InvMatrix(
+                                             dct.RawStrategy(), 0)[i]) *
+                  dequant_matrices.Matrix(dct.RawStrategy(), 0)[i];
+    }
+
+    slow_dct::IDCTSlow<16>(slow_coeffs);
+    dct.TransformToPixels(coeffs, pixels, 16);
+    for (size_t i = 0; i < 64 * 4; i++) {
+      EXPECT_NEAR(pixels[i], slow_coeffs[i], 1e-4);
+    }
+  }
+
+  // Check that all matrices have the same DC quantization, i.e. that they all
+  // have the same scaling.
+  for (size_t i = 0; i < AcStrategy::kNumValidStrategies; i++) {
+    EXPECT_NEAR(dequant_matrices.Matrix(i, 0)[0], kUniformQuant, 1e-6);
+  }
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/quantizer.cc b/jxl/quantizer.cc
new file mode 100644
index 000000000..65cb3bd75
--- /dev/null
+++ b/jxl/quantizer.cc
@@ -0,0 +1,299 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/quantizer.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/field_encodings.h"
+#include "jxl/fields.h"
+#include "jxl/image.h"
+#include "jxl/image_ops.h"
+#include "jxl/quant_weights.h"
+
+namespace jxl {
+
+static const int kDefaultQuant = 64;
+
+Quantizer::Quantizer(const DequantMatrices* dequant)
+    : Quantizer(dequant, kDefaultQuant, kGlobalScaleDenom / kDefaultQuant) {}
+
+Quantizer::Quantizer(const DequantMatrices* dequant, int quant_dc,
+                     int global_scale)
+    : global_scale_(global_scale), quant_dc_(quant_dc), dequant_(dequant) {
+  JXL_ASSERT(dequant_ != nullptr);
+  RecomputeFromGlobalScale();
+  inv_quant_dc_ = inv_global_scale_ / quant_dc_;
+
+  memcpy(zero_bias_, kZeroBiasDefault, sizeof(kZeroBiasDefault));
+}
+
+void Quantizer::ComputeGlobalScaleAndQuant(float quant_dc, float quant_median,
+                                           float quant_median_absd) {
+  // Target value for the median value in the quant field.
+  const float kQuantFieldTarget = 3.80987740592518214386f;
+  // We reduce the median of the quant field by the median absolute deviation:
+  // higher resolution on highly varying quant fields.
+  int new_global_scale =
+      static_cast<int>(kGlobalScaleDenom * (quant_median - quant_median_absd) /
+                       kQuantFieldTarget);
+  // Ensure that quant_dc_ will always be at least
+  // kGlobalScaleDenom/kGlobalScaleNumerator.
+  const int scaled_quant_dc =
+      static_cast<int>(quant_dc * kGlobalScaleNumerator);
+  if (new_global_scale > scaled_quant_dc) {
+    new_global_scale = scaled_quant_dc;
+  }
+  // Ensure that new_global_scale is positive and no more than 1<<15.
+  if (new_global_scale <= 0) new_global_scale = 1;
+  if (new_global_scale > (1 << 15)) new_global_scale = 1 << 15;
+  global_scale_ = new_global_scale;
+  // Code below uses inv_global_scale_.
+  RecomputeFromGlobalScale();
+
+  float fval = quant_dc * inv_global_scale_ + 0.5f;
+  fval = std::min<float>(1 << 16, fval);
+  const int new_quant_dc = static_cast<int>(fval);
+  quant_dc_ = new_quant_dc;
+
+  // quant_dc_ was updated, recompute values.
+  RecomputeFromGlobalScale();
+}
+
+void Quantizer::SetQuantField(const float quant_dc, const ImageF& qf,
+                              ImageI* JXL_RESTRICT raw_quant_field) {
+  JXL_CHECK(SameSize(*raw_quant_field, qf));
+  std::vector<float> data(qf.xsize() * qf.ysize());
+  for (size_t y = 0; y < qf.ysize(); ++y) {
+    const float* JXL_RESTRICT row_qf = qf.Row(y);
+    for (size_t x = 0; x < qf.xsize(); ++x) {
+      float quant = row_qf[x];
+      data[qf.xsize() * y + x] = quant;
+    }
+  }
+  const float quant_median = Median(&data);
+  const float quant_median_absd = MedianAbsoluteDeviation(data, quant_median);
+  ComputeGlobalScaleAndQuant(quant_dc, quant_median, quant_median_absd);
+  for (size_t y = 0; y < qf.ysize(); ++y) {
+    const float* JXL_RESTRICT row_qf = qf.Row(y);
+    int32_t* JXL_RESTRICT row_qi = raw_quant_field->Row(y);
+    for (size_t x = 0; x < qf.xsize(); ++x) {
+      int val = ClampVal(row_qf[x] * inv_global_scale_ + 0.5f);
+      row_qi[x] = val;
+    }
+  }
+}
+
+void Quantizer::SetQuant(float quant_dc, float quant_ac,
+                         ImageI* JXL_RESTRICT raw_quant_field) {
+  ComputeGlobalScaleAndQuant(quant_dc, quant_ac, 0);
+  int val = ClampVal(quant_ac * inv_global_scale_ + 0.5f);
+  FillImage(val, raw_quant_field);
+}
+
+HWY_ATTR void Quantizer::QuantizeBlockAC(
+    const bool error_diffusion, size_t c, int32_t quant, float qm_multiplier,
+    size_t quant_kind, size_t xsize, size_t ysize,
+    const float* JXL_RESTRICT block_in,
+    ac_qcoeff_t* JXL_RESTRICT block_out) const {
+  PROFILER_FUNC;
+  const float* JXL_RESTRICT qm = dequant_->InvMatrix(quant_kind, c);
+  const float qac = Scale() * quant;
+  // Not SIMD-fied for now.
+  float thres[4] = {0.5f, 0.6f, 0.6f, 0.65f};
+  if (c != 1) {
+    for (int i = 1; i < 4; ++i) {
+      thres[i] = 0.75f;
+    }
+  }
+
+  CoefficientLayout(&ysize, &xsize);
+
+  if (!error_diffusion) {
+    HWY_CAPPED(float, kBlockDim) df;
+    HWY_CAPPED(uint32_t, kBlockDim) du;
+    const auto quant = Set(df, qac * qm_multiplier);
+    const auto abs_mask = BitCast(df, Set(du, 0x7FFFFFFFu));
+
+    for (size_t y = 0; y < ysize * kBlockDim; y++) {
+      size_t yfix = static_cast<size_t>(y >= ysize * kBlockDim / 2) * 2;
+      const size_t off = y * kBlockDim * xsize;
+      for (size_t x = 0; x < xsize * kBlockDim; x += df.N) {
+        auto thr = Zero(df);
+        if (xsize == 1) {
+          HWY_ALIGN uint32_t kMask[kBlockDim] = {0,   0,   0,   0,
+                                                 ~0u, ~0u, ~0u, ~0u};
+          const auto mask = MaskFromVec(BitCast(df, Load(du, kMask + x)));
+          thr =
+              IfThenElse(mask, Set(df, thres[yfix + 1]), Set(df, thres[yfix]));
+        } else {
+          // Same for all lanes in the vector.
+          thr = Set(
+              df,
+              thres[yfix + static_cast<size_t>(x >= xsize * kBlockDim / 2)]);
+        }
+
+        const auto q = Load(df, qm + off + x) * quant;
+        const auto in = Load(df, block_in + off + x);
+        const auto val = q * in;
+        const auto nzero_mask = (abs_mask & val) >= thr;
+        const auto v = IfThenElseZero(nzero_mask, Round(val));
+        Store(v, df, block_out + off + x);
+      }
+    }
+    return;
+  }
+
+retry:
+  int hfNonZeros[4] = {};
+  float hfError[4] = {};
+  float hfMaxError[4] = {};
+  size_t hfMaxErrorIx[4] = {};
+  for (size_t y = 0; y < ysize * kBlockDim; y++) {
+    for (size_t x = 0; x < xsize * kBlockDim; x++) {
+      const size_t pos = y * kBlockDim * xsize + x;
+      if (x < xsize && y < ysize) {
+        // Ensure block is initialized
+        block_out[pos] = 0;
+        continue;
+      }
+      const size_t hfix = (static_cast<size_t>(y >= ysize * kBlockDim / 2) * 2 +
+                           static_cast<size_t>(x >= xsize * kBlockDim / 2));
+      const float val = block_in[pos] * (qm[pos] * qac * qm_multiplier);
+      float v = (std::abs(val) < thres[hfix]) ? 0 : std::rint(val);
+      const float error = std::abs(val) - std::abs(v);
+      hfError[hfix] += error;
+      if (hfMaxError[hfix] < error) {
+        hfMaxError[hfix] = error;
+        hfMaxErrorIx[hfix] = pos;
+      }
+      if (v != 0.0f) {
+        hfNonZeros[hfix] += std::abs(v);
+      }
+      block_out[pos] = static_cast<ac_qcoeff_t>(std::rint(v));
+    }
+  }
+  if (c != 1) return;
+  // TODO(veluca): include AFV?
+  const size_t kPartialBlockKinds =
+      (1 << AcStrategy::Type::IDENTITY) | (1 << AcStrategy::Type::DCT2X2) |
+      (1 << AcStrategy::Type::DCT4X4) | (1 << AcStrategy::Type::DCT4X8) |
+      (1 << AcStrategy::Type::DCT8X4);
+  if ((1 << quant_kind) & kPartialBlockKinds) return;
+  float hfErrorLimit = 0.1f * (xsize * ysize) * kDCTBlockSize * 0.25f;
+  bool goretry = false;
+  for (int i = 1; i < 4; ++i) {
+    if (hfError[i] >= hfErrorLimit &&
+        hfNonZeros[i] <= (xsize + ysize) * 0.25f) {
+      if (thres[i] >= 0.4f) {
+        thres[i] -= 0.01f;
+        goretry = true;
+      }
+    }
+  }
+  if (goretry) goto retry;
+  for (int i = 1; i < 4; ++i) {
+    if (hfError[i] >= hfErrorLimit && hfNonZeros[i] == 0) {
+      const size_t pos = hfMaxErrorIx[i];
+      if (hfMaxError[i] >= 0.4f) {
+        block_out[pos] = block_in[pos] > 0.0f ? 1.0f : -1.0f;
+      }
+    }
+  }
+}
+
+HWY_ATTR void Quantizer::QuantizeRoundtripYBlockAC(
+    const bool error_diffusion, int32_t quant, size_t quant_kind, size_t xsize,
+    size_t ysize, const float* JXL_RESTRICT biases,
+    const float* JXL_RESTRICT in, ac_qcoeff_t* JXL_RESTRICT quantized,
+    float* JXL_RESTRICT out) const {
+  QuantizeBlockAC(error_diffusion, 1, quant, 1.0f, quant_kind, xsize, ysize, in,
+                  quantized);
+
+  PROFILER_ZONE("enc quant adjust bias");
+  const float* JXL_RESTRICT dequant_matrix = DequantMatrix(quant_kind, 1);
+
+  HWY_CAPPED(float, kDCTBlockSize) df;
+  const auto inv_qac = Set(df, inv_quant_ac(quant));
+  for (size_t k = 0; k < kDCTBlockSize * xsize * ysize; k += df.N) {
+    const auto quant = Load(df, quantized + k);
+    const auto adj_quant = AdjustQuantBias(df, 1, quant, biases);
+    const auto dequantm = Load(df, dequant_matrix + k);
+    Store(adj_quant * dequantm * inv_qac, df, out + k);
+  }
+}
+
+struct QuantizerParams {
+  QuantizerParams() { Bundle::Init(this); }
+  static const char* Name() { return "QuantizerParams"; }
+
+  template <class Visitor>
+  Status VisitFields(Visitor* JXL_RESTRICT visitor) {
+    visitor->U32(Bits(11), BitsOffset(11, 2048), BitsOffset(12, 4096),
+                 BitsOffset(15, 8192), 0, &global_scale_minus_1);
+    visitor->U32(Val(15), Bits(5), Bits(8), Bits(16), 0, &quant_dc_minus_1);
+    return true;
+  }
+
+  uint32_t global_scale_minus_1;
+  uint32_t quant_dc_minus_1;
+};
+
+void TestQuantizerParams() {
+  for (uint32_t i = 1; i < 10000; ++i) {
+    QuantizerParams p;
+    p.global_scale_minus_1 = i - 1;
+    size_t extension_bits = 0, total_bits = 0;
+    JXL_CHECK(Bundle::CanEncode(p, &extension_bits, &total_bits));
+    JXL_CHECK(extension_bits == 0);
+    JXL_CHECK(total_bits >= 4);
+  }
+}
+
+Status Quantizer::Encode(BitWriter* writer, size_t layer,
+                         AuxOut* aux_out) const {
+  QuantizerParams params;
+  params.global_scale_minus_1 = global_scale_ - 1;
+  params.quant_dc_minus_1 = quant_dc_ - 1;
+  return Bundle::Write(params, writer, layer, aux_out);
+}
+
+Status Quantizer::Decode(BitReader* reader) {
+  QuantizerParams params;
+  JXL_RETURN_IF_ERROR(Bundle::Read(reader, &params));
+  global_scale_ = static_cast<int>(params.global_scale_minus_1 + 1);
+  quant_dc_ = static_cast<int>(params.quant_dc_minus_1 + 1);
+  RecomputeFromGlobalScale();
+  return true;
+}
+
+void Quantizer::DumpQuantizationMap(const ImageI& raw_quant_field) const {
+  printf("Global scale: %d (%.7f)\nDC quant: %d\n", global_scale_,
+         global_scale_ * 1.0 / kGlobalScaleDenom, quant_dc_);
+  printf("AC quantization Map:\n");
+  for (size_t y = 0; y < raw_quant_field.ysize(); ++y) {
+    for (size_t x = 0; x < raw_quant_field.xsize(); ++x) {
+      printf(" %3d", raw_quant_field.Row(y)[x]);
+    }
+    printf("\n");
+  }
+}
+
+static constexpr JXL_INLINE int QuantizeValue(float value, float inv_step) {
+  return static_cast<int>(value * inv_step + (value >= 0 ? .5f : -.5f));
+}
+
+}  // namespace jxl
diff --git a/jxl/quantizer.h b/jxl/quantizer.h
new file mode 100644
index 000000000..1511bbd14
--- /dev/null
+++ b/jxl/quantizer.h
@@ -0,0 +1,220 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_QUANTIZER_H_
+#define JXL_QUANTIZER_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <cmath>
+#include <hwy/static_targets.h>
+#include <utility>
+#include <vector>
+
+#include "jxl/ac_strategy.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/bits.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/robust_statistics.h"
+#include "jxl/base/status.h"
+#include "jxl/common.h"
+#include "jxl/dct_util.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/image.h"
+#include "jxl/linalg.h"
+#include "jxl/quant_weights.h"
+
+// Quantizes DC and AC coefficients, with separate quantization tables according
+// to the quant_kind (which is currently computed from the AC strategy and the
+// block index inside that strategy).
+
+namespace jxl {
+
+static constexpr int kGlobalScaleDenom = 1 << 16;
+static constexpr int kGlobalScaleNumerator = 4096;
+
+// zero-biases for quantizing channels X, Y, B
+static constexpr float kZeroBiasDefault[3] = {0.5f, 0.5f, 0.5f};
+
+// Returns adjusted version of a quantized integer, such that its value is
+// closer to the expected value of the original.
+// The residuals of AC coefficients that we quantize are not uniformly
+// distributed. Numerical experiments show that they have a distribution with
+// the "shape" of 1/(1+x^2) [up to some coefficients]. This means that the
+// expected value of a coefficient that gets quantized to x will not be x
+// itself, but (at least with reasonable approximation):
+// - 0 if x is 0
+// - x * biases[c] if x is 1 or -1
+// - x - biases[3]/x otherwise
+// This follows from computing the distribution of the quantization bias, which
+// can be approximated fairly well by <constant>/x when |x| is at least two.
+static constexpr float kBiasNumerator = 0.145f;
+
+static constexpr float kDefaultQuantBias[4] = {
+    1.0f - 0.05465007330715401f,
+    1.0f - 0.07005449891748593f,
+    1.0f - 0.049935103337343655f,
+    0.145f,
+};
+
+template <class DF>
+HWY_ATTR JXL_INLINE hwy::VT<DF> AdjustQuantBias(
+    DF df, const size_t c, const hwy::VT<DF> quant,
+    const float* JXL_RESTRICT biases) {
+  const hwy::Desc<int32_t, DF::LanesOr0()> di;
+
+  // Compare |quant|, keep sign bit for negating result.
+  const auto kSign = BitCast(df, Set(di, INT32_MIN));
+  const auto sign = quant & kSign;  // TODO(janwas): = abs ^ orig
+  const auto abs_quant = AndNot(kSign, quant);
+
+  // If |x| is 1, kZeroBias creates a different bias for each channel.
+  // We're implementing the following:
+  // if (quant == 0) return 0;
+  // if (quant == 1) return biases[c];
+  // if (quant == -1) return -biases[c];
+  // return quant - biases[3] / quant;
+
+  // Integer comparison is not helpful because Clang incurs bypass penalties
+  // from unnecessarily mixing integer and float.
+  const auto is_01 = abs_quant < Set(df, 1.125f);
+  const auto not_0 = abs_quant > Zero(df);
+
+  // Bitwise logic is faster than quant * biases[3].
+  const auto one_bias = IfThenElseZero(not_0, Set(df, biases[c]) ^ sign);
+
+  // About 2E-5 worse than ReciprocalNR or division.
+  const auto bias =
+      NegMulAdd(Set(df, biases[3]), ApproximateReciprocal(quant), quant);
+
+  return IfThenElse(is_01, one_bias, bias);
+}
+
+class Quantizer {
+ public:
+  explicit Quantizer(const DequantMatrices* dequant);
+  Quantizer(const DequantMatrices* dequant, int quant_dc, int global_scale);
+
+  static JXL_INLINE int ClampVal(float val) {
+    static const int kQuantMax = 256;
+    return std::min(static_cast<int>(std::max(1.0f, val)), kQuantMax);
+  }
+
+  // Recomputes other derived fields after global_scale_ has changed.
+  void RecomputeFromGlobalScale() {
+    global_scale_float_ = global_scale_ * (1.0 / kGlobalScaleDenom);
+    inv_global_scale_ = 1.0 * kGlobalScaleDenom / global_scale_;
+    inv_quant_dc_ = inv_global_scale_ / quant_dc_;
+    for (size_t c = 0; c < 3; c++) {
+      mul_dc_[c] = GetDcStep(c);
+      inv_mul_dc_[c] = GetInvDcStep(c);
+    }
+  }
+
+  // Returns scaling factor such that Scale() * (RawDC() or RawQuantField())
+  // pixels yields the same float values returned by GetQuantField.
+  JXL_INLINE float Scale() const { return global_scale_float_; }
+
+  // Reciprocal of Scale().
+  JXL_INLINE float InvGlobalScale() const { return inv_global_scale_; }
+
+  void SetQuantField(const float quant_dc, const ImageF& qf,
+                     ImageI* JXL_RESTRICT raw_quant_field);
+
+  void SetQuant(float quant_dc, float quant_ac,
+                ImageI* JXL_RESTRICT raw_quant_field);
+
+  // Returns the DC quantization base value, which is currently global (not
+  // adaptive). The actual scale factor used to dequantize pixels in channel c
+  // is: inv_quant_dc() * dequant_->DCQuant(c).
+  float inv_quant_dc() const { return inv_quant_dc_; }
+
+  // Dequantize by multiplying with this times dequant_matrix.
+  float inv_quant_ac(int32_t quant) const { return inv_global_scale_ / quant; }
+
+  // NOTE: caller takes care of extracting quant from rect of RawQuantField.
+  HWY_ATTR void QuantizeBlockAC(const bool error_diffusion, size_t c,
+                                int32_t quant, float qm_multiplier,
+                                size_t quant_kind, size_t xsize, size_t ysize,
+                                const float* JXL_RESTRICT block_in,
+                                ac_qcoeff_t* JXL_RESTRICT block_out) const;
+
+  // NOTE: caller takes care of extracting quant from rect of RawQuantField.
+  HWY_ATTR void QuantizeRoundtripYBlockAC(const bool error_diffusion,
+                                          int32_t quant, size_t quant_kind,
+                                          size_t xsize, size_t ysize,
+                                          const float* JXL_RESTRICT biases,
+                                          const float* JXL_RESTRICT in,
+                                          ac_qcoeff_t* JXL_RESTRICT quantized,
+                                          float* JXL_RESTRICT out) const;
+
+  Status Encode(BitWriter* writer, size_t layer, AuxOut* aux_out) const;
+
+  Status Decode(BitReader* reader);
+
+  void DumpQuantizationMap(const ImageI& raw_quant_field) const;
+
+  JXL_INLINE const float* DequantMatrix(size_t quant_kind, size_t c) const {
+    return dequant_->Matrix(quant_kind, c);
+  }
+
+  JXL_INLINE size_t DequantMatrixOffset(size_t quant_kind, size_t c) const {
+    return dequant_->MatrixOffset(quant_kind, c);
+  }
+
+  // Calculates DC quantization step.
+  JXL_INLINE float GetDcStep(size_t c) const {
+    return inv_quant_dc_ * dequant_->DCQuant(c);
+  }
+  JXL_INLINE float GetInvDcStep(size_t c) const {
+    return dequant_->InvDCQuant(c) * (global_scale_float_ * quant_dc_);
+  }
+
+  JXL_INLINE const float* MulDC() const { return mul_dc_; }
+  JXL_INLINE const float* InvMulDC() const { return inv_mul_dc_; }
+
+  JXL_INLINE void ClearDCMul() {
+    std::fill(mul_dc_, mul_dc_ + 4, 1);
+    std::fill(inv_mul_dc_, inv_mul_dc_ + 4, 1);
+  }
+
+ private:
+  void ComputeGlobalScaleAndQuant(float quant_dc, float quant_median,
+                                  float quant_absd);
+
+  // These are serialized:
+  int global_scale_;
+  int quant_dc_;
+
+  // These are derived from global_scale_:
+  float inv_global_scale_;
+  float global_scale_float_;  // reciprocal of inv_global_scale_
+  float inv_quant_dc_;
+
+  float zero_bias_[3];
+  HWY_ALIGN float mul_dc_[4];
+  HWY_ALIGN float inv_mul_dc_[4];
+  const DequantMatrices* dequant_;
+};
+
+void TestQuantizerParams();
+
+}  // namespace jxl
+
+#endif  // JXL_QUANTIZER_H_
diff --git a/jxl/quantizer_test.cc b/jxl/quantizer_test.cc
new file mode 100644
index 000000000..708800a8e
--- /dev/null
+++ b/jxl/quantizer_test.cc
@@ -0,0 +1,82 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/quantizer.h"
+
+#include <random>
+
+#include "gtest/gtest.h"
+#include "jxl/base/span.h"
+#include "jxl/common.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/image_ops.h"
+#include "jxl/image_test_utils.h"
+
+namespace jxl {
+namespace {
+
+void TestEquivalence(int qxsize, int qysize, const Quantizer& quantizer1,
+                     const Quantizer& quantizer2) {
+  ASSERT_NEAR(quantizer1.inv_quant_dc(), quantizer2.inv_quant_dc(), 1e-7);
+}
+
+TEST(QuantizerTest, QuantizerParams) { TestQuantizerParams(); }
+
+TEST(QuantizerTest, BitStreamRoundtripSameQuant) {
+  const int qxsize = 8;
+  const int qysize = 8;
+  DequantMatrices dequant;
+  Quantizer quantizer1(&dequant);
+  ImageI raw_quant_field(qxsize, qysize);
+  quantizer1.SetQuant(0.17f, 0.17f, &raw_quant_field);
+  BitWriter writer;
+  EXPECT_TRUE(quantizer1.Encode(&writer, 0, nullptr));
+  writer.ZeroPadToByte();
+  const size_t bits_written = writer.BitsWritten();
+  Quantizer quantizer2(&dequant, qxsize, qysize);
+  BitReader reader(writer.GetSpan());
+  EXPECT_TRUE(quantizer2.Decode(&reader));
+  EXPECT_TRUE(reader.JumpToByteBoundary());
+  EXPECT_EQ(reader.TotalBitsConsumed(), bits_written);
+  EXPECT_TRUE(reader.Close());
+  TestEquivalence(qxsize, qysize, quantizer1, quantizer2);
+}
+
+TEST(QuantizerTest, BitStreamRoundtripRandomQuant) {
+  const int qxsize = 8;
+  const int qysize = 8;
+  DequantMatrices dequant;
+  Quantizer quantizer1(&dequant);
+  ImageI raw_quant_field(qxsize, qysize);
+  quantizer1.SetQuant(0.17f, 0.17f, &raw_quant_field);
+  std::mt19937_64 rng;
+  std::uniform_int_distribution<> uniform(1, 256);
+  float quant_dc = 0.17f;
+  ImageF qf(qxsize, qysize);
+  RandomFillImage(&qf, 1.0f);
+  quantizer1.SetQuantField(quant_dc, qf, &raw_quant_field);
+  BitWriter writer;
+  EXPECT_TRUE(quantizer1.Encode(&writer, 0, nullptr));
+  writer.ZeroPadToByte();
+  const size_t bits_written = writer.BitsWritten();
+  Quantizer quantizer2(&dequant, qxsize, qysize);
+  BitReader reader(writer.GetSpan());
+  EXPECT_TRUE(quantizer2.Decode(&reader));
+  EXPECT_TRUE(reader.JumpToByteBoundary());
+  EXPECT_EQ(reader.TotalBitsConsumed(), bits_written);
+  EXPECT_TRUE(reader.Close());
+  TestEquivalence(qxsize, qysize, quantizer1, quantizer2);
+}
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/rational_polynomial.h b/jxl/rational_polynomial.h
new file mode 100644
index 000000000..a2a46904d
--- /dev/null
+++ b/jxl/rational_polynomial.h
@@ -0,0 +1,278 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_RATIONAL_POLYNOMIAL_H_
+#define JXL_RATIONAL_POLYNOMIAL_H_
+
+// Fast SIMD evaluation of rational polynomials for approximating functions.
+
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+
+// One Newton-Raphson iteration.
+template <class V>
+static HWY_ATTR HWY_INLINE V ReciprocalNR(const V x) {
+  const auto rcp = ApproximateReciprocal(x);
+  const auto sum = rcp + rcp;
+  const auto x_rcp = x * rcp;
+  return NegMulAdd(x_rcp, rcp, sum);
+}
+
+// Primary template: default to actual division.
+template <typename T, class V>
+struct FastDivision {
+  HWY_ATTR V operator()(const V n, const V d) const { return n / d; }
+};
+// Partial specialization for float vectors.
+template <class V>
+struct FastDivision<float, V> {
+  HWY_ATTR V operator()(const V n, const V d) const {
+    return n * ReciprocalNR(d);
+  }
+};
+
+// Approximates smooth functions via rational polynomials (i.e. dividing two
+// polynomials). Supports V = SIMD or Scalar<T> inputs.
+
+// Evaluates the polynomial using Horner's method, which is faster than
+// Clenshaw recurrence for Chebyshev polynomials.
+//
+// "kDeg" is the degree of the numerator and denominator polynomials;
+// kDegP == kDegQ + 1 = 3 or 4 is usually a good choice.
+template <class D, int kDegP, int kDegQ>
+class RationalPolynomial {
+  using T = typename D::T;
+  using V = hwy::VT<D>;
+  static_assert(kDegP <= 7, "Unroll more iterations");
+  static_assert(kDegQ <= 7, "Unroll more iterations");
+
+ public:
+  template <typename U>
+  HWY_ATTR void SetCoefficients(const U (&p)[kDegP + 1],
+                                const U (&q)[kDegQ + 1]) {
+    for (int i = 0; i <= kDegP; ++i) {
+      p_[i] = Set(D(), static_cast<T>(p[i]));
+    }
+    for (int i = 0; i <= kDegQ; ++i) {
+      q_[i] = Set(D(), static_cast<T>(q[i]));
+    }
+  }
+
+  HWY_ATTR void GetCoefficients(T (*p)[kDegP + 1], T (*q)[kDegQ + 1]) const {
+    for (int i = 0; i <= kDegP; ++i) {
+      p[i] = GetLane(p_[i]);
+    }
+    for (int i = 0; i <= kDegQ; ++i) {
+      q[i] = GetLane(q_[i]);
+    }
+  }
+
+  template <typename U>
+  HWY_ATTR RationalPolynomial(const U (&p)[kDegP + 1],
+                              const U (&q)[kDegQ + 1]) {
+    SetCoefficients(p, q);
+  }
+
+  // Evaluates the polynomial at x.
+  HWY_ATTR JXL_INLINE V operator()(const V x) const {
+    V yp = p_[kDegP];
+    V yq = q_[kDegQ];
+    JXL_COMPILER_FENCE;
+    if (kDegP >= 1) yp = MulAdd(yp, x, p_[kDegP - 1]);
+    if (kDegQ >= 1) yq = MulAdd(yq, x, q_[kDegQ - 1]);
+    JXL_COMPILER_FENCE;
+    if (kDegP >= 2) yp = MulAdd(yp, x, p_[kDegP - 2]);
+    if (kDegQ >= 2) yq = MulAdd(yq, x, q_[kDegQ - 2]);
+    JXL_COMPILER_FENCE;
+    if (kDegP >= 3) yp = MulAdd(yp, x, p_[kDegP - 3]);
+    if (kDegQ >= 3) yq = MulAdd(yq, x, q_[kDegQ - 3]);
+    JXL_COMPILER_FENCE;
+    if (kDegP >= 4) yp = MulAdd(yp, x, p_[kDegP - 4]);
+    if (kDegQ >= 4) yq = MulAdd(yq, x, q_[kDegQ - 4]);
+    JXL_COMPILER_FENCE;
+    if (kDegP >= 5) yp = MulAdd(yp, x, p_[kDegP - 5]);
+    if (kDegQ >= 5) yq = MulAdd(yq, x, q_[kDegQ - 5]);
+    JXL_COMPILER_FENCE;
+    if (kDegP >= 6) yp = MulAdd(yp, x, p_[kDegP - 6]);
+    if (kDegQ >= 6) yq = MulAdd(yq, x, q_[kDegQ - 6]);
+    JXL_COMPILER_FENCE;
+    if (kDegP >= 7) yp = MulAdd(yp, x, p_[kDegP - 7]);
+    if (kDegQ >= 7) yq = MulAdd(yq, x, q_[kDegQ - 7]);
+
+    // Division is faster for a single evaluation but the Triple below are
+    // much faster with NR, and we use the same approach to here so that we
+    // compute the same max error as reached below.
+    return FastDivision<T, V>()(yp, yq);
+  }
+
+ private:
+  // Horner coefficients in ascending order.
+  V p_[kDegP + 1];
+  V q_[kDegQ + 1];
+};
+
+// Evaluates a rational polynomial via Horner's scheme. Equivalent to
+// RationalPolynomial poly(p, q); return poly(x). This can be more efficient
+// because the coefficients are loaded directly from memory, whereas Set
+// can result in copying them from RIP+x to stack frame. LoadDup128 allows us
+// to specify constants (replicated 4x) independently of the lane count.
+template <int NP, int NQ, class V, typename T>
+HWY_ATTR JXL_INLINE V EvalRationalPolynomial(const V x, const T (&p)[NP],
+                                             const T (&q)[NQ]) {
+  const HWY_FULL(T) d;
+  constexpr int kDegP = NP / 4 - 1;
+  constexpr int kDegQ = NQ / 4 - 1;
+  auto yp = LoadDup128(d, &p[kDegP * 4]);
+  auto yq = LoadDup128(d, &q[kDegQ * 4]);
+  JXL_COMPILER_FENCE;
+  if (kDegP >= 1) yp = MulAdd(yp, x, LoadDup128(d, &p[(kDegP - 1) * 4]));
+  if (kDegQ >= 1) yq = MulAdd(yq, x, LoadDup128(d, &q[(kDegQ - 1) * 4]));
+  JXL_COMPILER_FENCE;
+  if (kDegP >= 2) yp = MulAdd(yp, x, LoadDup128(d, &p[(kDegP - 2) * 4]));
+  if (kDegQ >= 2) yq = MulAdd(yq, x, LoadDup128(d, &q[(kDegQ - 2) * 4]));
+  JXL_COMPILER_FENCE;
+  if (kDegP >= 3) yp = MulAdd(yp, x, LoadDup128(d, &p[(kDegP - 3) * 4]));
+  if (kDegQ >= 3) yq = MulAdd(yq, x, LoadDup128(d, &q[(kDegQ - 3) * 4]));
+  JXL_COMPILER_FENCE;
+  if (kDegP >= 4) yp = MulAdd(yp, x, LoadDup128(d, &p[(kDegP - 4) * 4]));
+  if (kDegQ >= 4) yq = MulAdd(yq, x, LoadDup128(d, &q[(kDegQ - 4) * 4]));
+  JXL_COMPILER_FENCE;
+  if (kDegP >= 5) yp = MulAdd(yp, x, LoadDup128(d, &p[(kDegP - 5) * 4]));
+  if (kDegQ >= 5) yq = MulAdd(yq, x, LoadDup128(d, &q[(kDegQ - 5) * 4]));
+  JXL_COMPILER_FENCE;
+  if (kDegP >= 6) yp = MulAdd(yp, x, LoadDup128(d, &p[(kDegP - 6) * 4]));
+  if (kDegQ >= 6) yq = MulAdd(yq, x, LoadDup128(d, &q[(kDegQ - 6) * 4]));
+  JXL_COMPILER_FENCE;
+  if (kDegP >= 7) yp = MulAdd(yp, x, LoadDup128(d, &p[(kDegP - 7) * 4]));
+  if (kDegQ >= 7) yq = MulAdd(yq, x, LoadDup128(d, &q[(kDegQ - 7) * 4]));
+
+  return FastDivision<T, V>()(yp, yq);
+}
+
+// Evaluates three at once for better FMA utilization and fewer loads.
+template <int NP, int NQ, class V, typename T>
+HWY_ATTR void EvalRationalPolynomialTriple(const V x0, const V x1, const V x2,
+                                           const T (&p)[NP], const T (&q)[NQ],
+                                           V* JXL_RESTRICT y0,
+                                           V* JXL_RESTRICT y1,
+                                           V* JXL_RESTRICT y2) {
+  // Computing both polynomials in parallel is slightly faster than sequential
+  // (better utilization of FMA slots despite higher register pressure).
+  const HWY_FULL(T) d;
+  constexpr int kDegP = NP / 4 - 1;
+  constexpr int kDegQ = NQ / 4 - 1;
+  V yp0 = LoadDup128(d, &p[kDegP * 4]);
+  V yq0 = LoadDup128(d, &q[kDegQ * 4]);
+  V yp1 = yp0;
+  V yq1 = yq0;
+  V yp2 = yp0;
+  V yq2 = yq0;
+  V c;
+  if (kDegP >= 1) {
+    c = LoadDup128(d, &p[(kDegP - 1) * 4]);
+    yp0 = MulAdd(yp0, x0, c);
+    yp1 = MulAdd(yp1, x1, c);
+    yp2 = MulAdd(yp2, x2, c);
+  }
+  if (kDegQ >= 1) {
+    c = LoadDup128(d, &q[(kDegQ - 1) * 4]);
+    yq0 = MulAdd(yq0, x0, c);
+    yq1 = MulAdd(yq1, x1, c);
+    yq2 = MulAdd(yq2, x2, c);
+  }
+  if (kDegP >= 2) {
+    c = LoadDup128(d, &p[(kDegP - 2) * 4]);
+    yp0 = MulAdd(yp0, x0, c);
+    yp1 = MulAdd(yp1, x1, c);
+    yp2 = MulAdd(yp2, x2, c);
+  }
+  if (kDegQ >= 2) {
+    c = LoadDup128(d, &q[(kDegQ - 2) * 4]);
+    yq0 = MulAdd(yq0, x0, c);
+    yq1 = MulAdd(yq1, x1, c);
+    yq2 = MulAdd(yq2, x2, c);
+  }
+  if (kDegP >= 3) {
+    c = LoadDup128(d, &p[(kDegP - 3) * 4]);
+    yp0 = MulAdd(yp0, x0, c);
+    yp1 = MulAdd(yp1, x1, c);
+    yp2 = MulAdd(yp2, x2, c);
+  }
+  if (kDegQ >= 3) {
+    c = LoadDup128(d, &q[(kDegQ - 3) * 4]);
+    yq0 = MulAdd(yq0, x0, c);
+    yq1 = MulAdd(yq1, x1, c);
+    yq2 = MulAdd(yq2, x2, c);
+  }
+  if (kDegP >= 4) {
+    c = LoadDup128(d, &p[(kDegP - 4) * 4]);
+    yp0 = MulAdd(yp0, x0, c);
+    yp1 = MulAdd(yp1, x1, c);
+    yp2 = MulAdd(yp2, x2, c);
+  }
+  if (kDegQ >= 4) {
+    c = LoadDup128(d, &q[(kDegQ - 4) * 4]);
+    yq0 = MulAdd(yq0, x0, c);
+    yq1 = MulAdd(yq1, x1, c);
+    yq2 = MulAdd(yq2, x2, c);
+  }
+  if (kDegP >= 5) {
+    c = LoadDup128(d, &p[(kDegP - 5) * 4]);
+    yp0 = MulAdd(yp0, x0, c);
+    yp1 = MulAdd(yp1, x1, c);
+    yp2 = MulAdd(yp2, x2, c);
+  }
+  if (kDegQ >= 5) {
+    c = LoadDup128(d, &q[(kDegQ - 5) * 4]);
+    yq0 = MulAdd(yq0, x0, c);
+    yq1 = MulAdd(yq1, x1, c);
+    yq2 = MulAdd(yq2, x2, c);
+  }
+  if (kDegP >= 6) {
+    c = LoadDup128(d, &p[(kDegP - 6) * 4]);
+    yp0 = MulAdd(yp0, x0, c);
+    yp1 = MulAdd(yp1, x1, c);
+    yp2 = MulAdd(yp2, x2, c);
+  }
+  if (kDegQ >= 6) {
+    c = LoadDup128(d, &q[(kDegQ - 6) * 4]);
+    yq0 = MulAdd(yq0, x0, c);
+    yq1 = MulAdd(yq1, x1, c);
+    yq2 = MulAdd(yq2, x2, c);
+  }
+  if (kDegP >= 7) {
+    c = LoadDup128(d, &p[(kDegP - 7) * 4]);
+    yp0 = MulAdd(yp0, x0, c);
+    yp1 = MulAdd(yp1, x1, c);
+    yp2 = MulAdd(yp2, x2, c);
+  }
+  if (kDegQ >= 7) {
+    c = LoadDup128(d, &q[(kDegQ - 7) * 4]);
+    yq0 = MulAdd(yq0, x0, c);
+    yq1 = MulAdd(yq1, x1, c);
+    yq2 = MulAdd(yq2, x2, c);
+  }
+
+  // Much faster than division when computing three at once.
+  *y0 = FastDivision<T, V>()(yp0, yq0);
+  *y1 = FastDivision<T, V>()(yp1, yq1);
+  *y2 = FastDivision<T, V>()(yp2, yq2);
+}
+
+}  // namespace jxl
+
+#endif  // JXL_RATIONAL_POLYNOMIAL_H_
diff --git a/jxl/rational_polynomial_test.cc b/jxl/rational_polynomial_test.cc
new file mode 100644
index 000000000..0af958e8a
--- /dev/null
+++ b/jxl/rational_polynomial_test.cc
@@ -0,0 +1,178 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/rational_polynomial.h"
+
+#include <stdio.h>
+
+#include <cmath>
+#include <hwy/static_targets.h>
+#include <string>
+
+#include "gtest/gtest.h"
+#include "jxl/base/descriptive_statistics.h"
+#include "jxl/common.h"
+
+namespace jxl {
+namespace {
+
+#if HWY_HAS_DOUBLE
+using T = double;
+#else
+using T = float;
+#endif
+using D = HWY_FULL(T);
+
+// Functions to approximate:
+
+T LinearToSrgb8Direct(T val) {
+  if (val < 0.0) return 0.0;
+  if (val >= 255.0) return 255.0;
+  if (val <= 10.0 / 12.92) return val * 12.92;
+  return 255.0 * (std::pow(val / 255.0, 1.0 / 2.4) * 1.055 - 0.055);
+}
+
+T SimpleGamma(T v) {
+  static const T kGamma = 0.387494322593;
+  static const T limit = 43.01745241042018;
+  T bright = v - limit;
+  if (bright >= 0) {
+    static const T mul = 0.0383723643799;
+    v -= bright * mul;
+  }
+  static const T limit2 = 94.68634353321337;
+  T bright2 = v - limit2;
+  if (bright2 >= 0) {
+    static const T mul = 0.22885405968;
+    v -= bright2 * mul;
+  }
+  static const T offset = 0.156775786057;
+  static const T scale = 8.898059160493739;
+  T retval = scale * (offset + pow(v, kGamma));
+  return retval;
+}
+
+// Runs CaratheodoryFejer and verifies the polynomial using a lot of samples to
+// return the biggest error.
+template <class Poly>
+HWY_ATTR T RunApproximation(T x0, T x1, const Poly& poly, T func_to_approx(T)) {
+  Stats err;
+
+  T lastPrint = 0;
+  // NOLINTNEXTLINE(clang-analyzer-security.FloatLoopCounter)
+  for (T x = x0; x <= x1; x += (x1 - x0) / 10000.0) {
+    const T f = func_to_approx(x);
+    const HWY_FULL(T) d;
+    HWY_ALIGN T g_lanes[d.N];
+    Store(poly(Set(d, x)), d, g_lanes);
+    const T g = g_lanes[0];
+    err.Notify(fabs(g - f));
+    if (x == x0 || x - lastPrint > (x1 - x0) / 20.0) {
+      printf("x: %11.6f, f: %11.6f, g: %11.6f, e: %11.6f\n", x, f, g,
+             fabs(g - f));
+      lastPrint = x;
+    }
+  }
+  printf("%s\n", err.ToString().c_str());
+
+  return err.Max();
+}
+
+HWY_ATTR void SimpleGammaImpl() {
+  const T p[6 + 1] = {-5.0646949363741811E-05, 6.7369380528439771E-05,
+                      8.9376652530412794E-05,  2.1153513301520462E-06,
+                      -6.9130322970386449E-08, 3.9424752749293728E-10,
+                      1.2360288207619576E-13};
+
+  const T q[6 + 1] = {-6.6389733798591366E-06, 1.3299859726565908E-05,
+                      3.8538748358398873E-06,  -2.8707687262928236E-08,
+                      -6.6897385800005434E-10, 6.1428748869186003E-12,
+                      -2.5475738169252870E-15};
+
+  const RationalPolynomial<D, 6, 6> poly(p, q);
+
+  const T err = RunApproximation(0.77, 274.579999999999984, poly, SimpleGamma);
+  EXPECT_LT(err, 0.05);
+}
+TEST(Approximation, SimpleGamma) { SimpleGammaImpl(); }
+
+HWY_ATTR void LinearToSrgb8DirectImpl() {
+  const T p[5 + 1] = {-9.5357499040105154E-05, 4.6761186249798248E-04,
+                      2.5708174333943594E-04,  1.5250087770436082E-05,
+                      1.1946768008931187E-07,  5.9916446295972850E-11};
+
+  const T q[4 + 1] = {1.8932479758079768E-05, 2.7312342474687321E-05,
+                      4.3901204783327006E-06, 1.0417787306920273E-07,
+                      3.0084206762140419E-10};
+
+  const RationalPolynomial<D, 5, 4> poly(p, q);
+  const T err = RunApproximation(0.77, 255, poly, LinearToSrgb8Direct);
+  EXPECT_LT(err, 0.05);
+}
+TEST(Approximation, LinearToSrgb8Direct) { LinearToSrgb8DirectImpl(); }
+
+HWY_ATTR void ExpImpl() {
+  const T p[2 + 1] = {9.6266879665530902E-01, 4.8961265681586763E-01,
+                      8.2619259189548433E-02};
+  const T q[2 + 1] = {9.6259895571622622E-01, -4.7272457588933831E-01,
+                      7.4802088567547664E-02};
+  const RationalPolynomial<D, 2, 2> poly(p, q);
+  const T err = RunApproximation(-1, 1, poly, [](T x) { return T(exp(x)); });
+  EXPECT_LT(err, 1E-4);
+}
+TEST(Approximation, Exp) { ExpImpl(); }
+
+HWY_ATTR void NegExpImpl() {
+  // 4,3 is the min required for monotonicity; max error in 0,10: 751 ppm
+  // no benefit for k>50.
+  const T p[4 + 1] = {5.9580258551150123E-02, -2.5073728806886408E-02,
+                      4.1561830213689248E-03, -3.1815408488900372E-04,
+                      9.3866690094906802E-06};
+  const T q[3 + 1] = {5.9579108238812878E-02, 3.4542074345478582E-02,
+                      8.7263562483501714E-03, 1.4095109143061216E-03
+
+  };
+  const RationalPolynomial<D, 4, 3> poly(p, q);
+
+  const T err = RunApproximation(0, 10, poly, [](T x) { return T(exp(-x)); });
+#if HWY_HAS_DOUBLE
+  EXPECT_LT(err, 2E-5);
+#else
+  EXPECT_LT(err, 3E-5);
+#endif
+}
+TEST(Approximation, NegExp) { NegExpImpl(); }
+
+HWY_ATTR void SinImpl() {
+  const T p[6 + 1] = {1.5518122109203780E-05,  2.3388958643675966E+00,
+                      -8.6705520940849157E-01, -1.9702294764873535E-01,
+                      1.2193404314472320E-01,  -1.7373966109788839E-02,
+                      7.8829435883034796E-04};
+  const T q[5 + 1] = {2.3394371422557279E+00, -8.7028221081288615E-01,
+                      2.0052872219658430E-01, -3.2460335995264836E-02,
+                      3.1546157932479282E-03, -1.6692542019380155E-04};
+  const RationalPolynomial<D, 6, 5> poly(p, q);
+
+  const T err =
+      RunApproximation(0, Pi<T>(1) * 2, poly, [](T x) { return T(sin(x)); });
+#if HWY_HAS_DOUBLE
+  EXPECT_LT(err, 5E-4);
+#else
+  EXPECT_LT(err, 7E-4);
+#endif
+}
+TEST(Approximation, Sin) { SinImpl(); }
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/robust_statistics_test.cc b/jxl/robust_statistics_test.cc
new file mode 100644
index 000000000..e79660a8b
--- /dev/null
+++ b/jxl/robust_statistics_test.cc
@@ -0,0 +1,159 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/base/robust_statistics.h"
+
+#include <stdio.h>
+
+#include <numeric>  // partial_sum
+#include <random>
+
+#include "gtest/gtest.h"
+#include "jxl/noise_distributions.h"
+
+namespace jxl {
+namespace {
+
+TEST(RobustStatisticsTest, TestMode) {
+  // Enough to populate bins. We have to sort this many values.
+  constexpr size_t kReps = 15000;
+  constexpr size_t kBins = 101;
+
+  std::mt19937 rng(65537);
+
+  // Place Poisson mean at 1/10, 2/10 .. 9/10 of the bin range.
+  for (int frac = 1; frac < 10; ++frac) {
+    printf("===========================frac %d\n", frac);
+
+    NoisePoisson noise(frac * kBins / 10);
+    std::vector<float> values;
+    values.reserve(kReps);
+
+    uint32_t bins[kBins] = {0};
+
+    std::uniform_real_distribution<float> jitter(-1E-3f, 1E-3f);
+    for (size_t rep = 0; rep < kReps; ++rep) {
+      // Scale back to integer, add jitter to avoid too many repeated values.
+      const float poisson = noise(0.0f, &rng) * 1E3f + jitter(rng);
+
+      values.push_back(poisson);
+
+      const int idx_bin = static_cast<int>(poisson);
+      if (idx_bin < kBins) {
+        bins[idx_bin] += 1;
+      }  // else skip instead of clamping to avoid bias
+    }
+
+    // // Print histogram
+    // for (const uint32_t b : bins) {
+    //   printf("%u\n", b);
+    // }
+
+    // (Smoothed) argmax and median for verification
+    float smoothed[kBins];
+    smoothed[0] = bins[0];
+    smoothed[kBins - 1] = bins[kBins - 1];
+    for (size_t i = 1; i < kBins - 1; ++i) {
+      smoothed[i] = (2 * bins[i] + bins[i - 1] + bins[i + 1]) * 0.25f;
+    }
+    const float argmax =
+        std::max_element(smoothed, smoothed + kBins) - smoothed;
+    const float median = Median(&values);
+
+    std::sort(values.begin(), values.end());
+    const float hsm = HalfSampleMode()(values.data(), values.size());
+
+    uint32_t cdf[kBins];
+    std::partial_sum(bins, bins + kBins, cdf);
+    const int hrm = HalfRangeMode()(cdf, kBins);
+
+    const auto is_near = [](const float expected, const float actual) {
+      return std::abs(expected - actual) <= 1.0f + 1E-5f;
+    };
+    EXPECT_TRUE(is_near(hsm, argmax) || is_near(hsm, median));
+    EXPECT_TRUE(is_near(hrm, argmax) || is_near(hrm, median));
+
+    printf("hsm %.1f hrm %d argmax %.1f median %f\n", hsm, hrm, argmax, median);
+    const int center = static_cast<int>(argmax);
+    printf("%d %d %d %d %d\n", bins[center - 2], bins[center - 1], bins[center],
+           bins[center + 1], bins[center + 2]);
+  }
+}
+
+// Ensures Median3/5 return the same results as Median.
+TEST(RobustStatisticsTest, TestMedian) {
+  std::vector<float> v3(3), v5(5);
+
+  std::uniform_real_distribution<float> dist(-100.0f, 100.0f);
+  std::mt19937 rng(129);
+
+#ifdef NDEBUG
+  constexpr size_t kReps = 100000;
+#else
+  constexpr size_t kReps = 100;
+#endif
+  for (size_t i = 0; i < kReps; ++i) {
+    v3[0] = dist(rng);
+    v3[1] = dist(rng);
+    v3[2] = dist(rng);
+    for (size_t j = 0; j < 5; ++j) {
+      v5[j] = dist(rng);
+    }
+
+    JXL_ASSERT(Median(&v3) == Median3(v3[0], v3[1], v3[2]));
+    JXL_ASSERT(Median(&v5) == Median5(v5[0], v5[1], v5[2], v5[3], v5[4]));
+  }
+}
+
+template <class Noise>
+void TestLine(const Noise& noise, float max_l1_limit, float mad_limit) {
+  std::vector<Bivariate> points;
+  Line perfect(0.6f, 2.0f);
+
+  // Random spacing of X (must be unique)
+  float x = -100.0f;
+  std::mt19937_64 rng(129);
+  std::uniform_real_distribution<float> x_dist(1E-6f, 10.0f);
+  for (size_t ix = 0; ix < 500; ++ix) {
+    x += x_dist(rng);
+    const float y = noise(perfect(x), &rng);
+    points.emplace_back(x, y);
+    // printf("%f,%f\n", x, y);
+  }
+
+  Line est(points);
+  float max_l1, mad;
+  EvaluateQuality(est, points, &max_l1, &mad);
+  printf("x %f  slope=%.2f b=%.2f  max_l1 %f mad %f\n", x, est.slope(),
+         est.intercept(), max_l1, mad);
+
+  EXPECT_LE(max_l1, max_l1_limit);
+  EXPECT_LE(mad, mad_limit);
+}
+
+TEST(RobustStatisticsTest, CleanLine) {
+  const NoiseNone noise;
+  TestLine(noise, 1E-6, 1E-7);
+}
+TEST(RobustStatisticsTest, Uniform) {
+  const NoiseUniform noise(-100.0f, 100.0f);
+  TestLine(noise, 107, 53);
+}
+TEST(RobustStatisticsTest, Gauss) {
+  const NoiseGaussian noise(10.0f);
+  TestLine(noise, 37, 7);
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/speed_tier_test.cc b/jxl/speed_tier_test.cc
new file mode 100644
index 000000000..6e4c25dd0
--- /dev/null
+++ b/jxl/speed_tier_test.cc
@@ -0,0 +1,112 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+#include "jxl/enc_butteraugli_comparator.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_file.h"
+#include "jxl/enc_params.h"
+#include "jxl/extras/codec.h"
+#include "jxl/image.h"
+#include "jxl/image_test_utils.h"
+#include "jxl/test_utils.h"
+#include "jxl/testdata_path.h"
+
+namespace jxl {
+namespace {
+
+struct SpeedTierTestParams {
+  explicit SpeedTierTestParams(const SpeedTier speed_tier,
+                               const bool shrink8 = false)
+      : speed_tier(speed_tier), shrink8(shrink8) {}
+  SpeedTier speed_tier;
+  bool shrink8;
+};
+
+std::ostream& operator<<(std::ostream& os, SpeedTierTestParams params) {
+  auto previous_flags = os.flags();
+  os << std::boolalpha;
+  os << "SpeedTierTestParams{" << SpeedTierName(params.speed_tier)
+     << ", /*shrink8=*/" << params.shrink8 << "}";
+  os.flags(previous_flags);
+  return os;
+}
+
+class SpeedTierTest : public testing::TestWithParam<SpeedTierTestParams> {};
+
+INSTANTIATE_TEST_SUITE_P(
+    SpeedTierTestInstantiation, SpeedTierTest,
+    testing::Values(SpeedTierTestParams{SpeedTier::kCheetah,
+                                        /*shrink8=*/true},
+                    SpeedTierTestParams{SpeedTier::kCheetah,
+                                        /*shrink8=*/false},
+                    SpeedTierTestParams{SpeedTier::kFalcon,
+                                        /*shrink8=*/true},
+                    SpeedTierTestParams{SpeedTier::kFalcon,
+                                        /*shrink8=*/false},
+                    SpeedTierTestParams{SpeedTier::kHare,
+                                        /*shrink8=*/true},
+                    SpeedTierTestParams{SpeedTier::kHare,
+                                        /*shrink8=*/false},
+                    SpeedTierTestParams{SpeedTier::kWombat,
+                                        /*shrink8=*/true},
+                    SpeedTierTestParams{SpeedTier::kWombat,
+                                        /*shrink8=*/false},
+                    SpeedTierTestParams{SpeedTier::kSquirrel,
+                                        /*shrink8=*/true},
+                    SpeedTierTestParams{SpeedTier::kSquirrel,
+                                        /*shrink8=*/false},
+                    SpeedTierTestParams{SpeedTier::kKitten,
+                                        /*shrink8=*/true},
+                    SpeedTierTestParams{SpeedTier::kKitten,
+                                        /*shrink8=*/false},
+                    // Only downscaled image for Tortoise mode.
+                    SpeedTierTestParams{SpeedTier::kTortoise,
+                                        /*shrink8=*/true}));
+
+TEST_P(SpeedTierTest, Roundtrip) {
+  const std::string pathname =
+      GetTestDataPath("wesaturate/500px/u76c0g_bliznaca_srgb8.png");
+  CodecInOut io;
+  ThreadPoolInternal pool(8);
+  ASSERT_TRUE(SetFromFile(pathname, &io, &pool));
+
+  const SpeedTierTestParams& params = GetParam();
+
+  if (params.shrink8) {
+    io.ShrinkTo(io.xsize() / 8, io.ysize() / 8);
+  }
+
+  CompressParams cparams;
+  cparams.speed_tier = params.speed_tier;
+  DecompressParams dparams;
+
+  CodecInOut io2;
+  test::Roundtrip(&io, cparams, dparams, nullptr, &io2);
+
+  EXPECT_LE(ButteraugliDistance(io, io2, cparams.hf_asymmetry,
+                                /*distmap=*/nullptr, /*pool=*/nullptr),
+            2);
+}
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/splines.cc b/jxl/splines.cc
new file mode 100644
index 000000000..6d9ce979a
--- /dev/null
+++ b/jxl/splines.cc
@@ -0,0 +1,499 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/splines.h"
+
+#include <algorithm>
+#include <hwy/static_targets.h>
+
+#include "jxl/ans_params.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/common.h"
+#include "jxl/dct.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/opsin_params.h"
+#include "jxl/splines_fastmath.h"
+
+namespace jxl {
+
+namespace {
+
+constexpr float kDesiredRenderingDistance = 1.f;
+
+template <int N>
+constexpr float DCTScale(int i) {
+  return DCTScales<N>()[i] * (i == 0 ? splines_internal::InvSqrt<N>()
+                                     : splines_internal::InvSqrt<N / 2>());
+}
+
+// X, Y, B, sigma.
+float ColorQuantizationWeight(const int32_t adjustment, const int channel,
+                              const int i) {
+  const float multiplier = adjustment >= 0 ? 1.f + .125f * adjustment
+                                           : 1.f / (1.f + .125f * -adjustment);
+
+  static constexpr float kChannelWeight[] = {0.025f, 0.45f, 0.42f, 2.f};
+
+  return multiplier / (DCTScale<32>(i) * kChannelWeight[channel]);
+}
+
+enum SplineEntropyContexts : size_t {
+  kQuantizationAdjustmentContext = 0,
+  kStartingPositionContext,
+  kNumSplinesContext,
+  kNumControlPointsContext,
+  kControlPointsContext,
+  kDCTContext,
+  kNumSplineContexts
+};
+
+HWY_ATTR void EncodeAllStartingPoints(const std::vector<Spline::Point>& points,
+                                      std::vector<Token>* tokens) {
+  int64_t last_x = 0;
+  int64_t last_y = 0;
+  for (size_t i = 0; i < points.size(); i++) {
+    const int64_t x = std::lround(points[i].x);
+    const int64_t y = std::lround(points[i].y);
+    if (i == 0) {
+      TokenizeHybridUint(kStartingPositionContext, x, tokens);
+      TokenizeHybridUint(kStartingPositionContext, y, tokens);
+    } else {
+      TokenizeHybridUint(kStartingPositionContext, PackSigned(x - last_x),
+                         tokens);
+      TokenizeHybridUint(kStartingPositionContext, PackSigned(y - last_y),
+                         tokens);
+    }
+    last_x = x;
+    last_y = y;
+  }
+}
+
+HWY_ATTR Status DecodeAllStartingPoints(
+    std::vector<Spline::Point>* const points, BitReader* const br,
+    ANSSymbolReader* reader, const std::vector<uint8_t>& context_map,
+    size_t num_splines) {
+  points->resize(num_splines);
+  int64_t last_x = 0;
+  int64_t last_y = 0;
+  for (size_t i = 0; i < points->size(); i++) {
+    int64_t x =
+        ReadHybridUint(kStartingPositionContext, br, reader, context_map);
+    int64_t y =
+        ReadHybridUint(kStartingPositionContext, br, reader, context_map);
+    if (i != 0) {
+      x = UnpackSigned(x) + last_x;
+      y = UnpackSigned(y) + last_y;
+    }
+    (*points)[i].x = x;
+    (*points)[i].y = y;
+    last_x = x;
+    last_y = y;
+  }
+  return true;
+}
+
+struct Vector {
+  float x, y;
+  Vector operator-() const { return {-x, -y}; }
+  Vector operator+(const Vector& other) const {
+    return {x + other.x, y + other.y};
+  }
+  float SquaredNorm() const { return x * x + y * y; }
+};
+Vector operator*(const float k, const Vector& vec) {
+  return {k * vec.x, k * vec.y};
+}
+
+Spline::Point operator+(const Spline::Point& p, const Vector& vec) {
+  return {p.x + vec.x, p.y + vec.y};
+}
+Spline::Point operator-(const Spline::Point& p, const Vector& vec) {
+  return p + -vec;
+}
+Vector operator-(const Spline::Point& a, const Spline::Point& b) {
+  return {a.x - b.x, a.y - b.y};
+}
+
+std::vector<Spline::Point> DrawCentripetalCatmullRomSpline(
+    std::vector<Spline::Point> points) {
+  if (points.size() <= 1) return points;
+  // Number of points to compute between each control point.
+  static constexpr int kNumPoints = 16;
+  std::vector<Spline::Point> result;
+  result.reserve((points.size() - 1) * kNumPoints + 1);
+  points.insert(points.begin(), points[0] + (points[0] - points[1]));
+  points.push_back(points[points.size() - 1] +
+                   (points[points.size() - 1] - points[points.size() - 2]));
+  // points has at least 4 elements at this point.
+  for (size_t start = 0; start < points.size() - 3; ++start) {
+    // 4 of them are used, and we draw from p[1] to p[2].
+    const Spline::Point* const p = &points[start];
+    result.push_back(p[1]);
+    float t[4] = {0};
+    for (int k = 1; k < 4; ++k) {
+      t[k] = std::sqrt(std::hypot(p[k].x - p[k - 1].x, p[k].y - p[k - 1].y)) +
+             t[k - 1];
+    }
+    for (int i = 1; i < kNumPoints; ++i) {
+      const float tt =
+          t[1] + (static_cast<float>(i) / kNumPoints) * (t[2] - t[1]);
+      Spline::Point a[3];
+      for (int k = 0; k < 3; ++k) {
+        a[k] = p[k] + ((tt - t[k]) / (t[k + 1] - t[k])) * (p[k + 1] - p[k]);
+      }
+      Spline::Point b[2];
+      for (int k = 0; k < 2; ++k) {
+        b[k] = a[k] + ((tt - t[k]) / (t[k + 2] - t[k])) * (a[k + 1] - a[k]);
+      }
+      result.push_back(b[0] + ((tt - t[1]) / (t[2] - t[1])) * (b[1] - b[0]));
+    }
+  }
+  result.push_back(points[points.size() - 2]);
+  return result;
+}
+
+// Given a set of DCT coefficients, this returns the result of performing cosine
+// interpolation on the original samples.
+template <int N>
+float ContinuousIDCT(const float dct[N], float t) {
+  float result = DCTScale<N>(0) * dct[0];
+  for (int i = 1; i < N; ++i) {
+    result += DCTScale<N>(i) * dct[i] *
+              splines_internal::Cos((kPi / N) * i * (t + 0.5f));
+  }
+  return result;
+}
+
+// Used for Gaussian splatting. This gives the intensity of the Gaussian for a
+// given distance from its center.
+float BrushIntensity(const float distance, const float sigma) {
+  const float one_dimensional_delta = (1.f / 1.4142135623730951f) * distance;
+  const float inv_sqrt2_times_sigma = 1.f / (1.4142135623730951f * sigma);
+  const float one_dimensional_factor =
+      splines_internal::Erf((one_dimensional_delta + .5f) *
+                            inv_sqrt2_times_sigma) -
+      splines_internal::Erf((one_dimensional_delta - .5f) *
+                            inv_sqrt2_times_sigma);
+  return .25f * sigma * one_dimensional_factor * one_dimensional_factor;
+}
+
+// Splats a single Gaussian on the image.
+void DrawGaussian(Image3F* const opsin, const Rect& opsin_rect,
+                  const Rect& image_rect, const Spline::Point& center,
+                  const float intensity, const float color[3],
+                  const float sigma) {
+  constexpr float kDistanceMultiplier = 4.605170185988091f;  // -2 * log(0.1)
+  // Distance beyond which exp(-d^2 / (2 * sigma^2)) drops below 0.1.
+  const float maximum_distance = sigma * sigma * kDistanceMultiplier;
+  const auto xbegin =
+      static_cast<size_t>(std::max(0.f, center.x - maximum_distance + .5f));
+  const auto xend =
+      std::min<size_t>(center.x + maximum_distance + .5f, opsin->xsize() - 1);
+  const auto ybegin =
+      static_cast<size_t>(std::max(0.f, center.y - maximum_distance + .5f));
+  const auto yend =
+      std::min<size_t>(center.y + maximum_distance + .5f, opsin->ysize() - 1);
+  for (size_t y = ybegin; y <= yend; ++y) {
+    if (y < image_rect.y0() || y >= image_rect.y0() + image_rect.ysize())
+      continue;
+    float* JXL_RESTRICT rows[3] = {
+        opsin_rect.PlaneRow(opsin, 0, y - image_rect.y0()),
+        opsin_rect.PlaneRow(opsin, 1, y - image_rect.y0()),
+        opsin_rect.PlaneRow(opsin, 2, y - image_rect.y0()),
+    };
+    for (size_t x = xbegin; x <= xend; ++x) {
+      if (x < image_rect.x0() || x >= image_rect.x0() + image_rect.xsize())
+        continue;
+      const Spline::Point point{static_cast<float>(x), static_cast<float>(y)};
+      const float distance = std::sqrt((point - center).SquaredNorm());
+      const float local_intensity = intensity * BrushIntensity(distance, sigma);
+      for (size_t c = 0; c < 3; ++c) {
+        rows[c][x - image_rect.x0()] += local_intensity * color[c];
+      }
+    }
+  }
+}
+
+// Move along the line segments defined by `points`, `kDesiredRenderingDistance`
+// pixels at a time, and call `functor` with each point and the actual distance
+// to the previous point (which will always be kDesiredRenderingDistance except
+// possibly for the very last point).
+template <typename Points, typename Functor>
+void ForEachEquallySpacedPoint(const Points& points, const Functor& functor) {
+  Spline::Point current = points.front();
+  functor(current, kDesiredRenderingDistance);
+  auto next = points.begin();
+  while (next != points.end()) {
+    const Spline::Point* previous = &current;
+    float arclength_from_previous = 0.f;
+    for (;;) {
+      const float arclength_to_next =
+          std::sqrt((*next - *previous).SquaredNorm());
+      if (next == points.end()) {
+        functor(*previous, arclength_from_previous);
+        return;
+      }
+      if (arclength_from_previous + arclength_to_next >=
+          kDesiredRenderingDistance) {
+        current =
+            *previous + ((kDesiredRenderingDistance - arclength_from_previous) /
+                         arclength_to_next) *
+                            (*next - *previous);
+        functor(current, kDesiredRenderingDistance);
+        break;
+      }
+      arclength_from_previous += arclength_to_next;
+      previous = &*next;
+      ++next;
+    }
+  }
+}
+
+}  // namespace
+
+QuantizedSpline::QuantizedSpline(const Spline& original,
+                                 const int32_t quantization_adjustment,
+                                 float ytox, float ytob) {
+  JXL_ASSERT(!original.control_points.empty());
+  control_points_.reserve(original.control_points.size() - 1);
+  const Spline::Point& starting_point = original.control_points.front();
+  int previous_x = static_cast<int>(std::round(starting_point.x)),
+      previous_y = static_cast<int>(std::round(starting_point.y));
+  int previous_delta_x = 0, previous_delta_y = 0;
+  for (auto it = original.control_points.begin() + 1;
+       it != original.control_points.end(); ++it) {
+    const int new_x = static_cast<int>(std::round(it->x));
+    const int new_y = static_cast<int>(std::round(it->y));
+    const int new_delta_x = new_x - previous_x;
+    const int new_delta_y = new_y - previous_y;
+    control_points_.emplace_back(new_delta_x - previous_delta_x,
+                                 new_delta_y - previous_delta_y);
+    previous_delta_x = new_delta_x;
+    previous_delta_y = new_delta_y;
+    previous_x = new_x;
+    previous_y = new_y;
+  }
+
+  for (int c = 0; c < 3; ++c) {
+    float factor = c == 0 ? ytox : c == 1 ? 0 : ytob;
+    for (int i = 0; i < 32; ++i) {
+      const float coefficient =
+          original.color_dct[c][i] -
+          factor * color_dct_[1][i] /
+              ColorQuantizationWeight(quantization_adjustment, 1, i);
+      color_dct_[c][i] = static_cast<int>(
+          std::round(coefficient *
+                     ColorQuantizationWeight(quantization_adjustment, c, i)));
+    }
+  }
+  for (int i = 0; i < 32; ++i) {
+    sigma_dct_[i] = static_cast<int>(
+        std::round(original.sigma_dct[i] *
+                   ColorQuantizationWeight(quantization_adjustment, 3, i)));
+  }
+}
+
+Spline QuantizedSpline::Dequantize(const Spline::Point& starting_point,
+                                   const int32_t quantization_adjustment,
+                                   float ytox, float ytob) const {
+  Spline result;
+
+  result.control_points.reserve(control_points_.size() + 1);
+  int current_x = static_cast<int>(std::round(starting_point.x)),
+      current_y = static_cast<int>(std::round(starting_point.y));
+  result.control_points.push_back(Spline::Point{static_cast<float>(current_x),
+                                                static_cast<float>(current_y)});
+  int current_delta_x = 0, current_delta_y = 0;
+  for (const auto& point : control_points_) {
+    current_delta_x += point.first;
+    current_delta_y += point.second;
+    current_x += current_delta_x;
+    current_y += current_delta_y;
+    result.control_points.push_back(Spline::Point{
+        static_cast<float>(current_x), static_cast<float>(current_y)});
+  }
+
+  for (int c = 0; c < 3; ++c) {
+    for (int i = 0; i < 32; ++i) {
+      result.color_dct[c][i] =
+          color_dct_[c][i] /
+          ColorQuantizationWeight(quantization_adjustment, c, i);
+    }
+  }
+  for (int i = 0; i < 32; ++i) {
+    result.color_dct[0][i] += ytox * result.color_dct[1][i];
+    result.color_dct[2][i] += ytob * result.color_dct[1][i];
+  }
+  for (int i = 0; i < 32; ++i) {
+    result.sigma_dct[i] =
+        sigma_dct_[i] / ColorQuantizationWeight(quantization_adjustment, 3, i);
+  }
+
+  return result;
+}
+
+void QuantizedSpline::Tokenize(std::vector<Token>* const tokens) const {
+  uint32_t nbits, bits;
+  TokenizeHybridUint(kNumControlPointsContext, control_points_.size(), tokens);
+  for (const auto& point : control_points_) {
+    TokenizeHybridUint(kControlPointsContext, PackSigned(point.first), tokens);
+    TokenizeHybridUint(kControlPointsContext, PackSigned(point.second), tokens);
+  }
+  const auto encode_dct = [tokens, &nbits, &bits](const int dct[32]) {
+    for (int i = 0; i < 32; ++i) {
+      EncodeVarLenInt(dct[i], &nbits, &bits);
+      tokens->emplace_back(kDCTContext, nbits, nbits, bits);
+    }
+  };
+  for (int c = 0; c < 3; ++c) {
+    encode_dct(color_dct_[c]);
+  }
+  encode_dct(sigma_dct_);
+}
+
+HWY_ATTR void QuantizedSpline::Decode(const std::vector<uint8_t>& context_map,
+                                      ANSSymbolReader* const decoder,
+                                      BitReader* const br) {
+  const size_t num_control_points =
+      ReadHybridUint(kNumControlPointsContext, br, decoder, context_map);
+  control_points_.resize(num_control_points);
+  for (std::pair<int64_t, int64_t>& control_point : control_points_) {
+    control_point.first = UnpackSigned(
+        ReadHybridUint(kControlPointsContext, br, decoder, context_map));
+    control_point.second = UnpackSigned(
+        ReadHybridUint(kControlPointsContext, br, decoder, context_map));
+  }
+
+  const auto decode_dct = [decoder, br, &context_map](int dct[32]) HWY_ATTR {
+    for (int i = 0; i < 32; ++i) {
+      const uint32_t s = decoder->ReadSymbol(context_map[kDCTContext], br);
+      const uint32_t bits = br->ReadBits(s);
+      dct[i] = DecodeVarLenInt(s, bits);
+    }
+  };
+  for (int c = 0; c < 3; ++c) {
+    decode_dct(color_dct_[c]);
+  }
+  decode_dct(sigma_dct_);
+}
+
+void Splines::Encode(BitWriter* writer, const size_t layer,
+                     AuxOut* aux_out) const {
+  JXL_ASSERT(HasAny());
+
+  std::vector<QuantizedSpline> splines = splines_;
+  std::vector<std::vector<Token>> tokens(1);
+  TokenizeHybridUint(kNumSplinesContext, splines.size() - 1, &tokens[0]);
+  EncodeAllStartingPoints(starting_points_, &tokens[0]);
+
+  TokenizeHybridUint(kQuantizationAdjustmentContext,
+                     PackSigned(quantization_adjustment_), &tokens[0]);
+
+  for (const QuantizedSpline& spline : splines) {
+    spline.Tokenize(&tokens[0]);
+  }
+
+  EntropyEncodingData codes;
+  std::vector<uint8_t> context_map;
+  BuildAndEncodeHistograms(HistogramParams(), kNumSplineContexts, tokens,
+                           &codes, &context_map, writer, layer, aux_out);
+  WriteTokens(tokens[0], codes, context_map, writer, layer, aux_out);
+}
+
+HWY_ATTR Status Splines::Decode(jxl::BitReader* br) {
+  std::vector<uint8_t> context_map;
+  ANSCode code;
+  JXL_RETURN_IF_ERROR(DecodeHistograms(
+      br, kNumSplineContexts, /*max_alphabet_size=*/64, &code, &context_map));
+  ANSSymbolReader decoder(&code, br);
+  const int num_splines =
+      1 + ReadHybridUint(kNumSplinesContext, br, &decoder, context_map);
+  JXL_RETURN_IF_ERROR(DecodeAllStartingPoints(&starting_points_, br, &decoder,
+                                              context_map, num_splines));
+
+  quantization_adjustment_ = UnpackSigned(ReadHybridUint(
+      kQuantizationAdjustmentContext, br, &decoder, context_map));
+
+  splines_.reserve(num_splines);
+  for (int i = 0; i < num_splines; ++i) {
+    QuantizedSpline spline;
+    spline.Decode(context_map, &decoder, br);
+    splines_.push_back(std::move(spline));
+  }
+
+  JXL_RETURN_IF_ERROR(decoder.CheckANSFinalState());
+
+  if (!HasAny()) {
+    return JXL_FAILURE("Decoded splines but got none");
+  }
+
+  return true;
+}
+
+void Splines::AddTo(Image3F* const opsin, const Rect& opsin_rect,
+                    const Rect& image_rect,
+                    const ColorCorrelationMap& cmap) const {
+  Apply</*add=*/true>(opsin, opsin_rect, image_rect, cmap);
+}
+
+void Splines::SubtractFrom(Image3F* const opsin,
+                           const ColorCorrelationMap& cmap) const {
+  Apply</*add=*/false>(opsin, Rect(*opsin), Rect(*opsin), cmap);
+}
+
+template <bool add>
+void Splines::Apply(Image3F* const opsin, const Rect& opsin_rect,
+                    const Rect& image_rect,
+                    const ColorCorrelationMap& cmap) const {
+  for (size_t i = 0; i < splines_.size(); ++i) {
+    const Spline spline = splines_[i].Dequantize(
+        starting_points_[i], quantization_adjustment_,
+        cmap.YtoXRatio(kColorOffset), cmap.YtoBRatio(kColorOffset));
+    std::vector<std::pair<Spline::Point, float>> points_to_draw;
+    ForEachEquallySpacedPoint(
+        DrawCentripetalCatmullRomSpline(spline.control_points),
+        [&](const Spline::Point& point, const float multiplier) {
+          points_to_draw.emplace_back(point, multiplier);
+        });
+    const float arc_length =
+        (points_to_draw.size() - 2) * kDesiredRenderingDistance +
+        points_to_draw.back().second;
+    int k = 0;
+    for (const auto& point_to_draw : points_to_draw) {
+      const Spline::Point& point = point_to_draw.first;
+      const float multiplier =
+          add ? point_to_draw.second : -point_to_draw.second;
+      const float progress_along_arc =
+          (k * kDesiredRenderingDistance) / arc_length;
+      ++k;
+      float color[3];
+      for (size_t c = 0; c < 3; ++c) {
+        color[c] = ContinuousIDCT<32>(spline.color_dct[c],
+                                      (32 - 1) * progress_along_arc);
+      }
+      const float sigma =
+          ContinuousIDCT<32>(spline.sigma_dct, (32 - 1) * progress_along_arc);
+      DrawGaussian(opsin, opsin_rect, image_rect, point, multiplier, color,
+                   sigma);
+    }
+  }
+}
+
+Splines FindSplines(const Image3F& opsin) {
+  // TODO: implement spline detection.
+  return {};
+}
+
+}  // namespace jxl
diff --git a/jxl/splines.h b/jxl/splines.h
new file mode 100644
index 000000000..c68eab3d3
--- /dev/null
+++ b/jxl/splines.h
@@ -0,0 +1,116 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_SPLINES_H_
+#define JXL_SPLINES_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <utility>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/status.h"
+#include "jxl/chroma_from_luma.h"
+#include "jxl/dec_ans.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_ans.h"
+#include "jxl/enc_bit_writer.h"
+#include "jxl/entropy_coder.h"
+#include "jxl/image.h"
+
+namespace jxl {
+
+struct Spline {
+  struct Point {
+    float x, y;
+  };
+  std::vector<Point> control_points;
+  // X, Y, B.
+  float color_dct[3][32];
+  // Splines are draws by normalized Gaussian splatting. This controls the
+  // Gaussian's parameter along the spline.
+  float sigma_dct[32];
+};
+
+class QuantizedSpline {
+ public:
+  QuantizedSpline() = default;
+  explicit QuantizedSpline(const Spline& original,
+                           int32_t quantization_adjustment, float ytox,
+                           float ytob);
+
+  Spline Dequantize(const Spline::Point& starting_point,
+                    int32_t quantization_adjustment, float ytox,
+                    float ytob) const;
+
+  void Tokenize(std::vector<Token>* tokens) const;
+
+  void Decode(const std::vector<uint8_t>& context_map, ANSSymbolReader* decoder,
+              BitReader* br);
+
+ private:
+  std::vector<std::pair<int64_t, int64_t>>
+      control_points_;  // Double delta-encoded.
+  int color_dct_[3][32] = {};
+  int sigma_dct_[32] = {};
+};
+
+class Splines {
+ public:
+  Splines() = default;
+  explicit Splines(const int32_t quantization_adjustment,
+                   std::vector<QuantizedSpline> splines,
+                   std::vector<Spline::Point> starting_points)
+      : quantization_adjustment_(quantization_adjustment),
+        splines_(std::move(splines)),
+        starting_points_(std::move(starting_points)) {}
+
+  bool HasAny() const { return !splines_.empty(); }
+
+  // Only call if HasAny().
+  void Encode(BitWriter* writer, size_t layer, AuxOut* aux_out) const;
+  Status Decode(BitReader* br);
+
+  void AddTo(Image3F* opsin, const Rect& opsin_rect, const Rect& image_rect,
+             const ColorCorrelationMap& cmap) const;
+  void SubtractFrom(Image3F* opsin, const ColorCorrelationMap& cmap) const;
+
+  const std::vector<QuantizedSpline>& TestOnlyQuantizedSplines() const {
+    return splines_;
+  }
+  const std::vector<Spline::Point>& TestOnlyStartingPoints() const {
+    return starting_points_;
+  }
+
+ private:
+  template <bool>
+  void Apply(Image3F* opsin, const Rect& opsin_rect, const Rect& image_rect,
+             const ColorCorrelationMap& cmap) const;
+
+  // If positive, quantization weights are multiplied by 1 + this/8, which
+  // increases precision. If negative, they are divided by 1 - this/8. If 0,
+  // they are unchanged.
+  int32_t quantization_adjustment_ = 0;
+  std::vector<QuantizedSpline> splines_;
+  std::vector<Spline::Point> starting_points_;
+};
+
+Splines FindSplines(const Image3F& image);
+
+}  // namespace jxl
+
+#endif  // JXL_SPLINES_H_
diff --git a/jxl/splines_fastmath.h b/jxl/splines_fastmath.h
new file mode 100644
index 000000000..f175b3a01
--- /dev/null
+++ b/jxl/splines_fastmath.h
@@ -0,0 +1,232 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_SPLINES_FASTMATH_H_
+#define JXL_SPLINES_FASTMATH_H_
+
+#include <cmath>
+
+#include "jxl/common.h"  // kPi
+
+namespace jxl {
+namespace splines_internal {
+
+template <int N>
+static constexpr float InvSqrt();
+template <>
+constexpr float InvSqrt<2>() {
+  return 0.7071067811865475f;
+}
+template <>
+constexpr float InvSqrt<4>() {
+  return 0.5f;
+}
+template <>
+constexpr float InvSqrt<8>() {
+  return 0.35355339059327373f;
+}
+template <>
+constexpr float InvSqrt<16>() {
+  return 0.25f;
+}
+template <>
+constexpr float InvSqrt<32>() {
+  return 0.17677669529663687f;
+}
+
+template <int Size>
+static float InterpolateLUT(const float LUT[Size], const float i) {
+  if (i >= Size - 1) {
+    return LUT[Size - 1];
+  }
+  const int i_int = static_cast<int>(i);
+  const float i_frac = i - i_int;
+  return (1 - i_frac) * LUT[i_int] + i_frac * LUT[i_int + 1];
+}
+
+// kCosLUT[i] = cos(i * (kPi / 2) / 63)
+static constexpr float kCosLUT[64] = {
+    1.f,
+    0.999689182000816f,
+    0.998756921218922f,
+    0.99720379718118f,
+    0.995030775365401f,
+    0.992239206600172f,
+    0.988830826225129f,
+    0.984807753012208f,
+    0.980172487848544f,
+    0.974927912181824f,
+    0.969077286229078f,
+    0.962624246950012f,
+    0.955572805786141f,
+    0.947927346167132f,
+    0.939692620785908f,
+    0.930873748644204f,
+    0.921476211870408f,
+    0.911505852311673f,
+    0.900968867902419f,
+    0.889871808811469f,
+    0.878221573370229f,
+    0.866025403784439f,
+    0.853290881632156f,
+    0.840025923150772f,
+    0.826238774315995f,
+    0.811938005715857f,
+    0.797132507222923f,
+    0.78183148246803f,
+    0.766044443118978f,
+    0.749781202967734f,
+    0.733051871829826f,
+    0.715866849259718f,
+    0.698236818086073f,
+    0.680172737770919f,
+    0.661685837596859f,
+    0.642787609686539f,
+    0.623489801858734f,
+    0.603804410325477f,
+    0.58374367223479f,
+    0.563320058063622f,
+    0.542546263865759f,
+    0.521435203379498f,
+    0.5f,
+    0.478253978621318f,
+    0.456210657353163f,
+    0.433883739117558f,
+    0.411287103130612f,
+    0.388434796274695f,
+    0.365341024366395f,
+    0.342020143325669f,
+    0.318486650251684f,
+    0.294755174410904f,
+    0.270840468143005f,
+    0.246757397690294f,
+    0.222520933956314f,
+    0.198146143199398f,
+    0.17364817766693f,
+    0.149042266176175f,
+    0.124343704647485f,
+    0.0995678465958167f,
+    0.0747300935864244f,
+    0.0498458856606972f,
+    0.024930691738073f,
+    0.f,
+};
+
+static constexpr float InaccurateFMod(const float x, const float y) {
+  return x - static_cast<int>(x / y) * y;
+}
+
+static inline float Cos(float x) {
+  if (x < 0) x = -x;
+  if (x >= 2 * kPi) x = InaccurateFMod(x, 2 * kPi);
+  if (x > kPi) x = 2 * kPi - x;
+  bool opposite = false;
+  if (x > kPi / 2) {
+    x = kPi - x;
+    opposite = true;
+  }
+  float result =
+      InterpolateLUT<64>(kCosLUT, x * static_cast<float>(63 / (kPi / 2)));
+  if (opposite) result = -result;
+  return result;
+}
+
+// kErfLUT[i] = erf(2 * i / 63)
+static constexpr float kErfLUT[64] = {
+    0.f,
+    0.0358095307155359f,
+    0.0715469677785195f,
+    0.107140652672516f,
+    0.142519792779007f,
+    0.177614883452066f,
+    0.212358117215062f,
+    0.246683776068649f,
+    0.280528603134456f,
+    0.313832150144761f,
+    0.346537097619992f,
+    0.378589544947352f,
+    0.409939267978589f,
+    0.440539942196093f,
+    0.470349329946658f,
+    0.499329430703871f,
+    0.52744659378564f,
+    0.554671593415279f,
+    0.580979666465684f,
+    0.606350513659535f,
+    0.630768265407845f,
+    0.65422141384884f,
+    0.676702712994044f,
+    0.698209049194451f,
+    0.718741284403263f,
+    0.738304074930581f,
+    0.756905668557882f,
+    0.774557683005487f,
+    0.791274868824678f,
+    0.807074859818627f,
+    0.821977914084574f,
+    0.836006648716235f,
+    0.849185771113207f,
+    0.861541809716806f,
+    0.873102846833217f,
+    0.883898256019451f,
+    0.893958446299829f,
+    0.903314615255194f,
+    0.911998512788506f,
+    0.92004221712334f,
+    0.927477924340631f,
+    0.934337752507907f,
+    0.94065356120808f,
+    0.946456787035268f,
+    0.951778295396072f,
+    0.956648248739137f,
+    0.961095991135758f,
+    0.965149948951748f,
+    0.968837547187f,
+    0.972185140915123f,
+    0.975217961131759f,
+    0.977960074216708f,
+    0.980434354131662f,
+    0.982662466411504f,
+    0.984664862961992f,
+    0.986460786649043f,
+    0.988068284653553f,
+    0.989504229569219f,
+    0.990784347237614f,
+    0.991923250343289f,
+    0.992934476830092f,
+    0.993830532246756f,
+    0.994622935183356f,
+    0.995322265018953f,
+};
+
+static inline float Erf(float x) {
+  bool opposite = false;
+  if (x < 0) {
+    x = -x;
+    opposite = true;
+  }
+  float result;
+  if (x > 2) {
+    result = 1.f;
+  } else {
+    result = InterpolateLUT<64>(kErfLUT, x * (.5f * 63));
+  }
+  if (opposite) result = -result;
+  return result;
+}
+
+}  // namespace splines_internal
+}  // namespace jxl
+
+#endif  // JXL_SPLINES_FASTMATH_H_
diff --git a/jxl/splines_test.cc b/jxl/splines_test.cc
new file mode 100644
index 000000000..224c6b870
--- /dev/null
+++ b/jxl/splines_test.cc
@@ -0,0 +1,202 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/splines.h"
+
+#include "gmock/gmock.h"
+#include "gtest/gtest.h"
+
+namespace jxl {
+
+std::ostream& operator<<(std::ostream& os, const Spline::Point& p) {
+  return os << "(" << p.x << ", " << p.y << ")";
+}
+
+std::ostream& operator<<(std::ostream& os, const Spline& spline) {
+  return os << "(spline with " << spline.control_points.size()
+            << " control points)";
+}
+
+namespace {
+
+using ::testing::AllOf;
+using ::testing::Field;
+using ::testing::FloatNear;
+using ::testing::Pointwise;
+
+constexpr int kQuantizationAdjustment = 0;
+constexpr float kYToX = 0;
+constexpr float kYToB = 0;
+
+constexpr float kTolerance = 0.1;
+
+std::vector<Spline> DequantizeSplines(const Splines& splines) {
+  const auto& quantized_splines = splines.TestOnlyQuantizedSplines();
+  const auto& starting_points = splines.TestOnlyStartingPoints();
+  JXL_ASSERT(quantized_splines.size() == starting_points.size());
+
+  std::vector<Spline> dequantized;
+  for (size_t i = 0; i < quantized_splines.size(); ++i) {
+    dequantized.push_back(quantized_splines[i].Dequantize(
+        starting_points[i], kQuantizationAdjustment, kYToX, kYToB));
+  }
+  return dequantized;
+}
+
+MATCHER(ControlPointIs, "") {
+  const Spline::Point& actual = std::get<0>(arg);
+  const Spline::Point& expected = std::get<1>(arg);
+  return testing::ExplainMatchResult(
+      AllOf(Field("x", &Spline::Point::x, FloatNear(expected.x, kTolerance)),
+            Field("y", &Spline::Point::y, FloatNear(expected.y, kTolerance))),
+      actual, result_listener);
+}
+
+MATCHER(ControlPointsMatch, "") {
+  const Spline& actual = std::get<0>(arg);
+  const Spline& expected = std::get<1>(arg);
+  return testing::ExplainMatchResult(
+      Field("control_points", &Spline::control_points,
+            Pointwise(ControlPointIs(), expected.control_points)),
+      actual, result_listener);
+}
+
+MATCHER(SplinesMatch, "") {
+  const Spline& actual = std::get<0>(arg);
+  const Spline& expected = std::get<1>(arg);
+  if (!testing::ExplainMatchResult(ControlPointsMatch(), arg,
+                                   result_listener)) {
+    return false;
+  }
+  for (int i = 0; i < 3; ++i) {
+    testing::StringMatchResultListener color_dct_listener;
+    if (!testing::ExplainMatchResult(
+            Pointwise(FloatNear(kTolerance), expected.color_dct[i]),
+            actual.color_dct[i], &color_dct_listener)) {
+      *result_listener << ", where color_dct[" << i << "] don't match, "
+                       << color_dct_listener.str();
+      return false;
+    }
+  }
+  if (!testing::ExplainMatchResult(
+          Field("sigma_dct", &Spline::sigma_dct,
+                Pointwise(FloatNear(kTolerance), expected.sigma_dct)),
+          actual, result_listener)) {
+    return false;
+  }
+  return true;
+}
+
+TEST(SplinesTest, Serialization) {
+  std::vector<Spline> spline_data = {
+      {/*control_points=*/{
+           {109, 54}, {218, 159}, {80, 3}, {110, 274}, {94, 185}, {17, 277}},
+       /*color_dct=*/
+       {{36.3, 39.7, 23.2, 67.5, 4.4,  71.5, 62.3, 32.3, 92.2, 10.1, 10.8,
+         9.2,  6.1,  10.5, 79.1, 7,    24.6, 90.8, 5.5,  84,   43.8, 49,
+         33.5, 78.9, 54.5, 77.9, 62.1, 51.4, 36.4, 14.3, 83.7, 35.4},
+        {9.4,  53.4, 9.5,  74.9, 72.7, 26.7, 7.9,  0.9, 84.9, 23.2, 26.5,
+         31.1, 91,   11.7, 74.1, 39.3, 23.7, 82.5, 4.8, 2.7,  61.2, 96.4,
+         13.7, 66.7, 62.9, 82.4, 5.9,  98.7, 21.5, 7.9, 51.7, 63.1},
+        {48,   39.3, 6.9,  26.3, 33.3, 6.2,  1.7,  98.9, 59.9, 59.6, 95,
+         61.3, 82.7, 53,   6.1,  30.4, 34.7, 96.9, 93.4, 17,   38.8, 80.8,
+         63,   18.6, 43.6, 32.3, 61,   20.2, 24.3, 28.3, 69.1, 62.4}},
+       /*sigma_dct=*/{32.7, 21.5, 44.4, 1.8,  45.8, 90.6, 29.3, 59.2,
+                      23.7, 85.2, 84.8, 27.2, 42.1, 84.1, 50.6, 17.6,
+                      93.7, 4.9,  2.6,  69.8, 94.9, 52,   24.3, 18.8,
+                      12.1, 95.7, 28.5, 81.4, 89.9, 31.4, 74.8, 52}},
+      {/*control_points=*/{{172, 309},
+                           {196, 277},
+                           {42, 238},
+                           {114, 350},
+                           {307, 290},
+                           {316, 269},
+                           {124, 66},
+                           {233, 267}},
+       /*color_dct=*/
+       {{15,   28.9, 22, 6.6,  41.8, 83,   8.6,  56.8, 68.9, 9.7,  5.4,
+         19.8, 70.8, 90, 52.5, 65.2, 7.8,  23.5, 26.4, 72.2, 64.7, 87.1,
+         1.3,  67.5, 46, 68.4, 65.4, 35.5, 29.1, 13,   41.6, 23.9},
+        {47.7, 79.4, 62.7, 29.1, 96.8, 18.5, 17.6, 15.2, 80.5, 56,  96.2,
+         59.9, 26.7, 96.1, 92.3, 42.1, 35.8, 54,   23.2, 55,   76,  35.8,
+         58.4, 88.7, 2.4,  78.1, 95.6, 27.5, 6.6,  78.5, 24.1, 69.8},
+        {43.8, 96.5, 0.9,  95.1, 49.1, 71.2, 25.1, 33.6, 75.2, 95,  82.1,
+         19.7, 10.5, 44.9, 50,   93.3, 83.5, 99.5, 64.6, 54,   3.5, 99.7,
+         45.3, 82.1, 22.4, 37.9, 60,   32.2, 12.6, 4.6,  65.5, 96.4}},
+       /*sigma_dct=*/{72.5, 2.6,  41.7, 2.2,  39.7, 79.1, 69.6, 19.9,
+                      92.3, 71.5, 41.9, 62.1, 30,   49.4, 70.3, 45.3,
+                      62.5, 47.2, 46.7, 41.2, 90.8, 46.8, 91.2, 55,
+                      8.1,  69.6, 25.4, 84.7, 61.7, 27.6, 3.7,  46.9}},
+      {/*control_points=*/{{100, 186},
+                           {257, 97},
+                           {170, 49},
+                           {25, 169},
+                           {309, 104},
+                           {232, 237},
+                           {385, 101},
+                           {122, 168},
+                           {26, 300},
+                           {390, 88}},
+       /*color_dct=*/
+       {{16.9, 64.8, 4.2,  10.6, 23.5, 17,   79.3, 5.7,  60.4, 16.6, 94.9,
+         63.7, 87.6, 10.5, 3.8,  61.1, 22.9, 81.9, 80.4, 40.5, 45.9, 25.4,
+         39.8, 30,   50.2, 90.4, 27.9, 93.7, 65.1, 48.2, 22.3, 43.9},
+        {24.9, 66,   3.5,  90.2, 97.1, 15.8, 35.6, 0.6,  68,   39.6, 24.4,
+         85.9, 57.7, 77.6, 47.5, 67.9, 4.3,  5.4,  91.2, 58.5, 0.1,  52.2,
+         3.5,  47.8, 63.2, 43.5, 85.8, 35.8, 50.2, 35.9, 19.2, 48.2},
+        {82.8, 44.9, 76.4, 39.5, 94.1, 14.3, 89.8, 10,   10.5, 74.5, 56.3,
+         65.8, 7.8,  23.3, 52.8, 99.3, 56.8, 46,   76.7, 13.5, 67,   22.4,
+         29.9, 43.3, 70.3, 26,   74.3, 53.9, 62,   19.1, 49.3, 46.7}},
+       /*sigma_dct=*/{83.5, 1.7,  25.1, 18.7, 46.5, 75.3, 28,   62.3,
+                      50.3, 23.3, 85.6, 96,   45.8, 33.1, 33.4, 52.9,
+                      26.3, 58.5, 19.6, 70,   92.6, 22.5, 57,   21.6,
+                      76.8, 87.5, 22.9, 66.3, 35.7, 35.6, 56.8, 67.2}},
+  };
+
+  std::vector<QuantizedSpline> quantized_splines;
+  std::vector<Spline::Point> starting_points;
+  for (const Spline& spline : spline_data) {
+    quantized_splines.emplace_back(spline, kQuantizationAdjustment, kYToX,
+                                   kYToB);
+    starting_points.push_back(spline.control_points.front());
+  }
+
+  Splines splines(kQuantizationAdjustment, std::move(quantized_splines),
+                  std::move(starting_points));
+  const std::vector<Spline> quantized_spline_data = DequantizeSplines(splines);
+  EXPECT_THAT(quantized_spline_data,
+              Pointwise(ControlPointsMatch(), spline_data));
+
+  BitWriter writer;
+  splines.Encode(&writer, kLayerSplines, nullptr);
+  writer.ZeroPadToByte();
+  const size_t bits_written = writer.BitsWritten();
+
+  printf("Wrote %zu bits of splines.\n", bits_written);
+
+  BitReader reader(writer.GetSpan());
+  Splines decoded_splines;
+  ASSERT_TRUE(decoded_splines.Decode(&reader));
+  ASSERT_TRUE(reader.JumpToByteBoundary());
+  EXPECT_EQ(reader.TotalBitsConsumed(), bits_written);
+  ASSERT_TRUE(reader.Close());
+
+  const std::vector<Spline> decoded_spline_data =
+      DequantizeSplines(decoded_splines);
+  EXPECT_THAT(decoded_spline_data,
+              Pointwise(SplinesMatch(), quantized_spline_data));
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/test_utils.h b/jxl/test_utils.h
new file mode 100644
index 000000000..ff40919f0
--- /dev/null
+++ b/jxl/test_utils.h
@@ -0,0 +1,80 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_TEST_UTILS_H_
+#define JXL_TEST_UTILS_H_
+
+// Macros and functions useful for tests.
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+#include "jxl/enc_file.h"
+#include "jxl/enc_params.h"
+
+#ifdef JXL_DISABLE_SLOW_TESTS
+#define JXL_SLOW_TEST(X) DISABLED_##X
+#else
+#define JXL_SLOW_TEST(X) X
+#endif  // JXL_DISABLE_SLOW_TESTS
+
+namespace jxl {
+namespace test {
+// Returns compressed size [bytes].
+size_t Roundtrip(CodecInOut* io, const CompressParams& cparams,
+                 const DecompressParams& dparams, ThreadPool* pool,
+                 CodecInOut* JXL_RESTRICT io2, AuxOut* aux_out = nullptr) {
+  const ImageBundle& ib1 = io->Main();
+  PaddedBytes compressed;
+
+  // Remember original encoding, will be returned by decoder.
+  const Primaries ext_pr = ib1.metadata()->color_encoding.primaries;
+  const CustomTransferFunction ext_tf = ib1.metadata()->color_encoding.tf;
+  // c_current should not change during encoding.
+  const Primaries cur_pr = ib1.c_current().primaries;
+  const CustomTransferFunction cur_tf = ib1.c_current().tf;
+
+  PassesEncoderState enc_state;
+  EXPECT_TRUE(EncodeFile(cparams, io, &enc_state, &compressed, aux_out, pool));
+
+  // Should still be in the same color space after encoding.
+  EXPECT_EQ(cur_pr, ib1.c_current().primaries);
+  EXPECT_TRUE(cur_tf.IsSame(ib1.c_current().tf));
+
+  EXPECT_TRUE(DecodeFile(dparams, compressed, io2, aux_out, pool));
+  const ImageBundle& ib2 = io2->Main();
+
+  if (!cparams.modular_group_mode) {
+    // Non-modular returns linear sRGB.
+    EXPECT_EQ(Primaries::kSRGB, ib2.c_current().primaries);
+    EXPECT_TRUE(ib2.c_current().tf.IsLinear());
+  } else {
+    // Modular returns the original color space.
+    EXPECT_EQ(ib1.c_current().primaries, ib2.c_current().primaries);
+    EXPECT_TRUE(ib1.c_current().tf.IsSame(ib2.c_current().tf));
+  }
+
+  // Decoder returns the originals passed to the encoder.
+  EXPECT_EQ(ext_pr, ib2.metadata()->color_encoding.primaries);
+  EXPECT_TRUE(ext_tf.IsSame(ib2.metadata()->color_encoding.tf));
+
+  return compressed.size();
+}
+
+}  // namespace test
+}  // namespace jxl
+
+#endif  // JXL_TEST_UTILS_H_
diff --git a/jxl/testdata_path.h b/jxl/testdata_path.h
new file mode 100644
index 000000000..c9e6d5064
--- /dev/null
+++ b/jxl/testdata_path.h
@@ -0,0 +1,29 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_TESTDATA_PATH_H_
+#define JXL_TESTDATA_PATH_H_
+
+#include <string>
+
+
+namespace jxl {
+
+static inline std::string GetTestDataPath(const std::string& filename) {
+  return std::string(TEST_DATA_PATH "/") + filename;
+}
+
+}  // namespace jxl
+
+#endif  // JXL_TESTDATA_PATH_H_
diff --git a/jxl/threads/thread_parallel_runner.cc b/jxl/threads/thread_parallel_runner.cc
new file mode 100644
index 000000000..1c2972cf9
--- /dev/null
+++ b/jxl/threads/thread_parallel_runner.cc
@@ -0,0 +1,191 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jpegxl/thread_parallel_runner.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <algorithm>  // max
+#include <atomic>
+#include <condition_variable>  //NOLINT
+#include <mutex>               //NOLINT
+#include <thread>              //NOLINT
+#include <vector>
+
+#include "jxl/base/bits.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+
+namespace jpegxl {
+
+// static
+JpegxlParallelRetCode ThreadParallelRunner::Runner(
+    void* runner_opaque, void* jpegxl_opaque, JpegxlParallelRunInit init,
+    JpegxlParallelRunFunction func, uint32_t start_range, uint32_t end_range) {
+  ThreadParallelRunner* self =
+      static_cast<ThreadParallelRunner*>(runner_opaque);
+  if (start_range > end_range) return -1;
+  if (start_range == end_range) return 0;
+
+  int ret = init(jpegxl_opaque, std::max<size_t>(self->num_worker_threads_, 1));
+  if (ret != 0) return ret;
+
+  // Use a sequential run when num_worker_threads_ is zero since we have no
+  // worker threads.
+  if (self->num_worker_threads_ == 0) {
+    const size_t thread = 0;
+    for (uint32_t task = start_range; task < end_range; ++task) {
+      func(jpegxl_opaque, task, thread);
+    }
+    return 0;
+  }
+
+  if (self->depth_.fetch_add(1, std::memory_order_acq_rel) != 0) {
+    return -1;  // Must not re-enter.
+  }
+
+  const WorkerCommand worker_command =
+      (static_cast<WorkerCommand>(start_range) << 32) + end_range;
+  // Ensure the inputs do not result in a reserved command.
+  JXL_ASSERT(worker_command != kWorkerWait);
+  JXL_ASSERT(worker_command != kWorkerOnce);
+  JXL_ASSERT(worker_command != kWorkerExit);
+
+  self->data_func_ = func;
+  self->jpegxl_opaque_ = jpegxl_opaque;
+  self->num_reserved_.store(0, std::memory_order_relaxed);
+
+  self->StartWorkers(worker_command);
+  self->WorkersReadyBarrier();
+
+  if (self->depth_.fetch_add(-1, std::memory_order_acq_rel) != 1) {
+    return -1;
+  }
+  return 0;
+}
+
+// static
+void ThreadParallelRunner::RunRange(ThreadParallelRunner* self,
+                                    const WorkerCommand command,
+                                    const int thread) {
+  const uint32_t begin = command >> 32;
+  const uint32_t end = command & 0xFFFFFFFF;
+  const uint32_t num_tasks = end - begin;
+  const uint32_t num_worker_threads = self->num_worker_threads_;
+
+  // OpenMP introduced several "schedule" strategies:
+  // "single" (static assignment of exactly one chunk per thread): slower.
+  // "dynamic" (allocates k tasks at a time): competitive for well-chosen k.
+  // "guided" (allocates k tasks, decreases k): computing k = remaining/n
+  //   is faster than halving k each iteration. We prefer this strategy
+  //   because it avoids user-specified parameters.
+
+  for (;;) {
+#if 0
+      // dynamic
+      const uint32_t my_size = std::max(num_tasks / (num_worker_threads * 4), 1);
+#else
+    // guided
+    const uint32_t num_reserved =
+        self->num_reserved_.load(std::memory_order_relaxed);
+    const uint32_t num_remaining = num_tasks - num_reserved;
+    const uint32_t my_size =
+        std::max(num_remaining / (num_worker_threads * 4), 1u);
+#endif
+    const uint32_t my_begin = begin + self->num_reserved_.fetch_add(
+                                          my_size, std::memory_order_relaxed);
+    const uint32_t my_end = std::min(my_begin + my_size, begin + num_tasks);
+    // Another thread already reserved the last task.
+    if (my_begin >= my_end) {
+      break;
+    }
+    for (uint32_t task = my_begin; task < my_end; ++task) {
+      self->data_func_(self->jpegxl_opaque_, task, thread);
+    }
+  }
+}
+
+// static
+void ThreadParallelRunner::ThreadFunc(ThreadParallelRunner* self,
+                                      const int thread) {
+  // Until kWorkerExit command received:
+  for (;;) {
+    std::unique_lock<std::mutex> lock(self->mutex_);
+    // Notify main thread that this thread is ready.
+    if (++self->workers_ready_ == self->num_threads_) {
+      self->workers_ready_cv_.notify_one();
+    }
+  RESUME_WAIT:
+    // Wait for a command.
+    self->worker_start_cv_.wait(lock);
+    const WorkerCommand command = self->worker_start_command_;
+    switch (command) {
+      case kWorkerWait:    // spurious wakeup:
+        goto RESUME_WAIT;  // lock still held, avoid incrementing ready.
+      case kWorkerOnce:
+        lock.unlock();
+        self->data_func_(self->jpegxl_opaque_, thread, thread);
+        break;
+      case kWorkerExit:
+        return;  // exits thread
+      default:
+        lock.unlock();
+        RunRange(self, command, thread);
+        break;
+    }
+  }
+}
+
+ThreadParallelRunner::ThreadParallelRunner(const int num_worker_threads)
+    : num_worker_threads_(num_worker_threads),
+      num_threads_(std::max(num_worker_threads, 1)) {
+  PROFILER_ZONE("ThreadParallelRunner ctor");
+
+  JXL_CHECK(num_worker_threads >= 0);
+  threads_.reserve(num_worker_threads);
+
+  // Suppress "unused-private-field" warning.
+  (void)padding1;
+  (void)padding2;
+
+  // Safely handle spurious worker wakeups.
+  worker_start_command_ = kWorkerWait;
+
+  for (int i = 0; i < num_worker_threads; ++i) {
+    threads_.emplace_back(ThreadFunc, this, i);
+  }
+
+  if (num_worker_threads_ != 0) {
+    WorkersReadyBarrier();
+  }
+
+  // Warm up profiler on worker threads so its expensive initialization
+  // doesn't count towards other timer measurements.
+  RunOnEachThread(
+      [](const int task, const int thread) { PROFILER_ZONE("@InitWorkers"); });
+}
+
+ThreadParallelRunner::~ThreadParallelRunner() {
+  if (num_worker_threads_ != 0) {
+    StartWorkers(kWorkerExit);
+  }
+
+  for (std::thread& thread : threads_) {
+    JXL_ASSERT(thread.joinable());
+    thread.join();
+  }
+}
+
+}  // namespace jpegxl
diff --git a/jxl/threads/thread_parallel_runner_test.cc b/jxl/threads/thread_parallel_runner_test.cc
new file mode 100644
index 000000000..a850bef78
--- /dev/null
+++ b/jxl/threads/thread_parallel_runner_test.cc
@@ -0,0 +1,124 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "gtest/gtest.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/thread_pool_internal.h"
+
+namespace jpegxl {
+namespace {
+
+int PopulationCount(uint64_t bits) {
+  int num_set = 0;
+  while (bits != 0) {
+    num_set += bits & 1;
+    bits >>= 1;
+  }
+  return num_set;
+}
+
+// Ensures task parameter is in bounds, every parameter is reached,
+// pool can be reused (multiple consecutive Run calls), pool can be destroyed
+// (joining with its threads), num_threads=0 works (runs on current thread).
+TEST(ThreadParallelRunnerTest, TestPool) {
+  for (int num_threads = 0; num_threads <= 18; ++num_threads) {
+    jxl::ThreadPoolInternal pool(num_threads);
+    for (int num_tasks = 0; num_tasks < 32; ++num_tasks) {
+      std::vector<int> mementos(num_tasks);
+      for (int begin = 0; begin < 32; ++begin) {
+        std::fill(mementos.begin(), mementos.end(), 0);
+        pool.Run(
+            begin, begin + num_tasks, jxl::ThreadPool::SkipInit(),
+            [begin, num_tasks, &mementos](const int task, const int thread) {
+              // Parameter is in the given range
+              EXPECT_GE(task, begin);
+              EXPECT_LT(task, begin + num_tasks);
+
+              // Store mementos to be sure we visited each task.
+              mementos.at(task - begin) = 1000 + task;
+            });
+        for (int task = begin; task < begin + num_tasks; ++task) {
+          EXPECT_EQ(1000 + task, mementos.at(task - begin));
+        }
+      }
+    }
+  }
+}
+
+// Verify "thread" parameter when processing few tasks.
+TEST(ThreadParallelRunnerTest, TestSmallAssignments) {
+  // WARNING: cumulative total threads must not exceed profiler.h kMaxThreads.
+  const int kMaxThreads = 8;
+  for (int num_threads = 1; num_threads <= kMaxThreads; ++num_threads) {
+    jxl::ThreadPoolInternal pool(num_threads);
+
+    // (Avoid mutex because it may perturb the worker thread scheduling)
+    std::atomic<uint64_t> id_bits{0};
+    std::atomic<int> num_calls{0};
+
+    pool.Run(
+        0, num_threads, jxl::ThreadPool::SkipInit(),
+        [&num_calls, num_threads, &id_bits](const int task, const int thread) {
+          num_calls.fetch_add(1, std::memory_order_relaxed);
+
+          EXPECT_LT(thread, num_threads);
+          uint64_t bits = id_bits.load(std::memory_order_relaxed);
+          while (
+              !id_bits.compare_exchange_weak(bits, bits | (1ULL << thread))) {
+          }
+        });
+
+    // Correct number of tasks.
+    EXPECT_EQ(num_threads, num_calls.load());
+
+    const int num_participants = PopulationCount(id_bits.load());
+    // Can't expect equality because other workers may have woken up too late.
+    EXPECT_LE(num_participants, num_threads);
+  }
+}
+
+struct Counter {
+  Counter() {
+    // Suppress "unused-field" warning.
+    (void)padding;
+  }
+  void Assimilate(const Counter& victim) { counter += victim.counter; }
+  int counter = 0;
+  int padding[31];
+};
+
+TEST(ThreadParallelRunnerTest, TestCounter) {
+  const int kNumThreads = 12;
+  jxl::ThreadPoolInternal pool(kNumThreads);
+  alignas(128) Counter counters[kNumThreads];
+
+  const int kNumTasks = kNumThreads * 19;
+  pool.Run(0, kNumTasks, jxl::ThreadPool::SkipInit(),
+           [&counters](const int task, const int thread) {
+             counters[thread].counter += task;
+           });
+
+  int expected = 0;
+  for (int i = 0; i < kNumTasks; ++i) {
+    expected += i;
+  }
+
+  for (int i = 1; i < kNumThreads; ++i) {
+    counters[0].Assimilate(counters[i]);
+  }
+  EXPECT_EQ(expected, counters[0].counter);
+}
+
+}  // namespace
+}  // namespace jpegxl
diff --git a/jxl/toc.cc b/jxl/toc.cc
new file mode 100644
index 000000000..81f45da73
--- /dev/null
+++ b/jxl/toc.cc
@@ -0,0 +1,110 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/toc.h"
+
+#include <stdint.h>
+
+#include "jxl/aux_out_fwd.h"
+#include "jxl/coeff_order.h"
+#include "jxl/coeff_order_fwd.h"
+#include "jxl/common.h"
+#include "jxl/field_encodings.h"
+#include "jxl/fields.h"
+
+namespace jxl {
+namespace {
+// (2+bits) = 2,3,4 bytes so encoders can patch TOC after encoding.
+// 30 is sufficient for 4K channels of uncompressed 16-bit samples.
+constexpr U32Enc kDist(Bits(10), BitsOffset(14, 1024), BitsOffset(22, 17408),
+                       BitsOffset(30, 4211712));
+
+static size_t MaxBits(const size_t num_sizes) {
+  const size_t entry_bits = U32Coder::MaxEncodedBits(kDist) * num_sizes;
+  // permutation bit (not its tokens!), padding, entries, padding.
+  return 1 + kBitsPerByte + entry_bits + kBitsPerByte;
+}
+
+}  // namespace
+
+HWY_ATTR Status ReadGroupOffsets(size_t toc_entries,
+                                 BitReader* JXL_RESTRICT reader,
+                                 std::vector<uint64_t>* JXL_RESTRICT offsets) {
+  std::vector<coeff_order_t> permutation;
+  if (reader->ReadFixedBits<1>() == 1) {
+    permutation.resize(toc_entries);
+    JXL_RETURN_IF_ERROR(
+        DecodePermutation(/*skip=*/0, toc_entries, permutation.data(), reader));
+  }
+
+  JXL_RETURN_IF_ERROR(reader->JumpToByteBoundary());
+  std::vector<uint32_t> sizes;
+  sizes.reserve(toc_entries);
+  for (size_t i = 0; i < toc_entries; ++i) {
+    sizes.push_back(U32Coder::Read(kDist, reader));
+  }
+  JXL_RETURN_IF_ERROR(reader->JumpToByteBoundary());
+
+  // Prefix sum starting with 0 and ending with total_size
+  offsets->reserve(toc_entries + 1);
+  offsets->push_back(0);
+  uint64_t offset = 0;
+  for (size_t i = 0; i < toc_entries; ++i) {
+    offset += sizes[i];
+    offsets->push_back(offset);
+  }
+
+  if (!permutation.empty()) {
+    std::vector<uint64_t> permuted_offsets;
+    permuted_offsets.reserve(offsets->size());
+    for (coeff_order_t index : permutation) {
+      permuted_offsets.push_back((*offsets)[index]);
+    }
+    std::swap(*offsets, permuted_offsets);
+  }
+
+  return true;
+}
+
+Status WriteGroupOffsets(const std::vector<BitWriter>& group_codes,
+                         BitWriter* JXL_RESTRICT writer, AuxOut* aux_out) {
+  BitWriter::Allotment allotment(writer, MaxBits(group_codes.size()));
+  writer->Write(1, 0);  // no permutation
+  // TODO(janwas): WriteTokens(permutation), already has an allotment.
+  writer->ZeroPadToByte();  // before TOC entries
+
+  for (size_t i = 0; i < group_codes.size(); i++) {
+    JXL_ASSERT(group_codes[i].BitsWritten() % kBitsPerByte == 0);
+    const size_t group_size = group_codes[i].BitsWritten() / kBitsPerByte;
+    JXL_RETURN_IF_ERROR(U32Coder::Write(kDist, group_size, writer));
+  }
+  writer->ZeroPadToByte();  // before first group
+  ReclaimAndCharge(writer, &allotment, kLayerTOC, aux_out);
+  return true;
+}
+
+Status WriteResponsiveOffsets(const std::vector<int>& offsets,
+                              BitWriter* JXL_RESTRICT writer, AuxOut* aux_out) {
+  BitWriter::Allotment allotment(writer, MaxBits(offsets.size()));
+  writer->Write(1, 0);      // no permutation
+  writer->ZeroPadToByte();  // before TOC entries
+  for (size_t i = 0; i < offsets.size(); i++) {
+    JXL_RETURN_IF_ERROR(U32Coder::Write(kDist, offsets[i], writer));
+  }
+  writer->ZeroPadToByte();  // before first group
+  ReclaimAndCharge(writer, &allotment, kLayerTOC, aux_out);
+  return true;
+}
+
+}  // namespace jxl
diff --git a/jxl/toc.h b/jxl/toc.h
new file mode 100644
index 000000000..f5f4200b7
--- /dev/null
+++ b/jxl/toc.h
@@ -0,0 +1,55 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_TOC_H_
+#define JXL_TOC_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/status.h"
+#include "jxl/dec_bit_reader.h"
+#include "jxl/enc_bit_writer.h"
+
+namespace jxl {
+
+static JXL_INLINE size_t AcGroupIndex(size_t pass, size_t group,
+                                      size_t num_groups, size_t num_dc_groups,
+                                      bool has_ac_global) {
+  return 1 + num_dc_groups + has_ac_global + pass * num_groups + group;
+}
+
+static JXL_INLINE size_t NumTocEntries(size_t num_groups, size_t num_dc_groups,
+                                       size_t num_passes, bool has_ac_global) {
+  if (num_groups == 1 && num_passes == 1) return 1;
+  return AcGroupIndex(0, 0, num_groups, num_dc_groups, has_ac_global) +
+         num_groups * num_passes;
+}
+
+Status ReadGroupOffsets(size_t toc_entries, BitReader* JXL_RESTRICT reader,
+                        std::vector<uint64_t>* JXL_RESTRICT group_offsets);
+
+Status WriteGroupOffsets(const std::vector<BitWriter>& group_codes,
+                         BitWriter* JXL_RESTRICT writer, AuxOut* aux_out);
+
+Status WriteResponsiveOffsets(const std::vector<int>& offsets,
+                              BitWriter* JXL_RESTRICT writer, AuxOut* aux_out);
+
+}  // namespace jxl
+
+#endif  // JXL_TOC_H_
diff --git a/jxl/toc_test.cc b/jxl/toc_test.cc
new file mode 100644
index 000000000..cc2a8f56a
--- /dev/null
+++ b/jxl/toc_test.cc
@@ -0,0 +1,73 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/toc.h"
+
+#include <random>
+
+#include "gtest/gtest.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/span.h"
+#include "jxl/common.h"
+
+namespace jxl {
+namespace {
+
+void Roundtrip(size_t num_entries, std::mt19937* rng) {
+  // Generate num_entries groups of random (byte-aligned) length
+  std::vector<BitWriter> group_codes(num_entries);
+  for (BitWriter& writer : group_codes) {
+    const size_t max_bits = (*rng)() & 0xFFF;
+    BitWriter::Allotment allotment(&writer, max_bits + kBitsPerByte);
+    size_t i = 0;
+    for (; i + BitWriter::kMaxBitsPerCall < max_bits;
+         i += BitWriter::kMaxBitsPerCall) {
+      writer.Write(BitWriter::kMaxBitsPerCall, 0);
+    }
+    for (; i < max_bits; i += 1) {
+      writer.Write(/*n_bits=*/1, 0);
+    }
+    writer.ZeroPadToByte();
+    AuxOut aux_out;
+    ReclaimAndCharge(&writer, &allotment, 0, &aux_out);
+  }
+
+  BitWriter writer;
+  AuxOut aux_out;
+  JXL_CHECK(WriteGroupOffsets(group_codes, &writer, &aux_out));
+
+  BitReader reader(writer.GetSpan());
+  std::vector<uint64_t> group_offsets;
+  ASSERT_TRUE(ReadGroupOffsets(num_entries, &reader, &group_offsets));
+  ASSERT_EQ(num_entries + 1, group_offsets.size());
+  EXPECT_TRUE(reader.Close());
+
+  EXPECT_EQ(0, group_offsets[0]);
+  size_t prefix_sum = 0;
+  for (size_t i = 1; i < num_entries + 1; ++i) {
+    EXPECT_TRUE(group_codes[i - 1].BitsWritten() % kBitsPerByte == 0);
+    prefix_sum += group_codes[i - 1].BitsWritten() / kBitsPerByte;
+    EXPECT_EQ(prefix_sum, group_offsets[i]);
+  }
+}
+
+TEST(TocTest, Test) {
+  std::mt19937 rng(12345);
+  for (size_t num_entries = 0; num_entries < 5; ++num_entries) {
+    Roundtrip(num_entries, &rng);
+  }
+}
+
+}  // namespace
+}  // namespace jxl
diff --git a/jxl/xorshift128plus.h b/jxl/xorshift128plus.h
new file mode 100644
index 000000000..25529a118
--- /dev/null
+++ b/jxl/xorshift128plus.h
@@ -0,0 +1,88 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_XORSHIFT128PLUS_H_
+#define JXL_XORSHIFT128PLUS_H_
+
+// Fast but weak random generator.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <hwy/compiler_specific.h>
+#include <hwy/static_targets.h>
+
+#include "jxl/base/compiler_specific.h"
+
+namespace jxl {
+
+// Adapted from https://github.com/vpxyz/xorshift/blob/master/xorshift128plus/
+// (MIT-license)
+class Xorshift128Plus {
+ public:
+  // 8 independent generators (= single iteration for AVX-512)
+  enum { N = 8 };
+
+  explicit Xorshift128Plus(const uint64_t seed) {
+    // Init state using SplitMix64 generator
+    s0_[0] = SplitMix64(seed + 0x9E3779B97F4A7C15ull);
+    s1_[0] = SplitMix64(s0_[0]);
+    for (size_t i = 1; i < N; ++i) {
+      s0_[i] = SplitMix64(s1_[i - 1]);
+      s1_[i] = SplitMix64(s0_[i]);
+    }
+  }
+
+  HWY_ATTR HWY_INLINE void Fill(uint64_t* JXL_RESTRICT random_bits) {
+#if HWY_HAS_INT64
+    const HWY_FULL(uint64_t) d;
+    for (size_t i = 0; i < N; i += d.N) {
+      auto s1 = Load(d, s1_ + i);
+      const auto s0 = Load(d, s0_ + i);
+      s1 ^= hwy::ShiftLeft<23>(s1);
+      const auto bits = s1 + s0;  // b, c
+      Store(s0, d, s0_ + i);
+      s1 ^= s0 ^ hwy::ShiftRight<18>(s1) ^ hwy::ShiftRight<5>(s0);
+      Store(bits, d, random_bits + i);
+      Store(s1, d, s1_ + i);
+    }
+#else
+    for (size_t i = 0; i < N; ++i) {
+      auto s1 = s1_[i];
+      const auto s0 = s0_[i];
+      s1 ^= s1 << 23;
+      const auto bits = s1 + s0;  // b, c
+      // TODO(eustas): does that make any sense?
+      s0_[i] = s0;
+      s1 ^= s0 ^ (s1 >> 18) ^ (s0 >> 18);
+      random_bits[i] = bits;
+      s1_[i] = s1;
+    }
+#endif
+  }
+
+ private:
+  static uint64_t SplitMix64(uint64_t z) {
+    z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull;
+    z = (z ^ (z >> 27)) * 0x94D049BB133111EBull;
+    return z ^ (z >> 31);
+  }
+
+  HWY_ALIGN uint64_t s0_[N];
+  HWY_ALIGN uint64_t s1_[N];
+};
+
+}  // namespace jxl
+
+#endif  // JXL_XORSHIFT128PLUS_H_
diff --git a/jxl/xorshift128plus_test.cc b/jxl/xorshift128plus_test.cc
new file mode 100644
index 000000000..70e41b777
--- /dev/null
+++ b/jxl/xorshift128plus_test.cc
@@ -0,0 +1,354 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/xorshift128plus.h"
+
+#include <stdint.h>
+
+#include <algorithm>
+#include <vector>
+
+#include "gtest/gtest.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/thread_pool_internal.h"
+
+namespace jxl {
+namespace HWY_NAMESPACE {
+namespace {
+
+// Define to nonzero in order to print the (new) golden outputs.
+#define PRINT_RESULTS 0
+
+const size_t kVectors = 64;
+
+#if PRINT_RESULTS
+
+template <int kNumLanes>
+void Print(const uint64_t (&result)[kNumLanes]) {
+  printf("{ ");
+  for (int i = 0; i < kNumLanes; ++i) {
+    if (i != 0) {
+      printf(", ");
+    }
+    printf("0x%016lXull", result[i]);
+  }
+  printf("},\n");
+}
+#endif  // PRINT_RESULTS
+
+const uint64_t kExpected[kVectors][Xorshift128Plus::N] = {
+    {0xE57D429BCBF7CBB1ull, 0xD409D608465DA2A2ull, 0xF33DAEF5C060AE99ull,
+     0xDD0768FA29FD6896ull, 0x70DE3CCD8DDCF532ull, 0x617908C4E4DE7E32ull,
+     0x9FE35280D25C0DD0ull, 0xFC0FC203EC34D836ull},
+    {0xABC72C7F0DFAFD55ull, 0x3DC7516EAEFC178Eull, 0xF9C0F467E92843C6ull,
+     0x106F209938F24441ull, 0x68C95808217F699Full, 0x419CFE6050EA062Eull,
+     0x1CD4976BEB1DB4A0ull, 0xDA8311DEF1AB2299ull},
+    {0xCC171C4D5510FD32ull, 0xD1059F8519FC387Bull, 0x0BF9802AB0CD2848ull,
+     0x0C94B2274A1DA664ull, 0x0348B2B0976EB019ull, 0x272092955EFD9330ull,
+     0xB387F47E321F4801ull, 0x9803026497BD9F18ull},
+    {0x38E5D5EC43AE6A10ull, 0x59C7999E07FE7ACEull, 0x926733F0F8371230ull,
+     0x4423EDCE6C8DC6F4ull, 0x6D3011EEC454FF1Bull, 0xF970E961F4F6CACDull,
+     0xBC844A3B01D942E0ull, 0xCD6E29C9F44E8915ull},
+    {0x25B70DD81E7738CAull, 0xCC90B3B446728FFAull, 0x1C74221A22BEB66Aull,
+     0x36BDE1375913FDA8ull, 0x6F471E35D83F1D4Cull, 0x0B10DD0B08AA76AFull,
+     0xB56CD19CD3AF8BF0ull, 0xFC53E99B9E66D3B4ull},
+    {0xF0AB794697B774CEull, 0x25AEF6A691010AE9ull, 0x2EF44B85C2C0638Eull,
+     0x6D603B0AF1E783D5ull, 0xC44BA15692234216ull, 0x7CC4AC08578451DEull,
+     0xEA2BCAACE5313675ull, 0x861F055431031B9Bull},
+    {0x8F782C99EB93931Aull, 0xB621588B882EEA9Eull, 0x083ED710D1E40E69ull,
+     0x95468E592E7A976Full, 0x9F79D6E0DDE76FD1ull, 0xA712ADE3916DCA79ull,
+     0xD584B73362038793ull, 0x06FD29ACF5CFF04Cull},
+    {0x0935DC4F3E1E59C7ull, 0xC2AEF358C86A3D1Eull, 0x930ED570BEA39FD6ull,
+     0xCEAB9F42FB1772A8ull, 0x5B2856D66DD0D725ull, 0x24B93595478B0F58ull,
+     0xB887CA012F1756B9ull, 0x9E4417FE78ECECBEull},
+    {0x20E3BC288C0085B9ull, 0x9A15C05159F39A8Full, 0x52378DDEA00C1439ull,
+     0xD1033445AB3E6656ull, 0x2C71F7E6C2ECD397ull, 0xC318AFA4AFCA5901ull,
+     0xD1FDAB980D7212D6ull, 0x74604CAE55F39313ull},
+    {0xE764EC148914275Full, 0x713BC1A5A6AB08BCull, 0x6EF34BC0C4072670ull,
+     0xF93539FD8BD6AEAEull, 0x0A48B717F684F17Eull, 0x26E1E4EFE1F6D547ull,
+     0x1DBB6220AFED9EE0ull, 0x9AD50F69A198F580ull},
+    {0xD6F811F172838CE2ull, 0x136FFBC56AC4A5DDull, 0xD9632E7A2E929D36ull,
+     0xB2B459999AD05D90ull, 0x78001609755916C1ull, 0x24A7E6869B972C75ull,
+     0x1E040ABAF2143A75ull, 0x95EEA900D6C93DFCull},
+    {0x0475CB5BD01CE1E3ull, 0xAAAB2443A332FBBBull, 0xBE3AAE00AC34C4DDull,
+     0x76A202BF5643935Cull, 0xB62E1C0916C84FF1ull, 0x3DC568DE07A226DFull,
+     0xC1AB6C7709270F5Bull, 0xF34E2950E8FF383Cull},
+    {0x4B5AB7E8D065929Cull, 0x4B303F81416817F8ull, 0x9191E7B479B0FC7Full,
+     0xEB86A57E8CF9ECF5ull, 0x50F28DEF8DD7BF1Dull, 0xDB01F2DE36250173ull,
+     0xEF9A526F9AE4C5B8ull, 0xD1F5AED58FD5416Aull},
+    {0xF05D0858E7402214ull, 0x1C9E8ACAD3636326ull, 0x0CF031BB5967B43Cull,
+     0xB2D7CEEF3393DF8Aull, 0xB27ECC7E343A33ADull, 0x94EBC2BFBBA22D44ull,
+     0x7C45214465A126EAull, 0x18D94F208C21F55Dull},
+    {0x232405F45B4197A2ull, 0x77DE013E9755B379ull, 0xFE7A7F642331A4AAull,
+     0x4E01955CBBD5BA8Full, 0x8626F42DB98C65B4ull, 0x30F4E6FCC82DE010ull,
+     0xD2592D51E8A9B8E8ull, 0x2FE77E6522E00477ull},
+    {0x0ABE8BC56E5B0513ull, 0xF2517C0B9841D243ull, 0x350E5A826B5AF9B1ull,
+     0x31CB7D9D31F93DB1ull, 0x0537129E66961239ull, 0xF6829BF45B116861ull,
+     0x16043F8604A4C62Cull, 0x60CF7DA19A2F4BADull},
+    {0x6F6A9DF8FE1E6D1Cull, 0x4CCF5272722770B4ull, 0xBEDA23DCA1FF32A4ull,
+     0x14AB8FA09305D907ull, 0xAF924FE7523018F9ull, 0xFF93C68C60EBB2D9ull,
+     0x5F6DE063BD1CCFEEull, 0x8933B97896111A93ull},
+    {0x52789E30890BE8E2ull, 0xCEC7468F25E878BAull, 0xBC5A3A9BB7738768ull,
+     0x1C350D17BB7855F5ull, 0xF1F50FCA0748146Bull, 0xC05A24EB51636BD8ull,
+     0xCF8C49D3FCDC384Eull, 0x78DCDD3E403D0E05ull},
+    {0xF36E34314FA1CB41ull, 0x71449EA6244FB584ull, 0x4964C689B76D7143ull,
+     0xC8CEA7E0EE6C12EBull, 0xD4DED2DD182967BFull, 0xE8BFC70338C37EF7ull,
+     0xABDF362FBF98BFDEull, 0xA0F59A3EAD936DA8ull},
+    {0x32C09999C30A1B98ull, 0x919CF62E9BE1B90Cull, 0x34AB8EAC92900753ull,
+     0x6BC1690777698728ull, 0x5E39E5CB0783D8A3ull, 0xBD1A1FBA3D6D6DF0ull,
+     0x88C97C8508454751ull, 0xC43EE86A4BFD0553ull},
+    {0xC607A8A4A7C7E83Bull, 0x586CAE6F11B19AC3ull, 0xE105DBDF9956C6BBull,
+     0x7EC474745F81F12Bull, 0x097583AAD464EBA8ull, 0xE80DA6B2E29F3FD1ull,
+     0x33FA8B7A2D7D0DAAull, 0x141CFBC57390A642ull},
+    {0x210CFF7D274CC26Bull, 0x6D69290A78725AA0ull, 0x197968442E5BCA29ull,
+     0xC72AB6EBB0C74932ull, 0xA5A5078C8C9BC4EBull, 0x277CA101F1AF4745ull,
+     0xA924A4B7487A09A3ull, 0x1D9D2FC8A827BB38ull},
+    {0xD7AE9CA43C7E4858ull, 0x302564BCE6D79113ull, 0xE42B126E8EE911FCull,
+     0xC5044B0019A40D78ull, 0xEE49138094ED59AEull, 0x2E64E870090AFA95ull,
+     0x6818A204E82DD36Bull, 0x2685C620728FB120ull},
+    {0x47D1EB9BD1715CDEull, 0xE3855B939C633DE9ull, 0x8676EA385D02520Dull,
+     0x0AD0BCC21E49A3E9ull, 0xA0276EEAA31AC707ull, 0x8132DB6DF80A9DACull,
+     0x41F448B11655620Eull, 0xC48EB17C2691CC1Aull},
+    {0x5C7B5C4D56C02D9Bull, 0x47819CA6AF8BDE76ull, 0xA6BEC45D58F79D82ull,
+     0xD8744220BD128C62ull, 0x78AB538D1731ED2Aull, 0x052F2E4C6FCB7ED5ull,
+     0x2403C0F4E1E82FB1ull, 0xB78559371438C620ull},
+    {0xAD456248EA5EBB8Dull, 0xED86CDA340A6012Full, 0x49D63F93D4DB0B08ull,
+     0x07EFDF5D55250D37ull, 0x20DE21454E436201ull, 0xEFC52935E813239Cull,
+     0x5AAE0E7EDFC49EA2ull, 0xF7F29F38AA741284ull},
+    {0x357086BF543F3283ull, 0x01267109C9C935D1ull, 0xC0505E3E42BAFC7Dull,
+     0x4B0C85D53D41F246ull, 0xDE3D5B4FEF83DE6Eull, 0xBDAAB24F078A5CE7ull,
+     0x2E39115A1A86AE38ull, 0xC1AAAA59DB3817AFull},
+    {0x6FEF7D4B573924F5ull, 0xD42E75460CF06368ull, 0x2068E4481AD02980ull,
+     0x184F3163DE3F173Full, 0xCB158DCEDB462B71ull, 0x2943B98DE4E17B71ull,
+     0x5B2C60D97D09AD73ull, 0xB38CEEE539DC5ED3ull},
+    {0xAB14C4B7D81C32C2ull, 0xC403C2C9F35A6518ull, 0xBA3D4EC5D73D4678ull,
+     0x69C06180321E4E87ull, 0xF9C1A9120CA423BDull, 0xAF49B2CEA398C375ull,
+     0x47FDADF1F4A6C69Full, 0x57D31E72078AF15Aull},
+    {0xA2DCD2450694853Dull, 0x8D615C90B34F5CC5ull, 0xABD4E2B4092FB864ull,
+     0xFC7B2A796227EC37ull, 0x8011950E3EF76F0Aull, 0x49E4DF900AB1CBDFull,
+     0x08EA8BF48FC7B35Cull, 0x70E9EE9E4F00A31Full},
+    {0x8BCD34C2CD790F20ull, 0xEE59008F7DE6F50Dull, 0x721797508237DBE4ull,
+     0x8DCF5025BAE4128Dull, 0x4D97EE955DA52E32ull, 0xF5B2ED46346A6D2Full,
+     0x7103AFF8D8A32007ull, 0x265DF60E92A206EDull},
+    {0x8FFCF9912B73FDE4ull, 0x6D992045B14F2F43ull, 0xC2390CFE4D081736ull,
+     0x2D6DCEA3C054D66Cull, 0x8B497E8838193695ull, 0x8DB2903176BF4151ull,
+     0x0BA2542407C8A145ull, 0xD8CC8007EE247247ull},
+    {0x6D039DFB5A4466A1ull, 0x2B7AC59391F987FBull, 0x8F0C8CB07AC346B0ull,
+     0x9AA2105D60039661ull, 0xE0C9058949A881B5ull, 0x67106B25782F02DDull,
+     0x4688DF0C63182B9Eull, 0x67AF5F64849C28D0ull},
+    {0xE5EF87BCEBD571D0ull, 0x8CDC03BCD571A74Eull, 0x2C99A956C89A8147ull,
+     0x729622DCA5236E78ull, 0x3AD35A6890590251ull, 0x437E8E3AE80241ADull,
+     0xA67E7AB9002EBAF1ull, 0xCCC448C4682B37E5ull},
+    {0x2E1A5D060B93FB3Cull, 0x66F0580ECA05D71Dull, 0x4417AD979075B32Aull,
+     0x5CC1F9A4FBE3E509ull, 0x4FFB6B5122E0AF21ull, 0x6E247D49ED6C3ED1ull,
+     0x09AA116B8B79F254ull, 0x50DF99ACB68461CCull},
+    {0xFFA5AF4A05473A61ull, 0x129B5536D74E002Bull, 0x7CBF21B7F4AAB340ull,
+     0x5D368F9646757428ull, 0xA3B6FFDD731B24EFull, 0x263F3A713B3600F0ull,
+     0xE636E0E0A07B1DDCull, 0x8C0D3DB62CE94B7Bull},
+    {0x339D01864E26F8D1ull, 0x7BC54480ED405173ull, 0xA73C4590FCFDDE21ull,
+     0x68486F8063E151EDull, 0x8084E97B1E754352ull, 0x7F556F3C4636525Bull,
+     0x70C598DEB3B6BD14ull, 0xEFF23E9DE385C733ull},
+    {0xD21377E73D5228A9ull, 0xD8E71E583CFB7EA7ull, 0xD247EC066D9DF9ADull,
+     0xCE2E5796DE66E8CDull, 0x4C15D9257CC03A09ull, 0xA0736B8215E8C5B0ull,
+     0xB33D8F6503FC9C93ull, 0x8752A30BF4A47E44ull},
+    {0x93536BAA8A4E3C64ull, 0xCF4A40768840F562ull, 0x4F67B9F9C69F3209ull,
+     0x1A4191EECCDAF107ull, 0x0913C01EC7682ABFull, 0xD6728915E6A4E1EEull,
+     0x4C0E895F65FCD13Bull, 0xC4DE18972061427Bull},
+    {0x19CD2286B787DD96ull, 0x9438FAF9E402F3D5ull, 0x2E26CBF57B18C3EDull,
+     0xF494799C5FD0F468ull, 0x6FF44B76887E09B3ull, 0xAE294393940F9EA1ull,
+     0x064D3BA251950313ull, 0x0F66056AAC458B51ull},
+    {0x340F853AFBF3E356ull, 0xEF3D39340621DFF2ull, 0x2580452508A04CE8ull,
+     0xFB5B5016193E4C84ull, 0x7927043E083219FDull, 0x7CAC7DC943C55AD9ull,
+     0xB45BE046B20C1E1Cull, 0xA2A26F6CD4981F72ull},
+    {0x7B318C926156EDB3ull, 0x11D250F130316AEDull, 0xCE90DAB86764678Cull,
+     0x048AE20531F2660Cull, 0xCA4BB8219AB468E6ull, 0x436AA30CA4FB0A8Eull,
+     0xB631D2B370C43B30ull, 0xA717551B4CC52546ull},
+    {0x89DF5A885FE53A7Full, 0x304B290B0A92326Eull, 0xF189B7BFDAF3259Aull,
+     0x3DDFC94D0B3A80A9ull, 0xCA42DA2BAEAB111Dull, 0xC9D9C056C41382ABull,
+     0x1683B5E15378EFEAull, 0xF4C94EF6FA84C585ull},
+    {0xB1AC121140D16E17ull, 0xBFFC45C22BB6D475ull, 0x6892B99251285291ull,
+     0x36CC7CF99098145Eull, 0xC4AB7BCC7EB1E3E8ull, 0xA684A120F05072D8ull,
+     0x0E6008652E79B25Dull, 0x72FF3B8D72B838B6ull},
+    {0xC8C91D3CBF42F23Aull, 0x8C5EA25B83C3DA57ull, 0xEAE6E5555B88040Full,
+     0x4DBA6D33F553E010ull, 0xCC9F2C0E1AD3351Full, 0xBD1D554DD7650FB0ull,
+     0xFCEC97B85BB9F615ull, 0x7AC77933D98D237Aull},
+    {0x268A991694A88E8Bull, 0x2954B370C5794E80ull, 0xAA9BE8A0B9289539ull,
+     0x359C2DCBC7288B7Dull, 0x2CC774E17B48FCF0ull, 0xADE07AD94EBE348Full,
+     0x4668225086AE65CDull, 0x9F4ADAFC19D2E1FFull},
+    {0xF5AB6C7F2438B113ull, 0xB8B9796BB1F8ABA5ull, 0xA2D5F3955BB52BB8ull,
+     0x759F72DA1A29C887ull, 0x9BFFC8BC2E6E6D3Cull, 0x26ABC4027348221Aull,
+     0x00C35B0456E7D008ull, 0xD7C3B53D4317B389ull},
+    {0xD7C06A9EB9C2AA67ull, 0x2B43D2BB4D8893A0ull, 0x448BA8089D6BBC82ull,
+     0xBDCA1E61406CE83Cull, 0x2346CB4ACD52A7FAull, 0xD8E8CEBE1223FD34ull,
+     0x1E276720DFBF519Bull, 0x6E86A68A604294CDull},
+    {0x4754F6FF7C5093B6ull, 0x134A362980B4C4C6ull, 0x02DBBBDEE06A0113ull,
+     0x3AD02F87FFCB0877ull, 0x95D242C0C99184A9ull, 0xEC3E78DF8E171883ull,
+     0x880766F8FB62E142ull, 0x5774BDBEEFE2ECEFull},
+    {0xFA8519C45B9AC144ull, 0xF1B5F11E955BE024ull, 0x81A79B0A80E52D42ull,
+     0x0B708908BEC35874ull, 0x39007F7D2AB4272Aull, 0xA2140C50BCAADE31ull,
+     0x567953E428C5AAB0ull, 0xA30A6793FACD70F8ull},
+    {0x88006BA8EEDDEC9Bull, 0xA7488475FB7801F9ull, 0xE287BCA951A6E934ull,
+     0x8E75D7DF95AAD7BDull, 0x8DDE60C72F00B2E1ull, 0xEF8582121EB36C60ull,
+     0x80BF8896DBA52C6Bull, 0xAE212787FAFA2E31ull},
+    {0x9837A02183186435ull, 0xFC3AC585D8B60BABull, 0x7513F58AD93C7992ull,
+     0x3EB5A96563AC9A26ull, 0x6A369CAE5860A637ull, 0x6F518B805A9B45EBull,
+     0xE6656B6E74591BE8ull, 0x03E2529E04D270DDull},
+    {0x72CFDDCE6FFC0F89ull, 0xCD7F97B19BB3F949ull, 0x82644123E65EC706ull,
+     0x5110936324CC613Dull, 0x2F4A3ACDB6E2E929ull, 0x31E1002A6DDEDF36ull,
+     0xB13A20047EDB0E18ull, 0xF274CDBF44301FABull},
+    {0x1DB2F773453926EEull, 0x38BB71AB95C29B6Aull, 0x92C1A0E9A1A2A4DAull,
+     0x0A37D050C9AD0A6Eull, 0xD7F1DAD7AE488E0Eull, 0x724A43415990C17Aull,
+     0xDF32F1C71003A688ull, 0xA6D181EA54CAC105ull},
+    {0x27069F86A0023C59ull, 0x4802D57205AE7ACDull, 0xA3B257DDCFE9597Dull,
+     0xFA1C90DB1D068B05ull, 0xBF58DF4988E56A62ull, 0xB1D05A43331C9A62ull,
+     0xD3885EACE06C6AE2ull, 0x645124896F3363C5ull},
+    {0x2CAC63167BB6DFB8ull, 0xEBC01D4E9CAD8F6Dull, 0xEE939E5DEB65A94Cull,
+     0xC1872BC1E8C7E597ull, 0xCBB3B48ABEA3F935ull, 0xD5E36FB19A4436CFull,
+     0xC3C7A129D4BADA16ull, 0x895006606ACAF7AAull},
+    {0x232AC9194D2634ECull, 0xD1EB2A7103B56D0Dull, 0xC77F706B0409425Aull,
+     0x7C1361A0974E3FD5ull, 0x4359422326151812ull, 0xDABF671CA5F3FC38ull,
+     0x0AEE8B4964C9A80Eull, 0xEFFB22551D3D225Full},
+    {0x6668F7DE6E4BA454ull, 0x33E1048790F618AFull, 0x0F9CDFBBDB62E113ull,
+     0x9FA1A91FC63FB239ull, 0x64595DEEAA8A2271ull, 0x35E71D20801E7483ull,
+     0xD82D7E79D142F612ull, 0xBA4B6600F25A33FEull},
+    {0x2C7E2B3FDFD58C27ull, 0x9B2E24A5EC33F85Dull, 0x5269136821B51004ull,
+     0xCAF3CBA5352DF391ull, 0x4D2380F14167F6ACull, 0x41B72E326872F5AFull,
+     0x5B43216C9D058B28ull, 0xD0C016BBB650292Aull},
+    {0x5467777A0EF33FCBull, 0xBAEAB899997E07DEull, 0x9851A1A8895ECAB6ull,
+     0x8CA5F6FDA29F1705ull, 0xCC4602E069497728ull, 0x3358603908632AECull,
+     0xBF7EDEEBAE872D03ull, 0xC84332C49E298700ull},
+    {0x10800F61FDC6FFEEull, 0x8E71F723FF36A58Aull, 0xF82EEB504F450A2Aull,
+     0x356037B45247E035ull, 0x69DC8BF6D5642C75ull, 0x1A7D71BA0C794DCBull,
+     0x717625EC7025D0C8ull, 0xC5B16D653B526F98ull},
+    {0xC3332F21FCF9353Eull, 0x410FB53C081C1E48ull, 0x80DDA11931F4DA34ull,
+     0x581647EA07D56297ull, 0x762DA15445101202ull, 0x7F38819F3FC36BD1ull,
+     0x07363A2FCFC75C2Bull, 0xAEDB7A939981DE72ull},
+    {0x099C4A9511D70019ull, 0xFE303D0DCD28A8C5ull, 0xE58634C0FFCB40F6ull,
+     0xDFF372E7DBD4B092ull, 0xA33A58219FCAD09Cull, 0xCD7C3241A60E7A96ull,
+     0x38219A6A1996A8B0ull, 0x99B6D144D7A60904ull},
+    {0x30E2377AC5E67EF5ull, 0x31E058DD8EA10F84ull, 0x5FB57E154B9E64BFull,
+     0xE4E585D3AF94EF97ull, 0x8F9AE0A30DD3A9B1ull, 0x289D730C3FDAB25Eull,
+     0xBFE76D6C3255A7BFull, 0x6391A94D4FAD0EFBull},
+};
+
+HWY_ATTR void Xorshift128PlusTest_TestGolden() {
+  HWY_ALIGN Xorshift128Plus rng(12345);
+  for (uint64_t vector = 0; vector < kVectors; ++vector) {
+    HWY_ALIGN uint64_t lanes[Xorshift128Plus::N];
+    rng.Fill(lanes);
+#if PRINT_RESULTS
+    Print(lanes);
+#else
+    for (size_t i = 0; i < Xorshift128Plus::N; ++i) {
+      ASSERT_EQ(kExpected[vector][i], lanes[i])
+          << "Where vector=" << vector << " i=" << i;
+    }
+#endif
+  }
+}
+// Ensures Xorshift128+ returns consistent and unchanging values.
+TEST(Xorshift128PlusTest, TestGolden) { Xorshift128PlusTest_TestGolden(); }
+
+// Output changes when given different seeds
+HWY_ATTR void Xorshift128PlusTest_TestSeedChanges() {
+  HWY_ALIGN uint64_t lanes[Xorshift128Plus::N];
+
+  std::vector<uint64_t> first;
+  constexpr size_t kNumSeeds = 16384;
+  first.reserve(kNumSeeds);
+
+  // All 14-bit seeds
+  for (size_t seed = 0; seed < kNumSeeds; ++seed) {
+    HWY_ALIGN Xorshift128Plus rng(seed);
+
+    rng.Fill(lanes);
+    first.push_back(lanes[0]);
+  }
+
+  // All outputs are unique
+  ASSERT_EQ(kNumSeeds, first.size());
+  std::sort(first.begin(), first.end());
+  first.erase(std::unique(first.begin(), first.end()), first.end());
+  EXPECT_EQ(kNumSeeds, first.size());
+}
+
+TEST(Xorshift128PlusTest, TestSeedChanges) {
+  Xorshift128PlusTest_TestSeedChanges();
+}
+
+HWY_ATTR void Xorshift128PlusTest_TestFloat() {
+  ThreadPoolInternal pool(8);
+
+  // All 14-bit seeds
+  pool.Run(0, 16384, ThreadPool::SkipInit(),
+           [](const int seed, const int /*thread*/) HWY_ATTR {
+             HWY_ALIGN Xorshift128Plus rng(seed);
+
+             const HWY_FULL(uint32_t) du;
+             const HWY_FULL(float) df;
+             HWY_ALIGN uint64_t batch[Xorshift128Plus::N];
+             HWY_ALIGN float lanes[df.N];
+             double sum = 0.0;
+             size_t count = 0;
+             const size_t kReps = 2000;
+             for (size_t reps = 0; reps < kReps; ++reps) {
+               rng.Fill(batch);
+               for (size_t i = 0; i < Xorshift128Plus::N * 2; i += df.N) {
+                 const auto bits =
+                     Load(du, reinterpret_cast<const uint32_t*>(batch) + i);
+                 // 1.0 + 23 random mantissa bits = [1, 2)
+                 const auto rand12 = BitCast(
+                     df, hwy::ShiftRight<9>(bits) | Set(du, 0x3F800000));
+                 const auto rand01 = rand12 - Set(df, 1.0f);
+                 Store(rand01, df, lanes);
+                 for (float lane : lanes) {
+                   sum += lane;
+                   count += 1;
+                   EXPECT_LE(lane, 1.0f);
+                   EXPECT_GE(lane, 0.0f);
+                 }
+               }
+             }
+
+             // Verify average (uniform distribution)
+             EXPECT_NEAR(0.5, sum / count, 0.02);
+           });
+}
+
+// Not more than one 64-bit zero
+TEST(Xorshift128PlusTest, TestNotZero) {
+  ThreadPoolInternal pool(8);
+
+  pool.Run(0, 2000, ThreadPool::SkipInit(),
+           [](const int task, const int /*thread*/) HWY_ATTR {
+             HWY_ALIGN uint64_t lanes[Xorshift128Plus::N];
+
+             HWY_ALIGN Xorshift128Plus rng(task);
+             size_t num_zero = 0;
+             for (size_t vectors = 0; vectors < 10000; ++vectors) {
+               rng.Fill(lanes);
+               for (uint64_t lane : lanes) {
+                 num_zero += static_cast<size_t>(lane == 0);
+               }
+             }
+             EXPECT_LE(num_zero, 1);
+           });
+}
+
+TEST(Xorshift128PlusTest, TestFloat) { Xorshift128PlusTest_TestFloat(); }
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace jxl
diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt
new file mode 100644
index 000000000..8b8b55ccc
--- /dev/null
+++ b/plugins/CMakeLists.txt
@@ -0,0 +1,17 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_subdirectory(gdk-pixbuf)
+add_subdirectory(gimp)
+add_subdirectory(mime)
diff --git a/plugins/gdk-pixbuf/CMakeLists.txt b/plugins/gdk-pixbuf/CMakeLists.txt
new file mode 100644
index 000000000..8073a9c4a
--- /dev/null
+++ b/plugins/gdk-pixbuf/CMakeLists.txt
@@ -0,0 +1,39 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+find_package(PkgConfig)
+pkg_check_modules(Gdk-Pixbuf IMPORTED_TARGET gdk-pixbuf-2.0>=2.38)
+
+if (NOT Gdk-Pixbuf_FOUND)
+  message(WARNING "GDK Pixbuf development libraries not found, \
+                   the Gdk-Pixbuf plugin will not be built")
+  return ()
+endif ()
+
+add_library(pixbufloader-jxl SHARED pixbufloader-jxl.c c_interop.cc  c_interop.h)
+target_link_libraries(pixbufloader-jxl jpegxl-static PkgConfig::Gdk-Pixbuf)
+
+install(TARGETS pixbufloader-jxl LIBRARY DESTINATION /usr/lib/x86_64-linux-gnu/gdk-pixbuf-2.0/2.10.0/loaders)
+
+find_program(GDK_PIXBUF_QUERY_LOADERS_PROG gdk-pixbuf-query-loaders /usr/lib/x86_64-linux-gnu/gdk-pixbuf-2.0)
+if(NOT ${GDK_PIXBUF_QUERY_LOADERS_PROG} STREQUAL "GDK_PIXBUF_QUERY_LOADERS_PROG-NOTFOUND")
+  install(CODE "execute_process(COMMAND ${GDK_PIXBUF_QUERY_LOADERS_PROG} --update-cache)")
+else()
+  message(WARNING "gdk-pixbuf-query-loeaders not found")
+endif()
+
+# Instead of the following, we might instead add the
+# mime type image/x-jxl to
+# /usr/share/thumbnailers/gdk-pixbuf-thumbnailer.thumbnailer
+install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/jxl.thumbnailer DESTINATION /usr/share/thumbnailers/)
diff --git a/plugins/gdk-pixbuf/c_interop.cc b/plugins/gdk-pixbuf/c_interop.cc
new file mode 100644
index 000000000..01b27b79c
--- /dev/null
+++ b/plugins/gdk-pixbuf/c_interop.cc
@@ -0,0 +1,99 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "c_interop.h"
+
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/dec_file.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+extern "C" uint8_t *JxlMemoryToPixels(const uint8_t *data, size_t size,
+                                      size_t *stride, size_t *xsize,
+                                      size_t *ysize, int *has_alpha) {
+  jxl::PaddedBytes bytes(size);
+  memcpy(bytes.data(), data, size);
+  jxl::DecompressParams params;
+  jxl::ThreadPoolInternal pool(4);
+  jxl::CodecInOut io;
+
+  if (jxl::DecodeFile(params, bytes, &io, nullptr, &pool) == false)
+    return nullptr;
+
+  jxl::Image3B converted;
+
+  if (!io.Main().CopyToSRGB(jxl::Rect(io), &converted, &pool)) {
+    return nullptr;
+  }
+
+  size_t io_stride = io.Main().color().PixelsPerRow();
+
+  if (io.Main().HasAlpha()) {
+    uint8_t *image = new uint8_t[4 * io_stride * io.ysize()];
+
+    if (image == nullptr) {
+      return nullptr;
+    }
+
+    *stride = 4 * io_stride;
+    *xsize = io.xsize();
+    *ysize = io.ysize();
+    *has_alpha = 1;
+    const int alpha_right_shift_amount =
+        static_cast<int>(io.metadata.alpha_bits) - 8;
+    for (int y = 0; y < *ysize; ++y) {
+      uint8_t *JXL_RESTRICT const row = image + y * *stride;
+      const uint16_t *const alpha_row = io.Main().alpha().ConstRow(y);
+      const uint8_t *JXL_RESTRICT const red_row = converted.ConstPlaneRow(0, y);
+      const uint8_t *JXL_RESTRICT const green_row =
+          converted.ConstPlaneRow(1, y);
+      const uint8_t *JXL_RESTRICT const blue_row =
+          converted.ConstPlaneRow(2, y);
+      for (int x = 0; x < *xsize; ++x) {
+        row[4 * x] = red_row[x];
+        row[4 * x + 1] = green_row[x];
+        row[4 * x + 2] = blue_row[x];
+        row[4 * x + 3] = alpha_row[x] >> alpha_right_shift_amount;
+      }
+    }
+    return image;
+  } else {
+    uint8_t *image = new uint8_t[3 * io_stride * io.ysize()];
+
+    if (image == nullptr) {
+      return nullptr;
+    }
+
+    *stride = 3 * io_stride;
+    *xsize = io.xsize();
+    *ysize = io.ysize();
+    *has_alpha = 0;
+    for (int y = 0; y < *ysize; ++y) {
+      uint8_t *JXL_RESTRICT const row = image + y * *stride;
+      const uint8_t *JXL_RESTRICT const red_row = converted.ConstPlaneRow(0, y);
+      const uint8_t *JXL_RESTRICT const green_row =
+          converted.ConstPlaneRow(1, y);
+      const uint8_t *JXL_RESTRICT const blue_row =
+          converted.ConstPlaneRow(2, y);
+      for (int x = 0; x < *xsize; ++x) {
+        row[3 * x] = red_row[x];
+        row[3 * x + 1] = green_row[x];
+        row[3 * x + 2] = blue_row[x];
+      }
+    }
+    return image;
+  }
+}
+
+extern "C" void JxlFreePixels(uint8_t *pixels) { delete[] pixels; }
diff --git a/plugins/gdk-pixbuf/c_interop.h b/plugins/gdk-pixbuf/c_interop.h
new file mode 100644
index 000000000..c632c40de
--- /dev/null
+++ b/plugins/gdk-pixbuf/c_interop.h
@@ -0,0 +1,42 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_C_INTEROP
+#define JXL_C_INTEROP
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#ifdef __cplusplus
+#define CLINK extern "C"
+#else
+#define CLINK
+#endif
+
+// Input: byte array of length size.
+// Returns a pointer to an owned chunk of memory, containing interleaved R, G,
+// B, A channels (null there was a decoding error). If decoding was successful,
+// writes to stride the stride of the returned buffer, to xsize and ysize the
+// size of the image in pixels, and 0/1 to has_alpha if the image doesn't/has an
+// alpha channel. Returned pointer should be freed with JxlFreePixels.
+// TODO(firsching): remove this file and use the generic C API directly.
+CLINK uint8_t *JxlMemoryToPixels(const uint8_t *data, size_t size,
+                                 size_t *stride, size_t *xsize, size_t *ysize,
+                                 int *has_alpha);
+
+CLINK void JxlFreePixels(uint8_t *pixels);
+
+#undef CLINK
+
+#endif
diff --git a/plugins/gdk-pixbuf/jxl.thumbnailer b/plugins/gdk-pixbuf/jxl.thumbnailer
new file mode 100644
index 000000000..0ce169a17
--- /dev/null
+++ b/plugins/gdk-pixbuf/jxl.thumbnailer
@@ -0,0 +1,4 @@
+[Thumbnailer Entry]
+TryExec=/usr/bin/gdk-pixbuf-thumbnailer
+Exec=/usr/bin/gdk-pixbuf-thumbnailer -s %s %u %o
+MimeType=image/x-jxl;
diff --git a/plugins/gdk-pixbuf/pixbufloader-jxl.c b/plugins/gdk-pixbuf/pixbufloader-jxl.c
new file mode 100644
index 000000000..8227e26f3
--- /dev/null
+++ b/plugins/gdk-pixbuf/pixbufloader-jxl.c
@@ -0,0 +1,207 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#define GDK_PIXBUF_ENABLE_BACKEND
+#include <gdk-pixbuf/gdk-pixbuf.h>
+#undef GDK_PIXBUF_ENABLE_BACKEND
+
+#include "c_interop.h"
+
+static void DestroyPixels(guchar *pixels, gpointer data) {
+  JxlFreePixels(pixels);
+}
+
+typedef struct {
+  GdkPixbufModuleSizeFunc size_func;
+  GdkPixbufModuleUpdatedFunc update_func;
+  GdkPixbufModulePreparedFunc prepare_func;
+  gpointer user_data;
+  GdkPixbuf *pixbuf;
+  GError **error;
+
+  FILE *increment_buffer;
+  char *increment_buffer_ptr;
+  size_t increment_buffer_size;
+
+} JxlContext;
+
+static GdkPixbuf *gdk_pixbuf__jxl_image_load(FILE *f, GError **error) {
+  size_t data_size;
+  int status;
+  gpointer data;
+
+  // Get data size
+  status = fseek(f, 0, SEEK_END);
+  if (status) {
+    g_set_error(error, GDK_PIXBUF_ERROR, GDK_PIXBUF_ERROR_FAILED,
+                "Failed to find end of file");
+  }
+  data_size = ftell(f);
+  fseek(f, 0, SEEK_SET);
+  status = fseek(f, 0, SEEK_SET);
+  if (status) {
+    g_set_error(error, GDK_PIXBUF_ERROR, GDK_PIXBUF_ERROR_FAILED,
+                "Failed to set pointer to beginning of file");
+  }
+
+  // Get data
+  data = g_malloc(data_size);
+  status = (fread(data, data_size, 1, f) == 1);
+  if (!status) {
+    g_set_error(error, GDK_PIXBUF_ERROR, GDK_PIXBUF_ERROR_FAILED,
+                "Failed to read file");
+    g_free(data);
+    return NULL;
+  }
+  size_t xsize, ysize, stride;
+  int has_alpha;
+  uint8_t *decoded =
+      JxlMemoryToPixels(data, data_size, &stride, &xsize, &ysize, &has_alpha);
+  g_free(data);
+  if (!decoded) {
+    g_set_error(error, GDK_PIXBUF_ERROR, GDK_PIXBUF_ERROR_FAILED,
+                "Failed to decode data");
+    return NULL;
+  }
+
+  GdkPixbuf *pixbuf =
+      gdk_pixbuf_new_from_data(decoded, GDK_COLORSPACE_RGB, has_alpha, 8, xsize,
+                               ysize, stride, &DestroyPixels, NULL);
+
+  if (!pixbuf) {
+    g_set_error(error, GDK_PIXBUF_ERROR, GDK_PIXBUF_ERROR_FAILED,
+                "Failed to create output pixbuf");
+    JxlFreePixels(decoded);
+
+    return NULL;
+  }
+
+  return pixbuf;
+}
+
+static gpointer gdk_pixbuf__jxl_image_begin_load(
+    GdkPixbufModuleSizeFunc size_func, GdkPixbufModulePreparedFunc prepare_func,
+    GdkPixbufModuleUpdatedFunc update_func, gpointer user_data,
+    GError **error) {
+  JxlContext *context = g_new(JxlContext, 1);
+  context->size_func = size_func;
+  context->prepare_func = prepare_func;
+  context->update_func = update_func;
+  context->user_data = user_data;
+  context->error = error;
+
+  context->increment_buffer = open_memstream(&context->increment_buffer_ptr,
+                                             &context->increment_buffer_size);
+
+  if (!context->increment_buffer) {
+    perror("Cannot create increment buffer.");
+    g_free(context);
+    return NULL;
+  }
+
+  return context;
+}
+
+static gboolean gdk_pixbuf__jxl_image_stop_load(gpointer user_context,
+                                                GError **error) {
+  JxlContext *context = (JxlContext *)user_context;
+
+  int status = fflush(context->increment_buffer);
+  status |= fseek(context->increment_buffer, 0L, SEEK_SET);
+
+  if (status != 0) {
+    perror("Cannot flush and rewind increment buffer.");
+    fclose(context->increment_buffer);
+    free(context->increment_buffer_ptr);
+    g_free(context);
+    return FALSE;
+  }
+
+  context->pixbuf =
+      gdk_pixbuf__jxl_image_load(context->increment_buffer, error);
+
+  gint width = gdk_pixbuf_get_width(context->pixbuf);
+  gint height = gdk_pixbuf_get_height(context->pixbuf);
+  if (context->size_func) {
+    context->size_func(&width, &height, context->user_data);
+  }
+
+  if (context->prepare_func) {
+    (*context->prepare_func)(context->pixbuf, NULL, context->user_data);
+  }
+
+  if (context->update_func) {
+    (*context->update_func)(
+        context->pixbuf, 0, 0, gdk_pixbuf_get_width(context->pixbuf),
+        gdk_pixbuf_get_height(context->pixbuf), context->user_data);
+  }
+
+  fclose(context->increment_buffer);
+
+  free(context->increment_buffer_ptr);
+
+  g_object_unref(context->pixbuf);
+  g_free(context);
+
+  return TRUE;
+}
+
+static gboolean gdk_pixbuf__jxl_image_load_increment(gpointer user_context,
+                                                     const guchar *buf,
+                                                     guint size,
+                                                     GError **error) {
+  JxlContext *context = (JxlContext *)user_context;
+
+  int status = fwrite(buf, size, sizeof(guchar), context->increment_buffer);
+
+  if (status != sizeof(guchar)) {
+    g_set_error(error, GDK_PIXBUF_ERROR, GDK_PIXBUF_ERROR_FAILED,
+                "Can't write to increment buffer.");
+    return FALSE;
+  }
+
+  status = fflush(context->increment_buffer);
+
+  if (status != 0) {
+    g_set_error(error, GDK_PIXBUF_ERROR, GDK_PIXBUF_ERROR_FAILED,
+                "Can't flush the increment buffer.");
+    return FALSE;
+  }
+
+  return TRUE;
+}
+
+void fill_vtable(GdkPixbufModule *module) {
+  module->load = gdk_pixbuf__jxl_image_load;
+  module->begin_load = gdk_pixbuf__jxl_image_begin_load;
+  module->stop_load = gdk_pixbuf__jxl_image_stop_load;
+  module->load_increment = gdk_pixbuf__jxl_image_load_increment;
+}
+
+void fill_info(GdkPixbufFormat *info) {
+  static GdkPixbufModulePattern signature[] = {
+      {"\xd7\x4c\x4d\x0a", "    ", 100}, {NULL, NULL, 0}};
+
+  static gchar *mime_types[] = {"image/x-jxl", NULL};
+
+  static gchar *extensions[] = {"jxl", "j", NULL};
+
+  info->name = "JPEG XL";
+  info->signature = signature;
+  info->description = "JPEG XL image";
+  info->mime_types = mime_types;
+  info->extensions = extensions;
+  info->flags = GDK_PIXBUF_FORMAT_THREADSAFE;
+  info->license = "Apache 2";
+}
diff --git a/plugins/gimp/CMakeLists.txt b/plugins/gimp/CMakeLists.txt
new file mode 100644
index 000000000..74c5e759f
--- /dev/null
+++ b/plugins/gimp/CMakeLists.txt
@@ -0,0 +1,36 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+find_package(PkgConfig)
+pkg_check_modules(Gimp IMPORTED_TARGET gimp-2.0>=2.10 gimpui-2.0>=2.10)
+
+if (NOT Gimp_FOUND)
+  message(WARNING "Gimp development libraries not found, the Gimp plugin will not be built")
+  return ()
+endif ()
+
+option(JPEGXL_ENABLE_GIMP_SAVING "Enable saving to JPEG XL from GIMP" OFF)
+
+add_executable(file-j
+  common.h
+  file-j-load.cc
+  file-j-load.h
+  file-j-save.cc
+  file-j-save.h
+  file-j.cc)
+if (JPEGXL_ENABLE_GIMP_SAVING)
+  target_compile_definitions(file-j PRIVATE -DJPEGXL_ENABLE_GIMP_SAVING=1)
+endif ()
+target_link_libraries(file-j jpegxl-static jpegxl_threads PkgConfig::Gimp)
+install(TARGETS file-j RUNTIME DESTINATION lib/gimp/2.0/plug-ins/file-j)
diff --git a/plugins/gimp/common.h b/plugins/gimp/common.h
new file mode 100644
index 000000000..d26a726ea
--- /dev/null
+++ b/plugins/gimp/common.h
@@ -0,0 +1,51 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PLUGINS_GIMP_COMMON_H_
+#define PLUGINS_GIMP_COMMON_H_
+
+#include <libgimp/gimp.h>
+
+namespace jxl {
+
+// FromFloat expects a value between 0 and 255, and ToFloat returns such values
+// from GIMP values.
+template <GimpPrecision>
+struct BufferFormat;
+template <>
+struct BufferFormat<GIMP_PRECISION_U8_GAMMA> {
+  using Sample = uint8_t;
+  static Sample FromFloat(const float x) {
+    return static_cast<Sample>(std::round(x));
+  }
+  static float ToFloat(const Sample s) { return s; }
+};
+template <>
+struct BufferFormat<GIMP_PRECISION_U16_GAMMA> {
+  using Sample = uint16_t;
+  static Sample FromFloat(const float x) {
+    return static_cast<Sample>(std::round(x * (65535.f / 255.f)));
+  }
+  static float ToFloat(const Sample s) { return s * (255.f / 65535.f); }
+};
+template <>
+struct BufferFormat<GIMP_PRECISION_FLOAT_GAMMA> {
+  using Sample = float;
+  static Sample FromFloat(const float x) { return x * (1.f / 255.f); }
+  static float ToFloat(const Sample s) { return s * 255.f; }
+};
+
+}  // namespace jxl
+
+#endif  // PLUGINS_GIMP_COMMON_H_
diff --git a/plugins/gimp/file-j-load.cc b/plugins/gimp/file-j-load.cc
new file mode 100644
index 000000000..07e6980f7
--- /dev/null
+++ b/plugins/gimp/file-j-load.cc
@@ -0,0 +1,162 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "plugins/gimp/file-j-load.h"
+
+// Defined by both FUIF and glib.
+#undef MAX
+#undef MIN
+#undef CLAMP
+
+#include "jxl/base/file_io.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/dec_file.h"
+#include "plugins/gimp/common.h"
+
+namespace jxl {
+
+namespace {
+
+template <GimpPrecision precision, bool has_alpha, size_t num_channels>
+void FillBuffer(
+    const CodecInOut& io,
+    std::vector<typename BufferFormat<precision>::Sample>* const pixel_data) {
+  pixel_data->reserve(io.xsize() * io.ysize() * (num_channels + has_alpha));
+  const float alpha_multiplier =
+      has_alpha ? 255.f / ((1u << io.metadata.alpha_bits) - 1.f) : 0.f;
+  for (size_t y = 0; y < io.ysize(); ++y) {
+    const float* rows[num_channels];
+    for (size_t c = 0; c < num_channels; ++c) {
+      rows[c] = io.Main().color().ConstPlaneRow(c, y);
+    }
+    const uint16_t* const alpha_row =
+        has_alpha ? io.Main().alpha().ConstRow(y) : nullptr;
+    for (size_t x = 0; x < io.xsize(); ++x) {
+      for (const float* const row : rows) {
+        pixel_data->push_back(BufferFormat<precision>::FromFloat(
+            std::max(0.f, std::min(255.f, row[x]))));
+      }
+      if (has_alpha) {
+        pixel_data->push_back(BufferFormat<precision>::FromFloat(
+            alpha_row[x] * alpha_multiplier));
+      }
+    }
+  }
+}
+
+template <GimpPrecision precision>
+Status FillGimpLayer(const gint32 layer, const CodecInOut& io,
+                     GimpImageType layer_type) {
+  std::vector<typename BufferFormat<precision>::Sample> pixel_data;
+  switch (layer_type) {
+    case GIMP_GRAY_IMAGE:
+      FillBuffer<precision, /*has_alpha=*/false, /*num_channels=*/1>(
+          io, &pixel_data);
+      break;
+    case GIMP_GRAYA_IMAGE:
+      FillBuffer<precision, /*has_alpha=*/true, /*num_channels=*/1>(
+          io, &pixel_data);
+      break;
+    case GIMP_RGB_IMAGE:
+      FillBuffer<precision, /*has_alpha=*/false, /*num_channels=*/3>(
+          io, &pixel_data);
+      break;
+    case GIMP_RGBA_IMAGE:
+      FillBuffer<precision, /*has_alpha=*/true, /*num_channels=*/3>(
+          io, &pixel_data);
+      break;
+    default:
+      return false;
+  }
+
+  GeglBuffer* buffer = gimp_drawable_get_buffer(layer);
+  gegl_buffer_set(buffer, GEGL_RECTANGLE(0, 0, io.xsize(), io.ysize()), 0,
+                  nullptr, pixel_data.data(), GEGL_AUTO_ROWSTRIDE);
+  g_clear_object(&buffer);
+  return true;
+}
+
+}  // namespace
+
+Status LoadJpegXlImage(const gchar* const filename, gint32* const image_id) {
+  PaddedBytes compressed;
+  JXL_RETURN_IF_ERROR(ReadFile(filename, &compressed));
+
+  // TODO(deymo): Use C API instead of the ThreadPoolInternal.
+  ThreadPoolInternal pool;
+  DecompressParams dparams;
+  CodecInOut io;
+  JXL_RETURN_IF_ERROR(
+      DecodeFile(dparams, compressed, &io, /*aux_out=*/nullptr, &pool));
+
+  JXL_RETURN_IF_ERROR(io.TransformTo(io.metadata.color_encoding, &pool));
+
+  const PaddedBytes& icc = io.metadata.color_encoding.icc;
+  GimpColorProfile* profile =
+      gimp_color_profile_new_from_icc_profile(icc.data(), icc.size(),
+                                              /*error=*/nullptr);
+  if (profile == nullptr) {
+    return JXL_FAILURE(
+        "Failed to create GIMP color profile from %zu bytes of ICC data",
+        icc.size());
+  }
+
+  GimpImageBaseType image_type;
+  GimpImageType layer_type;
+
+  if (io.Main().IsGray()) {
+    image_type = GIMP_GRAY;
+    if (io.Main().HasAlpha()) {
+      layer_type = GIMP_GRAYA_IMAGE;
+    } else {
+      layer_type = GIMP_GRAY_IMAGE;
+    }
+  } else {
+    image_type = GIMP_RGB;
+    if (io.Main().HasAlpha()) {
+      layer_type = GIMP_RGBA_IMAGE;
+    } else {
+      layer_type = GIMP_RGB_IMAGE;
+    }
+  }
+
+  GimpPrecision precision;
+  Status (*fill_layer)(gint32 layer, const CodecInOut& io, GimpImageType);
+  if (io.metadata.bits_per_sample <= 8) {
+    precision = GIMP_PRECISION_U8_GAMMA;
+    fill_layer = &FillGimpLayer<GIMP_PRECISION_U8_GAMMA>;
+  } else if (io.metadata.bits_per_sample <= 16) {
+    precision = GIMP_PRECISION_U16_GAMMA;
+    fill_layer = &FillGimpLayer<GIMP_PRECISION_U16_GAMMA>;
+  } else {
+    precision = GIMP_PRECISION_FLOAT_GAMMA;
+    fill_layer = &FillGimpLayer<GIMP_PRECISION_FLOAT_GAMMA>;
+  }
+
+  *image_id = gimp_image_new_with_precision(io.xsize(), io.ysize(), image_type,
+                                            precision);
+  gimp_image_set_color_profile(*image_id, profile);
+  g_clear_object(&profile);
+  const gint32 layer = gimp_layer_new(
+      *image_id, "image", io.xsize(), io.ysize(), layer_type, /*opacity=*/100,
+      gimp_image_get_default_new_layer_mode(*image_id));
+  gimp_image_set_filename(*image_id, filename);
+  gimp_image_insert_layer(*image_id, layer, /*parent_id=*/-1, /*position=*/0);
+
+  JXL_RETURN_IF_ERROR(fill_layer(layer, io, layer_type));
+
+  return true;
+}
+
+}  // namespace jxl
diff --git a/plugins/gimp/file-j-load.h b/plugins/gimp/file-j-load.h
new file mode 100644
index 000000000..a17e8b83e
--- /dev/null
+++ b/plugins/gimp/file-j-load.h
@@ -0,0 +1,28 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PLUGINS_GIMP_FILE_J_LOAD_H_
+#define PLUGINS_GIMP_FILE_J_LOAD_H_
+
+#include <libgimp/gimp.h>
+
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+Status LoadJpegXlImage(const gchar* filename, gint32* image_id);
+
+}
+
+#endif
diff --git a/plugins/gimp/file-j-save.cc b/plugins/gimp/file-j-save.cc
new file mode 100644
index 000000000..afc4d3af5
--- /dev/null
+++ b/plugins/gimp/file-j-save.cc
@@ -0,0 +1,130 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "plugins/gimp/file-j-save.h"
+
+// Defined by both FUIF and glib.
+#undef MAX
+#undef MIN
+#undef CLAMP
+
+#include "jxl/base/file_io.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/enc_file.h"
+#include "plugins/gimp/common.h"
+
+namespace jxl {
+
+namespace {
+
+template <bool has_alpha, size_t alpha_bits = 16>
+Status ReadBuffer(const size_t xsize, const size_t ysize,
+                  const std::vector<float>& pixel_data, PaddedBytes icc,
+                  CodecInOut* const io) {
+  constexpr float alpha_multiplier =
+      has_alpha ? ((1u << alpha_bits) - 1.f) / 255.f : 0.f;
+  Image3F image(xsize, ysize);
+  ImageU alpha;
+  if (has_alpha) {
+    alpha = ImageU(xsize, ysize);
+  }
+  const float* current_sample = pixel_data.data();
+  for (size_t y = 0; y < ysize; ++y) {
+    float* rows[3];
+    for (size_t c = 0; c < 3; ++c) {
+      rows[c] = image.PlaneRow(c, y);
+    }
+    uint16_t* const alpha_row = has_alpha ? alpha.Row(y) : nullptr;
+    for (size_t x = 0; x < xsize; ++x) {
+      for (float* const row : rows) {
+        row[x] = BufferFormat<GIMP_PRECISION_FLOAT_GAMMA>::ToFloat(
+            *current_sample++);
+      }
+      if (has_alpha) {
+        alpha_row[x] = static_cast<uint16_t>(
+            std::round(alpha_multiplier *
+                       BufferFormat<GIMP_PRECISION_FLOAT_GAMMA>::ToFloat(
+                           *current_sample++)));
+      }
+    }
+  }
+
+  ColorEncoding color_encoding;
+  JXL_RETURN_IF_ERROR(
+      ColorManagement::SetProfile(std::move(icc), &color_encoding));
+  io->metadata.color_encoding = color_encoding;
+  io->SetFromImage(std::move(image), color_encoding);
+  if (has_alpha) {
+    io->metadata.alpha_bits = alpha_bits;
+    io->Main().SetAlpha(std::move(alpha));
+  }
+  return true;
+}
+
+}  // namespace
+
+Status SaveJpegXlImage(const gint32 image_id, const gint32 drawable_id,
+                       const gint32 orig_image_id,
+                       const gchar* const filename) {
+  GimpColorProfile* profile =
+      gimp_image_get_effective_color_profile(orig_image_id);
+  gsize icc_size;
+  const guint8* const icc_bytes =
+      gimp_color_profile_get_icc_profile(profile, &icc_size);
+  PaddedBytes icc;
+  icc.assign(icc_bytes, icc_bytes + icc_size);
+  g_clear_object(&profile);
+
+  const Babl* format;
+  Status (*read_buffer)(size_t, size_t, const std::vector<float>&, PaddedBytes,
+                        CodecInOut*);
+  const bool has_alpha = gimp_drawable_has_alpha(drawable_id);
+  if (has_alpha) {
+    format = babl_format("R'G'B'A float");
+    read_buffer = &ReadBuffer</*has_alpha=*/true>;
+  } else {
+    format = babl_format("R'G'B' float");
+    read_buffer = &ReadBuffer</*has_alpha=*/false>;
+  }
+
+  CodecInOut io;
+
+  GeglBuffer* gegl_buffer = gimp_drawable_get_buffer(drawable_id);
+
+  const Babl* native_format = gegl_buffer_get_format(gegl_buffer);
+  io.metadata.bits_per_sample = 8 *
+                                babl_format_get_bytes_per_pixel(native_format) /
+                                babl_format_get_n_components(native_format);
+
+  const GeglRectangle rect = *gegl_buffer_get_extent(gegl_buffer);
+  std::vector<float> pixel_data(rect.width * rect.height * (3 + has_alpha));
+  gegl_buffer_get(gegl_buffer, &rect, 1., format, pixel_data.data(),
+                  GEGL_AUTO_ROWSTRIDE, GEGL_ABYSS_NONE);
+  g_clear_object(&gegl_buffer);
+
+  JXL_RETURN_IF_ERROR(
+      read_buffer(rect.width, rect.height, pixel_data, std::move(icc), &io));
+  CompressParams params;
+  PassesEncoderState encoder_state;
+  PaddedBytes compressed;
+  ThreadPoolInternal pool;
+  params.butteraugli_distance = 1.f;
+  JXL_RETURN_IF_ERROR(EncodeFile(params, &io, &encoder_state, &compressed,
+                                 /*aux_out=*/nullptr, &pool));
+  JXL_RETURN_IF_ERROR(WriteFile(compressed, filename));
+
+  return true;
+}
+
+}  // namespace jxl
diff --git a/plugins/gimp/file-j-save.h b/plugins/gimp/file-j-save.h
new file mode 100644
index 000000000..40d7c2cfc
--- /dev/null
+++ b/plugins/gimp/file-j-save.h
@@ -0,0 +1,29 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef PLUGINS_GIMP_FILE_J_SAVE_H_
+#define PLUGINS_GIMP_FILE_J_SAVE_H_
+
+#include <libgimp/gimp.h>
+
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+Status SaveJpegXlImage(gint32 image_id, gint32 drawable_id,
+                       gint32 orig_image_id, const gchar* filename);
+
+}
+
+#endif
diff --git a/plugins/gimp/file-j.cc b/plugins/gimp/file-j.cc
new file mode 100644
index 000000000..e77ac79bd
--- /dev/null
+++ b/plugins/gimp/file-j.cc
@@ -0,0 +1,179 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <glib/gi18n.h>
+#include <libgimp/gimp.h>
+#include <libgimp/gimpui.h>
+#include <string.h>
+
+#include <string>
+
+#include "plugins/gimp/file-j-load.h"
+#include "plugins/gimp/file-j-save.h"
+
+namespace jxl {
+namespace {
+
+constexpr char kLoadProc[] = "file-j-load";
+constexpr char kSaveProc[] = "file-j-save";
+
+void Query() {
+  {
+    static char run_mode_name[] = "run-mode";
+    static char run_mode_description[] = "Run mode";
+    static char filename_name[] = "filename";
+    static char filename_description[] = "The name of the file to load";
+    static char raw_filename_name[] = "raw-filename";
+    static char raw_filename_description[] =
+        "The name of the file, as entered by the user";
+    static const GimpParamDef load_args[] = {
+        {GIMP_PDB_INT32, run_mode_name, run_mode_description},
+        {GIMP_PDB_STRING, filename_name, filename_description},
+        {GIMP_PDB_STRING, raw_filename_name, raw_filename_description},
+    };
+    static char image_name[] = "image";
+    static char image_description[] = "Loaded image";
+    static const GimpParamDef load_return_vals[] = {
+        {GIMP_PDB_IMAGE, image_name, image_description},
+    };
+
+    gimp_install_procedure(
+        /*name=*/kLoadProc, /*blurb=*/"Loads JPEG XL image files",
+        /*help=*/"Loads JPEG XL image files", /*author=*/"JPEG XL Project",
+        /*copyright=*/"JPEG XL Project", /*date=*/"2019",
+        /*menu_label=*/"JPEG XL image", /*image_types=*/nullptr,
+        /*type=*/GIMP_PLUGIN, /*n_params=*/G_N_ELEMENTS(load_args),
+        /*n_return_vals=*/G_N_ELEMENTS(load_return_vals), /*params=*/load_args,
+        /*return_vals=*/load_return_vals);
+    gimp_register_file_handler_mime(kLoadProc, "image/j");
+    gimp_register_magic_load_handler(kLoadProc, "j", "",
+                                     "0,string,\x0A\x04\x42\xD2\xD5\x4E\x12,0,"
+                                     "string,\xFF\x0A,0,string,\xFF\x58");
+  }
+
+  {
+    static char run_mode_name[] = "run-mode";
+    static char run_mode_description[] = "Run mode";
+    static char image_name[] = "image";
+    static char image_description[] = "Input image";
+    static char drawable_name[] = "drawable";
+    static char drawable_description[] = "Drawable to save";
+    static char filename_name[] = "filename";
+    static char filename_description[] = "The name of the file to save";
+    static char raw_filename_name[] = "raw-filename";
+    static char raw_filename_description[] = "The name of the file to save";
+    static const GimpParamDef save_args[] = {
+        {GIMP_PDB_INT32, run_mode_name, run_mode_description},
+        {GIMP_PDB_IMAGE, image_name, image_description},
+        {GIMP_PDB_DRAWABLE, drawable_name, drawable_description},
+        {GIMP_PDB_STRING, filename_name, filename_description},
+        {GIMP_PDB_STRING, raw_filename_name, raw_filename_description},
+    };
+
+    gimp_install_procedure(
+        /*name=*/kSaveProc, /*blurb=*/"Saves JPEG XL image files",
+        /*help=*/"Saves JPEG XL image files", /*author=*/"JPEG XL Project",
+        /*copyright=*/"JPEG XL Project", /*date=*/"2019",
+        /*menu_label=*/"JPEG XL image", /*image_types=*/"RGB*, GRAY*",
+        /*type=*/GIMP_PLUGIN, /*n_params=*/G_N_ELEMENTS(save_args),
+        /*n_return_vals=*/0, /*params=*/save_args,
+        /*return_vals=*/nullptr);
+    gimp_register_file_handler_mime(kSaveProc, "image/j");
+    gimp_register_save_handler(kSaveProc, "j", "");
+  }
+}
+
+void Run(const gchar* const name, const gint nparams,
+         const GimpParam* const params, gint* const nreturn_vals,
+         GimpParam** const return_vals) {
+  gegl_init(nullptr, nullptr);
+
+  static GimpParam values[2];
+
+  *nreturn_vals = 1;
+  *return_vals = values;
+
+  values[0].type = GIMP_PDB_STATUS;
+  values[0].data.d_status = GIMP_PDB_EXECUTION_ERROR;
+
+  if (strcmp(name, kLoadProc) == 0) {
+    if (nparams != 3) {
+      values[0].data.d_status = GIMP_PDB_CALLING_ERROR;
+      return;
+    }
+
+    const gchar* const filename = params[1].data.d_string;
+    gint32 image_id;
+    if (!LoadJpegXlImage(filename, &image_id)) {
+      values[0].data.d_status = GIMP_PDB_EXECUTION_ERROR;
+      return;
+    }
+
+    *nreturn_vals = 2;
+    values[0].data.d_status = GIMP_PDB_SUCCESS;
+    values[1].type = GIMP_PDB_IMAGE;
+    values[1].data.d_image = image_id;
+  } else if (strcmp(name, kSaveProc) == 0) {
+#if !JPEGXL_ENABLE_GIMP_SAVING
+    *nreturn_vals = 2;
+    values[0].data.d_status = GIMP_PDB_EXECUTION_ERROR;
+    values[1].type = GIMP_PDB_STRING;
+    static gchar compatibility_message[] =
+        "Saving is disabled in this build of the JPEG XL plugin. Rebuild it "
+        "with -DJPEGXL_ENABLE_GIMP_SAVING=1 to enable the functionality, but "
+        "be aware that files created in this fashion may not work in future "
+        "versions of the decoder.";
+    values[1].data.d_string = compatibility_message;
+    return;
+#endif
+    if (nparams != 5) {
+      values[0].data.d_status = GIMP_PDB_CALLING_ERROR;
+      return;
+    }
+
+    gint32 image_id = params[1].data.d_image;
+    gint32 drawable_id = params[2].data.d_drawable;
+    const gchar* const filename = params[3].data.d_string;
+    const gint32 orig_image_id = image_id;
+    const GimpExportReturn export_result = gimp_export_image(
+        &image_id, &drawable_id, "JPEG XL",
+        static_cast<GimpExportCapabilities>(GIMP_EXPORT_CAN_HANDLE_RGB |
+                                            GIMP_EXPORT_CAN_HANDLE_ALPHA));
+    switch (export_result) {
+      case GIMP_EXPORT_CANCEL:
+        values[0].data.d_status = GIMP_PDB_CANCEL;
+        return;
+      case GIMP_EXPORT_IGNORE:
+        break;
+      case GIMP_EXPORT_EXPORT:
+        break;
+    }
+    gimp_progress_init_printf(_("Saving JPEG XL file \"%s\""), filename);
+    if (!SaveJpegXlImage(image_id, drawable_id, orig_image_id, filename)) {
+      return;
+    }
+    if (image_id != orig_image_id) {
+      gimp_image_delete(image_id);
+    }
+    values[0].data.d_status = GIMP_PDB_SUCCESS;
+  }
+}
+
+}  // namespace
+}  // namespace jxl
+
+static const GimpPlugInInfo PLUG_IN_INFO = {nullptr, nullptr, &jxl::Query,
+                                            &jxl::Run};
+
+MAIN()
diff --git a/plugins/mime/CMakeLists.txt b/plugins/mime/CMakeLists.txt
new file mode 100644
index 000000000..0270034bd
--- /dev/null
+++ b/plugins/mime/CMakeLists.txt
@@ -0,0 +1,22 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+find_program(XDG_MIME_PROG xdg-mime)
+if(NOT ${XDG_MIME_PROG} STREQUAL "XDG_MIME_PROG-NOTFOUND")
+  install(CODE "execute_process(COMMAND ${XDG_MIME_PROG} install --novendor \
+                              ${CMAKE_CURRENT_SOURCE_DIR}/image-x-jxl.xml)")
+else()
+  message(WARNING "xdg-mime not found")
+endif()
+# Can be uninstalled by `xdg-mine uninstall image-x-jxl.xml`.
diff --git a/plugins/mime/image-x-jxl.xml b/plugins/mime/image-x-jxl.xml
new file mode 100644
index 000000000..6e86c926c
--- /dev/null
+++ b/plugins/mime/image-x-jxl.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<mime-info xmlns="http://www.freedesktop.org/standards/shared-mime-info">
+  <mime-type type="image/x-jxl">
+    <comment>JPEG XL image</comment>
+    <glob pattern="*.jxl"/>
+    <glob pattern="*.j"/>
+  </mime-type>
+</mime-info>
diff --git a/third_party/CMakeLists.txt b/third_party/CMakeLists.txt
new file mode 100644
index 000000000..79ccdbe2f
--- /dev/null
+++ b/third_party/CMakeLists.txt
@@ -0,0 +1,59 @@
+# Copyright (c) the JPEG XL Project
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+# Highway
+set(HWY_USE_GTEST ON CACHE BOOL "HWY_USE_GTEST")
+add_subdirectory(highway)
+
+# lodepng
+if( NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/lodepng/.git" )
+  message(SEND_ERROR "Please run git submodule update --init")
+endif()
+include(lodepng.cmake)
+
+# brotli
+if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/brotli/c/include/brotli/decode.h")
+  message(SEND_ERROR "Brotli is required to build, run \"git submodule update --init\" to get it, then run cmake again")
+endif ()
+add_subdirectory(brotli EXCLUDE_FROM_ALL)
+
+# *cms
+if (JPEGXL_ENABLE_SKCMS)
+  if( NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/skcms/.git" )
+    message(SEND_ERROR "Please run git submodule update --init")
+  endif()
+  include(skcms.cmake)
+else ()
+  if( NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/lcms/.git" )
+    message(SEND_ERROR "Please run git submodule update --init")
+  endif()
+  include(lcms2.cmake)
+endif()
+
+# gperftools (TCMalloc)
+if("${JPEGXL_ENABLE_TCMALLOC}")
+  if( NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/gperftools/.git" )
+    message(SEND_ERROR "Please run git submodule update --init")
+  endif()
+  include(gperftools.cmake)
+endif()  # JPEGXL_ENABLE_TCMALLOC
+
+add_subdirectory(googletest EXCLUDE_FROM_ALL)
+
+# brunsli
+# depends both on brotli and googletest, so should be added after them.
+if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/brunsli/c/include/brunsli/brunsli_decode.h")
+  message(SEND_ERROR "Brunsli is required to build, run \"git submodule update --init\" to get it, then run cmake again")
+endif ()
+add_subdirectory(brunsli EXCLUDE_FROM_ALL)
+
+# sjpeg
+if (JPEGXL_ENABLE_SJPEG)
+  if (NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/sjpeg/.git")
+    message(SEND_ERROR "Please run git submodule update --init")
+  endif()
+  include(sjpeg.cmake)
+endif ()
diff --git a/third_party/brotli b/third_party/brotli
new file mode 160000
index 000000000..35ef5c554
--- /dev/null
+++ b/third_party/brotli
@@ -0,0 +1 @@
+Subproject commit 35ef5c554d888bef217d449346067de05e269b30
diff --git a/third_party/brunsli b/third_party/brunsli
new file mode 160000
index 000000000..3eaaff1a1
--- /dev/null
+++ b/third_party/brunsli
@@ -0,0 +1 @@
+Subproject commit 3eaaff1a135c31903fb5bf114858d3b8bd6e68ee
diff --git a/third_party/dirent.cc b/third_party/dirent.cc
new file mode 100644
index 000000000..81e164af9
--- /dev/null
+++ b/third_party/dirent.cc
@@ -0,0 +1,140 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#if defined(_WIN32) || defined(_WIN64)
+#include "third_party/dirent.h"
+
+#include "jxl/base/status.h"
+
+#define NOMINMAX
+#include <Windows.h>
+
+#include <memory>
+#include <string>
+
+int mkdir(const char* path, mode_t /*mode*/) {
+  const LPSECURITY_ATTRIBUTES sec = nullptr;
+  if (!CreateDirectory(path, sec)) {
+    JXL_NOTIFY_ERROR("Failed to create directory %s", path);
+    return -1;
+  }
+  return 0;
+}
+
+// Modified from code bearing the following notice:
+// https://trac.wildfiregames.com/browser/ps/trunk/source/lib/sysdep/os/
+/* Copyright (C) 2010 Wildfire Games.
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included
+ * in all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+ * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+ * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+ * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+ * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+ */
+
+struct DIR {
+  HANDLE hFind;
+
+  WIN32_FIND_DATA findData;  // indeterminate if hFind == INVALID_HANDLE_VALUE
+
+  // readdir will return the address of this member.
+  // (must be stored in DIR to allow multiple independent
+  // opendir/readdir sequences).
+  dirent ent;
+
+  // used by readdir to skip the first FindNextFile.
+  size_t numCalls = 0;
+};
+
+static bool IsValidDirectory(const char* path) {
+  const DWORD fileAttributes = GetFileAttributes(path);
+
+  // path not found
+  if (fileAttributes == INVALID_FILE_ATTRIBUTES) return false;
+
+  // not a directory
+  if ((fileAttributes & FILE_ATTRIBUTE_DIRECTORY) == 0) return false;
+
+  return true;
+}
+
+DIR* opendir(const char* path) {
+  if (!IsValidDirectory(path)) {
+    errno = ENOENT;
+    return nullptr;
+  }
+
+  std::unique_ptr<DIR> d(new DIR);
+
+  // NB: "c:\\path" only returns information about that directory;
+  // trailing slashes aren't allowed. append "\\*" to retrieve its entries.
+  std::string searchPath(path);
+  if (searchPath.back() != '/' && searchPath.back() != '\\') {
+    searchPath += '\\';
+  }
+  searchPath += '*';
+
+  // (we don't defer FindFirstFile until readdir because callers
+  // expect us to return 0 if directory reading will/did fail.)
+  d->hFind = FindFirstFile(searchPath.c_str(), &d->findData);
+  if (d->hFind != INVALID_HANDLE_VALUE) return d.release();
+  if (GetLastError() == ERROR_NO_MORE_FILES) return d.release();  // empty
+
+  JXL_NOTIFY_ERROR("Failed to open directory %s", searchPath.c_str());
+  return nullptr;
+}
+
+int closedir(DIR* dir) {
+  delete dir;
+  return 0;
+}
+
+dirent* readdir(DIR* d) {
+  // "empty" case from opendir
+  if (d->hFind == INVALID_HANDLE_VALUE) return nullptr;
+
+  // until end of directory or a valid entry was found:
+  for (;;) {
+    if (d->numCalls++ != 0)  // (skip first call to FindNextFile - see opendir)
+    {
+      if (!FindNextFile(d->hFind, &d->findData)) {
+        JXL_ASSERT(GetLastError() == ERROR_NO_MORE_FILES);
+        SetLastError(0);
+        return nullptr;  // end of directory or error
+      }
+    }
+
+    // only return non-hidden and non-system entries
+    if ((d->findData.dwFileAttributes &
+         (FILE_ATTRIBUTE_HIDDEN | FILE_ATTRIBUTE_SYSTEM)) == 0) {
+      d->ent.d_name = d->findData.cFileName;
+      return &d->ent;
+    }
+  }
+}
+
+#endif  // #if defined(_WIN32) || defined(_WIN64)
diff --git a/third_party/dirent.h b/third_party/dirent.h
new file mode 100644
index 000000000..d6a2758c1
--- /dev/null
+++ b/third_party/dirent.h
@@ -0,0 +1,42 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef JXL_THIRD_PARTY_DIRENT_H_
+#define JXL_THIRD_PARTY_DIRENT_H_
+
+// Emulates POSIX readdir for Windows
+
+#if defined(_WIN32) || defined(_WIN64)
+
+#include <sys/stat.h>  // S_IFREG
+
+typedef unsigned int mode_t;
+int mkdir(const char* path, mode_t mode);
+
+struct dirent {
+  char* d_name;  // no path
+};
+
+#define stat _stat64
+
+#define S_ISDIR(m) (m & S_IFDIR)
+#define S_ISREG(m) (m & S_IFREG)
+
+struct DIR;
+DIR* opendir(const char* path);
+int closedir(DIR* dir);
+dirent* readdir(DIR* d);
+
+#endif  // #if defined(_WIN32) || defined(_WIN64)
+#endif  // JXL_THIRD_PARTY_DIRENT_H_
diff --git a/third_party/googletest b/third_party/googletest
new file mode 160000
index 000000000..0ea2d8f8f
--- /dev/null
+++ b/third_party/googletest
@@ -0,0 +1 @@
+Subproject commit 0ea2d8f8fa1601abb9ce713b7414e7b86f90bc61
diff --git a/third_party/gperftools b/third_party/gperftools
new file mode 160000
index 000000000..9e5b16287
--- /dev/null
+++ b/third_party/gperftools
@@ -0,0 +1 @@
+Subproject commit 9e5b1628737c67b4587f937164572774592978c4
diff --git a/third_party/gperftools.cmake b/third_party/gperftools.cmake
new file mode 100644
index 000000000..4929b5728
--- /dev/null
+++ b/third_party/gperftools.cmake
@@ -0,0 +1,128 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_library(gperftools STATIC
+  gperftools/src/addressmap-inl.h
+  gperftools/src/base/atomicops-internals-arm-generic.h
+  gperftools/src/base/atomicops-internals-arm-v6plus.h
+  gperftools/src/base/atomicops-internals-gcc.h
+  gperftools/src/base/atomicops-internals-linuxppc.h
+  gperftools/src/base/atomicops-internals-macosx.h
+  gperftools/src/base/atomicops-internals-mips.h
+  gperftools/src/base/atomicops-internals-windows.h
+  gperftools/src/base/atomicops-internals-x86.h
+  gperftools/src/base/atomicops.h
+  gperftools/src/base/basictypes.h
+  gperftools/src/base/commandlineflags.h
+  gperftools/src/base/dynamic_annotations.c
+  gperftools/src/base/dynamic_annotations.h
+  gperftools/src/base/elfcore.h
+  gperftools/src/base/googleinit.h
+  gperftools/src/base/linux_syscall_support.h
+  gperftools/src/base/linuxthreads.cc
+  gperftools/src/base/linuxthreads.h
+  gperftools/src/base/logging.cc
+  gperftools/src/base/logging.h
+  gperftools/src/base/low_level_alloc.cc
+  gperftools/src/base/spinlock.cc
+  gperftools/src/base/spinlock.h
+  gperftools/src/base/spinlock_internal.cc
+  gperftools/src/base/spinlock_internal.h
+  gperftools/src/base/spinlock_linux-inl.h
+  gperftools/src/base/spinlock_posix-inl.h
+  gperftools/src/base/spinlock_win32-inl.h
+  gperftools/src/base/stl_allocator.h
+  gperftools/src/base/sysinfo.cc
+  gperftools/src/base/sysinfo.h
+  gperftools/src/base/thread_annotations.h
+  gperftools/src/base/thread_lister.c
+  gperftools/src/base/thread_lister.h
+  gperftools/src/central_freelist.cc
+  gperftools/src/central_freelist.h
+  gperftools/src/common.cc
+  gperftools/src/common.h
+  gperftools/src/emergency_malloc.cc
+  gperftools/src/emergency_malloc.h
+  gperftools/src/emergency_malloc_for_stacktrace.cc
+  gperftools/src/fake_stacktrace_scope.cc
+  gperftools/src/gperftools/heap-checker.h
+  gperftools/src/gperftools/heap-profiler.h
+  gperftools/src/gperftools/malloc_extension.h
+  gperftools/src/gperftools/malloc_extension_c.h
+  gperftools/src/gperftools/malloc_hook.h
+  gperftools/src/gperftools/malloc_hook_c.h
+  gperftools/src/gperftools/nallocx.h
+  gperftools/src/stacktrace.cc
+  gperftools/src/gperftools/stacktrace.h
+  gperftools/src/heap-checker-bcad.cc
+  gperftools/src/heap-checker.cc
+  gperftools/src/heap-profile-stats.h
+  gperftools/src/heap-profile-table.cc
+  gperftools/src/heap-profile-table.h
+  gperftools/src/heap-profiler.cc
+  gperftools/src/internal_logging.cc
+  gperftools/src/internal_logging.h
+  gperftools/src/libc_override.h
+  gperftools/src/libc_override_gcc_and_weak.h
+  gperftools/src/libc_override_glibc.h
+  gperftools/src/libc_override_osx.h
+  gperftools/src/libc_override_redefine.h
+  gperftools/src/linked_list.h
+  gperftools/src/malloc_extension.cc
+  gperftools/src/malloc_hook-inl.h
+  gperftools/src/malloc_hook.cc
+  gperftools/src/malloc_hook_mmap_freebsd.h
+  gperftools/src/malloc_hook_mmap_linux.h
+  gperftools/src/maybe_emergency_malloc.h
+  gperftools/src/maybe_threads.cc
+  gperftools/src/maybe_threads.h
+  gperftools/src/memfs_malloc.cc
+  gperftools/src/memory_region_map.cc
+  gperftools/src/packed-cache-inl.h
+  gperftools/src/page_heap.cc
+  gperftools/src/page_heap.h
+  gperftools/src/page_heap_allocator.h
+  gperftools/src/pagemap.h
+  gperftools/src/raw_printer.cc
+  gperftools/src/raw_printer.h
+  gperftools/src/sampler.cc
+  gperftools/src/sampler.h
+  gperftools/src/span.cc
+  gperftools/src/span.h
+  gperftools/src/stack_trace_table.cc
+  gperftools/src/stack_trace_table.h
+  gperftools/src/static_vars.cc
+  gperftools/src/static_vars.h
+  gperftools/src/symbolize.cc
+  gperftools/src/symbolize.h
+  gperftools/src/system-alloc.cc
+  gperftools/src/system-alloc.h
+  gperftools/src/tcmalloc.cc
+  gperftools/src/tcmalloc_guard.h
+  gperftools/src/third_party/valgrind.h
+  gperftools/src/thread_cache.cc
+  gperftools/src/thread_cache.hgperftools/
+)
+target_include_directories(gperftools
+  PRIVATE "${CMAKE_CURRENT_LIST_DIR}/gperftools/src"
+  PRIVATE "${CMAKE_CURRENT_LIST_DIR}/gperftools_config")
+
+# Force every translation unit to include the fixup header.
+target_compile_options(gperftools
+    PRIVATE -include "${CMAKE_CURRENT_LIST_DIR}/gperftools_fixes.h" -Wall -Werror)
+
+set_source_files_properties(gperftools/src/tcmalloc.cc PROPERTIES
+    COMPILE_FLAGS "-include ${CMAKE_CURRENT_LIST_DIR}/gperftools_fixes.h -Wno-unused-function")
+
+set_property(TARGET gperftools PROPERTY POSITION_INDEPENDENT_CODE ON)
diff --git a/third_party/gperftools_config/config.h b/third_party/gperftools_config/config.h
new file mode 100644
index 000000000..2f38130d0
--- /dev/null
+++ b/third_party/gperftools_config/config.h
@@ -0,0 +1,323 @@
+/* src/config.h.  Generated from config.h.in by configure.  */
+/* src/config.h.in.  Generated from configure.ac by autoheader.  */
+
+
+#ifndef GPERFTOOLS_CONFIG_H_
+#define GPERFTOOLS_CONFIG_H_
+
+
+/* enable aggressive decommit by default */
+/* #undef ENABLE_AGGRESSIVE_DECOMMIT_BY_DEFAULT */
+
+/* Build new/delete operators for overaligned types */
+#define ENABLE_ALIGNED_NEW_DELETE 0  // avoid align_val_t
+
+/* Build runtime detection for sized delete */
+/* #undef ENABLE_DYNAMIC_SIZED_DELETE */
+
+/* report large allocation */
+/* #undef ENABLE_LARGE_ALLOC_REPORT */
+
+/* Build sized deletion operators */
+/* #undef ENABLE_SIZED_DELETE */
+
+/* Define to 1 if you have the <asm/ptrace.h> header file. */
+#define HAVE_ASM_PTRACE_H 1
+
+/* Define to 1 if compiler supports __builtin_stack_pointer */
+/* #undef HAVE_BUILTIN_STACK_POINTER */
+
+/* Define to 1 if you have the <conflict-signal.h> header file. */
+/* #undef HAVE_CONFLICT_SIGNAL_H */
+
+/* define if the compiler supports basic C++11 syntax */
+#define HAVE_CXX11 1
+
+/* Define to 1 if you have the <cygwin/signal.h> header file. */
+/* #undef HAVE_CYGWIN_SIGNAL_H */
+
+/* Define to 1 if you have the declaration of `backtrace', and to 0 if you
+   don't. */
+/* #undef HAVE_DECL_BACKTRACE */
+
+/* Define to 1 if you have the declaration of `cfree', and to 0 if you don't.
+   */
+#define HAVE_DECL_CFREE 1
+
+/* Define to 1 if you have the declaration of `memalign', and to 0 if you
+   don't. */
+#define HAVE_DECL_MEMALIGN 1
+
+/* Define to 1 if you have the declaration of `nanosleep', and to 0 if you
+   don't. */
+/* #undef HAVE_DECL_NANOSLEEP */
+
+/* Define to 1 if you have the declaration of `posix_memalign', and to 0 if
+   you don't. */
+#define HAVE_DECL_POSIX_MEMALIGN 1
+
+/* Define to 1 if you have the declaration of `pvalloc', and to 0 if you
+   don't. */
+#define HAVE_DECL_PVALLOC 1
+
+/* Define to 1 if you have the declaration of `sleep', and to 0 if you don't.
+   */
+/* #undef HAVE_DECL_SLEEP */
+
+/* Define to 1 if you have the declaration of `uname', and to 0 if you don't.
+   */
+#define HAVE_DECL_UNAME 1
+
+/* Define to 1 if you have the declaration of `valloc', and to 0 if you don't.
+   */
+#define HAVE_DECL_VALLOC 1
+
+/* Define to 1 if you have the <dlfcn.h> header file. */
+#define HAVE_DLFCN_H 1
+
+/* Define to 1 if the system has the type `Elf32_Versym'. */
+#define HAVE_ELF32_VERSYM 1
+
+/* Define to 1 if you have the <execinfo.h> header file. */
+#define HAVE_EXECINFO_H 1
+
+/* Define to 1 if you have the <fcntl.h> header file. */
+#define HAVE_FCNTL_H 1
+
+/* Define to 1 if you have the <features.h> header file. */
+#define HAVE_FEATURES_H 1
+
+/* Define to 1 if you have the `fork' function. */
+#define HAVE_FORK 1
+
+/* Define to 1 if you have the `geteuid' function. */
+#define HAVE_GETEUID 1
+
+/* Define to 1 if you have the `getpagesize' function. */
+#define HAVE_GETPAGESIZE 1
+
+/* Define to 1 if you have the <glob.h> header file. */
+#define HAVE_GLOB_H 1
+
+/* Define to 1 if you have the <grp.h> header file. */
+#define HAVE_GRP_H 1
+
+/* Define to 1 if you have the <inttypes.h> header file. */
+#define HAVE_INTTYPES_H 1
+
+/* Define to 1 if you have the <libunwind.h> header file. */
+/* #undef HAVE_LIBUNWIND_H */
+
+/* Define to 1 if you have the <linux/ptrace.h> header file. */
+#define HAVE_LINUX_PTRACE_H 1
+
+/* Define if this is Linux that has SIGEV_THREAD_ID */
+#define HAVE_LINUX_SIGEV_THREAD_ID 1
+
+/* Define to 1 if you have the <malloc.h> header file. */
+#define HAVE_MALLOC_H 1
+
+/* Define to 1 if you have the <memory.h> header file. */
+#define HAVE_MEMORY_H 1
+
+/* Define to 1 if you have a working `mmap' system call. */
+#define HAVE_MMAP 1
+
+/* define if the compiler implements namespaces */
+#define HAVE_NAMESPACES 1
+
+/* Define to 1 if you have the <poll.h> header file. */
+#define HAVE_POLL_H 1
+
+/* define if libc has program_invocation_name */
+#define HAVE_PROGRAM_INVOCATION_NAME 1
+
+/* Define if you have POSIX threads libraries and header files. */
+#define HAVE_PTHREAD 1
+
+/* defined to 1 if pthread symbols are exposed even without include pthread.h
+   */
+/* #undef HAVE_PTHREAD_DESPITE_ASKING_FOR */
+
+/* Define to 1 if you have the <pwd.h> header file. */
+#define HAVE_PWD_H 1
+
+/* Define to 1 if you have the `sbrk' function. */
+#define HAVE_SBRK 1
+
+/* Define to 1 if you have the <sched.h> header file. */
+#define HAVE_SCHED_H 1
+
+/* Define to 1 if you have the <stdint.h> header file. */
+#define HAVE_STDINT_H 1
+
+/* Define to 1 if you have the <stdlib.h> header file. */
+#define HAVE_STDLIB_H 1
+
+/* Define to 1 if you have the <strings.h> header file. */
+#define HAVE_STRINGS_H 1
+
+/* Define to 1 if you have the <string.h> header file. */
+#define HAVE_STRING_H 1
+
+/* Define to 1 if the system has the type `struct mallinfo'. */
+#define HAVE_STRUCT_MALLINFO 1
+
+/* Define to 1 if you have the <sys/cdefs.h> header file. */
+#define HAVE_SYS_CDEFS_H 1
+
+/* Define to 1 if you have the <sys/param.h> header file. */
+#define HAVE_SYS_PARAM_H 1
+
+/* Define to 1 if you have the <sys/prctl.h> header file. */
+#define HAVE_SYS_PRCTL_H 1
+
+/* Define to 1 if you have the <sys/resource.h> header file. */
+#define HAVE_SYS_RESOURCE_H 1
+
+/* Define to 1 if you have the <sys/socket.h> header file. */
+#define HAVE_SYS_SOCKET_H 1
+
+/* Define to 1 if you have the <sys/stat.h> header file. */
+#define HAVE_SYS_STAT_H 1
+
+/* Define to 1 if you have the <sys/syscall.h> header file. */
+#define HAVE_SYS_SYSCALL_H 1
+
+/* Define to 1 if you have the <sys/types.h> header file. */
+#define HAVE_SYS_TYPES_H 1
+
+/* Define to 1 if you have the <sys/ucontext.h> header file. */
+#define HAVE_SYS_UCONTEXT_H 1
+
+/* Define to 1 if you have the <sys/wait.h> header file. */
+#define HAVE_SYS_WAIT_H 1
+
+/* Define to 1 if compiler supports __thread */
+#define HAVE_TLS 1
+
+/* Define to 1 if you have the <ucontext.h> header file. */
+#define HAVE_UCONTEXT_H 1
+
+/* Define to 1 if you have the <unistd.h> header file. */
+#define HAVE_UNISTD_H 1
+
+/* Whether <unwind.h> contains _Unwind_Backtrace */
+#define HAVE_UNWIND_BACKTRACE 1
+
+/* Define to 1 if you have the <unwind.h> header file. */
+#define HAVE_UNWIND_H 1
+
+/* Define to 1 if you have the <valgrind.h> header file. */
+/* #undef HAVE_VALGRIND_H */
+
+/* define if your compiler has __attribute__ */
+#define HAVE___ATTRIBUTE__ 1
+
+/* define if your compiler supports alignment of functions */
+#define HAVE___ATTRIBUTE__ALIGNED_FN 1
+
+/* Define to 1 if compiler supports __environ */
+#define HAVE___ENVIRON 1
+
+/* Define to 1 if the system has the type `__int64'. */
+/* #undef HAVE___INT64 */
+
+/* Define to 1 if you have the `__sbrk' function. */
+#define HAVE___SBRK 1
+
+/* prefix where we look for installed files */
+#define INSTALL_PREFIX "/usr/local"
+
+/* Define to 1 if int32_t is equivalent to intptr_t */
+/* #undef INT32_EQUALS_INTPTR */
+
+/* Define to the sub-directory where libtool stores uninstalled libraries. */
+#define LT_OBJDIR ".libs/"
+
+/* Name of package */
+#define PACKAGE "gperftools"
+
+/* Define to the address where bug reports for this package should be sent. */
+#define PACKAGE_BUGREPORT "gperftools@googlegroups.com"
+
+/* Define to the full name of this package. */
+#define PACKAGE_NAME "gperftools"
+
+/* Define to the full name and version of this package. */
+#define PACKAGE_STRING "gperftools 2.7"
+
+/* Define to the one symbol short name of this package. */
+#define PACKAGE_TARNAME "gperftools"
+
+/* Define to the home page for this package. */
+#define PACKAGE_URL ""
+
+/* Define to the version of this package. */
+#define PACKAGE_VERSION "2.7"
+
+/* How to access the PC from a struct ucontext */
+#define PC_FROM_UCONTEXT uc_mcontext.gregs[REG_RIP]
+
+/* Always the empty-string on non-windows systems. On windows, should be
+   "__declspec(dllexport)". This way, when we compile the dll, we export our
+   functions/classes. It's safe to define this here because config.h is only
+   used internally, to compile the DLL, and every DLL source file #includes
+   "config.h" before anything else. */
+#define PERFTOOLS_DLL_DECL /**/
+
+/* printf format code for printing a size_t and ssize_t */
+#define PRIdS "ld"
+
+/* printf format code for printing a size_t and ssize_t */
+#define PRIuS "lu"
+
+/* printf format code for printing a size_t and ssize_t */
+#define PRIxS "lx"
+
+/* Mark the systems where we know it's bad if pthreads runs too
+   early before main (before threads are initialized, presumably).  */
+#ifdef __FreeBSD__
+#define PTHREADS_CRASHES_IF_RUN_TOO_EARLY 1
+#endif
+
+/* Define to necessary symbol if this constant uses a non-standard name on
+   your system. */
+/* #undef PTHREAD_CREATE_JOINABLE */
+
+/* Define to 1 if you have the ANSI C header files. */
+#define STDC_HEADERS 1
+
+/* the namespace where STL code like vector<> is defined */
+#define STL_NAMESPACE std
+
+/* Define 32K of internal pages size for tcmalloc */
+/* #undef TCMALLOC_32K_PAGES */
+
+/* Define 64K of internal pages size for tcmalloc */
+/* #undef TCMALLOC_64K_PAGES */
+
+/* Define 8 bytes of allocation alignment for tcmalloc */
+/* #undef TCMALLOC_ALIGN_8BYTES */
+
+/* Version number of package */
+#define VERSION "2.7"
+
+/* C99 says: define this to get the PRI... macros from stdint.h */
+#ifndef __STDC_FORMAT_MACROS
+# define __STDC_FORMAT_MACROS 1
+#endif
+
+/* Define to `__inline__' or `__inline' if that's what the C compiler
+   calls it, or to nothing if 'inline' is not supported under any name.  */
+#ifndef __cplusplus
+/* #undef inline */
+#endif
+
+
+#ifdef __MINGW32__
+#include "windows/mingw.h"
+#endif
+
+#endif  /* #ifndef GPERFTOOLS_CONFIG_H_ */
+
diff --git a/third_party/gperftools_config/gperftools/tcmalloc.h b/third_party/gperftools_config/gperftools/tcmalloc.h
new file mode 100644
index 000000000..79350817b
--- /dev/null
+++ b/third_party/gperftools_config/gperftools/tcmalloc.h
@@ -0,0 +1,163 @@
+// -*- Mode: C; c-basic-offset: 2; indent-tabs-mode: nil -*-
+/* Copyright (c) 2003, Google Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ *     * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ *     * Redistributions in binary form must reproduce the above
+ * copyright notice, this list of conditions and the following disclaimer
+ * in the documentation and/or other materials provided with the
+ * distribution.
+ *     * Neither the name of Google Inc. nor the names of its
+ * contributors may be used to endorse or promote products derived from
+ * this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * ---
+ * Author: Sanjay Ghemawat <opensource@google.com>
+ *         .h file by Craig Silverstein <opensource@google.com>
+ */
+
+#ifndef TCMALLOC_TCMALLOC_H_
+#define TCMALLOC_TCMALLOC_H_
+
+#include <stddef.h>                     /* for size_t */
+#ifdef __cplusplus
+#include <new>                          /* for std::nothrow_t, std::align_val_t */
+#endif
+
+/* Define the version number so folks can check against it */
+#define TC_VERSION_MAJOR  2
+#define TC_VERSION_MINOR  7
+#define TC_VERSION_PATCH  ""
+#define TC_VERSION_STRING "gperftools 2.7"
+
+/* For struct mallinfo, if it's defined. */
+#if 1
+# include <malloc.h>
+#endif
+
+#ifndef PERFTOOLS_NOTHROW
+
+#if __cplusplus >= 201103L
+#define PERFTOOLS_NOTHROW noexcept
+#elif defined(__cplusplus)
+#define PERFTOOLS_NOTHROW throw()
+#else
+# ifdef __GNUC__
+#  define PERFTOOLS_NOTHROW __attribute__((__nothrow__))
+# else
+#  define PERFTOOLS_NOTHROW
+# endif
+#endif
+
+#endif
+
+#ifndef PERFTOOLS_DLL_DECL
+# ifdef _WIN32
+#   define PERFTOOLS_DLL_DECL  __declspec(dllimport)
+# else
+#   define PERFTOOLS_DLL_DECL
+# endif
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+  /*
+   * Returns a human-readable version string.  If major, minor,
+   * and/or patch are not NULL, they are set to the major version,
+   * minor version, and patch-code (a string, usually "").
+   */
+  PERFTOOLS_DLL_DECL const char* tc_version(int* major, int* minor,
+                                            const char** patch) PERFTOOLS_NOTHROW;
+
+  PERFTOOLS_DLL_DECL void* tc_malloc(size_t size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void* tc_malloc_skip_new_handler(size_t size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_free(void* ptr) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_free_sized(void *ptr, size_t size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void* tc_realloc(void* ptr, size_t size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void* tc_calloc(size_t nmemb, size_t size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_cfree(void* ptr) PERFTOOLS_NOTHROW;
+
+  PERFTOOLS_DLL_DECL void* tc_memalign(size_t __alignment,
+                                       size_t __size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL int tc_posix_memalign(void** ptr,
+                                           size_t align, size_t size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void* tc_valloc(size_t __size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void* tc_pvalloc(size_t __size) PERFTOOLS_NOTHROW;
+
+  PERFTOOLS_DLL_DECL void tc_malloc_stats(void) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL int tc_mallopt(int cmd, int value) PERFTOOLS_NOTHROW;
+#if 1
+  PERFTOOLS_DLL_DECL struct mallinfo tc_mallinfo(void) PERFTOOLS_NOTHROW;
+#endif
+
+  /*
+   * This is an alias for MallocExtension::instance()->GetAllocatedSize().
+   * It is equivalent to
+   *    OS X: malloc_size()
+   *    glibc: malloc_usable_size()
+   *    Windows: _msize()
+   */
+  PERFTOOLS_DLL_DECL size_t tc_malloc_size(void* ptr) PERFTOOLS_NOTHROW;
+
+#ifdef __cplusplus
+  PERFTOOLS_DLL_DECL int tc_set_new_mode(int flag) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void* tc_new(size_t size);
+  PERFTOOLS_DLL_DECL void* tc_new_nothrow(size_t size,
+                                          const std::nothrow_t&) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_delete(void* p) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_delete_sized(void* p, size_t size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_delete_nothrow(void* p,
+                                            const std::nothrow_t&) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void* tc_newarray(size_t size);
+  PERFTOOLS_DLL_DECL void* tc_newarray_nothrow(size_t size,
+                                               const std::nothrow_t&) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_deletearray(void* p) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_deletearray_sized(void* p, size_t size) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_deletearray_nothrow(void* p,
+                                                 const std::nothrow_t&) PERFTOOLS_NOTHROW;
+
+#if 1 && __cplusplus >= 201703L
+  PERFTOOLS_DLL_DECL void* tc_new_aligned(size_t size, std::align_val_t al);
+  PERFTOOLS_DLL_DECL void* tc_new_aligned_nothrow(size_t size, std::align_val_t al,
+                                          const std::nothrow_t&) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_delete_aligned(void* p, std::align_val_t al) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_delete_sized_aligned(void* p, size_t size, std::align_val_t al) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_delete_aligned_nothrow(void* p, std::align_val_t al,
+                                            const std::nothrow_t&) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void* tc_newarray_aligned(size_t size, std::align_val_t al);
+  PERFTOOLS_DLL_DECL void* tc_newarray_aligned_nothrow(size_t size, std::align_val_t al,
+                                               const std::nothrow_t&) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_deletearray_aligned(void* p, std::align_val_t al) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_deletearray_sized_aligned(void* p, size_t size, std::align_val_t al) PERFTOOLS_NOTHROW;
+  PERFTOOLS_DLL_DECL void tc_deletearray_aligned_nothrow(void* p, std::align_val_t al,
+                                                 const std::nothrow_t&) PERFTOOLS_NOTHROW;
+#endif
+}
+#endif
+
+/* We're only un-defining for public */
+#if !defined(GPERFTOOLS_CONFIG_H_)
+
+#undef PERFTOOLS_NOTHROW
+
+#endif /* GPERFTOOLS_CONFIG_H_ */
+
+#endif  /* #ifndef TCMALLOC_TCMALLOC_H_ */
diff --git a/third_party/gperftools_fixes.h b/third_party/gperftools_fixes.h
new file mode 100644
index 000000000..508a4152c
--- /dev/null
+++ b/third_party/gperftools_fixes.h
@@ -0,0 +1,31 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef __THIRD_PARTY_GPERFTOOLS_FIXES_H__
+#define __THIRD_PARTY_GPERFTOOLS_FIXES_H__
+
+#include <stdarg.h>  // for thread_lister.h
+
+#ifdef __cplusplus
+#include <cstddef>  // std::size_t
+namespace std {
+
+#if __cplusplus < 201703L
+enum class align_val_t : std::size_t {};
+#endif
+
+}  // namespace std
+#endif  // __cplusplus
+
+#endif  // __THIRD_PARTY_GPERFTOOLS_FIXES_H__
diff --git a/third_party/highway/CMakeLists.txt b/third_party/highway/CMakeLists.txt
new file mode 100644
index 000000000..b61e00944
--- /dev/null
+++ b/third_party/highway/CMakeLists.txt
@@ -0,0 +1,174 @@
+# Copyright (c) the JPEG XL Project
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+
+cmake_minimum_required(VERSION 3.2)
+set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_EXTENSIONS OFF)
+set(CMAKE_CXX_STANDARD_REQUIRED YES)
+
+option(HWY_USE_GTEST "Links tests with GoogleTest")
+if(HWY_USE_GTEST)
+  find_package(GTest)
+endif()
+
+include(CheckCXXSourceCompiles)
+check_cxx_source_compiles(
+   "int main() {
+      #if !defined(__EMSCRIPTEN__)
+      static_assert(false, \"__EMSCRIPTEN__ is not defined\");
+      #endif
+      return 0;
+    }"
+  HWY_EMSCRIPTEN
+)
+
+if (NOT CMAKE_BUILD_TYPE)
+  set(CMAKE_BUILD_TYPE RelWithDebInfo)
+endif()
+
+set(HWY_SOURCES
+  hwy/arch.h
+  hwy/arm64_neon.h
+  hwy/cache_control.h
+  hwy/compiler_specific.h
+  hwy/foreach_target.h
+  hwy/include_headers.h
+  hwy/runtime_dispatch.cc
+  hwy/runtime_dispatch.h
+  hwy/runtime_targets.h
+  hwy/scalar.h
+  hwy/shared.h
+  hwy/static_targets.h
+  hwy/targets.h
+  hwy/tests/test_target_util.h
+  hwy/tests/test_util.h
+  hwy/type_traits.h
+  hwy/x86/avx2.h
+  hwy/x86/avx512.h
+  hwy/x86/sse4.h
+  hwy/wasm.h
+)
+
+set(HWY_FLAGS
+  # Avoid changing binaries based on the current time and date.
+  -Wno-builtin-macro-redefined
+  -D__DATE__="redacted"
+  -D__TIMESTAMP__="redacted"
+  -D__TIME__="redacted"
+
+  # Optimizations
+  -fmerge-all-constants
+
+  # Warnings
+  -Wall
+  -Wextra
+  -Wformat-security
+  -Wno-unused-function
+  -Wnon-virtual-dtor
+  -Woverloaded-virtual
+  -Wvla
+)
+
+if(${CMAKE_CXX_COMPILER_ID} MATCHES "Clang")
+  list(APPEND HWY_FLAGS
+    -Wc++2a-extensions
+    -Wfloat-overflow-conversion
+    -Wfloat-zero-conversion
+    -Wfor-loop-analysis
+    -Wgnu-redeclared-enum
+    -Winfinite-recursion
+    -Wself-assign
+    -Wstring-conversion
+    -Wtautological-overlap-compare
+    -Wthread-safety-analysis
+    -Wundefined-func-template
+
+    -fno-cxx-exceptions
+    -fno-slp-vectorize
+    -fno-vectorize
+
+    # Use color in messages
+    -Xclang -fdiagnostics-show-option -fcolor-diagnostics
+  )
+endif()
+
+if (WIN32)
+  list(APPEND HWY_FLAGS
+    -Wno-c++98-compat-pedantic
+    -Wno-cast-align
+    -Wno-double-promotion
+    -Wno-float-equal
+    -Wno-format-nonliteral
+    -Wno-global-constructors
+    -Wno-language-extension-token
+    -Wno-missing-prototypes
+    -Wno-shadow
+    -Wno-shadow-field-in-constructor
+    -Wno-sign-conversion
+    -Wno-unused-member-function
+    -Wno-unused-template
+    -Wno-used-but-marked-unused
+    -Wno-zero-as-null-pointer-constant
+  )
+else()
+  list(APPEND HWY_FLAGS
+    -fmath-errno
+    -fno-exceptions
+  )
+endif()
+
+if(HWY_USE_GTEST)
+  list(APPEND HWY_FLAGS -DHWY_USE_GTEST)
+endif()
+
+if(HWY_EMSCRIPTEN)
+  list(APPEND HWY_FLAGS -s SIMD=1)
+endif()
+
+add_library(hwy STATIC ${HWY_SOURCES})
+
+target_compile_options(hwy PRIVATE ${HWY_FLAGS})
+target_include_directories(hwy PUBLIC ${CMAKE_CURRENT_LIST_DIR})
+
+if(HWY_EMSCRIPTEN)
+  target_link_libraries(hwy PUBLIC "-s SIMD=1")
+endif()
+
+# -------------------------------------------------------- Tests
+
+enable_testing()
+
+set(HWY_TEST_FILES
+  hwy/tests/arithmetic_test.cc
+  hwy/tests/compare_test.cc
+  hwy/tests/convert_test.cc
+  hwy/tests/hwy_test.cc
+  hwy/tests/logical_test.cc
+  hwy/tests/memory_test.cc
+  hwy/tests/swizzle_test.cc
+)
+
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests)
+foreach (TESTFILE IN LISTS HWY_TEST_FILES)
+  # The TESTNAME is the name without the extension or directory.
+  get_filename_component(TESTNAME ${TESTFILE} NAME_WE)
+  add_executable(${TESTNAME} ${TESTFILE})
+  target_compile_options(${TESTNAME} PRIVATE ${HWY_FLAGS})
+
+  target_link_libraries(${TESTNAME} hwy)
+  # Output test targets in the test directory.
+  set_target_properties(${TESTNAME} PROPERTIES PREFIX "tests/")
+
+  if(HWY_USE_GTEST)
+    target_link_libraries(${TESTNAME} gtest gtest_main)
+  endif()
+
+  # gtest_discover_tests requires CMake 3.10 and compiling with gtest.
+  # gtest_add_tests runs cmake every time a test file changes and requires
+  # find_package(GTest) or include(GoogleTest). We know there is only a
+  # single test case (due to runtime dispatch), so add manually.
+  add_test(NAME ${TESTNAME} COMMAND ${TESTNAME})
+endforeach ()
diff --git a/third_party/highway/LICENSE b/third_party/highway/LICENSE
new file mode 100644
index 000000000..624c6befd
--- /dev/null
+++ b/third_party/highway/LICENSE
@@ -0,0 +1,20 @@
+Copyright (c) the JPEG XL Project
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/third_party/highway/Makefile b/third_party/highway/Makefile
new file mode 100644
index 000000000..801b7175c
--- /dev/null
+++ b/third_party/highway/Makefile
@@ -0,0 +1,30 @@
+.DELETE_ON_ERROR:
+
+OBJS := runtime_dispatch.o nanobenchmark.o
+TEST_NAMES := arithmetic_test compare_test convert_test hwy_test logical_test memory_test nanobenchmark_test swizzle_test
+TESTS := $(foreach i, $(TEST_NAMES), bin/$(i))
+
+CXXFLAGS += -I. -fmerge-all-constants -std=c++17 -O2 \
+    -Wno-builtin-macro-redefined -D__DATE__="redacted" \
+    -D__TIMESTAMP__="redacted" -D__TIME__="redacted"  \
+    -Wall -Wextra -Wformat-security -Wno-unused-function \
+    -Wnon-virtual-dtor -Woverloaded-virtual -Wvla
+
+.PHONY: all
+all: $(TESTS) benchmark test
+
+.PHONY: clean
+clean: ; @rm -rf $(OBJS) bin/ benchmark.o
+
+$(OBJS): %.o: hwy/%.cc
+	$(CXX) -c $(CXXFLAGS) $< -o $@
+
+benchmark: $(OBJS) hwy/benchmark.cc
+	mkdir -p bin && $(CXX) $(CXXFLAGS) $(OBJS) hwy/benchmark.cc -o bin/benchmark
+
+$(TESTS): $(OBJS)
+	mkdir -p bin && $(CXX) $(CXXFLAGS) $(subst bin/,hwy/tests/,$@).cc $(OBJS) -o $@
+
+.PHONY: test
+test: $(TESTS)
+	for name in $^; do echo ---------------------$$name && $$name; done
diff --git a/third_party/highway/README.md b/third_party/highway/README.md
new file mode 100644
index 000000000..6750d81cc
--- /dev/null
+++ b/third_party/highway/README.md
@@ -0,0 +1,350 @@
+## Efficient and performance-portable SIMD wrapper
+
+This library provides type-safe and source-code portable wrappers over existing
+platform-specific intrinsics. Its design aims for simplicity, reliable
+efficiency across platforms, and immediate usability with current compilers.
+
+## Current status
+
+Implemented for scalar/SSE4/AVX2/AVX-512/ARMv8 targets, each with unit tests.
+
+Tested on GCC 8.3.0 and Clang 6.0.1.
+
+A [quick-reference page](quick_reference.md) briefly lists all operations
+and their parameters.
+
+## Design philosophy
+
+*   Performance is important but not the sole consideration. Anyone who goes to
+    the trouble of using SIMD clearly cares about speed. However, portability,
+    maintainability and readability also matter, otherwise we would write in
+    assembly. We aim for performance within 10-20% of a hand-written assembly
+    implementation on the development platform.
+
+*   The guiding principles of C++ are "pay only for what you use" and "leave no
+    room for a lower-level language below C++". We apply these by defining a
+    SIMD API that ensures operation costs are visible, predictable and minimal.
+
+*   Performance portability is important, i.e. the API should be efficient on
+    all target platforms. Unfortunately, common idioms for one platform can be
+    inefficient on others. For example: summing lanes horizontally versus
+    shuffling. Documenting which operations are expensive does not prevent their
+    use, as evidenced by widespread use of `HADDPS`. Performance acceptance
+    tests may detect large regressions, but do not help choose the approach
+    during initial development. Analysis tools can warn about some potential
+    inefficiencies, but likely not all. We instead provide [a carefully chosen
+    set of vector types and operations that are efficient on all target
+    platforms][instmtx] (PPC8, SSE4/AVX2+, ARMv8), plus some useful but less
+    performance-portable operations in an `ext` namespace to make their cost
+    visible.
+
+*   Future SIMD hardware features are difficult to predict. For example, AVX2
+    came with surprising semantics (almost no interaction between 128-bit
+    blocks) and AVX-512 added two kinds of predicates (writemask and zeromask).
+    To ensure the API reflects hardware realities, we suggest a flexible
+    approach that adds new operations as they become commonly available.
+
+*   Masking is not yet widely supported on current CPUs. It is difficult to
+    define an interface that provides access to all platform features while
+    retaining performance portability. The P0214R5 proposal lacks support for
+    AVX-512/ARM SVE zeromasks. We suggest limiting usage of masks to the
+    IfThen[Zero]Else[Zero] functions until the community has gained more
+    experience with them.
+
+*   "Width-agnostic" SIMD is more future-proof than user-specified fixed sizes.
+    For example, valarray-like code can iterate over a 1D array with a
+    library-specified vector width. This will result in better code when vector
+    sizes increase, and matches the direction taken by
+    [ARM SVE](https://alastairreid.github.io/papers/sve-ieee-micro-2017.pdf) and
+    RiscV hardware as well as Agner Fog's
+    [ForwardCom instruction set proposal](https://goo.gl/CFizWu). However, some
+    applications may require fixed sizes, so we also guarantee support for
+    128-bit vectors in each instruction set.
+
+*   The API and its implementation should be usable and efficient with commonly
+    used compilers. Some of our open-source users cannot upgrade, so we need to
+    support ~4 year old compilers. For example, we write `ShiftLeft<3>(v)`
+    instead of `v << 3` because MSVC 2017 (ARM64) does not propagate the literal
+    (https://godbolt.org/g/rKx5Ga). However, we do require function-specific
+    target attributes, supported by GCC 4.9 / Clang 3.9 / MSVC 2015.
+
+*   Efficient and safe runtime dispatch is important. Modules such as image or
+    video codecs are typically embedded into larger applications such as
+    browsers, so they cannot require separate binaries for each CPU. Libraries
+    also cannot predict whether the application already uses AVX2 (and pays the
+    frequency throttling cost), so this decision must be left to the
+    application. Using only the lowest-common denominator instructions
+    sacrifices too much performance.
+    Therefore, we provide code paths for multiple instruction sets and choose
+    the most suitable at runtime. To reduce overhead, dispatch should be hoisted
+    to higher layers instead of checking inside every low-level function.
+    Generating each code path from the same source reduces implementation- and
+    debugging cost.
+
+*   Not every CPU need be supported. For example, pre-SSE4.1 CPUs are
+    increasingly rare and the AVX instruction set is limited to floating-point
+    operations. To reduce code size and compile time, we provide specializations
+    for SSE4, AVX2 and AVX-512 instruction sets on x86.
+
+*   Access to platform-specific intrinsics is necessary for acceptance in
+    performance-critical projects. We provide conversions to and from intrinsics
+    to allow utilizing specialized platform-specific functionality, and simplify
+    incremental porting of existing code.
+
+*   The core API should be compact and easy to learn. We provide only the few
+    dozen operations which are necessary and sufficient for most of the 150+
+    SIMD applications we examined.
+
+## Differences versus [P0214R5 proposal](https://goo.gl/zKW4SA)
+
+1.  Adding widely used and portable operations such as `AndNot`, `AverageRound`,
+    bit-shift by immediates and `IfThenElse`.
+
+1.  Designing the API to avoid or minimize overhead on AVX2/AVX-512 caused by
+    crossing 128-bit 'block' boundaries.
+
+1.  Avoiding the need for non-native vectors. By contrast, P0214R5's `simd_cast`
+    returns `fixed_size<>` vectors which are more expensive to access because
+    they reside on the stack. We can avoid this plus additional overhead on
+    ARM/AVX2 by defining width-expanding operations as functions of a vector
+    part, e.g. promoting half a vector of `uint8_t` lanes to one full vector of
+    `uint16_t`, or demoting full vectors to half vectors with half-width lanes.
+
+1.  Guaranteeing access to the underlying intrinsic vector type. This ensures
+    all platform-specific capabilities can be used. P0214R5 instead only
+    'encourages' implementations to provide access.
+
+1.  Enabling safe runtime dispatch and inlining in the same binary. P0214R5 is
+    based on the Vc library, which does not provide assistance for linking
+    multiple instruction sets into the same binary. The Vc documentation
+    suggests compiling separate executables for each instruction set or using
+    GCC's ifunc (indirect functions). The latter is compiler-specific and risks
+    crashes due to ODR violations when compiling the same function with
+    different compiler flags. We solve this problem via target-specific
+    attributes (see HOWTO section below). We also permit a mix of static
+    target selection and runtime dispatch for hotspots that may benefit from
+    newer instruction sets if available.
+
+1.  Using built-in PPC vector types without a wrapper class. This leads to much
+    better code generation with GCC 6.3: https://godbolt.org/z/pd2PNP.
+    By contrast, P0214R5 requires a wrapper. We avoid this by using only the
+    member operators provided by the PPC vectors; all other functions and
+    typedefs are non-members. 2019-04 update: Clang power64le does not have
+    this issue, so we simplified get_part(d, v) to GetLane(v).
+
+*   Omitting inefficient or non-performance-portable operations such as `hmax`,
+    `operator[]`, and unsupported integer comparisons. Applications can often
+    replace these operations at lower cost than emulating that exact behavior.
+
+*   Omitting `long double` types: these are not commonly available in hardware.
+
+*   Ensuring signed integer overflow has well-defined semantics (wraparound).
+
+*   Simple header-only implementation and less than a tenth of the size of the
+    Vc library from which P0214 was derived (98,000 lines in
+    https://github.com/VcDevel/Vc according to the gloc Chrome extension).
+
+*   Avoiding hidden performance costs. P0214R5 allows implicit conversions from
+    integer to float, which costs 3-4 cycles on x86. We make these conversions
+    explicit to ensure their cost is visible.
+
+## Prior API designs
+
+The author has been writing SIMD code since 2002: first via assembly language,
+then intrinsics, later Intel's `F32vec4` wrapper, followed by three generations
+of custom vector classes. The first used macros to generate the classes, which
+reduces duplication but also readability. The second used templates instead.
+The third (used in highwayhash and PIK) added support for AVX2 and runtime
+dispatch. The current design (used in JPEG XL) enables code generation for
+multiple platforms and/or instruction sets from the same source, and improves
+runtime dispatch.
+
+## Other related work
+
+*   [Neat SIMD](http://ieeexplore.ieee.org/stamp/stamp.jsp?arnumber=7568423)
+    adopts a similar approach with interchangeable vector/scalar types and
+    a compact interface. It allows access to the underlying intrinsics, but
+    does not appear to be designed for other platforms than x86.
+
+*   UME::SIMD ([code](https://goo.gl/yPeVZx), [paper](https://goo.gl/2xpZrk))
+    also adopts an explicit vectorization model with vector classes.
+    However, it exposes the union of all platform capabilities, which makes the
+    API harder to learn (209-page spec) and implement (the estimated LOC count
+    is [500K](https://goo.gl/1THFRi)). The API is less performance-portable
+    because it allows applications to use operations that are inefficient on
+    other platforms.
+
+*   Inastemp ([code](https://goo.gl/hg3USM), [paper](https://goo.gl/YcTU7S))
+    is a vector library for scientific computing with some innovative features:
+    automatic FLOPS counting, and "if/else branches" using lambda functions.
+    It supports IBM Power8, but only provides float and double types.
+
+## Overloaded function API
+
+Most C++ vector APIs rely on class templates. However, two PPC compilers
+including GCC 6.3 generate inefficient code for classes with a SIMD vector
+member: an [extra load/store for every function argument/return
+value](https://godbolt.org/z/pd2PNP). To avoid this overhead, we use built-in
+vector types on PPC. These provide overloaded arithmetic operators but do not
+allow member functions/typedefs such as `size()` or `value_type`. We instead
+rely on overloaded functions.
+
+Because vectors of various lengths are the same type on PPC, we need an
+additional tag argument for disambiguation. Any function template with multiple
+return types uses a descriptor argument to specify the return type. For example,
+the return type of `Zero(Desc<T, N>)` is `VT<Desc<T, N>>`. For brevity,
+`Desc` is abbreviated to `D` for template arguments and `d` in lvalues.
+
+It may seem preferable to write `Zero<D>()` rather than `Zero(D())`, but
+there are technical difficulties. We prefer generic implementations where
+possible rather than overloading for every single `T`. Because C++ does not
+allow partial specialization of function templates, we need multiple overloads:
+one primary template per target. Thus, functions cannot be invoked using
+template syntax. Can we instead add a wrapper function template that calls the
+appropriate overload? Unfortunately, the compiler mechanism for avoiding
+dangerous per-file `-mavx2` requires per-function annotations, and these
+attributes are not generic. Thus, a wrapper into which SIMD functions are
+inlined cannot be a function, because it would also need a target-specific
+attribute. A macro `ZERO(D)` could work, but this is hardly more clear than a
+normal function with arguments. Note that descriptors occur often, so user code
+can define a `const HWY_FULL(float) d;` and then write `Zero(d)`.
+
+## Masks
+
+AVX-512 introduced a major change to the SIMD interface: special mask registers
+(one bit per lane) that serve as predicates. It would be expensive to force
+AVX-512 implementations to conform to the prior model of full vectors with lanes
+set to all one or all zero bits. We instead provide a Mask type that emulates
+a subset of this functionality on other platforms at zero cost.
+
+Masks are returned by comparisons and `TestBit`; they serve as the input to
+`IfThen*`. We provide conversions between masks and vector lanes. For clarity
+and safety, we use FF..FF as the definition of true. To also benefit from
+x86 instructions that only require the sign bit of floating-point inputs to be
+set, we provide a special `ZeroIfNegative` function.
+
+## Use cases and HOWTO
+
+Whenever possible, use full-width descriptors for maximum performance
+portability: `HWY_FULL(float)`. When necessary, applications may also rely on
+128-bit vectors: `Desc<float, 4>`. There is also the option of a vector of up
+to N lanes: `HWY_CAPPED(float, N)` (the length is always a power of two).
+
+*   Single instruction set per platform: use normal C++ functions with
+    `HWY_ATTR` annotation and `#include "static_targets.h"`. Note that this
+    can co-exist with runtime dispatch in the same translation unit.
+
+*   Runtime dispatch means calling the best available implementation:
+
+```c++
+  #include "jxl/hwy/runtime_dispatch.h"
+  Dispatch(TargetBitfield().Best(), Module()/*, args*/);
+```
+
+The module is declared in a header as a non-const member function:
+```c++
+  struct Module {
+    HWY_DECLARE(void, (args))
+  };
+```
+
+The module.cc is automatically compiled multiple times, once per target CPU:
+```c++
+  #ifndef HWY_TARGET_INCLUDE
+  #define HWY_TARGET_INCLUDE "path/module.cc"
+  #include "path/module.h"
+  #endif  // end of non-SIMD code
+
+  #include "jxl/hwy/foreach_target.h"
+
+  namespace HWY_NAMESPACE { namespace {
+    HWY_ATTR void Implementation() {}
+  }}
+  HWY_ATTR void Module::HWY_FUNC(/*args*/) {
+    HWY_NAMESPACE::Implementation();
+  }
+```
+
+## Example source code
+
+```c++
+void FloorLog2(const uint8_t* HWY_RESTRICT values,
+               uint8_t* HWY_RESTRICT log2) {
+  // Descriptors for all required data types:
+  const HWY_FULL(int32_t) d32;
+  const HWY_FULL(float) df;
+  const HWY_CAPPED(uint8_t, d32.N) d8;
+
+  const auto u8 = Load(d8, values);
+  const auto bits = BitCast(d32, ConvertTo(df, ConvertTo(d32, u8)));
+  const auto exponent = ShiftRight<23>(bits) - Set(d32, 127);
+  Store(ConvertTo(d8, exponent), d8, log2);
+}
+```
+
+This generates the following SSE4 and AVX2 code, as shown by IACA:
+
+```
+ p0  p1  p5
+|   |   | 1 | CP | pmovzxbd xmm1, dword [rsp+0x25c]
+|   | 1 |   |    | cvtdq2ps xmm1, xmm1
+| 1 |   |   |    | psrad xmm1, 0x17
+|   | 1 |   |    | paddd xmm1, xmm0
+|   |   | 1 | CP | packusdw xmm1, xmm0
+|   |   | 1 | CP | packuswb xmm1, xmm0
+|   |   |   |    | movd [rsp+0x45c], xmm1
+
+|   |   | 1 | CP | vpmovzxbd ymm1, qword [rsp+0x228]
+|   | 1 |   |    | vcvtdq2ps ymm1, ymm1
+| 1 |   |   |    | vpsrad ymm1, ymm1, 0x17
+|   | 1 |   |    | vpaddd ymm1, ymm1, ymm0
+|   |   | 1 | CP | vpackusdw ymm1, ymm1, ymm0
+|   |   | 1 | CP | vpermq ymm1, ymm1, 0xe8
+|   |   | 1 | CP | vpackuswb xmm1, xmm1, xmm0
+|   |   |   |    | vmovq [rsp+0x448], xmm1
+```
+
+```c++
+void Copy(const uint8_t* HWY_RESTRICT from, const size_t size,
+          uint8_t* HWY_RESTRICT to) {
+  // Width-agnostic (library-specified N)
+  const HWY_FULL(uint8_t) d;
+  const Scalar<uint8_t> ds;
+  size_t i = 0;
+  for (; i + d.N <= size; i += d.N) {
+    const auto bytes = Load(d, from + i);
+    Store(bytes, d, to + i);
+  }
+
+  for (; i < size; ++i) {
+    // (Same loop body as above, could factor into a shared template)
+    const auto bytes = Load(ds, from + i);
+    Store(bytes, ds, to + i);
+  }
+}
+```
+
+```c++
+void MulAdd(const T* HWY_RESTRICT mul_array, const T* HWY_RESTRICT add_array,
+            const size_t size, T* HWY_RESTRICT x_array) {
+  // Type-agnostic (caller-specified lane type) and width-agnostic (uses
+  // best available instruction set).
+  const HWY_FULL(T) d;
+  for (size_t i = 0; i < size; i += d.N) {
+    const auto mul = Load(d, mul_array + i);
+    const auto add = Load(d, add_array + i);
+    auto x = Load(d, x_array + i);
+    x = MulAdd(mul, x, add);
+    Store(x, d, x_array + i);
+  }
+}
+```
+
+## Additional resources
+
+*   [Overview of instructions per operation on different architectures][instmtx]
+
+[instmtx]: instruction_matrix.pdf
+
+This is not an officially supported Google product.
+Contact: janwas@google.com
diff --git a/third_party/highway/hwy/arch.h b/third_party/highway/hwy/arch.h
new file mode 100644
index 000000000..ab052ceeb
--- /dev/null
+++ b/third_party/highway/hwy/arch.h
@@ -0,0 +1,53 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_ARCH_H_
+#define HWY_ARCH_H_
+
+// Sets HWY_ARCH to one of the following based on predefined macros:
+
+#include <stddef.h>
+
+#define HWY_ARCH_X86 8
+#define HWY_ARCH_PPC 9
+#define HWY_ARCH_ARM 0xA
+#define HWY_ARCH_SCALAR 0xB
+#define HWY_ARCH_WASM 0xC
+
+#if defined(__wasm_simd128__)
+#define HWY_ARCH HWY_ARCH_WASM
+
+#elif defined(__i386__) || defined(__x86_64__) || defined(_M_X64)
+#define HWY_ARCH HWY_ARCH_X86
+
+#elif defined(__powerpc64__) || defined(_M_PPC)
+#define HWY_ARCH HWY_ARCH_PPC
+
+#elif defined(__ARM_NEON__) || defined(__ARM_NEON)
+#define HWY_ARCH HWY_ARCH_ARM
+
+#elif defined(__EMSCRIPTEN__)
+#define HWY_ARCH HWY_ARCH_SCALAR
+
+#else
+#error "Unsupported platform"
+#endif
+
+#if HWY_ARCH == HWY_ARCH_X86
+static constexpr size_t kMaxVectorSize = 64;  // AVX512
+#else
+static constexpr size_t kMaxVectorSize = 16;
+#endif
+
+#endif  // HWY_ARCH_H_
diff --git a/third_party/highway/hwy/arm64_neon.h b/third_party/highway/hwy/arm64_neon.h
new file mode 100644
index 000000000..0300c648b
--- /dev/null
+++ b/third_party/highway/hwy/arm64_neon.h
@@ -0,0 +1,2703 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_ARM64_NEON_H_
+#define HWY_ARM64_NEON_H_
+
+// 128-bit ARM64 NEON vectors and operations.
+
+#include <arm_neon.h>
+#include <stddef.h>
+
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+#include <stdio.h>
+#endif
+
+#include "hwy/shared.h"
+
+#define HWY_ATTR_ARM8 HWY_TARGET_ATTR("crypto")
+
+#define HWY_CONCAT_IMPL(a, b) a##b
+#define HWY_CONCAT(a, b) HWY_CONCAT_IMPL(a, b)
+
+// Macros used to define single and double function calls for multiple types
+// for full and half vectors. These macros are undefined at the end of the file.
+
+// HWY_NEON_BUILD_TPL_* is the template<...> prefix to the function.
+#define HWY_NEON_BUILD_TPL_1
+#define HWY_NEON_BUILD_TPL_2
+#define HWY_NEON_BUILD_TPL_3
+
+// HWY_NEON_BUILD_RET_* is return type.
+#define HWY_NEON_BUILD_RET_1(type, size) Vec128<type, size>
+#define HWY_NEON_BUILD_RET_2(type, size) Vec128<type, size>
+#define HWY_NEON_BUILD_RET_3(type, size) Vec128<type, size>
+
+// HWY_NEON_BUILD_PARAM_* is the list of parameters the function receives.
+#define HWY_NEON_BUILD_PARAM_1(type, size) const Vec128<type, size> a
+#define HWY_NEON_BUILD_PARAM_2(type, size) \
+  const Vec128<type, size> a, const Vec128<type, size> b
+#define HWY_NEON_BUILD_PARAM_3(type, size)                \
+  const Vec128<type, size> a, const Vec128<type, size> b, \
+      const Vec128<type, size> c
+
+// HWY_NEON_BUILD_ARG_* is the list of arguments passed to the underlying
+// function.
+#define HWY_NEON_BUILD_ARG_1 a.raw
+#define HWY_NEON_BUILD_ARG_2 a.raw, b.raw
+#define HWY_NEON_BUILD_ARG_3 a.raw, b.raw, c.raw
+
+// We use HWY_NEON_EVAL(func, ...) to delay the evaluation of func until after
+// the __VA_ARGS__ have been expanded. This allows "func" to be a macro on
+// itself like with some of the library "functions" such as vshlq_u8. For
+// example, HWY_NEON_EVAL(vshlq_u8, MY_PARAMS) where MY_PARAMS is defined as
+// "a, b" (without the quotes) will end up expanding "vshlq_u8(a, b)" if needed.
+// Directly writing vshlq_u8(MY_PARAMS) would fail since vshlq_u8() macro
+// expects two arguments.
+#define HWY_NEON_EVAL(func, ...) func(__VA_ARGS__)
+
+// Main macro definition that defines a single function for the given type and
+// size of vector, using the underlying (prefix##infix##suffix) function and
+// the template, return type, parameters and arguments defined by the "args"
+// parameters passed here (see HWY_NEON_BUILD_* macros defined before).
+#define HWY_NEON_DEF_FUNCTION(type, size, name, prefix, infix, suffix, args) \
+  HWY_CONCAT(HWY_NEON_BUILD_TPL_, args)                                      \
+  HWY_INLINE HWY_CONCAT(HWY_NEON_BUILD_RET_, args)(type, size)               \
+      name(HWY_CONCAT(HWY_NEON_BUILD_PARAM_, args)(type, size)) {            \
+    return HWY_CONCAT(HWY_NEON_BUILD_RET_, args)(type, size)(                \
+        HWY_NEON_EVAL(prefix##infix##suffix, HWY_NEON_BUILD_ARG_##args));    \
+  }
+
+// The HWY_NEON_DEF_FUNCTION_* macros define all the variants of a function
+// called "name" using the set of neon functions starting with the given
+// "prefix" for all the variants of certain types, as specified next to each
+// macro. For example, the prefix "vsub" can be used to define the operator-
+// using args=2.
+
+// uint8_t
+#define HWY_NEON_DEF_FUNCTION_UINT_8(name, prefix, infix, args)        \
+  HWY_NEON_DEF_FUNCTION(uint8_t, 16, name, prefix##q, infix, u8, args) \
+  HWY_NEON_DEF_FUNCTION(uint8_t, 8, name, prefix, infix, u8, args)     \
+  HWY_NEON_DEF_FUNCTION(uint8_t, 4, name, prefix, infix, u8, args)     \
+  HWY_NEON_DEF_FUNCTION(uint8_t, 2, name, prefix, infix, u8, args)
+
+// int8_t
+#define HWY_NEON_DEF_FUNCTION_INT_8(name, prefix, infix, args)        \
+  HWY_NEON_DEF_FUNCTION(int8_t, 16, name, prefix##q, infix, s8, args) \
+  HWY_NEON_DEF_FUNCTION(int8_t, 8, name, prefix, infix, s8, args)     \
+  HWY_NEON_DEF_FUNCTION(int8_t, 4, name, prefix, infix, s8, args)     \
+  HWY_NEON_DEF_FUNCTION(int8_t, 2, name, prefix, infix, s8, args)
+
+// uint16_t
+#define HWY_NEON_DEF_FUNCTION_UINT_16(name, prefix, infix, args)        \
+  HWY_NEON_DEF_FUNCTION(uint16_t, 8, name, prefix##q, infix, u16, args) \
+  HWY_NEON_DEF_FUNCTION(uint16_t, 4, name, prefix, infix, u16, args)    \
+  HWY_NEON_DEF_FUNCTION(uint16_t, 2, name, prefix, infix, u16, args)    \
+  HWY_NEON_DEF_FUNCTION(uint16_t, 1, name, prefix, infix, u16, args)
+
+// int16_t
+#define HWY_NEON_DEF_FUNCTION_INT_16(name, prefix, infix, args)        \
+  HWY_NEON_DEF_FUNCTION(int16_t, 8, name, prefix##q, infix, s16, args) \
+  HWY_NEON_DEF_FUNCTION(int16_t, 4, name, prefix, infix, s16, args)    \
+  HWY_NEON_DEF_FUNCTION(int16_t, 2, name, prefix, infix, s16, args)    \
+  HWY_NEON_DEF_FUNCTION(int16_t, 1, name, prefix, infix, s16, args)
+
+// uint32_t
+#define HWY_NEON_DEF_FUNCTION_UINT_32(name, prefix, infix, args)        \
+  HWY_NEON_DEF_FUNCTION(uint32_t, 4, name, prefix##q, infix, u32, args) \
+  HWY_NEON_DEF_FUNCTION(uint32_t, 2, name, prefix, infix, u32, args)    \
+  HWY_NEON_DEF_FUNCTION(uint32_t, 1, name, prefix, infix, u32, args)
+
+// int32_t
+#define HWY_NEON_DEF_FUNCTION_INT_32(name, prefix, infix, args)        \
+  HWY_NEON_DEF_FUNCTION(int32_t, 4, name, prefix##q, infix, s32, args) \
+  HWY_NEON_DEF_FUNCTION(int32_t, 2, name, prefix, infix, s32, args)    \
+  HWY_NEON_DEF_FUNCTION(int32_t, 1, name, prefix, infix, s32, args)
+
+// uint64_t
+#define HWY_NEON_DEF_FUNCTION_UINT_64(name, prefix, infix, args)        \
+  HWY_NEON_DEF_FUNCTION(uint64_t, 2, name, prefix##q, infix, u64, args) \
+  HWY_NEON_DEF_FUNCTION(uint64_t, 1, name, prefix, infix, u64, args)
+
+// int64_t
+#define HWY_NEON_DEF_FUNCTION_INT_64(name, prefix, infix, args)        \
+  HWY_NEON_DEF_FUNCTION(int64_t, 2, name, prefix##q, infix, s64, args) \
+  HWY_NEON_DEF_FUNCTION(int64_t, 1, name, prefix, infix, s64, args)
+
+// float and double
+#if defined(__aarch64__)
+#define HWY_NEON_DEF_FUNCTION_ALL_FLOATS(name, prefix, infix, args)   \
+  HWY_NEON_DEF_FUNCTION(float, 4, name, prefix##q, infix, f32, args)  \
+  HWY_NEON_DEF_FUNCTION(float, 2, name, prefix, infix, f32, args)     \
+  HWY_NEON_DEF_FUNCTION(float, 1, name, prefix, infix, f32, args)     \
+  HWY_NEON_DEF_FUNCTION(double, 2, name, prefix##q, infix, f64, args) \
+  HWY_NEON_DEF_FUNCTION(double, 1, name, prefix, infix, f64, args)
+#else
+#define HWY_NEON_DEF_FUNCTION_ALL_FLOATS(name, prefix, infix, args)  \
+  HWY_NEON_DEF_FUNCTION(float, 4, name, prefix##q, infix, f32, args) \
+  HWY_NEON_DEF_FUNCTION(float, 2, name, prefix, infix, f32, args)    \
+  HWY_NEON_DEF_FUNCTION(float, 1, name, prefix, infix, f32, args)
+#endif
+
+// Helper macros to define for more than one type.
+// uint8_t, uint16_t and uint32_t
+#define HWY_NEON_DEF_FUNCTION_UINT_8_16_32(name, prefix, infix, args) \
+  HWY_NEON_DEF_FUNCTION_UINT_8(name, prefix, infix, args)             \
+  HWY_NEON_DEF_FUNCTION_UINT_16(name, prefix, infix, args)            \
+  HWY_NEON_DEF_FUNCTION_UINT_32(name, prefix, infix, args)
+
+// int8_t, int16_t and int32_t
+#define HWY_NEON_DEF_FUNCTION_INT_8_16_32(name, prefix, infix, args) \
+  HWY_NEON_DEF_FUNCTION_INT_8(name, prefix, infix, args)             \
+  HWY_NEON_DEF_FUNCTION_INT_16(name, prefix, infix, args)            \
+  HWY_NEON_DEF_FUNCTION_INT_32(name, prefix, infix, args)
+
+// uint8_t, uint16_t, uint32_t and uint64_t
+#define HWY_NEON_DEF_FUNCTION_UINTS(name, prefix, infix, args)  \
+  HWY_NEON_DEF_FUNCTION_UINT_8_16_32(name, prefix, infix, args) \
+  HWY_NEON_DEF_FUNCTION_UINT_64(name, prefix, infix, args)
+
+// int8_t, int16_t, int32_t and int64_t
+#define HWY_NEON_DEF_FUNCTION_INTS(name, prefix, infix, args)  \
+  HWY_NEON_DEF_FUNCTION_INT_8_16_32(name, prefix, infix, args) \
+  HWY_NEON_DEF_FUNCTION_INT_64(name, prefix, infix, args)
+
+// All int*_t and uint*_t up to 64
+#define HWY_NEON_DEF_FUNCTION_INTS_UINTS(name, prefix, infix, args) \
+  HWY_NEON_DEF_FUNCTION_INTS(name, prefix, infix, args)             \
+  HWY_NEON_DEF_FUNCTION_UINTS(name, prefix, infix, args)
+
+// All previous types.
+#define HWY_NEON_DEF_FUNCTION_ALL_TYPES(name, prefix, infix, args) \
+  HWY_NEON_DEF_FUNCTION_INTS_UINTS(name, prefix, infix, args)      \
+  HWY_NEON_DEF_FUNCTION_ALL_FLOATS(name, prefix, infix, args)
+
+// Emulation of some intrinsics on armv7.
+#if !defined(__aarch64__)
+#define vuzp1_s8(x, y) vuzp_s8(x, y).val[0]
+#define vuzp1_u8(x, y) vuzp_u8(x, y).val[0]
+#define vuzp1_s16(x, y) vuzp_s16(x, y).val[0]
+#define vuzp1_u16(x, y) vuzp_u16(x, y).val[0]
+#define vuzp1_s32(x, y) vuzp_s32(x, y).val[0]
+#define vuzp1_u32(x, y) vuzp_u32(x, y).val[0]
+#define vuzp1_f32(x, y) vuzp_f32(x, y).val[0]
+#define vuzp1q_s8(x, y) vuzpq_s8(x, y).val[0]
+#define vuzp1q_u8(x, y) vuzpq_u8(x, y).val[0]
+#define vuzp1q_s16(x, y) vuzpq_s16(x, y).val[0]
+#define vuzp1q_u16(x, y) vuzpq_u16(x, y).val[0]
+#define vuzp1q_s32(x, y) vuzpq_s32(x, y).val[0]
+#define vuzp1q_u32(x, y) vuzpq_u32(x, y).val[0]
+#define vuzp1q_f32(x, y) vuzpq_f32(x, y).val[0]
+#define vuzp2_s8(x, y) vuzp_s8(x, y).val[1]
+#define vuzp2_u8(x, y) vuzp_u8(x, y).val[1]
+#define vuzp2_s16(x, y) vuzp_s16(x, y).val[1]
+#define vuzp2_u16(x, y) vuzp_u16(x, y).val[1]
+#define vuzp2_s32(x, y) vuzp_s32(x, y).val[1]
+#define vuzp2_u32(x, y) vuzp_u32(x, y).val[1]
+#define vuzp2_f32(x, y) vuzp_f32(x, y).val[1]
+#define vuzp2q_s8(x, y) vuzpq_s8(x, y).val[1]
+#define vuzp2q_u8(x, y) vuzpq_u8(x, y).val[1]
+#define vuzp2q_s16(x, y) vuzpq_s16(x, y).val[1]
+#define vuzp2q_u16(x, y) vuzpq_u16(x, y).val[1]
+#define vuzp2q_s32(x, y) vuzpq_s32(x, y).val[1]
+#define vuzp2q_u32(x, y) vuzpq_u32(x, y).val[1]
+#define vuzp2q_f32(x, y) vuzpq_f32(x, y).val[1]
+#define vzip1_s8(x, y) vzip_s8(x, y).val[0]
+#define vzip1_u8(x, y) vzip_u8(x, y).val[0]
+#define vzip1_s16(x, y) vzip_s16(x, y).val[0]
+#define vzip1_u16(x, y) vzip_u16(x, y).val[0]
+#define vzip1_f32(x, y) vzip_f32(x, y).val[0]
+#define vzip1_u32(x, y) vzip_u32(x, y).val[0]
+#define vzip1_s32(x, y) vzip_s32(x, y).val[0]
+#define vzip1q_s8(x, y) vzipq_s8(x, y).val[0]
+#define vzip1q_u8(x, y) vzipq_u8(x, y).val[0]
+#define vzip1q_s16(x, y) vzipq_s16(x, y).val[0]
+#define vzip1q_u16(x, y) vzipq_u16(x, y).val[0]
+#define vzip1q_s32(x, y) vzipq_s32(x, y).val[0]
+#define vzip1q_u32(x, y) vzipq_u32(x, y).val[0]
+#define vzip1q_f32(x, y) vzipq_f32(x, y).val[0]
+#define vzip2_s8(x, y) vzip_s8(x, y).val[1]
+#define vzip2_u8(x, y) vzip_u8(x, y).val[1]
+#define vzip2_s16(x, y) vzip_s16(x, y).val[1]
+#define vzip2_u16(x, y) vzip_u16(x, y).val[1]
+#define vzip2_s32(x, y) vzip_s32(x, y).val[1]
+#define vzip2_u32(x, y) vzip_u32(x, y).val[1]
+#define vzip2_f32(x, y) vzip_f32(x, y).val[1]
+#define vzip2q_s8(x, y) vzipq_s8(x, y).val[1]
+#define vzip2q_u8(x, y) vzipq_u8(x, y).val[1]
+#define vzip2q_s16(x, y) vzipq_s16(x, y).val[1]
+#define vzip2q_u16(x, y) vzipq_u16(x, y).val[1]
+#define vzip2q_s32(x, y) vzipq_s32(x, y).val[1]
+#define vzip2q_u32(x, y) vzipq_u32(x, y).val[1]
+#define vzip2q_f32(x, y) vzipq_f32(x, y).val[1]
+#endif
+
+namespace hwy {
+
+template <typename T, size_t N>
+struct Raw128;
+
+// 128
+template <>
+struct Raw128<uint8_t, 16> {
+  using type = uint8x16_t;
+};
+
+template <>
+struct Raw128<uint16_t, 8> {
+  using type = uint16x8_t;
+};
+
+template <>
+struct Raw128<uint32_t, 4> {
+  using type = uint32x4_t;
+};
+
+template <>
+struct Raw128<uint64_t, 2> {
+  using type = uint64x2_t;
+};
+
+template <>
+struct Raw128<int8_t, 16> {
+  using type = int8x16_t;
+};
+
+template <>
+struct Raw128<int16_t, 8> {
+  using type = int16x8_t;
+};
+
+template <>
+struct Raw128<int32_t, 4> {
+  using type = int32x4_t;
+};
+
+template <>
+struct Raw128<int64_t, 2> {
+  using type = int64x2_t;
+};
+
+template <>
+struct Raw128<float, 4> {
+  using type = float32x4_t;
+};
+
+#if defined(__aarch64__)
+template <>
+struct Raw128<double, 2> {
+  using type = float64x2_t;
+};
+#endif
+
+// 64
+template <>
+struct Raw128<uint8_t, 8> {
+  using type = uint8x8_t;
+};
+
+template <>
+struct Raw128<uint16_t, 4> {
+  using type = uint16x4_t;
+};
+
+template <>
+struct Raw128<uint32_t, 2> {
+  using type = uint32x2_t;
+};
+
+template <>
+struct Raw128<uint64_t, 1> {
+  using type = uint64x1_t;
+};
+
+template <>
+struct Raw128<int8_t, 8> {
+  using type = int8x8_t;
+};
+
+template <>
+struct Raw128<int16_t, 4> {
+  using type = int16x4_t;
+};
+
+template <>
+struct Raw128<int32_t, 2> {
+  using type = int32x2_t;
+};
+
+template <>
+struct Raw128<int64_t, 1> {
+  using type = int64x1_t;
+};
+
+template <>
+struct Raw128<float, 2> {
+  using type = float32x2_t;
+};
+
+#if defined(__aarch64__)
+template <>
+struct Raw128<double, 1> {
+  using type = float64x1_t;
+};
+#endif
+
+// 32 (same as 64)
+template <>
+struct Raw128<uint8_t, 4> {
+  using type = uint8x8_t;
+};
+
+template <>
+struct Raw128<uint16_t, 2> {
+  using type = uint16x4_t;
+};
+
+template <>
+struct Raw128<uint32_t, 1> {
+  using type = uint32x2_t;
+};
+
+template <>
+struct Raw128<int8_t, 4> {
+  using type = int8x8_t;
+};
+
+template <>
+struct Raw128<int16_t, 2> {
+  using type = int16x4_t;
+};
+
+template <>
+struct Raw128<int32_t, 1> {
+  using type = int32x2_t;
+};
+
+template <>
+struct Raw128<float, 1> {
+  using type = float32x2_t;
+};
+
+// 16 (same as 64)
+template <>
+struct Raw128<uint8_t, 2> {
+  using type = uint8x8_t;
+};
+
+template <>
+struct Raw128<uint16_t, 1> {
+  using type = uint16x4_t;
+};
+
+template <>
+struct Raw128<int8_t, 2> {
+  using type = int8x8_t;
+};
+
+template <>
+struct Raw128<int16_t, 1> {
+  using type = int16x4_t;
+};
+
+template <typename T>
+using Full128 = Desc<T, 16 / sizeof(T)>;
+
+template <typename T, size_t N = 16 / sizeof(T)>
+class Vec128 {
+  using Raw = typename Raw128<T, N>::type;
+
+ public:
+  HWY_INLINE Vec128() {}
+  Vec128(const Vec128&) = default;
+  Vec128& operator=(const Vec128&) = default;
+  HWY_INLINE explicit Vec128(const Raw raw) : raw(raw) {}
+
+  // Compound assignment. Only usable if there is a corresponding non-member
+  // binary operator overload. For example, only f32 and f64 support division.
+  HWY_INLINE Vec128& operator*=(const Vec128 other) {
+    return *this = (*this * other);
+  }
+  HWY_INLINE Vec128& operator/=(const Vec128 other) {
+    return *this = (*this / other);
+  }
+  HWY_INLINE Vec128& operator+=(const Vec128 other) {
+    return *this = (*this + other);
+  }
+  HWY_INLINE Vec128& operator-=(const Vec128 other) {
+    return *this = (*this - other);
+  }
+  HWY_INLINE Vec128& operator&=(const Vec128 other) {
+    return *this = (*this & other);
+  }
+  HWY_INLINE Vec128& operator|=(const Vec128 other) {
+    return *this = (*this | other);
+  }
+  HWY_INLINE Vec128& operator^=(const Vec128 other) {
+    return *this = (*this ^ other);
+  }
+
+  Raw raw;
+};
+
+// FF..FF or 0, also for floating-point - see README.
+template <typename T, size_t N = 16 / sizeof(T)>
+class Mask128 {
+  using Raw = typename Raw128<T, N>::type;
+
+ public:
+  HWY_INLINE Mask128() {}
+  Mask128(const Mask128&) = default;
+  Mask128& operator=(const Mask128&) = default;
+  HWY_INLINE explicit Mask128(const Raw raw) : raw(raw) {}
+
+  Raw raw;
+};
+
+// ------------------------------ Cast
+
+// cast_to_u8
+
+// Converts from Vec128<T, N> to Vec128<uint8_t, N * sizeof(T)> using the
+// vreinterpret*_u8_*() set of functions.
+#define HWY_NEON_BUILD_TPL_HWY_CAST_TO_U8
+#define HWY_NEON_BUILD_RET_HWY_CAST_TO_U8(type, size) \
+  Vec128<uint8_t, size * sizeof(type)>
+#define HWY_NEON_BUILD_PARAM_HWY_CAST_TO_U8(type, size) Vec128<type, size> v
+#define HWY_NEON_BUILD_ARG_HWY_CAST_TO_U8 v.raw
+
+// Special case of u8 to u8 since vreinterpret*_u8_u8 is obviously not defined.
+template <size_t N>
+HWY_INLINE Vec128<uint8_t, N> cast_to_u8(Vec128<uint8_t, N> v) {
+  return v;
+}
+
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(cast_to_u8, vreinterpret, _u8_, HWY_CAST_TO_U8)
+HWY_NEON_DEF_FUNCTION_INTS(cast_to_u8, vreinterpret, _u8_, HWY_CAST_TO_U8)
+HWY_NEON_DEF_FUNCTION_UINT_16(cast_to_u8, vreinterpret, _u8_, HWY_CAST_TO_U8)
+HWY_NEON_DEF_FUNCTION_UINT_32(cast_to_u8, vreinterpret, _u8_, HWY_CAST_TO_U8)
+HWY_NEON_DEF_FUNCTION_UINT_64(cast_to_u8, vreinterpret, _u8_, HWY_CAST_TO_U8)
+
+#undef HWY_NEON_BUILD_TPL_HWY_CAST_TO_U8
+#undef HWY_NEON_BUILD_RET_HWY_CAST_TO_U8
+#undef HWY_NEON_BUILD_PARAM_HWY_CAST_TO_U8
+#undef HWY_NEON_BUILD_ARG_HWY_CAST_TO_U8
+
+// cast_u8_to
+
+template <size_t N>
+HWY_INLINE Vec128<uint8_t, N> cast_u8_to(Desc<uint8_t, N> /* tag */,
+                                         Vec128<uint8_t, N> v) {
+  return v;
+}
+
+// 64-bit full/part:
+
+// Differentiate between <= 64 bit and (64, 128] (for q suffix on NEON)
+#define HWY_IF64(T, N) EnableIf<N != 0 && (N * sizeof(T) <= 8)>* = nullptr
+
+// .. 8-bit
+template <size_t N, HWY_IF64(int8_t, N)>
+HWY_INLINE Vec128<int8_t, N> cast_u8_to(Desc<int8_t, N> /* tag */,
+                                        Vec128<uint8_t, N> v) {
+  return Vec128<int8_t, N>(vreinterpret_s8_u8(v.raw));
+}
+// .. 16-bit
+template <size_t N, HWY_IF64(uint16_t, N)>
+HWY_INLINE Vec128<uint16_t, N> cast_u8_to(Desc<uint16_t, N> /* tag */,
+                                          Vec128<uint8_t, N * 2> v) {
+  return Vec128<uint16_t, N>(vreinterpret_u16_u8(v.raw));
+}
+template <size_t N, HWY_IF64(int16_t, N)>
+HWY_INLINE Vec128<int16_t, N> cast_u8_to(Desc<int16_t, N> /* tag */,
+                                         Vec128<uint8_t, N * 2> v) {
+  return Vec128<int16_t, N>(vreinterpret_s16_u8(v.raw));
+}
+// .. 32-bit
+template <size_t N, HWY_IF64(uint32_t, N)>
+HWY_INLINE Vec128<uint32_t, N> cast_u8_to(Desc<uint32_t, N> /* tag */,
+                                          Vec128<uint8_t, N * 4> v) {
+  return Vec128<uint32_t, N>(vreinterpret_u32_u8(v.raw));
+}
+template <size_t N, HWY_IF64(int32_t, N)>
+HWY_INLINE Vec128<int32_t, N> cast_u8_to(Desc<int32_t, N> /* tag */,
+                                         Vec128<uint8_t, N * 4> v) {
+  return Vec128<int32_t, N>(vreinterpret_s32_u8(v.raw));
+}
+template <size_t N, HWY_IF64(float, N)>
+HWY_INLINE Vec128<float, N> cast_u8_to(Desc<float, N> /* tag */,
+                                       Vec128<uint8_t, N * 4> v) {
+  return Vec128<float, N>(vreinterpret_f32_u8(v.raw));
+}
+// .. 64-bit
+HWY_INLINE Vec128<uint64_t, 1> cast_u8_to(Desc<uint64_t, 1> /* tag */,
+                                          Vec128<uint8_t, 1 * 8> v) {
+  return Vec128<uint64_t, 1>(vreinterpret_u64_u8(v.raw));
+}
+HWY_INLINE Vec128<int64_t, 1> cast_u8_to(Desc<int64_t, 1> /* tag */,
+                                         Vec128<uint8_t, 1 * 8> v) {
+  return Vec128<int64_t, 1>(vreinterpret_s64_u8(v.raw));
+}
+#if defined(__aarch64__)
+HWY_INLINE Vec128<double, 1> cast_u8_to(Desc<double, 1> /* tag */,
+                                        Vec128<uint8_t, 1 * 8> v) {
+  return Vec128<double, 1>(vreinterpret_f64_u8(v.raw));
+}
+#endif
+
+// 128-bit full/part:
+
+#define HWY_IF128_Q(T, N) EnableIf<(N * sizeof(T) > 8)>* = nullptr
+
+// .. 8-bit
+template <size_t N, HWY_IF128_Q(int8_t, N)>
+HWY_INLINE Vec128<int8_t, N> cast_u8_to(Desc<int8_t, N> /* tag */,
+                                        Vec128<uint8_t, N> v) {
+  return Vec128<int8_t, N>(vreinterpretq_s8_u8(v.raw));
+}
+// .. 16-bit
+template <size_t N, HWY_IF128_Q(uint16_t, N)>
+HWY_INLINE Vec128<uint16_t, N> cast_u8_to(Desc<uint16_t, N> /* tag */,
+                                          Vec128<uint8_t, N * 2> v) {
+  return Vec128<uint16_t, N>(vreinterpretq_u16_u8(v.raw));
+}
+template <size_t N, HWY_IF128_Q(int16_t, N)>
+HWY_INLINE Vec128<int16_t, N> cast_u8_to(Desc<int16_t, N> /* tag */,
+                                         Vec128<uint8_t, N * 2> v) {
+  return Vec128<int16_t, N>(vreinterpretq_s16_u8(v.raw));
+}
+// .. 32-bit
+template <size_t N, HWY_IF128_Q(uint32_t, N)>
+HWY_INLINE Vec128<uint32_t, N> cast_u8_to(Desc<uint32_t, N> /* tag */,
+                                          Vec128<uint8_t, N * 4> v) {
+  return Vec128<uint32_t, N>(vreinterpretq_u32_u8(v.raw));
+}
+template <size_t N, HWY_IF128_Q(int32_t, N)>
+HWY_INLINE Vec128<int32_t, N> cast_u8_to(Desc<int32_t, N> /* tag */,
+                                         Vec128<uint8_t, N * 4> v) {
+  return Vec128<int32_t, N>(vreinterpretq_s32_u8(v.raw));
+}
+template <size_t N, HWY_IF128_Q(float, N)>
+HWY_INLINE Vec128<float, N> cast_u8_to(Desc<float, N> /* tag */,
+                                       Vec128<uint8_t, N * 4> v) {
+  return Vec128<float, N>(vreinterpretq_f32_u8(v.raw));
+}
+// .. 64-bit
+HWY_INLINE Vec128<uint64_t, 2> cast_u8_to(Desc<uint64_t, 2> /* tag */,
+                                          Vec128<uint8_t, 2 * 8> v) {
+  return Vec128<uint64_t, 2>(vreinterpretq_u64_u8(v.raw));
+}
+HWY_INLINE Vec128<int64_t, 2> cast_u8_to(Desc<int64_t, 2> /* tag */,
+                                         Vec128<uint8_t, 2 * 8> v) {
+  return Vec128<int64_t, 2>(vreinterpretq_s64_u8(v.raw));
+}
+
+#if defined(__aarch64__)
+HWY_INLINE Vec128<double, 2> cast_u8_to(Desc<double, 2> /* tag */,
+                                        Vec128<uint8_t, 2 * 8> v) {
+  return Vec128<double, 2>(vreinterpretq_f64_u8(v.raw));
+}
+#endif
+
+// BitCast
+template <typename T, size_t N, typename FromT>
+HWY_INLINE Vec128<T, N> BitCast(
+    Desc<T, N> d, Vec128<FromT, N * sizeof(T) / sizeof(FromT)> v) {
+  const auto u8 = cast_to_u8(v);
+  return cast_u8_to(d, u8);
+}
+
+// ------------------------------ Set
+
+// Returns a vector with all lanes set to "t".
+#define HWY_NEON_BUILD_TPL_HWY_SET1
+#define HWY_NEON_BUILD_RET_HWY_SET1(type, size) Vec128<type, size>
+#define HWY_NEON_BUILD_PARAM_HWY_SET1(type, size) \
+  Desc<type, size> /* tag */, const type t
+#define HWY_NEON_BUILD_ARG_HWY_SET1 t
+
+HWY_NEON_DEF_FUNCTION_ALL_TYPES(Set, vdup, _n_, HWY_SET1)
+
+#undef HWY_NEON_BUILD_TPL_HWY_SET1
+#undef HWY_NEON_BUILD_RET_HWY_SET1
+#undef HWY_NEON_BUILD_PARAM_HWY_SET1
+#undef HWY_NEON_BUILD_ARG_HWY_SET1
+
+// Returns an all-zero vector.
+template <typename T, size_t N>
+HWY_INLINE Vec128<T, N> Zero(Desc<T, N> d) {
+  return Set(d, 0);
+}
+
+// Returns a vector with lane i=[0, N) set to "first" + i. Unique per-lane
+// values are required to detect lane-crossing bugs.
+template <typename T, size_t N, typename T2>
+HWY_INLINE Vec128<T, N> Iota(Desc<T, N> d, const T2 first) {
+  alignas(16) T lanes[N];
+  for (size_t i = 0; i < N; ++i) {
+    lanes[i] = first + i;
+  }
+  return Load(d, lanes);
+}
+
+// Returns a vector with uninitialized elements.
+template <typename T, size_t N>
+HWY_INLINE Vec128<T, N> Undefined(Desc<T, N> /*d*/) {
+  HWY_DIAGNOSTICS(push)
+  HWY_DIAGNOSTICS_OFF(disable : 4701, ignored "-Wuninitialized")
+  typename Raw128<T, N>::type a;
+  return Vec128<T, N>(a);
+  HWY_DIAGNOSTICS(pop)
+}
+
+// ================================================== ARITHMETIC
+
+// ------------------------------ Addition
+HWY_NEON_DEF_FUNCTION_ALL_TYPES(operator+, vadd, _, 2)
+
+// ------------------------------ Subtraction
+HWY_NEON_DEF_FUNCTION_ALL_TYPES(operator-, vsub, _, 2)
+
+// ------------------------------ Saturating addition and subtraction
+// Only defined for uint8_t, uint16_t and their signed versions, as in other
+// architectures.
+
+// Returns a + b clamped to the destination range.
+HWY_NEON_DEF_FUNCTION_INT_8(SaturatedAdd, vqadd, _, 2)
+HWY_NEON_DEF_FUNCTION_INT_16(SaturatedAdd, vqadd, _, 2)
+HWY_NEON_DEF_FUNCTION_UINT_8(SaturatedAdd, vqadd, _, 2)
+HWY_NEON_DEF_FUNCTION_UINT_16(SaturatedAdd, vqadd, _, 2)
+
+// Returns a - b clamped to the destination range.
+HWY_NEON_DEF_FUNCTION_INT_8(SaturatedSub, vqsub, _, 2)
+HWY_NEON_DEF_FUNCTION_INT_16(SaturatedSub, vqsub, _, 2)
+HWY_NEON_DEF_FUNCTION_UINT_8(SaturatedSub, vqsub, _, 2)
+HWY_NEON_DEF_FUNCTION_UINT_16(SaturatedSub, vqsub, _, 2)
+
+// ------------------------------ Average
+
+// Returns (a + b + 1) / 2
+
+// Unsigned
+HWY_NEON_DEF_FUNCTION_UINT_8(AverageRound, vrhadd, _, 2)
+HWY_NEON_DEF_FUNCTION_UINT_16(AverageRound, vrhadd, _, 2)
+
+// ------------------------------ Absolute value
+
+// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1.
+template <size_t N>
+HWY_INLINE Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) {
+  return Vec128<int8_t, N>(vabsq_s8(v.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) {
+  return Vec128<int16_t, N>(vabsq_s16(v.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) {
+  return Vec128<int32_t, N>(vabsq_s32(v.raw));
+}
+
+// ------------------------------ Shift lanes by constant #bits
+
+// Only defined for ints and uints, except for signed i64 shr.
+#define HWY_NEON_BUILD_TPL_HWY_SHIFT template <int kBits>
+#define HWY_NEON_BUILD_RET_HWY_SHIFT(type, size) Vec128<type, size>
+#define HWY_NEON_BUILD_PARAM_HWY_SHIFT(type, size) const Vec128<type, size> v
+#define HWY_NEON_BUILD_ARG_HWY_SHIFT v.raw, kBits
+
+HWY_NEON_DEF_FUNCTION_INTS_UINTS(ShiftLeft, vshl, _n_, HWY_SHIFT)
+
+HWY_NEON_DEF_FUNCTION_UINTS(ShiftRight, vshr, _n_, HWY_SHIFT)
+HWY_NEON_DEF_FUNCTION_INT_8_16_32(ShiftRight, vshr, _n_, HWY_SHIFT)
+
+#undef HWY_NEON_BUILD_TPL_HWY_SHIFT
+#undef HWY_NEON_BUILD_RET_HWY_SHIFT
+#undef HWY_NEON_BUILD_PARAM_HWY_SHIFT
+#undef HWY_NEON_BUILD_ARG_HWY_SHIFT
+
+// ------------------------------ Shift lanes by independent variable #bits
+
+// Unsigned (no u8,u16)
+template <size_t N>
+HWY_INLINE Vec128<uint32_t, N> operator<<(const Vec128<uint32_t, N> v,
+                                          const Vec128<uint32_t, N> bits) {
+  return Vec128<uint32_t, N>(vshlq_u32(v.raw, bits.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<uint32_t, N> operator>>(const Vec128<uint32_t, N> v,
+                                          const Vec128<uint32_t, N> bits) {
+  return Vec128<uint32_t, N>(
+      vshlq_u32(v.raw, vnegq_s32(vreinterpretq_s32_u32(bits.raw))));
+}
+template <size_t N>
+HWY_INLINE Vec128<uint64_t, N> operator<<(const Vec128<uint64_t, N> v,
+                                          const Vec128<uint64_t, N> bits) {
+  return Vec128<uint64_t, N>(vshlq_u64(v.raw, bits.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<uint64_t, N> operator>>(const Vec128<uint64_t, N> v,
+                                          const Vec128<uint64_t, N> bits) {
+#if !defined(__aarch64__)
+  // A32 doesn't have vnegq_s64().
+  return Vec128<uint64_t, N>(
+      vshlq_u64(v.raw, vsubq_s64(Set(Desc<int64_t, N>(), 0).raw,
+                                 vreinterpretq_s64_u64(bits.raw))));
+#else
+  return Vec128<uint64_t, N>(
+      vshlq_u64(v.raw, vnegq_s64(vreinterpretq_s64_u64(bits.raw))));
+#endif
+}
+
+// Signed (no i8,i16)
+template <size_t N>
+HWY_INLINE Vec128<int32_t, N> operator<<(const Vec128<int32_t, N> v,
+                                         const Vec128<int32_t, N> bits) {
+  return Vec128<int32_t, N>(vshlq_s32(v.raw, bits.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<int32_t, N> operator>>(const Vec128<int32_t, N> v,
+                                         const Vec128<int32_t, N> bits) {
+  return Vec128<int32_t, N>(vshlq_s32(v.raw, vnegq_s32(bits.raw)));
+}
+template <size_t N>
+HWY_INLINE Vec128<int64_t, N> operator<<(const Vec128<int64_t, N> v,
+                                         const Vec128<int64_t, N> bits) {
+  return Vec128<int64_t, N>(vshlq_s64(v.raw, bits.raw));
+}
+
+// ------------------------------ Minimum
+
+// Unsigned (no u64)
+HWY_NEON_DEF_FUNCTION_UINT_8_16_32(Min, vmin, _, 2)
+
+// Signed (no i64)
+HWY_NEON_DEF_FUNCTION_INT_8_16_32(Min, vmin, _, 2)
+
+// Float
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Min, vmin, _, 2)
+
+// ------------------------------ Maximum
+
+// Unsigned (no u64)
+HWY_NEON_DEF_FUNCTION_UINT_8_16_32(Max, vmax, _, 2)
+
+// Signed (no i64)
+HWY_NEON_DEF_FUNCTION_INT_8_16_32(Max, vmax, _, 2)
+
+// Float
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Max, vmax, _, 2)
+
+// ------------------------------ Clamping
+template <typename T, size_t N>
+HWY_INLINE Vec128<T, N> Clamp(const Vec128<T, N> v, const Vec128<T, N> lo,
+                              const Vec128<T, N> hi) {
+  return Min(Max(lo, v), hi);
+}
+// ------------------------------ Integer multiplication
+
+// Unsigned
+template <size_t N>
+HWY_INLINE Vec128<uint16_t, N> operator*(const Vec128<uint16_t, N> a,
+                                         const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>(vmulq_u16(a.raw, b.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<uint32_t, N> operator*(const Vec128<uint32_t, N> a,
+                                         const Vec128<uint32_t, N> b) {
+  return Vec128<uint32_t, N>(vmulq_u32(a.raw, b.raw));
+}
+
+// Signed
+template <size_t N>
+HWY_INLINE Vec128<int16_t, N> operator*(const Vec128<int16_t, N> a,
+                                        const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>(vmulq_s16(a.raw, b.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<int32_t, N> operator*(const Vec128<int32_t, N> a,
+                                        const Vec128<int32_t, N> b) {
+  return Vec128<int32_t, N>(vmulq_s32(a.raw, b.raw));
+}
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// Returns the upper 16 bits of a * b in each lane.
+template <size_t N>
+HWY_INLINE Vec128<int16_t, N> MulHigh(const Vec128<int16_t, N> a,
+                                      const Vec128<int16_t, N> b) {
+  int32x4_t rlo = vmull_s16(vget_low_s16(a.raw), vget_low_s16(b.raw));
+#if defined(__aarch64__)
+  int32x4_t rhi = vmull_high_s16(a.raw, b.raw);
+#else
+  int32x4_t rhi = vmull_s16(vget_high_s16(a.raw), vget_high_s16(b.raw));
+#endif
+  return Vec128<int16_t, N>(
+      vuzp2q_s16(vreinterpretq_s16_s32(rlo), vreinterpretq_s16_s32(rhi)));
+}
+template <size_t N>
+HWY_INLINE Vec128<uint16_t, N> MulHigh(const Vec128<uint16_t, N> a,
+                                       const Vec128<uint16_t, N> b) {
+  int32x4_t rlo = vmull_u16(vget_low_u16(a.raw), vget_low_u16(b.raw));
+#if defined(__aarch64__)
+  int32x4_t rhi = vmull_high_u16(a.raw, b.raw);
+#else
+  int32x4_t rhi = vmull_u16(vget_high_u16(a.raw), vget_high_u16(b.raw));
+#endif
+  return Vec128<uint16_t, N>(
+      vuzp2q_u16(vreinterpretq_u16_u32(rlo), vreinterpretq_u16_u32(rhi)));
+}
+
+}  // namespace ext
+
+// Multiplies even lanes (0, 2 ..) and places the double-wide result into
+// even and the upper half into its odd neighbor lane.
+HWY_INLINE Vec128<int64_t> MulEven(const Vec128<int32_t> a,
+                                   const Vec128<int32_t> b) {
+  int32x4_t a_packed = vuzp1q_s32(a.raw, a.raw);
+  int32x4_t b_packed = vuzp1q_s32(b.raw, b.raw);
+  return Vec128<int64_t>(
+      vmull_s32(vget_low_s32(a_packed), vget_low_s32(b_packed)));
+}
+HWY_INLINE Vec128<uint64_t> MulEven(const Vec128<uint32_t> a,
+                                    const Vec128<uint32_t> b) {
+  uint32x4_t a_packed = vuzp1q_u32(a.raw, a.raw);
+  uint32x4_t b_packed = vuzp1q_u32(b.raw, b.raw);
+  return Vec128<uint64_t>(
+      vmull_u32(vget_low_u32(a_packed), vget_low_u32(b_packed)));
+}
+
+// ------------------------------ Floating-point negate
+
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Neg, vneg, _, 1)
+
+// ------------------------------ Floating-point mul / div
+
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator*, vmul, _, 2)
+
+// Approximate reciprocal
+HWY_INLINE Vec128<float, 4> ApproximateReciprocal(const Vec128<float, 4> v) {
+  return Vec128<float, 4>(vrecpeq_f32(v.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<float, N> ApproximateReciprocal(const Vec128<float, N> v) {
+  return Vec128<float, N>(vrecpe_f32(v.raw));
+}
+
+#if defined(__aarch64__)
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator/, vdiv, _, 2)
+#else
+// Emulated with approx reciprocal + Newton-Raphson + mul
+template <size_t N>
+HWY_INLINE Vec128<float, N> operator/(const Vec128<float, N> a,
+                                      const Vec128<float, N> b) {
+  auto x = ApproximateReciprocal(b);
+  // Newton-Raphson on 1/x - b
+  const auto two = Set(Desc<float, N>(), 2);
+  x = x * (two - b * x);
+  x = x * (two - b * x);
+  x = x * (two - b * x);
+  return a * x;
+}
+#endif
+
+namespace ext {
+// Absolute value of difference.
+template <size_t N, HWY_IF64(int8_t, N)>
+HWY_INLINE Vec128<float, N> AbsDiff(const Vec128<float, 2> a,
+                                    const Vec128<float, 2> b) {
+  return Vec128<float, N>(vabd_f32(a.raw, b.raw));
+}
+HWY_INLINE Vec128<float, 4> AbsDiff(const Vec128<float, 4> a,
+                                    const Vec128<float, 4> b) {
+  return Vec128<float, 4>(vabdq_f32(a.raw, b.raw));
+}
+}  // namespace ext
+
+// ------------------------------ Floating-point multiply-add variants
+
+// Returns add + mul * x
+#if defined(__aarch64__)
+HWY_INLINE Vec128<float, 4> MulAdd(const Vec128<float, 4> mul,
+                                   const Vec128<float, 4> x,
+                                   const Vec128<float, 4> add) {
+  return Vec128<float, 4>(vfmaq_f32(add.raw, mul.raw, x.raw));
+}
+HWY_INLINE Vec128<float, 2> MulAdd(const Vec128<float, 2> mul,
+                                   const Vec128<float, 2> x,
+                                   const Vec128<float, 2> add) {
+  return Vec128<float, 2>(vfma_f32(add.raw, mul.raw, x.raw));
+}
+HWY_INLINE Vec128<double, 2> MulAdd(const Vec128<double, 2> mul,
+                                    const Vec128<double, 2> x,
+                                    const Vec128<double, 2> add) {
+  return Vec128<double, 2>(vfmaq_f64(add.raw, mul.raw, x.raw));
+}
+HWY_INLINE Vec128<double, 1> MulAdd(const Vec128<double, 1> mul,
+                                    const Vec128<double, 1> x,
+                                    const Vec128<double, 1> add) {
+  return Vec128<double, 1>(vfma_f64(add.raw, mul.raw, x.raw));
+}
+#else
+// Emulate FMA for floats.
+template <size_t N>
+HWY_INLINE Vec128<float, N> MulAdd(const Vec128<float, N> mul,
+                                   const Vec128<float, N> x,
+                                   const Vec128<float, N> add) {
+  return mul * x + add;
+}
+#endif
+
+// Returns add - mul * x
+#if defined(__aarch64__)
+template <size_t N>
+HWY_INLINE Vec128<float, N> NegMulAdd(const Vec128<float, N> mul,
+                                      const Vec128<float, N> x,
+                                      const Vec128<float, N> add) {
+  return Vec128<float, N>(vfmsq_f32(add.raw, mul.raw, x.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<double, N> NegMulAdd(const Vec128<double, N> mul,
+                                       const Vec128<double, N> x,
+                                       const Vec128<double, N> add) {
+  return Vec128<double, N>(vfmsq_f64(add.raw, mul.raw, x.raw));
+}
+#else
+// Emulate FMA for floats.
+template <size_t N>
+HWY_INLINE Vec128<float, N> NegMulAdd(const Vec128<float, N> mul,
+                                      const Vec128<float, N> x,
+                                      const Vec128<float, N> add) {
+  return add - mul * x;
+}
+#endif
+
+// Slightly more expensive (extra negate)
+namespace ext {
+
+// Returns mul * x - sub
+template <size_t N>
+HWY_INLINE Vec128<float, N> MulSub(const Vec128<float, N> mul,
+                                   const Vec128<float, N> x,
+                                   const Vec128<float, N> sub) {
+  return Neg(NegMulAdd(mul, x, sub));
+}
+template <size_t N>
+HWY_INLINE Vec128<double, N> MulSub(const Vec128<double, N> mul,
+                                    const Vec128<double, N> x,
+                                    const Vec128<double, N> sub) {
+  return Neg(NegMulAdd(mul, x, sub));
+}
+
+// Returns -mul * x - sub
+template <size_t N>
+HWY_INLINE Vec128<float, N> NegMulSub(const Vec128<float, N> mul,
+                                      const Vec128<float, N> x,
+                                      const Vec128<float, N> sub) {
+  return Neg(MulAdd(mul, x, sub));
+}
+template <size_t N>
+HWY_INLINE Vec128<double, N> NegMulSub(const Vec128<double, N> mul,
+                                       const Vec128<double, N> x,
+                                       const Vec128<double, N> sub) {
+  return Neg(MulAdd(mul, x, sub));
+}
+
+}  // namespace ext
+
+// ------------------------------ Floating-point square root
+
+// Approximate reciprocal square root
+template <size_t N>
+HWY_INLINE Vec128<float, N> ApproximateReciprocalSqrt(
+    const Vec128<float, N> v) {
+  return Vec128<float, N>(vrsqrteq_f32(v.raw));
+}
+
+// Full precision square root
+#if defined(__aarch64__)
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Sqrt, vsqrt, _, 1)
+#else
+// Not defined on armv7: emulate with approx reciprocal sqrt + Goldschmidt.
+template <size_t N>
+HWY_INLINE Vec128<float, N> Sqrt(const Vec128<float, N> v) {
+  auto b = v;
+  auto Y = ApproximateReciprocalSqrt(v);
+  auto x = v * Y;
+  const auto half = Set(Desc<float, N>(), 0.5);
+  const auto oneandhalf = Set(Desc<float, N>(), 1.5);
+  for (size_t i = 0; i < 3; i++) {
+    b = b * Y * Y;
+    Y = oneandhalf - half * b;
+    x = x * Y;
+  }
+  return IfThenZeroElse(v == Zero(Desc<float, N>()), x);
+}
+#endif
+
+// ------------------------------ Floating-point rounding
+
+#if defined(__aarch64__)
+// Toward nearest integer
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Round, vrndn, _, 1)
+
+// Toward zero, aka truncate
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Trunc, vrnd, _, 1)
+
+// Toward +infinity, aka ceiling
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Ceil, vrndp, _, 1)
+
+// Toward -infinity, aka floor
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(Floor, vrndm, _, 1)
+#else
+
+template <size_t N>
+HWY_INLINE Vec128<float, N> Trunc(const Vec128<float, N> v) {
+  const Desc<uint32_t, N> du;
+  const Desc<int32_t, N> di;
+  const Desc<float, N> df;
+  const auto v_bits = BitCast(du, v);
+  const auto biased_exp = ShiftRight<23>(v_bits) & Set(du, 0xFF);
+  const auto bits_to_remove =
+      Set(du, 150) - Max(Min(biased_exp, Set(du, 150)), Set(du, 127));
+  const auto mask = (Set(du, 1) << bits_to_remove) - Set(du, 1);
+  return BitCast(df, IfThenZeroElse(BitCast(di, biased_exp) < Set(di, 127),
+                                    BitCast(di, AndNot(mask, v_bits))));
+}
+
+// WARNING: does not quite have the same semantics as what NEON does on
+// aarch64. In particular, does not break ties to even.
+template <size_t N>
+HWY_INLINE Vec128<float, N> Round(const Vec128<float, N> v) {
+  const Desc<uint32_t, N> du;
+  const Desc<float, N> df;
+  const auto sign_mask = BitCast(df, Set(du, 0x80000000u));
+  // move 0.5f away from 0 and call truncate.
+  return Trunc(v + ((v & sign_mask) | Set(df, 0.5f)));
+}
+
+template <size_t N>
+HWY_INLINE Vec128<float, N> Ceil(const Vec128<float, N> v) {
+  const Desc<uint32_t, N> du;
+  const Desc<int32_t, N> di;
+  const Desc<float, N> df;
+  const auto sign_mask = Set(du, 0x80000000u);
+  const auto v_bits = BitCast(du, v);
+  const auto biased_exp = ShiftRight<23>(v_bits) & Set(du, 0xFF);
+  const auto bits_to_remove =
+      Set(du, 150) - Max(Min(biased_exp, Set(du, 150)), Set(du, 127));
+  const auto high_bit = Set(du, 1) << bits_to_remove;
+  const auto mask = high_bit - Set(du, 1);
+  const auto removed_bits = mask & v_bits;
+  // number is positive and at least one bit was set in the mantissa
+  const auto should_round_up = MaskFromVec(
+      BitCast(df, AndNot(VecFromMask(removed_bits == Zero(du)),
+                         VecFromMask(Zero(du) == (v_bits & sign_mask)))));
+  const auto add_one = IfThenElseZero(should_round_up, Set(df, 1.0f));
+  const auto rounded =
+      BitCast(df, IfThenZeroElse(BitCast(di, biased_exp) < Set(di, 127),
+                                 BitCast(di, AndNot(mask, v_bits))));
+  return rounded + add_one;
+}
+
+template <size_t N>
+HWY_INLINE Vec128<float, N> Floor(const Vec128<float, N> v) {
+  const Desc<float, N> df;
+  const auto zero = Zero(df);
+  return zero - Ceil(zero - v);
+}
+
+#endif
+
+// ================================================== COMPARE
+
+#define HWY_NEON_BUILD_TPL_HWY_COMPARE
+#define HWY_NEON_BUILD_RET_HWY_COMPARE(type, size) Mask128<type, size>
+#define HWY_NEON_BUILD_PARAM_HWY_COMPARE(type, size) \
+  const Vec128<type, size> a, const Vec128<type, size> b
+#define HWY_NEON_BUILD_ARG_HWY_COMPARE a.raw, b.raw
+
+// Comparisons fill a lane with 1-bits if the condition is true, else 0.
+
+// ------------------------------ Equality
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator==, vceq, _, HWY_COMPARE)
+#if defined(__aarch64__)
+HWY_NEON_DEF_FUNCTION_INTS_UINTS(operator==, vceq, _, HWY_COMPARE);
+#else
+// No 64-bit comparisons on armv7: emulate them.
+HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator==, vceq, _, HWY_COMPARE)
+HWY_NEON_DEF_FUNCTION_UINT_8_16_32(operator==, vceq, _, HWY_COMPARE)
+
+HWY_INLINE Mask128<int64_t, 2> operator==(const Vec128<int64_t, 2> a,
+                                          const Vec128<int64_t, 2> b) {
+  const Full128<int32_t> d;
+  const Full128<int64_t> d64;
+  auto a32 = BitCast(d, a);
+  auto b32 = BitCast(d, b);
+  auto cmp = a32 == b32;
+  auto cmp_and = Vec128<int32_t, 4>(vandq_u32(cmp.raw, vrev64q_u32(cmp.raw)));
+  return Mask128<int64_t, 2>(BitCast(d64, cmp_and).raw);
+}
+
+HWY_INLINE Mask128<uint64_t, 2> operator==(const Vec128<uint64_t, 2> a,
+                                           const Vec128<uint64_t, 2> b) {
+  const Full128<uint32_t> d;
+  const Full128<uint64_t> d64;
+  auto a32 = BitCast(d, a);
+  auto b32 = BitCast(d, b);
+  auto cmp = a32 == b32;
+  auto cmp_and = Vec128<uint32_t, 4>(vandq_u32(cmp.raw, vrev64q_u32(cmp.raw)));
+  return Mask128<uint64_t, 2>(BitCast(d64, cmp_and).raw);
+}
+
+#endif
+
+// ------------------------------ Strict inequality
+
+// Signed/float < (no unsigned)
+#if defined(__aarch64__)
+HWY_NEON_DEF_FUNCTION_INTS(operator<, vclt, _, HWY_COMPARE)
+#else
+HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator<, vclt, _, HWY_COMPARE)
+#endif
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator<, vclt, _, HWY_COMPARE)
+
+// Signed/float > (no unsigned)
+#if defined(__aarch64__)
+HWY_NEON_DEF_FUNCTION_INTS(operator>, vcgt, _, HWY_COMPARE)
+#else
+HWY_NEON_DEF_FUNCTION_INT_8_16_32(operator>, vcgt, _, HWY_COMPARE)
+#endif
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator>, vcgt, _, HWY_COMPARE)
+
+// ------------------------------ Weak inequality
+
+// Float <= >=
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator<=, vcle, _, HWY_COMPARE)
+HWY_NEON_DEF_FUNCTION_ALL_FLOATS(operator>=, vcge, _, HWY_COMPARE)
+
+#undef HWY_NEON_BUILD_TPL_HWY_COMPARE
+#undef HWY_NEON_BUILD_RET_HWY_COMPARE
+#undef HWY_NEON_BUILD_PARAM_HWY_COMPARE
+#undef HWY_NEON_BUILD_ARG_HWY_COMPARE
+
+// ================================================== LOGICAL
+
+// ------------------------------ Bitwise AND
+HWY_NEON_DEF_FUNCTION_INTS_UINTS(operator&, vand, _, 2)
+// These operator& rely on the special cases for uint32_t and uint64_t just
+// defined by HWY_NEON_DEF_FUNCTION_INTS_UINTS() macro.
+template <size_t N>
+HWY_INLINE Vec128<float, N> operator&(const Vec128<float, N> a,
+                                      const Vec128<float, N> b) {
+  const Full128<uint32_t> d;
+  return BitCast(Full128<float>(), BitCast(d, a) & BitCast(d, b));
+}
+template <size_t N>
+HWY_INLINE Vec128<double, N> operator&(const Vec128<double, N> a,
+                                       const Vec128<double, N> b) {
+  const Full128<uint64_t> d;
+  return BitCast(Full128<double>(), BitCast(d, a) & BitCast(d, b));
+}
+
+// ------------------------------ Bitwise AND-NOT
+
+namespace internal {
+// reversed_andnot returns a & ~b.
+HWY_NEON_DEF_FUNCTION_INTS_UINTS(reversed_andnot, vbic, _, 2)
+}  // namespace internal
+
+// Returns ~not_mask & mask.
+template <typename T, size_t N>
+HWY_INLINE Vec128<T, N> AndNot(const Vec128<T, N> not_mask,
+                               const Vec128<T, N> mask) {
+  return internal::reversed_andnot(mask, not_mask);
+}
+
+// These AndNot() rely on the special cases for uint32_t and uint64_t just
+// defined by HWY_NEON_DEF_FUNCTION_INTS_UINTS() macro.
+
+template <size_t N>
+HWY_INLINE Vec128<float, N> AndNot(const Vec128<float, N> not_mask,
+                                   const Vec128<float, N> mask) {
+  const Desc<uint32_t, N> du;
+  Vec128<uint32_t, N> ret =
+      internal::reversed_andnot(BitCast(du, mask), BitCast(du, not_mask));
+  return BitCast(Desc<float, N>(), ret);
+}
+
+#if defined(__aarch64__)
+template <size_t N>
+HWY_INLINE Vec128<double, N> AndNot(const Vec128<double, N> not_mask,
+                                    const Vec128<double, N> mask) {
+  const Desc<uint64_t, N> du;
+  Vec128<uint64_t, N> ret =
+      internal::reversed_andnot(BitCast(du, mask), BitCast(du, not_mask));
+  return BitCast(Desc<double, N>(), ret);
+}
+#endif
+
+// ------------------------------ Bitwise OR
+
+HWY_NEON_DEF_FUNCTION_INTS_UINTS(operator|, vorr, _, 2)
+
+// These operator| rely on the special cases for uint32_t and uint64_t just
+// defined by HWY_NEON_DEF_FUNCTION_INTS_UINTS() macro.
+template <size_t N>
+HWY_INLINE Vec128<float, N> operator|(const Vec128<float, N> a,
+                                      const Vec128<float, N> b) {
+  const Desc<uint32_t, N> d;
+  return BitCast(Full128<float>(), BitCast(d, a) | BitCast(d, b));
+}
+template <size_t N>
+HWY_INLINE Vec128<double, N> operator|(const Vec128<double, N> a,
+                                       const Vec128<double, N> b) {
+  const Desc<uint64_t, N> d;
+  return BitCast(Full128<double>(), BitCast(d, a) | BitCast(d, b));
+}
+
+// ------------------------------ Bitwise XOR
+
+HWY_NEON_DEF_FUNCTION_INTS_UINTS(operator^, veor, _, 2)
+
+// These operator| rely on the special cases for uint32_t and uint64_t just
+// defined by HWY_NEON_DEF_FUNCTION_INTS_UINTS() macro.
+template <size_t N>
+HWY_INLINE Vec128<float, N> operator^(const Vec128<float, N> a,
+                                      const Vec128<float, N> b) {
+  const Desc<uint32_t, N> d;
+  return BitCast(Full128<float>(), BitCast(d, a) ^ BitCast(d, b));
+}
+template <size_t N>
+HWY_INLINE Vec128<double, N> operator^(const Vec128<double, N> a,
+                                       const Vec128<double, N> b) {
+  const Desc<uint64_t, N> d;
+  return BitCast(Full128<double>(), BitCast(d, a) ^ BitCast(d, b));
+}
+
+// ------------------------------ Make mask
+
+template <typename T, size_t N>
+HWY_INLINE Mask128<T, N> TestBit(Vec128<T, N> v, Vec128<T, N> bit) {
+  static_assert(!IsFloat<T>(), "Only integer vectors supported");
+  return (v & bit) == bit;
+}
+
+// Mask and Vec are the same (true = FF..FF).
+template <typename T, size_t N>
+HWY_INLINE Mask128<T, N> MaskFromVec(const Vec128<T, N> v) {
+  return Mask128<T, N>(v.raw);
+}
+
+template <typename T, size_t N>
+HWY_INLINE Vec128<T, N> VecFromMask(const Mask128<T, N> v) {
+  return Vec128<T, N>(v.raw);
+}
+
+// IfThenElse(mask, yes, no)
+// Returns mask ? b : a.
+#define HWY_NEON_BUILD_TPL_HWY_IF
+#define HWY_NEON_BUILD_RET_HWY_IF(type, size) Vec128<type, size>
+#define HWY_NEON_BUILD_PARAM_HWY_IF(type, size)                 \
+  const Mask128<type, size> mask, const Vec128<type, size> yes, \
+      const Vec128<type, size> no
+#define HWY_NEON_BUILD_ARG_HWY_IF mask.raw, yes.raw, no.raw
+
+HWY_NEON_DEF_FUNCTION_ALL_TYPES(IfThenElse, vbsl, _, HWY_IF)
+
+#undef HWY_NEON_BUILD_TPL_HWY_IF
+#undef HWY_NEON_BUILD_RET_HWY_IF
+#undef HWY_NEON_BUILD_PARAM_HWY_IF
+#undef HWY_NEON_BUILD_ARG_HWY_IF
+
+// mask ? yes : 0
+template <typename T, size_t N>
+HWY_INLINE Vec128<T, N> IfThenElseZero(const Mask128<T, N> mask,
+                                       const Vec128<T, N> yes) {
+  return yes & VecFromMask(mask);
+}
+
+// mask ? 0 : no
+template <typename T, size_t N>
+HWY_INLINE Vec128<T, N> IfThenZeroElse(const Mask128<T, N> mask,
+                                       const Vec128<T, N> no) {
+  return AndNot(VecFromMask(mask), no);
+}
+
+template <typename T, size_t N>
+HWY_INLINE Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) {
+  const auto zero = Zero(Desc<T, N>());
+  return Max(zero, v);
+}
+
+// ================================================== MEMORY
+
+// ------------------------------ Load 128
+
+HWY_INLINE Vec128<uint8_t> LoadU(Full128<uint8_t> /* tag */,
+                                 const uint8_t* HWY_RESTRICT aligned) {
+  return Vec128<uint8_t>(vld1q_u8(aligned));
+}
+HWY_INLINE Vec128<uint16_t> LoadU(Full128<uint16_t> /* tag */,
+                                  const uint16_t* HWY_RESTRICT aligned) {
+  return Vec128<uint16_t>(vld1q_u16(aligned));
+}
+HWY_INLINE Vec128<uint32_t> LoadU(Full128<uint32_t> /* tag */,
+                                  const uint32_t* HWY_RESTRICT aligned) {
+  return Vec128<uint32_t>(vld1q_u32(aligned));
+}
+HWY_INLINE Vec128<uint64_t> LoadU(Full128<uint64_t> /* tag */,
+                                  const uint64_t* HWY_RESTRICT aligned) {
+  return Vec128<uint64_t>(vld1q_u64(aligned));
+}
+HWY_INLINE Vec128<int8_t> LoadU(Full128<int8_t> /* tag */,
+                                const int8_t* HWY_RESTRICT aligned) {
+  return Vec128<int8_t>(vld1q_s8(aligned));
+}
+HWY_INLINE Vec128<int16_t> LoadU(Full128<int16_t> /* tag */,
+                                 const int16_t* HWY_RESTRICT aligned) {
+  return Vec128<int16_t>(vld1q_s16(aligned));
+}
+HWY_INLINE Vec128<int32_t> LoadU(Full128<int32_t> /* tag */,
+                                 const int32_t* HWY_RESTRICT aligned) {
+  return Vec128<int32_t>(vld1q_s32(aligned));
+}
+HWY_INLINE Vec128<int64_t> LoadU(Full128<int64_t> /* tag */,
+                                 const int64_t* HWY_RESTRICT aligned) {
+  return Vec128<int64_t>(vld1q_s64(aligned));
+}
+HWY_INLINE Vec128<float> LoadU(Full128<float> /* tag */,
+                               const float* HWY_RESTRICT aligned) {
+  return Vec128<float>(vld1q_f32(aligned));
+}
+#if defined(__aarch64__)
+HWY_INLINE Vec128<double> LoadU(Full128<double> /* tag */,
+                                const double* HWY_RESTRICT aligned) {
+  return Vec128<double>(vld1q_f64(aligned));
+}
+#endif
+
+template <typename T>
+HWY_INLINE Vec128<T> Load(Full128<T> d, const T* HWY_RESTRICT p) {
+  return LoadU(d, p);
+}
+
+// 128-bit SIMD => nothing to duplicate, same as an unaligned load.
+template <typename T>
+HWY_INLINE Vec128<T> LoadDup128(Full128<T> d, const T* const HWY_RESTRICT p) {
+  return LoadU(d, p);
+}
+
+// ------------------------------ Load 64
+
+HWY_INLINE Vec128<uint8_t, 8> Load(Desc<uint8_t, 8> /* tag */,
+                                   const uint8_t* HWY_RESTRICT p) {
+  return Vec128<uint8_t, 8>(vld1_u8(p));
+}
+HWY_INLINE Vec128<uint16_t, 4> Load(Desc<uint16_t, 4> /* tag */,
+                                    const uint16_t* HWY_RESTRICT p) {
+  return Vec128<uint16_t, 4>(vld1_u16(p));
+}
+HWY_INLINE Vec128<uint32_t, 2> Load(Desc<uint32_t, 2> /* tag */,
+                                    const uint32_t* HWY_RESTRICT p) {
+  return Vec128<uint32_t, 2>(vld1_u32(p));
+}
+HWY_INLINE Vec128<uint64_t, 1> Load(Desc<uint64_t, 1> /* tag */,
+                                    const uint64_t* HWY_RESTRICT p) {
+  return Vec128<uint64_t, 1>(vld1_u64(p));
+}
+HWY_INLINE Vec128<int8_t, 8> Load(Desc<int8_t, 8> /* tag */,
+                                  const int8_t* HWY_RESTRICT p) {
+  return Vec128<int8_t, 8>(vld1_s8(p));
+}
+HWY_INLINE Vec128<int16_t, 4> Load(Desc<int16_t, 4> /* tag */,
+                                   const int16_t* HWY_RESTRICT p) {
+  return Vec128<int16_t, 4>(vld1_s16(p));
+}
+HWY_INLINE Vec128<int32_t, 2> Load(Desc<int32_t, 2> /* tag */,
+                                   const int32_t* HWY_RESTRICT p) {
+  return Vec128<int32_t, 2>(vld1_s32(p));
+}
+HWY_INLINE Vec128<int64_t, 1> Load(Desc<int64_t, 1> /* tag */,
+                                   const int64_t* HWY_RESTRICT p) {
+  return Vec128<int64_t, 1>(vld1_s64(p));
+}
+HWY_INLINE Vec128<float, 2> Load(Desc<float, 2> /* tag */,
+                                 const float* HWY_RESTRICT p) {
+  return Vec128<float, 2>(vld1_f32(p));
+}
+#if defined(__aarch64__)
+HWY_INLINE Vec128<double, 1> Load(Desc<double, 1> /* tag */,
+                                  const double* HWY_RESTRICT p) {
+  return Vec128<double, 1>(vld1_f64(p));
+}
+#endif
+
+// ------------------------------ Load 32
+
+// In the following load functions, |a| is purposely undefined.
+// It is a required parameter to the intrinsic, however
+// we don't actually care what is in it, and we don't want
+// to introduce extra overhead by initializing it to something.
+
+HWY_INLINE Vec128<uint8_t, 4> Load(Desc<uint8_t, 4> d,
+                                   const uint8_t* HWY_RESTRICT p) {
+  uint32x2_t a = Undefined(d).raw;
+  uint32x2_t b = vld1_lane_u32(reinterpret_cast<const uint32_t*>(p), a, 0);
+  return Vec128<uint8_t, 4>(vreinterpret_u8_u32(b));
+}
+HWY_INLINE Vec128<uint16_t, 2> Load(Desc<uint16_t, 2> d,
+                                    const uint16_t* HWY_RESTRICT p) {
+  uint32x2_t a = Undefined(d).raw;
+  uint32x2_t b = vld1_lane_u32(reinterpret_cast<const uint32_t*>(p), a, 0);
+  return Vec128<uint16_t, 2>(vreinterpret_u16_u32(b));
+}
+HWY_INLINE Vec128<uint32_t, 1> Load(Desc<uint32_t, 1> d,
+                                    const uint32_t* HWY_RESTRICT p) {
+  uint32x2_t a = Undefined(d).raw;
+  uint32x2_t b = vld1_lane_u32(p, a, 0);
+  return Vec128<uint32_t, 1>(b);
+}
+HWY_INLINE Vec128<int8_t, 4> Load(Desc<int8_t, 4> d,
+                                  const int8_t* HWY_RESTRICT p) {
+  int32x2_t a = Undefined(d).raw;
+  int32x2_t b = vld1_lane_s32(reinterpret_cast<const int32_t*>(p), a, 0);
+  return Vec128<int8_t, 4>(vreinterpret_s8_s32(b));
+}
+HWY_INLINE Vec128<int16_t, 2> Load(Desc<int16_t, 2> d,
+                                   const int16_t* HWY_RESTRICT p) {
+  int32x2_t a = Undefined(d).raw;
+  int32x2_t b = vld1_lane_s32(reinterpret_cast<const int32_t*>(p), a, 0);
+  return Vec128<int16_t, 2>(vreinterpret_s16_s32(b));
+}
+HWY_INLINE Vec128<int32_t, 1> Load(Desc<int32_t, 1> d,
+                                   const int32_t* HWY_RESTRICT p) {
+  int32x2_t a = Undefined(d).raw;
+  int32x2_t b = vld1_lane_s32(p, a, 0);
+  return Vec128<int32_t, 1>(b);
+}
+HWY_INLINE Vec128<float, 1> Load(Desc<float, 1> d,
+                                 const float* HWY_RESTRICT p) {
+  float32x2_t a = Undefined(d).raw;
+  float32x2_t b = vld1_lane_f32(p, a, 0);
+  return Vec128<float, 1>(b);
+}
+
+// ------------------------------ Store 128
+
+HWY_INLINE void StoreU(const Vec128<uint8_t> v, Full128<uint8_t> /* tag */,
+                       uint8_t* HWY_RESTRICT aligned) {
+  vst1q_u8(aligned, v.raw);
+}
+HWY_INLINE void StoreU(const Vec128<uint16_t> v, Full128<uint16_t> /* tag */,
+                       uint16_t* HWY_RESTRICT aligned) {
+  vst1q_u16(aligned, v.raw);
+}
+HWY_INLINE void StoreU(const Vec128<uint32_t> v, Full128<uint32_t> /* tag */,
+                       uint32_t* HWY_RESTRICT aligned) {
+  vst1q_u32(aligned, v.raw);
+}
+HWY_INLINE void StoreU(const Vec128<uint64_t> v, Full128<uint64_t> /* tag */,
+                       uint64_t* HWY_RESTRICT aligned) {
+  vst1q_u64(aligned, v.raw);
+}
+HWY_INLINE void StoreU(const Vec128<int8_t> v, Full128<int8_t> /* tag */,
+                       int8_t* HWY_RESTRICT aligned) {
+  vst1q_s8(aligned, v.raw);
+}
+HWY_INLINE void StoreU(const Vec128<int16_t> v, Full128<int16_t> /* tag */,
+                       int16_t* HWY_RESTRICT aligned) {
+  vst1q_s16(aligned, v.raw);
+}
+HWY_INLINE void StoreU(const Vec128<int32_t> v, Full128<int32_t> /* tag */,
+                       int32_t* HWY_RESTRICT aligned) {
+  vst1q_s32(aligned, v.raw);
+}
+HWY_INLINE void StoreU(const Vec128<int64_t> v, Full128<int64_t> /* tag */,
+                       int64_t* HWY_RESTRICT aligned) {
+  vst1q_s64(aligned, v.raw);
+}
+HWY_INLINE void StoreU(const Vec128<float> v, Full128<float> /* tag */,
+                       float* HWY_RESTRICT aligned) {
+  vst1q_f32(aligned, v.raw);
+}
+#if defined(__aarch64__)
+HWY_INLINE void StoreU(const Vec128<double> v, Full128<double> /* tag */,
+                       double* HWY_RESTRICT aligned) {
+  vst1q_f64(aligned, v.raw);
+}
+#endif
+
+template <typename T, size_t N>
+HWY_INLINE void Store(Vec128<T, N> v, Desc<T, N> d, T* HWY_RESTRICT p) {
+  StoreU(v, d, p);
+}
+
+// ------------------------------ Store 64
+
+HWY_INLINE void Store(const Vec128<uint8_t, 8> v, Desc<uint8_t, 8>,
+                      uint8_t* HWY_RESTRICT p) {
+  vst1_u8(p, v.raw);
+}
+HWY_INLINE void Store(const Vec128<uint16_t, 4> v, Desc<uint16_t, 4>,
+                      uint16_t* HWY_RESTRICT p) {
+  vst1_u16(p, v.raw);
+}
+HWY_INLINE void Store(const Vec128<uint32_t, 2> v, Desc<uint32_t, 2>,
+                      uint32_t* HWY_RESTRICT p) {
+  vst1_u32(p, v.raw);
+}
+HWY_INLINE void Store(const Vec128<uint64_t, 1> v, Desc<uint64_t, 1>,
+                      uint64_t* HWY_RESTRICT p) {
+  vst1_u64(p, v.raw);
+}
+HWY_INLINE void Store(const Vec128<int8_t, 8> v, Desc<int8_t, 8>,
+                      int8_t* HWY_RESTRICT p) {
+  vst1_s8(p, v.raw);
+}
+HWY_INLINE void Store(const Vec128<int16_t, 4> v, Desc<int16_t, 4>,
+                      int16_t* HWY_RESTRICT p) {
+  vst1_s16(p, v.raw);
+}
+HWY_INLINE void Store(const Vec128<int32_t, 2> v, Desc<int32_t, 2>,
+                      int32_t* HWY_RESTRICT p) {
+  vst1_s32(p, v.raw);
+}
+HWY_INLINE void Store(const Vec128<int64_t, 1> v, Desc<int64_t, 1>,
+                      int64_t* HWY_RESTRICT p) {
+  vst1_s64(p, v.raw);
+}
+HWY_INLINE void Store(const Vec128<float, 2> v, Desc<float, 2>,
+                      float* HWY_RESTRICT p) {
+  vst1_f32(p, v.raw);
+}
+#if defined(__aarch64__)
+HWY_INLINE void Store(const Vec128<double, 1> v, Desc<double, 1>,
+                      double* HWY_RESTRICT p) {
+  vst1_f64(p, v.raw);
+}
+#endif
+
+// ------------------------------ Store 32
+
+HWY_INLINE void Store(const Vec128<uint8_t, 4> v, Desc<uint8_t, 4>,
+                      uint8_t* HWY_RESTRICT p) {
+  uint32x2_t a = vreinterpret_u32_u8(v.raw);
+  vst1_lane_u32(p, a, 0);
+}
+HWY_INLINE void Store(const Vec128<uint16_t, 2> v, Desc<uint16_t, 2>,
+                      uint16_t* HWY_RESTRICT p) {
+  uint32x2_t a = vreinterpret_u32_u16(v.raw);
+  vst1_lane_u32(p, a, 0);
+}
+HWY_INLINE void Store(const Vec128<uint32_t, 1> v, Desc<uint32_t, 1>,
+                      uint32_t* HWY_RESTRICT p) {
+  vst1_lane_u32(p, v.raw, 0);
+}
+HWY_INLINE void Store(const Vec128<int8_t, 4> v, Desc<int8_t, 4>,
+                      int8_t* HWY_RESTRICT p) {
+  int32x2_t a = vreinterpret_s32_s8(v.raw);
+  vst1_lane_s32(p, a, 0);
+}
+HWY_INLINE void Store(const Vec128<int16_t, 2> v, Desc<int16_t, 2>,
+                      int16_t* HWY_RESTRICT p) {
+  int32x2_t a = vreinterpret_s32_s16(v.raw);
+  vst1_lane_s32(p, a, 0);
+}
+HWY_INLINE void Store(const Vec128<int32_t, 1> v, Desc<int32_t, 1>,
+                      int32_t* HWY_RESTRICT p) {
+  vst1_lane_s32(p, v.raw, 0);
+}
+HWY_INLINE void Store(const Vec128<float, 1> v, Desc<float, 1>,
+                      float* HWY_RESTRICT p) {
+  vst1_lane_f32(p, v.raw, 0);
+}
+
+// ------------------------------ Non-temporal stores
+
+// Same as aligned stores on non-x86.
+
+template <typename T>
+HWY_INLINE void Stream(const Vec128<T> v, Full128<T> d,
+                       T* HWY_RESTRICT aligned) {
+  Store(v, d, aligned);
+}
+
+// ================================================== CONVERT
+
+// ------------------------------ Promotions (part w/ narrow lanes -> full)
+
+// Unsigned: zero-extend.
+HWY_INLINE Vec128<uint16_t> ConvertTo(Full128<uint16_t> /* tag */,
+                                      const Vec128<uint8_t, 8> v) {
+  return Vec128<uint16_t>(vmovl_u8(v.raw));
+}
+HWY_INLINE Vec128<uint32_t> ConvertTo(Full128<uint32_t> /* tag */,
+                                      const Vec128<uint8_t, 4> v) {
+  uint16x8_t a = vmovl_u8(v.raw);
+  return Vec128<uint32_t>(vmovl_u16(vget_low_u16(a)));
+}
+HWY_INLINE Vec128<uint32_t> ConvertTo(Full128<uint32_t> /* tag */,
+                                      const Vec128<uint16_t, 4> v) {
+  return Vec128<uint32_t>(vmovl_u16(v.raw));
+}
+HWY_INLINE Vec128<uint64_t> ConvertTo(Full128<uint64_t> /* tag */,
+                                      const Vec128<uint32_t, 2> v) {
+  return Vec128<uint64_t>(vmovl_u32(v.raw));
+}
+HWY_INLINE Vec128<int16_t> ConvertTo(Full128<int16_t> /* tag */,
+                                     const Vec128<uint8_t, 8> v) {
+  return Vec128<int16_t>(vmovl_u8(v.raw));
+}
+HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                     const Vec128<uint8_t, 4> v) {
+  uint16x8_t a = vmovl_u8(v.raw);
+  return Vec128<int32_t>(vreinterpretq_s32_u16(vmovl_u16(vget_low_u16(a))));
+}
+HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                     const Vec128<uint16_t, 4> v) {
+  return Vec128<int32_t>(vmovl_u16(v.raw));
+}
+
+HWY_INLINE Vec128<uint32_t> U32FromU8(const Vec128<uint8_t> v) {
+  return Vec128<uint32_t>(
+      vmovl_u16(vget_low_u16(vmovl_u8(vget_low_u8(v.raw)))));
+}
+
+// Signed: replicate sign bit.
+HWY_INLINE Vec128<int16_t> ConvertTo(Full128<int16_t> /* tag */,
+                                     const Vec128<int8_t, 8> v) {
+  return Vec128<int16_t>(vmovl_s8(v.raw));
+}
+HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                     const Vec128<int8_t, 4> v) {
+  int16x8_t a = vmovl_s8(v.raw);
+  return Vec128<int32_t>(vmovl_s16(vget_low_s16(a)));
+}
+HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                     const Vec128<int16_t, 4> v) {
+  return Vec128<int32_t>(vmovl_s16(v.raw));
+}
+HWY_INLINE Vec128<int64_t> ConvertTo(Full128<int64_t> /* tag */,
+                                     const Vec128<int32_t, 2> v) {
+  return Vec128<int64_t>(vmovl_s32(v.raw));
+}
+
+#if defined(__aarch64__)
+HWY_INLINE Vec128<double> ConvertTo(Full128<double> /* tag */,
+                                    const Vec128<float, 2> v) {
+  // vcvt_high_f64_f32 takes a float32x4_t even it only uses 2 of the values.
+  const float32x4_t w = vcombine_f32(v.raw, v.raw);
+  return Vec128<double>(vcvt_high_f64_f32(w));
+}
+#endif
+
+// ------------------------------ Demotions (full -> part w/ narrow lanes)
+
+template <size_t N>
+HWY_INLINE Vec128<uint16_t, N> ConvertTo(Desc<uint16_t, N> /* tag */,
+                                         const Vec128<int32_t> v) {
+  return Vec128<uint16_t, N>(vqmovun_s32(v.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<uint8_t, N> ConvertTo(Desc<uint8_t, N> /* tag */,
+                                        const Vec128<uint16_t> v) {
+  return Vec128<uint8_t, N>(vqmovn_u16(v.raw));
+}
+
+template <size_t N>
+HWY_INLINE Vec128<uint8_t, N> ConvertTo(Desc<uint8_t, N> /* tag */,
+                                        const Vec128<int16_t> v) {
+  return Vec128<uint8_t, N>(vqmovun_s16(v.raw));
+}
+
+template <size_t N>
+HWY_INLINE Vec128<int16_t, N> ConvertTo(Desc<int16_t, N> /* tag */,
+                                        const Vec128<int32_t> v) {
+  return Vec128<int16_t, N>(vqmovn_s32(v.raw));
+}
+template <size_t N>
+HWY_INLINE Vec128<int8_t, N> ConvertTo(Desc<int8_t, N> /* tag */,
+                                       const Vec128<int16_t> v) {
+  return Vec128<int8_t, N>(vqmovn_s16(v.raw));
+}
+
+HWY_INLINE Vec128<uint8_t, 4> U8FromU32(const Vec128<uint32_t> v) {
+  const uint8x16_t org_v = cast_to_u8(v).raw;
+  const uint8x16_t w = vuzp1q_u8(org_v, org_v);
+  return Vec128<uint8_t, 4>(vget_low_u8(vuzp1q_u8(w, w)));
+}
+
+// In the following ConvertTo functions, |b| is purposely undefined.
+// The value a needs to be extended to 128 bits so that vqmovn can be
+// used and |b| is undefined so that no extra overhead is introduced.
+HWY_DIAGNOSTICS(push)
+HWY_DIAGNOSTICS_OFF(disable : 4701, ignored "-Wuninitialized")
+
+template <size_t N>
+HWY_INLINE Vec128<uint8_t, N> ConvertTo(Desc<uint8_t, N> /* tag */,
+                                        const Vec128<int32_t> v) {
+  Vec128<uint16_t, N> a = ConvertTo(Desc<uint16_t, N>(), v);
+  Vec128<uint16_t, N> b;
+  uint16x8_t c = vcombine_u16(a.raw, b.raw);
+  return Vec128<uint8_t, N>(vqmovn_u16(c));
+}
+
+template <size_t N>
+HWY_INLINE Vec128<int8_t, N> ConvertTo(Desc<int8_t, N> /* tag */,
+                                       const Vec128<int32_t> v) {
+  Vec128<int16_t, N> a = ConvertTo(Desc<int16_t, N>(), v);
+  Vec128<int16_t, N> b;
+  uint16x8_t c = vcombine_s16(a.raw, b.raw);
+  return Vec128<int8_t, N>(vqmovn_s16(c));
+}
+
+HWY_DIAGNOSTICS(pop)
+
+// ------------------------------ Convert i32 <=> f32
+
+template <size_t N>
+HWY_INLINE Vec128<float, N> ConvertTo(Desc<float, N> /* tag */,
+                                      const Vec128<int32_t, N> v) {
+  return Vec128<float, N>(vcvtq_f32_s32(v.raw));
+}
+// Truncates (rounds toward zero).
+template <size_t N>
+HWY_INLINE Vec128<int32_t, N> ConvertTo(Desc<int32_t, N> /* tag */,
+                                        const Vec128<float, N> v) {
+  return Vec128<int32_t, N>(vcvtq_s32_f32(v.raw));
+}
+
+#if defined(__aarch64__)
+template <size_t N>
+HWY_INLINE Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) {
+  return Vec128<int32_t, N>(vcvtnq_s32_f32(v.raw));
+}
+#else
+template <size_t N>
+HWY_INLINE Vec128<int32_t, N> NearestInt(const Vec128<float, N> v) {
+  return ConvertTo(Desc<int32_t, N>(), Round(v));
+}
+#endif
+
+// ================================================== SWIZZLE
+
+// ------------------------------ Extract lane
+
+HWY_INLINE uint8_t GetLane(const Vec128<uint8_t, 16> v) {
+  return vget_lane_u8(vget_low_u8(v.raw), 0);
+}
+template <size_t N>
+HWY_INLINE uint8_t GetLane(const Vec128<uint8_t, N> v) {
+  return vget_lane_u8(v.raw, 0);
+}
+
+HWY_INLINE int8_t GetLane(const Vec128<int8_t, 16> v) {
+  return vget_lane_s8(vget_low_s8(v.raw), 0);
+}
+template <size_t N>
+HWY_INLINE int8_t GetLane(const Vec128<int8_t, N> v) {
+  return vget_lane_s8(v.raw, 0);
+}
+
+HWY_INLINE uint16_t GetLane(const Vec128<uint16_t, 8> v) {
+  return vget_lane_u16(vget_low_u16(v.raw), 0);
+}
+template <size_t N>
+HWY_INLINE uint16_t GetLane(const Vec128<uint16_t, N> v) {
+  return vget_lane_u16(v.raw, 0);
+}
+
+HWY_INLINE int16_t GetLane(const Vec128<int16_t, 8> v) {
+  return vget_lane_s16(vget_low_s16(v.raw), 0);
+}
+template <size_t N>
+HWY_INLINE int16_t GetLane(const Vec128<int16_t, N> v) {
+  return vget_lane_s16(v.raw, 0);
+}
+
+HWY_INLINE uint32_t GetLane(const Vec128<uint32_t, 4> v) {
+  return vget_lane_u32(vget_low_u32(v.raw), 0);
+}
+template <size_t N>
+HWY_INLINE uint32_t GetLane(const Vec128<uint32_t, N> v) {
+  return vget_lane_u32(v.raw, 0);
+}
+
+HWY_INLINE int32_t GetLane(const Vec128<int32_t, 4> v) {
+  return vget_lane_s32(vget_low_s32(v.raw), 0);
+}
+template <size_t N>
+HWY_INLINE int32_t GetLane(const Vec128<int32_t, N> v) {
+  return vget_lane_s32(v.raw, 0);
+}
+
+#if defined(__aarch64__)
+HWY_INLINE uint64_t GetLane(const Vec128<uint64_t, 2> v) {
+  return vget_lane_u64(vget_low_u64(v.raw), 0);
+}
+HWY_INLINE uint64_t GetLane(const Vec128<uint64_t, 1> v) {
+  return vget_lane_u64(v.raw, 0);
+}
+HWY_INLINE int64_t GetLane(const Vec128<int64_t, 2> v) {
+  return vget_lane_s64(vget_low_s64(v.raw), 0);
+}
+HWY_INLINE int64_t GetLane(const Vec128<int64_t, 1> v) {
+  return vget_lane_s64(v.raw, 0);
+}
+#endif
+
+HWY_INLINE float GetLane(const Vec128<float, 4> v) {
+  return vget_lane_f32(vget_low_f32(v.raw), 0);
+}
+HWY_INLINE float GetLane(const Vec128<float, 2> v) {
+  return vget_lane_f32(v.raw, 0);
+}
+HWY_INLINE float GetLane(const Vec128<float, 1> v) {
+  return vget_lane_f32(v.raw, 0);
+}
+#if defined(__aarch64__)
+HWY_INLINE double GetLane(const Vec128<double, 2> v) {
+  return vget_lane_f64(vget_low_f64(v.raw), 0);
+}
+HWY_INLINE double GetLane(const Vec128<double, 1> v) {
+  return vget_lane_f64(v.raw, 0);
+}
+#endif
+
+// ------------------------------ Extract half
+
+// <= 64 bit: just return different type
+template <typename T, size_t N, HWY_IF64(uint8_t, N)>
+HWY_INLINE Vec128<T, N / 2> LowerHalf(const Vec128<T, N> v) {
+  return Vec128<T, N / 2>(v.raw);
+}
+
+template <size_t N, HWY_IF128_Q(uint8_t, N)>
+HWY_INLINE Vec128<uint8_t, N / 2> LowerHalf(const Vec128<uint8_t, N> v) {
+  return Vec128<uint8_t, N / 2>(vget_low_u8(v.raw));
+}
+template <size_t N, HWY_IF128_Q(uint16_t, N)>
+HWY_INLINE Vec128<uint16_t, N / 2> LowerHalf(const Vec128<uint16_t, N> v) {
+  return Vec128<uint16_t, N / 2>(vget_low_u16(v.raw));
+}
+template <size_t N, HWY_IF128_Q(uint32_t, N)>
+HWY_INLINE Vec128<uint32_t, N / 2> LowerHalf(const Vec128<uint32_t, N> v) {
+  return Vec128<uint32_t, N / 2>(vget_low_u32(v.raw));
+}
+HWY_INLINE Vec128<uint64_t, 1> LowerHalf(const Vec128<uint64_t> v) {
+  return Vec128<uint64_t, 1>(vget_low_u64(v.raw));
+}
+template <size_t N, HWY_IF128_Q(int8_t, N)>
+HWY_INLINE Vec128<int8_t, N / 2> LowerHalf(const Vec128<int8_t, N> v) {
+  return Vec128<int8_t, N / 2>(vget_low_s8(v.raw));
+}
+template <size_t N, HWY_IF128_Q(int16_t, N)>
+HWY_INLINE Vec128<int16_t, N / 2> LowerHalf(const Vec128<int16_t, N> v) {
+  return Vec128<int16_t, N / 2>(vget_low_s16(v.raw));
+}
+template <size_t N, HWY_IF128_Q(int32_t, N)>
+HWY_INLINE Vec128<int32_t, N / 2> LowerHalf(const Vec128<int32_t, N> v) {
+  return Vec128<int32_t, N / 2>(vget_low_s32(v.raw));
+}
+HWY_INLINE Vec128<int64_t, 1> LowerHalf(const Vec128<int64_t> v) {
+  return Vec128<int64_t, 1>(vget_low_s64(v.raw));
+}
+template <size_t N, HWY_IF128_Q(float, N)>
+HWY_INLINE Vec128<float, N / 2> LowerHalf(const Vec128<float, N> v) {
+  return Vec128<float, N / 2>(vget_low_f32(v.raw));
+}
+#if defined(__aarch64__)
+HWY_INLINE Vec128<double, 1> LowerHalf(const Vec128<double> v) {
+  return Vec128<double, 1>(vget_low_f64(v.raw));
+}
+#endif
+
+HWY_INLINE Vec128<uint8_t, 8> UpperHalf(const Vec128<uint8_t> v) {
+  return Vec128<uint8_t, 8>(vget_high_u8(v.raw));
+}
+HWY_INLINE Vec128<uint16_t, 4> UpperHalf(const Vec128<uint16_t> v) {
+  return Vec128<uint16_t, 4>(vget_high_u16(v.raw));
+}
+HWY_INLINE Vec128<uint32_t, 2> UpperHalf(const Vec128<uint32_t> v) {
+  return Vec128<uint32_t, 2>(vget_high_u32(v.raw));
+}
+HWY_INLINE Vec128<uint64_t, 1> UpperHalf(const Vec128<uint64_t> v) {
+  return Vec128<uint64_t, 1>(vget_high_u64(v.raw));
+}
+HWY_INLINE Vec128<int8_t, 8> UpperHalf(const Vec128<int8_t> v) {
+  return Vec128<int8_t, 8>(vget_high_s8(v.raw));
+}
+HWY_INLINE Vec128<int16_t, 4> UpperHalf(const Vec128<int16_t> v) {
+  return Vec128<int16_t, 4>(vget_high_s16(v.raw));
+}
+HWY_INLINE Vec128<int32_t, 2> UpperHalf(const Vec128<int32_t> v) {
+  return Vec128<int32_t, 2>(vget_high_s32(v.raw));
+}
+HWY_INLINE Vec128<int64_t, 1> UpperHalf(const Vec128<int64_t> v) {
+  return Vec128<int64_t, 1>(vget_high_s64(v.raw));
+}
+HWY_INLINE Vec128<float, 2> UpperHalf(const Vec128<float> v) {
+  return Vec128<float, 2>(vget_high_f32(v.raw));
+}
+#if defined(__aarch64__)
+HWY_INLINE Vec128<double, 1> UpperHalf(const Vec128<double> v) {
+  return Vec128<double, 1>(vget_high_f64(v.raw));
+}
+#endif
+
+template <typename T>
+HWY_INLINE Vec128<T, 8 / sizeof(T)> GetHalf(Lower /* tag */, Vec128<T> v) {
+  return LowerHalf(v);
+}
+template <typename T>
+HWY_INLINE Vec128<T, 8 / sizeof(T)> GetHalf(Upper /* tag */, Vec128<T> v) {
+  return UpperHalf(v);
+}
+
+// ------------------------------ Extract from 2x 128-bit at constant offset
+
+// Extracts 128 bits from <hi, lo> by skipping the least-significant kBytes.
+template <int kBytes, typename T>
+HWY_INLINE Vec128<T> CombineShiftRightBytes(const Vec128<T> hi,
+                                            const Vec128<T> lo) {
+  static_assert(0 < kBytes && kBytes < 16, "kBytes must be in [1, 15]");
+  const Full128<uint8_t> d8;
+  return BitCast(Full128<T>(),
+                 Vec128<uint8_t>(vextq_u8(BitCast(d8, lo).raw,
+                                          BitCast(d8, hi).raw, kBytes)));
+}
+
+// ------------------------------ Shift vector by constant #bytes
+
+// 0x01..0F, kBytes = 1 => 0x02..0F00
+template <int kBytes, typename T, size_t N>
+HWY_INLINE Vec128<T, N> ShiftLeftBytes(const Vec128<T, N> v) {
+  return CombineShiftRightBytes<16 - kBytes>(v, Zero(Full128<T>()));
+}
+
+template <int kLanes, typename T>
+HWY_INLINE Vec128<T> ShiftLeftLanes(const Vec128<T> v) {
+  return ShiftLeftBytes<kLanes * sizeof(T)>(v);
+}
+
+// 0x01..0F, kBytes = 1 => 0x0001..0E
+template <int kBytes, typename T, size_t N>
+HWY_INLINE Vec128<T, N> ShiftRightBytes(const Vec128<T, N> v) {
+  return CombineShiftRightBytes<kBytes>(Zero(Full128<T>()), v);
+}
+
+template <int kLanes, typename T>
+HWY_INLINE Vec128<T> ShiftRightLanes(const Vec128<T> v) {
+  return ShiftRightBytes<kLanes * sizeof(T)>(v);
+}
+
+// ------------------------------ Broadcast/splat any lane
+
+#if defined(__aarch64__)
+// Unsigned
+template <int kLane>
+HWY_INLINE Vec128<uint16_t> Broadcast(const Vec128<uint16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  return Vec128<uint16_t>(vdupq_laneq_u16(v.raw, kLane));
+}
+template <int kLane, size_t N>
+HWY_INLINE Vec128<uint16_t> Broadcast(const Vec128<uint16_t, N> v) {
+  static_assert(0 <= kLane && kLane < N, "Invalid lane");
+  return Vec128<uint16_t>(vdupq_laneq_u16(vcombine_u16(v.raw, v.raw), kLane));
+}
+template <int kLane>
+HWY_INLINE Vec128<uint32_t> Broadcast(const Vec128<uint32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec128<uint32_t>(vdupq_laneq_u32(v.raw, kLane));
+}
+template <int kLane>
+HWY_INLINE Vec128<uint64_t> Broadcast(const Vec128<uint64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec128<uint64_t>(vdupq_laneq_u64(v.raw, kLane));
+}
+
+// Signed
+template <int kLane>
+HWY_INLINE Vec128<int16_t> Broadcast(const Vec128<int16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  return Vec128<int16_t>(vdupq_laneq_s16(v.raw, kLane));
+}
+template <int kLane, size_t N>
+HWY_INLINE Vec128<int16_t> Broadcast(const Vec128<int16_t, N> v) {
+  static_assert(0 <= kLane && kLane < N, "Invalid lane");
+  return Vec128<int16_t>(vdupq_laneq_s16(vcombine_s16(v.raw, v.raw), kLane));
+}
+template <int kLane>
+HWY_INLINE Vec128<int32_t> Broadcast(const Vec128<int32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec128<int32_t>(vdupq_laneq_s32(v.raw, kLane));
+}
+template <int kLane>
+HWY_INLINE Vec128<int64_t> Broadcast(const Vec128<int64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec128<int64_t>(vdupq_laneq_s64(v.raw, kLane));
+}
+
+// Float
+template <int kLane>
+HWY_INLINE Vec128<float> Broadcast(const Vec128<float> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec128<float>(vdupq_laneq_f32(v.raw, kLane));
+}
+template <int kLane>
+HWY_INLINE Vec128<double> Broadcast(const Vec128<double> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec128<double>(vdupq_laneq_f64(v.raw, kLane));
+}
+#else
+// No vdupq_laneq_* on armv7: use vgetq_lane_* + vdupq_n_*.
+
+// Unsigned
+template <int kLane>
+HWY_INLINE Vec128<uint16_t> Broadcast(const Vec128<uint16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  return Vec128<uint16_t>(vdupq_n_u16(vgetq_lane_u16(v.raw, kLane)));
+}
+template <int kLane, size_t N>
+HWY_INLINE Vec128<uint16_t> Broadcast(const Vec128<uint16_t, N> v) {
+  static_assert(0 <= kLane && kLane < N, "Invalid lane");
+  return Vec128<uint16_t>(
+      vdupq_n_u16(vgetq_lane_u16(vcombine_u16(v.raw, v.raw), kLane)));
+}
+template <int kLane>
+HWY_INLINE Vec128<uint32_t> Broadcast(const Vec128<uint32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec128<uint32_t>(vdupq_n_u32(vgetq_lane_u32(v.raw, kLane)));
+}
+template <int kLane>
+HWY_INLINE Vec128<uint64_t> Broadcast(const Vec128<uint64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec128<uint64_t>(vdupq_n_u64(vgetq_lane_u64(v.raw, kLane)));
+}
+
+// Signed
+template <int kLane>
+HWY_INLINE Vec128<int16_t> Broadcast(const Vec128<int16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  return Vec128<int16_t>(vdupq_n_s16(vgetq_lane_s16(v.raw, kLane)));
+}
+template <int kLane, size_t N>
+HWY_INLINE Vec128<int16_t> Broadcast(const Vec128<int16_t, N> v) {
+  static_assert(0 <= kLane && kLane < N, "Invalid lane");
+  return Vec128<int16_t>(
+      vdupq_n_s16(vgetq_lane_s16(vcombine_s16(v.raw, v.raw), kLane)));
+}
+template <int kLane>
+HWY_INLINE Vec128<int32_t> Broadcast(const Vec128<int32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec128<int32_t>(vdupq_n_s32(vgetq_lane_s32(v.raw, kLane)));
+}
+template <int kLane>
+HWY_INLINE Vec128<int64_t> Broadcast(const Vec128<int64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec128<int64_t>(vdupq_n_s64(vgetq_lane_s64(v.raw, kLane)));
+}
+
+// Float
+template <int kLane>
+HWY_INLINE Vec128<float> Broadcast(const Vec128<float> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec128<float>(vdupq_n_f32(vgetq_lane_f32(v.raw, kLane)));
+}
+#endif
+
+// ------------------------------ Shuffle bytes with variable indices
+
+// Returns vector of bytes[from[i]]. "from" is also interpreted as bytes:
+// either valid indices in [0, 16) or >= 0x80 to zero the i-th output byte.
+template <typename T, typename TI>
+HWY_INLINE Vec128<T> TableLookupBytes(const Vec128<T> bytes,
+                                      const Vec128<TI> from) {
+  const Full128<uint8_t> d8;
+#if defined(__aarch64__)
+  return BitCast(Full128<T>(),
+                 Vec128<uint8_t>(vqtbl1q_u8(BitCast(d8, bytes).raw,
+                                            BitCast(d8, from).raw)));
+#else
+  uint8x16_t table0 = BitCast(d8, bytes).raw;
+  uint8x8x2_t table;
+  table.val[0] = vget_low_u8(table0);
+  table.val[1] = vget_high_u8(table0);
+  uint8x16_t idx = BitCast(d8, from).raw;
+  uint8x8_t low = vtbl2_u8(table, vget_low_u8(idx));
+  uint8x8_t hi = vtbl2_u8(table, vget_high_u8(idx));
+  return BitCast(Full128<T>(), Vec128<uint8_t>(vcombine_u8(low, hi)));
+#endif
+}
+
+// ------------------------------ Hard-coded shuffles
+
+// Notation: let Vec128<int32_t> have lanes 3,2,1,0 (0 is least-significant).
+// Shuffle0321 rotates one lane to the right (the previous least-significant
+// lane is now most-significant). These could also be implemented via
+// CombineShiftRightBytes but the shuffle_abcd notation is more convenient.
+
+// Swap 32-bit halves in 64-bits
+template <typename T>
+HWY_INLINE Vec128<T> Shuffle2301(const Vec128<T> v) {
+  static constexpr uint8_t bytes[16] = {4,  5,  6,  7,  0, 1, 2,  3,
+                                        12, 13, 14, 15, 8, 9, 10, 11};
+  return TableLookupBytes(v, Load(Full128<uint8_t>(), bytes));
+}
+
+// Swap 64-bit halves
+template <typename T>
+HWY_INLINE Vec128<T> Shuffle1032(const Vec128<T> v) {
+  return CombineShiftRightBytes<8>(v, v);
+}
+template <typename T>
+HWY_INLINE Vec128<T> Shuffle01(const Vec128<T> v) {
+  return CombineShiftRightBytes<8>(v, v);
+}
+
+// Rotate right 32 bits
+template <typename T>
+HWY_INLINE Vec128<T> Shuffle0321(const Vec128<T> v) {
+  return CombineShiftRightBytes<4>(v, v);
+}
+
+// Rotate left 32 bits
+template <typename T>
+HWY_INLINE Vec128<T> Shuffle2103(const Vec128<T> v) {
+  return CombineShiftRightBytes<12>(v, v);
+}
+
+// Reverse
+template <typename T>
+HWY_INLINE Vec128<T> Shuffle0123(const Vec128<T> v) {
+  static_assert(sizeof(T) == 4,
+                "Shuffle0123 should only be applied to 32-bit types");
+  // TODO(janwas): more efficient implementation?,
+  // It is possible to use two instructions (vrev64q_u32 and vcombine_u32 of the
+  // high/low parts) instead of the extra memory and load.
+  static constexpr uint8_t bytes[16] = {12, 13, 14, 15, 8, 9, 10, 11,
+                                        4,  5,  6,  7,  0, 1, 2,  3};
+  return TableLookupBytes(v, Load(Full128<uint8_t>(), bytes));
+}
+
+// ------------------------------ Permute (runtime variable)
+
+// Returned by SetTableIndices for use by TableLookupLanes.
+template <typename T>
+struct Permute128 {
+  uint8x16_t raw;
+};
+
+template <typename T>
+HWY_INLINE Permute128<T> SetTableIndices(const Full128<T> d,
+                                         const int32_t* idx) {
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+  for (size_t i = 0; i < d.N; ++i) {
+    if (idx[i] >= static_cast<int32_t>(d.N)) {
+      printf("SetTableIndices [%zu] = %d >= %zu\n", i, idx[i], d.N);
+      HWY_TRAP();
+    }
+  }
+#else
+  (void)d;
+#endif
+
+  const Full128<uint8_t> d8;
+  alignas(16) uint8_t control[d8.N];
+  for (size_t idx_byte = 0; idx_byte < d8.N; ++idx_byte) {
+    const size_t idx_lane = idx_byte / sizeof(T);
+    const size_t mod = idx_byte % sizeof(T);
+    control[idx_byte] = idx[idx_lane] * sizeof(T) + mod;
+  }
+  return Permute128<T>{Load(d8, control).raw};
+}
+
+HWY_INLINE Vec128<uint32_t> TableLookupLanes(const Vec128<uint32_t> v,
+                                             const Permute128<uint32_t> idx) {
+  return TableLookupBytes(v, Vec128<uint8_t>(idx.raw));
+}
+HWY_INLINE Vec128<int32_t> TableLookupLanes(const Vec128<int32_t> v,
+                                            const Permute128<int32_t> idx) {
+  return TableLookupBytes(v, Vec128<uint8_t>(idx.raw));
+}
+HWY_INLINE Vec128<float> TableLookupLanes(const Vec128<float> v,
+                                          const Permute128<float> idx) {
+  const Full128<int32_t> di;
+  const Full128<float> df;
+  return BitCast(df,
+                 TableLookupBytes(BitCast(di, v), Vec128<uint8_t>(idx.raw)));
+}
+
+// ------------------------------ Interleave lanes
+
+// Interleaves lanes from halves of the 128-bit blocks of "a" (which provides
+// the least-significant lane) and "b". To concatenate two half-width integers
+// into one, use ZipLo/hi instead (also works with scalar).
+HWY_NEON_DEF_FUNCTION_INT_8_16_32(InterleaveLo, vzip1, _, 2)
+HWY_NEON_DEF_FUNCTION_UINT_8_16_32(InterleaveLo, vzip1, _, 2)
+
+HWY_NEON_DEF_FUNCTION_INT_8_16_32(InterleaveHi, vzip2, _, 2)
+HWY_NEON_DEF_FUNCTION_UINT_8_16_32(InterleaveHi, vzip2, _, 2)
+
+#if defined(__aarch64__)
+// For 64 bit types, we only have the "q" version of the function defined as
+// interleaving 64-wide registers with 64-wide types in them makes no sense.
+HWY_INLINE Vec128<uint64_t> InterleaveLo(const Vec128<uint64_t> a,
+                                         const Vec128<uint64_t> b) {
+  return Vec128<uint64_t>(vzip1q_u64(a.raw, b.raw));
+}
+HWY_INLINE Vec128<int64_t> InterleaveLo(const Vec128<int64_t> a,
+                                        const Vec128<int64_t> b) {
+  return Vec128<int64_t>(vzip1q_s64(a.raw, b.raw));
+}
+
+HWY_INLINE Vec128<uint64_t> InterleaveHi(const Vec128<uint64_t> a,
+                                         const Vec128<uint64_t> b) {
+  return Vec128<uint64_t>(vzip2q_u64(a.raw, b.raw));
+}
+HWY_INLINE Vec128<int64_t> InterleaveHi(const Vec128<int64_t> a,
+                                        const Vec128<int64_t> b) {
+  return Vec128<int64_t>(vzip2q_s64(a.raw, b.raw));
+}
+#else
+// ARMv7 emulation.
+HWY_INLINE Vec128<uint64_t> InterleaveLo(const Vec128<uint64_t> a,
+                                         const Vec128<uint64_t> b) {
+  auto flip = CombineShiftRightBytes<8>(a, a);
+  return CombineShiftRightBytes<8>(b, flip);
+}
+HWY_INLINE Vec128<int64_t> InterleaveLo(const Vec128<int64_t> a,
+                                        const Vec128<int64_t> b) {
+  auto flip = CombineShiftRightBytes<8>(a, a);
+  return CombineShiftRightBytes<8>(b, flip);
+}
+
+HWY_INLINE Vec128<uint64_t> InterleaveHi(const Vec128<uint64_t> a,
+                                         const Vec128<uint64_t> b) {
+  auto flip = CombineShiftRightBytes<8>(b, b);
+  return CombineShiftRightBytes<8>(flip, a);
+}
+HWY_INLINE Vec128<int64_t> InterleaveHi(const Vec128<int64_t> a,
+                                        const Vec128<int64_t> b) {
+  auto flip = CombineShiftRightBytes<8>(b, b);
+  return CombineShiftRightBytes<8>(flip, a);
+}
+#endif
+
+// Floats
+HWY_INLINE Vec128<float> InterleaveLo(const Vec128<float> a,
+                                      const Vec128<float> b) {
+  return Vec128<float>(vzip1q_f32(a.raw, b.raw));
+}
+#if defined(__aarch64__)
+HWY_INLINE Vec128<double> InterleaveLo(const Vec128<double> a,
+                                       const Vec128<double> b) {
+  return Vec128<double>(vzip1q_f64(a.raw, b.raw));
+}
+#endif
+
+HWY_INLINE Vec128<float> InterleaveHi(const Vec128<float> a,
+                                      const Vec128<float> b) {
+  return Vec128<float>(vzip2q_f32(a.raw, b.raw));
+}
+#if defined(__aarch64__)
+HWY_INLINE Vec128<double> InterleaveHi(const Vec128<double> a,
+                                       const Vec128<double> b) {
+  return Vec128<double>(vzip2q_s64(a.raw, b.raw));
+}
+#endif
+
+// ------------------------------ Zip lanes
+
+// Same as interleave_*, except that the return lanes are double-width integers;
+// this is necessary because the single-lane scalar cannot return two values.
+
+HWY_INLINE Vec128<uint16_t> ZipLo(const Vec128<uint8_t> a,
+                                  const Vec128<uint8_t> b) {
+  return Vec128<uint16_t>(vzip1q_u8(a.raw, b.raw));
+}
+HWY_INLINE Vec128<uint32_t> ZipLo(const Vec128<uint16_t> a,
+                                  const Vec128<uint16_t> b) {
+  return Vec128<uint32_t>(vzip1q_u16(a.raw, b.raw));
+}
+HWY_INLINE Vec128<uint64_t> ZipLo(const Vec128<uint32_t> a,
+                                  const Vec128<uint32_t> b) {
+  return Vec128<uint64_t>(vzip1q_u32(a.raw, b.raw));
+}
+
+HWY_INLINE Vec128<int16_t> ZipLo(const Vec128<int8_t> a,
+                                 const Vec128<int8_t> b) {
+  return Vec128<int16_t>(vzip1q_s8(a.raw, b.raw));
+}
+HWY_INLINE Vec128<int32_t> ZipLo(const Vec128<int16_t> a,
+                                 const Vec128<int16_t> b) {
+  return Vec128<int32_t>(vzip1q_s16(a.raw, b.raw));
+}
+HWY_INLINE Vec128<int64_t> ZipLo(const Vec128<int32_t> a,
+                                 const Vec128<int32_t> b) {
+  return Vec128<int64_t>(vzip1q_s32(a.raw, b.raw));
+}
+
+HWY_INLINE Vec128<uint16_t> ZipHi(const Vec128<uint8_t> a,
+                                  const Vec128<uint8_t> b) {
+  return Vec128<uint16_t>(vzip2q_u8(a.raw, b.raw));
+}
+HWY_INLINE Vec128<uint32_t> ZipHi(const Vec128<uint16_t> a,
+                                  const Vec128<uint16_t> b) {
+  return Vec128<uint32_t>(vzip2q_u16(a.raw, b.raw));
+}
+HWY_INLINE Vec128<uint64_t> ZipHi(const Vec128<uint32_t> a,
+                                  const Vec128<uint32_t> b) {
+  return Vec128<uint64_t>(vzip2q_u32(a.raw, b.raw));
+}
+
+HWY_INLINE Vec128<int16_t> ZipHi(const Vec128<int8_t> a,
+                                 const Vec128<int8_t> b) {
+  return Vec128<int16_t>(vzip2q_s8(a.raw, b.raw));
+}
+HWY_INLINE Vec128<int32_t> ZipHi(const Vec128<int16_t> a,
+                                 const Vec128<int16_t> b) {
+  return Vec128<int32_t>(vzip2q_s16(a.raw, b.raw));
+}
+HWY_INLINE Vec128<int64_t> ZipHi(const Vec128<int32_t> a,
+                                 const Vec128<int32_t> b) {
+  return Vec128<int64_t>(vzip2q_s32(a.raw, b.raw));
+}
+
+// ------------------------------ Blocks
+
+// hiH,hiL loH,loL |-> hiL,loL (= lower halves)
+template <typename T>
+HWY_INLINE Vec128<T> ConcatLoLo(const Vec128<T> hi, const Vec128<T> lo) {
+  const Full128<uint64_t> d64;
+  return BitCast(Full128<T>(),
+                 InterleaveLo(BitCast(d64, lo), BitCast(d64, hi)));
+}
+
+// hiH,hiL loH,loL |-> hiH,loH (= upper halves)
+template <typename T>
+HWY_INLINE Vec128<T> ConcatHiHi(const Vec128<T> hi, const Vec128<T> lo) {
+  const Full128<uint64_t> d64;
+  return BitCast(Full128<T>(),
+                 InterleaveHi(BitCast(d64, lo), BitCast(d64, hi)));
+}
+
+// hiH,hiL loH,loL |-> hiL,loH (= inner halves)
+template <typename T>
+HWY_INLINE Vec128<T> ConcatLoHi(const Vec128<T> hi, const Vec128<T> lo) {
+  return CombineShiftRightBytes<8>(hi, lo);
+}
+
+// hiH,hiL loH,loL |-> hiH,loL (= outer halves)
+template <typename T>
+HWY_INLINE Vec128<T> ConcatHiLo(const Vec128<T> hi, const Vec128<T> lo) {
+  // TODO(janwas): more efficient implementation?
+  alignas(16) const uint8_t kBytes[16] = {
+      0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0, 0, 0, 0, 0, 0, 0, 0};
+  const auto vec = BitCast(Full128<T>(), Load(Full128<uint8_t>(), kBytes));
+  return IfThenElse(MaskFromVec(vec), lo, hi);
+}
+
+// ------------------------------ Odd/even lanes
+
+template <typename T>
+HWY_INLINE Vec128<T> OddEven(const Vec128<T> a, const Vec128<T> b) {
+  alignas(16) constexpr uint8_t kBytes[16] = {
+      ((0 / sizeof(T)) & 1) ? 0 : 0xFF,  ((1 / sizeof(T)) & 1) ? 0 : 0xFF,
+      ((2 / sizeof(T)) & 1) ? 0 : 0xFF,  ((3 / sizeof(T)) & 1) ? 0 : 0xFF,
+      ((4 / sizeof(T)) & 1) ? 0 : 0xFF,  ((5 / sizeof(T)) & 1) ? 0 : 0xFF,
+      ((6 / sizeof(T)) & 1) ? 0 : 0xFF,  ((7 / sizeof(T)) & 1) ? 0 : 0xFF,
+      ((8 / sizeof(T)) & 1) ? 0 : 0xFF,  ((9 / sizeof(T)) & 1) ? 0 : 0xFF,
+      ((10 / sizeof(T)) & 1) ? 0 : 0xFF, ((11 / sizeof(T)) & 1) ? 0 : 0xFF,
+      ((12 / sizeof(T)) & 1) ? 0 : 0xFF, ((13 / sizeof(T)) & 1) ? 0 : 0xFF,
+      ((14 / sizeof(T)) & 1) ? 0 : 0xFF, ((15 / sizeof(T)) & 1) ? 0 : 0xFF,
+  };
+  const auto vec = BitCast(Full128<T>(), Load(Full128<uint8_t>(), kBytes));
+  return IfThenElse(MaskFromVec(vec), b, a);
+}
+
+// ================================================== MISC
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// ------------------------------ movemask
+
+// Returns a bit array of the most significant bit of each byte in "v", i.e.
+// sum_i=0..15 of (v[i] >> 7) << i; v[0] is the least-significant byte of "v".
+// This is useful for testing/branching based on comparison results.
+HWY_INLINE uint64_t movemask(const Vec128<uint8_t> v) {
+  static constexpr uint8x16_t kCollapseMask = {
+      1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80, 1, 2, 4, 8, 0x10, 0x20, 0x40, 0x80,
+  };
+  int8x16_t signed_v = vreinterpretq_s8_u8(v.raw);
+  int8x16_t signed_mask = vshrq_n_s8(signed_v, 7);
+  uint8x16_t values = vreinterpretq_u8_s8(signed_mask) & kCollapseMask;
+
+#if defined(__aarch64__)
+  uint8x8_t c0 = vget_low_u8(vpaddq_u8(values, values));
+  uint8x8_t c1 = vpadd_u8(c0, c0);
+  uint8x8_t c2 = vpadd_u8(c1, c1);
+#else
+  uint8x16x2_t v0 = vuzpq_u8(values, values);
+  uint8x8_t c0 = vget_low_u8(vorrq_u8(v0.val[0], v0.val[1]));
+  uint8x8x2_t v1 = vuzp_u8(c0, c0);
+  uint8x8_t c1 = vorr_u8(v1.val[0], v1.val[1]);
+  uint8x8x2_t v2 = vuzp_u8(c1, c1);
+  uint8x8_t c2 = vorr_u8(v2.val[0], v2.val[1]);
+#endif
+
+  return vreinterpret_u16_u8(c2)[0];
+}
+
+// Returns the most significant bit of each float/double lane (see above).
+HWY_INLINE uint64_t movemask(const Vec128<float> v) {
+  static constexpr uint32x4_t kCollapseMask = {1, 2, 4, 8};
+  int32x4_t signed_v = vreinterpretq_s32_f32(v.raw);
+  int32x4_t signed_mask = vshrq_n_s32(signed_v, 31);
+  uint32x4_t values = vreinterpretq_u32_s32(signed_mask) & kCollapseMask;
+#if defined(__aarch64__)
+  return vaddvq_u32(values);
+#else
+  uint32x4x2_t v0 = vuzpq_u32(values, values);
+  uint32x2_t c0 = vget_low_u32(vorrq_u32(v0.val[0], v0.val[1]));
+  uint32x2x2_t v1 = vuzp_u32(c0, c0);
+  uint32x2_t c1 = vorr_u32(v1.val[0], v1.val[1]);
+  return c1[0];
+#endif
+}
+#if defined(__aarch64__)
+HWY_INLINE uint64_t movemask(const Vec128<double> v) {
+  static constexpr uint64x2_t kCollapseMask = {1, 2};
+  int64x2_t signed_v = vreinterpretq_s64_f64(v.raw);
+  int64x2_t signed_mask = vshrq_n_s64(signed_v, 63);
+  uint64x2_t values = vreinterpretq_u64_s64(signed_mask) & kCollapseMask;
+  return vaddvq_u64(values);
+}
+#endif
+
+// ------------------------------ minpos
+
+// Returns index and min value in lanes 1 and 0.
+HWY_INLINE Vec128<uint16_t> minpos(const Vec128<uint16_t> v) {
+  // v = ABCDEFGH
+  uint16x8_t mask = {0, 1, 2, 3, 4, 5, 6, 7};
+  uint32x4_t a = vreinterpretq_u32_u16(vzip1q_u16(mask, v.raw));  // 0A1B2C3D
+  uint32x4_t b = vreinterpretq_u32_u16(vzip2q_u16(mask, v.raw));  // 4E5F6G7H
+  a = vminq_u32(a, b);
+#if defined(__aarch64__)
+  uint32_t pos_min = vminvq_u32(a);
+  a = Set(Full128<uint32_t>(), pos_min).raw;
+#else
+  uint32x4x2_t v0 = vuzpq_u32(a, a);
+  uint32x4_t c0 = vminq_u32(v0.val[0], v0.val[1]);
+  uint32x4x2_t v1 = vuzpq_u32(c0, c0);
+  a = vminq_u32(v1.val[0], v1.val[1]);
+#endif
+  // TODO(veluca): use CombineShiftRightBytes?
+  // At the end the minimum position is in the first 16-bit lane and the minimum
+  // value in the second 16-bit lane; but we need them switched. Since every
+  // 32-bit lane contains the same value we can just right-shift a 64-bit lane
+  // 16-bits.
+  return Vec128<uint16_t>(
+      vreinterpretq_u16_u64(vshrq_n_u64(vreinterpretq_u64_u32(a), 16)));
+}
+
+// ------------------------------ Horizontal sum (reduction)
+
+// Returns 64-bit sums of 8-byte groups.
+HWY_INLINE Vec128<uint64_t> SumsOfU8x8(const Vec128<uint8_t> v) {
+  uint16x8_t a = vpaddlq_u8(v.raw);
+  uint32x4_t b = vpaddlq_u16(a);
+  return Vec128<uint64_t>(vpaddlq_u32(b));
+}
+
+#if defined(__aarch64__)
+// Supported for 32b and 64b vector types. Returns the sum in each lane.
+HWY_INLINE Vec128<uint32_t> SumOfLanes(const Vec128<uint32_t> v) {
+  return Vec128<uint32_t>(vdupq_n_u32(vaddvq_u32(v.raw)));
+}
+HWY_INLINE Vec128<int32_t> SumOfLanes(const Vec128<int32_t> v) {
+  return Vec128<int32_t>(vdupq_n_s32(vaddvq_s32(v.raw)));
+}
+HWY_INLINE Vec128<float> SumOfLanes(const Vec128<float> v) {
+  return Vec128<float>(vdupq_n_f32(vaddvq_f32(v.raw)));
+}
+HWY_INLINE Vec128<uint64_t> SumOfLanes(const Vec128<uint64_t> v) {
+  return Vec128<uint64_t>(vdupq_n_u64(vaddvq_u64(v.raw)));
+}
+HWY_INLINE Vec128<int64_t> SumOfLanes(const Vec128<int64_t> v) {
+  return Vec128<int64_t>(vdupq_n_s64(vaddvq_s64(v.raw)));
+}
+HWY_INLINE Vec128<double> SumOfLanes(const Vec128<double> v) {
+  return Vec128<double>(vdupq_n_f64(vaddvq_f64(v.raw)));
+}
+#else
+// ARMv7 version for everything except doubles.
+HWY_INLINE Vec128<uint32_t> SumOfLanes(const Vec128<uint32_t> v) {
+  uint32x4x2_t v0 = vuzpq_u32(v.raw, v.raw);
+  uint32x4_t c0 = vaddq_u32(v0.val[0], v0.val[1]);
+  uint32x4x2_t v1 = vuzpq_u32(c0, c0);
+  return Vec128<uint32_t>(vaddq_u32(v1.val[0], v1.val[1]));
+}
+HWY_INLINE Vec128<int32_t> SumOfLanes(const Vec128<int32_t> v) {
+  int32x4x2_t v0 = vuzpq_s32(v.raw, v.raw);
+  int32x4_t c0 = vaddq_s32(v0.val[0], v0.val[1]);
+  int32x4x2_t v1 = vuzpq_s32(c0, c0);
+  return Vec128<int32_t>(vaddq_s32(v1.val[0], v1.val[1]));
+}
+HWY_INLINE Vec128<float> SumOfLanes(const Vec128<float> v) {
+  float32x4x2_t v0 = vuzpq_f32(v.raw, v.raw);
+  float32x4_t c0 = vaddq_f32(v0.val[0], v0.val[1]);
+  float32x4x2_t v1 = vuzpq_f32(c0, c0);
+  return Vec128<float>(vaddq_f32(v1.val[0], v1.val[1]));
+}
+HWY_INLINE Vec128<uint64_t> SumOfLanes(const Vec128<uint64_t> v) {
+  return v + CombineShiftRightBytes<8>(v, v);
+}
+HWY_INLINE Vec128<int64_t> SumOfLanes(const Vec128<int64_t> v) {
+  return v + CombineShiftRightBytes<8>(v, v);
+}
+#endif
+
+// ------------------------------ mask
+
+template <typename T>
+HWY_INLINE bool AllFalse(const Mask128<T> v) {
+  const auto v64 = BitCast(Full128<uint64_t>(), VecFromMask(v));
+  uint32x2_t a = vqmovn_u64(v64.raw);
+  return vreinterpret_u64_u32(a)[0] == 0;
+}
+
+template <typename T>
+HWY_INLINE bool AllTrue(const Mask128<T> v) {
+  return AllFalse(VecFromMask(v) == Zero(Full128<T>()));
+}
+
+template <typename T>
+HWY_INLINE size_t CountTrue(const Mask128<T> v) {
+  // Integer vectors: only have movemask for u8, so divide by number of bytes.
+  const auto bytes = BitCast(Full128<uint8_t>(), VecFromMask(v));
+  return PopCount(movemask(bytes)) / sizeof(T);
+}
+HWY_INLINE size_t CountTrue(const Mask128<float> v) {
+  return PopCount(movemask(VecFromMask(v)));
+}
+#if defined(__aarch64__)
+HWY_INLINE size_t CountTrue(const Mask128<double> v) {
+  return PopCount(movemask(VecFromMask(v)));
+}
+#endif
+
+// TODO(user): wrappers for all intrinsics (in neon namespace).
+}  // namespace ext
+
+}  // namespace hwy
+
+#if !defined(__aarch64__)
+#undef vuzp1_s8
+#undef vuzp1_u8
+#undef vuzp1_s16
+#undef vuzp1_u16
+#undef vuzp1_s32
+#undef vuzp1_u32
+#undef vuzp1_f32
+#undef vuzp1q_s8
+#undef vuzp1q_u8
+#undef vuzp1q_s16
+#undef vuzp1q_u16
+#undef vuzp1q_s32
+#undef vuzp1q_u32
+#undef vuzp1q_f32
+#undef vuzp2_s8
+#undef vuzp2_u8
+#undef vuzp2_s16
+#undef vuzp2_u16
+#undef vuzp2_s32
+#undef vuzp2_u32
+#undef vuzp2_f32
+#undef vuzp2q_s8
+#undef vuzp2q_u8
+#undef vuzp2q_s16
+#undef vuzp2q_u16
+#undef vuzp2q_s32
+#undef vuzp2q_u32
+#undef vuzp2q_f32
+#undef vzip1_s8
+#undef vzip1_u8
+#undef vzip1_s16
+#undef vzip1_u16
+#undef vzip1_s32
+#undef vzip1_u32
+#undef vzip1_f32
+#undef vzip1q_s8
+#undef vzip1q_u8
+#undef vzip1q_s16
+#undef vzip1q_u16
+#undef vzip1q_s32
+#undef vzip1q_u32
+#undef vzip1q_f32
+#undef vzip2_s8
+#undef vzip2_u8
+#undef vzip2_s16
+#undef vzip2_u16
+#undef vzip2_s32
+#undef vzip2_u32
+#undef vzip2_f32
+#undef vzip2q_s8
+#undef vzip2q_u8
+#undef vzip2q_s16
+#undef vzip2q_u16
+#undef vzip2q_s32
+#undef vzip2q_u32
+#undef vzip2q_f32
+#endif
+
+#undef HWY_NEON_BUILD_ARG_1
+#undef HWY_NEON_BUILD_ARG_2
+#undef HWY_NEON_BUILD_ARG_3
+#undef HWY_NEON_BUILD_PARAM_1
+#undef HWY_NEON_BUILD_PARAM_2
+#undef HWY_NEON_BUILD_PARAM_3
+#undef HWY_NEON_BUILD_RET_1
+#undef HWY_NEON_BUILD_RET_2
+#undef HWY_NEON_BUILD_RET_3
+#undef HWY_NEON_BUILD_TPL_1
+#undef HWY_NEON_BUILD_TPL_2
+#undef HWY_NEON_BUILD_TPL_3
+#undef HWY_NEON_DEF_FUNCTION
+#undef HWY_NEON_DEF_FUNCTION_ALL_FLOATS
+#undef HWY_NEON_DEF_FUNCTION_ALL_TYPES
+#undef HWY_NEON_DEF_FUNCTION_INT_8
+#undef HWY_NEON_DEF_FUNCTION_INT_16
+#undef HWY_NEON_DEF_FUNCTION_INT_32
+#undef HWY_NEON_DEF_FUNCTION_INT_8_16_32
+#undef HWY_NEON_DEF_FUNCTION_INTS
+#undef HWY_NEON_DEF_FUNCTION_INTS_UINTS
+#undef HWY_NEON_DEF_FUNCTION_TPL
+#undef HWY_NEON_DEF_FUNCTION_UINT_8
+#undef HWY_NEON_DEF_FUNCTION_UINT_16
+#undef HWY_NEON_DEF_FUNCTION_UINT_32
+#undef HWY_NEON_DEF_FUNCTION_UINT_8_16_32
+#undef HWY_NEON_DEF_FUNCTION_UINTS
+#undef HWY_NEON_EVAL
+
+#undef HWY_CONCAT_IMPL
+#undef HWY_CONCAT
+
+#endif  // HWY_ARM64_NEON_H_
diff --git a/third_party/highway/hwy/benchmark.cc b/third_party/highway/hwy/benchmark.cc
new file mode 100644
index 000000000..1bf916422
--- /dev/null
+++ b/third_party/highway/hwy/benchmark.cc
@@ -0,0 +1,261 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "hwy/benchmark.cc"
+
+// First time this file is included: include headers as usual.
+#include <stddef.h>
+#include <stdio.h>
+
+#include <cmath>
+#include <memory>
+#include <new>
+#include <numeric>  // iota
+
+#include "hwy/nanobenchmark.h"
+#include "hwy/runtime_dispatch.h"
+
+struct RunBenchmarks {
+  // Benchmark relies on this being one, AND the compiler not knowing it.
+  HWY_DECLARE(void, (int unpredictable1))
+};
+
+int main(int argc, char** /*argv*/) {
+  // No buffering (helps if running remotely)
+  setvbuf(stdin, nullptr, _IONBF, 0);
+
+  const hwy::TargetBitfield targets;  // supported by this CPU
+  // No need to return the bitfield of targets - benchmark already prints each.
+  // != 999 is an expression that evaluates to 1, but unknown to the compiler.
+  (void)targets.Foreach(RunBenchmarks(), argc != 999);
+}
+
+#endif  // HWY_TARGET_INCLUDE
+// Generates RunBenchmarks implementations for all targets.
+#include "hwy/foreach_target.h"
+
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+class TwoArray {
+ public:
+  // Passed to ctor as a value NOT known to the compiler. Must be a multiple of
+  // the vector lane count * 8.
+  static size_t NumItems() { return 3456; }
+
+  explicit TwoArray(const size_t num_items)
+      : a_(new (std::align_val_t(kMaxVectorSize)) float[num_items * 2]),
+        b_(a_.get() + num_items) {
+    const float init = num_items / NumItems();  // 1, but compiler doesn't know
+    std::iota(a_.get(), a_.get() + num_items, init);
+    std::iota(b_, b_ + num_items, init);
+  }
+
+ protected:
+  std::unique_ptr<float[]> a_;
+  float* b_;
+};
+
+// Calls operator() of the given closure (lambda function).
+template <class Closure>
+static FuncOutput CallClosure(const Closure* f, const FuncInput input) {
+  return (*f)(input);
+}
+
+// Same as Measure, except "closure" is typically a lambda function of
+// FuncInput -> FuncOutput with a capture list.
+template <class Closure>
+static inline size_t MeasureClosure(const Closure& closure,
+                                    const FuncInput* inputs,
+                                    const size_t num_inputs, Result* results,
+                                    const Params& p = Params()) {
+  return Measure(reinterpret_cast<Func>(&CallClosure<Closure>),
+                 reinterpret_cast<const uint8_t*>(&closure), inputs, num_inputs,
+                 results, p);
+}
+
+// Measures durations, verifies results, prints timings.
+template <class Benchmark>
+void RunBenchmark(const char* caption, const int unpredictable1) {
+  printf("%10s: ", caption);
+  const size_t kNumInputs = 1;
+  const uint64_t inputs[kNumInputs] = {Benchmark::NumItems() * unpredictable1};
+  Result results[kNumInputs];
+
+  const size_t num_items = Benchmark::NumItems() * unpredictable1;
+  Benchmark benchmark(num_items);
+
+  Params p;
+  p.verbose = false;
+  p.max_evals = 7;
+  p.target_rel_mad = 0.002;
+  const size_t num_results = MeasureClosure(
+      [&benchmark](const uint64_t input) { return benchmark(input); }, inputs,
+      kNumInputs, results, p);
+  if (num_results != kNumInputs) {
+    fprintf(stderr, "MeasureClosure failed.\n");
+  }
+
+  benchmark.Verify(num_items);
+
+  for (size_t i = 0; i < num_results; ++i) {
+    const double cycles_per_item = results[i].ticks / results[i].input;
+    const double mad = results[i].variability * cycles_per_item;
+    printf("%6zu: %6.3f (+/- %5.3f)\n", results[i].input, cycles_per_item, mad);
+  }
+}
+
+HWY_ATTR void Intro() {
+  HWY_ALIGN const float in[16] = {1, 2, 3, 4, 5, 6};
+  HWY_ALIGN float out[16];
+  HWY_FULL(float) d;  // largest possible vector
+  for (size_t i = 0; i < 16; i += d.N) {
+    const auto vec = Load(d, in + i);  // aligned!
+    auto result = vec * vec;
+    result += result;  // can update if not const
+    Store(result, d, out + i);
+  }
+  printf("F(x)->2*x^2, F(%.0f) = %.1f\n", in[2], out[2]);
+}
+
+// BEGINNER: dot product
+// 0.4 cyc/float = bronze, 0.25 = silver, 0.15 = gold!
+class BenchmarkDot : public TwoArray {
+ public:
+  explicit BenchmarkDot(size_t num_items) : TwoArray(num_items), dot_{-1.0f} {}
+
+  HWY_ATTR uint64_t operator()(const size_t num_items) {
+    HWY_FULL(float) d;
+    using V = decltype(Zero(d));
+    constexpr int unroll = 8;
+    // Compiler doesn't make independent sum* accumulators, so unroll manually.
+    // Some older compilers might not be able to fit the 8 arrays in registers,
+    // so manual unrolling can be helpfull if you run into this issue.
+    // 2 FMA ports * 4 cycle latency = 8x unrolled.
+    V sum[unroll];
+    for (int i = 0; i < unroll; ++i) {
+      sum[i] = Zero(d);
+    }
+    const float* const HWY_RESTRICT pa = &a_[0];
+    const float* const HWY_RESTRICT pb = b_;
+    for (size_t i = 0; i < num_items; i += unroll * d.N) {
+      for (int j = 0; j < unroll; ++j) {
+        const auto a = Load(d, pa + i + j * d.N);
+        const auto b = Load(d, pb + i + j * d.N);
+        sum[j] = MulAdd(a, b, sum[j]);
+      }
+    }
+    // Reduction tree: sum of all accumulators by pairs into sum[0], then the
+    // lanes.
+    for (int power = 1; power < unroll; power *= 2) {
+      for (int i = 0; i < unroll; i += 2 * power) {
+        sum[i] += sum[i + power];
+      }
+    }
+    return dot_ = GetLane(ext::SumOfLanes(sum[0]));
+  }
+  void Verify(size_t num_items) {
+    if (dot_ == -1.0f) {
+      fprintf(stderr, "Dot: must call Verify after benchmark");
+      abort();
+    }
+
+    const float expected =
+        std::inner_product(a_.get(), a_.get() + num_items, b_, 0.0f);
+    const float rel_err = std::abs(expected - dot_) / expected;
+    if (rel_err > 1.1E-6f) {
+      fprintf(stderr, "Dot: expected %e actual %e (%e)\n", expected, dot_,
+              rel_err);
+      abort();
+    }
+  }
+
+ private:
+  float dot_;  // for Verify
+};
+
+// INTERMEDIATE: delta coding
+// 1.0 cycles/float = bronze, 0.7 = silver, 0.4 = gold!
+struct BenchmarkDelta : public TwoArray {
+  explicit BenchmarkDelta(size_t num_items) : TwoArray(num_items) {}
+
+  HWY_ATTR uint64_t operator()(const size_t num_items) const {
+#if HWY_BITS == 0
+    b_[0] = a_[0];
+    for (size_t i = 1; i < num_items; ++i) {
+      b_[i] = a_[i] - a_[i - 1];
+    }
+#elif HWY_BITS == 128
+    // Slightly better than unaligned loads
+    const HWY_CAPPED(float, 4) df;
+    const HWY_CAPPED(int32_t, 4) di;
+    size_t i;
+    b_[0] = a_[0];
+    for (i = 1; i < df.N; ++i) {
+      b_[i] = a_[i] - a_[i - 1];
+    }
+    auto prev = Load(df, &a_[0]);
+    for (; i < num_items; i += df.N) {
+      const auto a = Load(df, &a_[i]);
+      constexpr int kBytes = (df.N - 1) * sizeof(float);
+      const auto shifted = BitCast(df, CombineShiftRightBytes<kBytes>(
+                                           BitCast(di, a), BitCast(di, prev)));
+      prev = a;
+      Store(a - shifted, df, &b_[i]);
+    }
+#else
+    // Larger vectors are split into 128-bit blocks, easiest to use the
+    // unaligned load support to shift between them.
+    const HWY_FULL(float) df;
+    size_t i;
+    b_[0] = a_[0];
+    for (i = 1; i < df.N; ++i) {
+      b_[i] = a_[i] - a_[i - 1];
+    }
+    for (; i < num_items; i += df.N) {
+      const auto a = Load(df, &a_[i]);
+      const auto shifted = LoadU(df, &a_[i - 1]);
+      Store(a - shifted, df, &b_[i]);
+    }
+#endif
+    return b_[num_items - 1];
+  }
+
+  void Verify(size_t num_items) {
+    for (size_t i = 0; i < num_items; ++i) {
+      const float expected = (i == 0) ? a_[0] : a_[i] - a_[i - 1];
+      const float err = std::abs(expected - b_[i]);
+      if (err > 1E-6f) {
+        fprintf(stderr, "Delta: expected %e, actual %e\n", expected, b_[i]);
+      }
+    }
+  }
+};
+
+void RunBenchmarks(int unpredictable1) {
+  Intro();
+  printf("------------------------ %d-bit vectors\n", HWY_BITS);
+  RunBenchmark<BenchmarkDot>("dot", unpredictable1);
+  RunBenchmark<BenchmarkDelta>("delta", unpredictable1);
+}
+
+}  // namespace
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+
+void RunBenchmarks::HWY_FUNC(int unpredictable1) {
+  hwy::HWY_NAMESPACE::RunBenchmarks(unpredictable1);
+}
diff --git a/third_party/highway/hwy/cache_control.h b/third_party/highway/hwy/cache_control.h
new file mode 100644
index 000000000..96c10c1d5
--- /dev/null
+++ b/third_party/highway/hwy/cache_control.h
@@ -0,0 +1,103 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_CACHE_CONTROL_H_
+#define HWY_CACHE_CONTROL_H_
+
+#include <stdint.h>
+#include <string.h>  // memcpy
+
+#include "hwy/arch.h"
+#include "hwy/compiler_specific.h"
+
+#if HWY_ARCH == HWY_ARCH_X86
+#include <emmintrin.h>
+#endif
+
+namespace hwy {
+
+HWY_INLINE void Stream(const uint32_t t, uint32_t* HWY_RESTRICT aligned) {
+#if HWY_ARCH == HWY_ARCH_X86
+  _mm_stream_si32(reinterpret_cast<int*>(aligned), t);
+#else
+  memcpy(aligned, &t, sizeof(t));
+#endif
+}
+
+HWY_INLINE void Stream(const uint64_t t, uint64_t* HWY_RESTRICT aligned) {
+#if defined(__x86_64__) || defined(_M_X64)
+  // NOLINTNEXTLINE(google-runtime-int)
+  _mm_stream_si64(reinterpret_cast<long long*>(aligned), t);
+#elif HWY_ARCH == HWY_ARCH_X86  // i386 case
+  _mm_stream_si32(reinterpret_cast<int*>(aligned), static_cast<uint32_t>(t));
+  _mm_stream_si32(reinterpret_cast<int*>(aligned) + 1, t >> 32u);
+#else
+  memcpy(aligned, &t, sizeof(t));
+#endif
+}
+
+// Delays subsequent loads until prior loads are visible. On Intel CPUs, also
+// serves as a full fence (waits for all prior instructions to complete).
+// No effect on non-x86.
+HWY_INLINE void LoadFence() {
+#if HWY_ARCH == HWY_ARCH_X86
+#ifdef __SSE2__
+  _mm_lfence();
+#elif HWY_COMPILER_MSVC
+  MemoryBarrier();
+#else
+  __sync_synchronize();
+#endif
+#endif
+}
+
+// Ensures previous weakly-ordered stores are visible. No effect on non-x86.
+HWY_INLINE void StoreFence() {
+#if HWY_ARCH == HWY_ARCH_X86
+#ifdef __SSE2__
+  _mm_sfence();
+#elif HWY_COMPILER_MSVC
+  MemoryBarrier();
+#else
+  __sync_synchronize();
+#endif
+#endif
+}
+
+// Begins loading the cache line containing "p".
+template <typename T>
+HWY_INLINE void Prefetch(const T* p) {
+#if HWY_COMPILER_GCC || HWY_COMPILER_CLANG
+  // Hint=0 (NTA) behavior differs, but skipping outer caches is probably not
+  // desirable, so use the default 3 (keep in caches).
+  __builtin_prefetch(p, /*write=*/0, /*hint=*/3);
+#elif HWY_ARCH == HWY_ARCH_X86
+  _mm_prefetch(p, _MM_HINT_T0);
+#else
+  (void)p;
+#endif
+}
+
+// Invalidates and flushes the cache line containing "p". No effect on non-x86.
+HWY_INLINE void FlushCacheline(const void* p) {
+#if HWY_ARCH == HWY_ARCH_X86
+  _mm_clflush(p);
+#else
+  (void)p;
+#endif
+}
+
+}  // namespace hwy
+
+#endif  // HWY_CACHE_CONTROL_H_
diff --git a/third_party/highway/hwy/compiler_specific.h b/third_party/highway/hwy/compiler_specific.h
new file mode 100644
index 000000000..2f9842645
--- /dev/null
+++ b/third_party/highway/hwy/compiler_specific.h
@@ -0,0 +1,85 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_COMPILER_SPECIFIC_H_
+#define HWY_COMPILER_SPECIFIC_H_
+
+// Compiler-specific includes and definitions.
+
+#ifdef _MSC_VER
+#define HWY_COMPILER_MSVC _MSC_VER
+#else
+#define HWY_COMPILER_MSVC 0
+#endif
+
+#ifdef __GNUC__
+#define HWY_COMPILER_GCC (__GNUC__ * 100 + __GNUC_MINOR__)
+#else
+#define HWY_COMPILER_GCC 0
+#endif
+
+// Clang can masquerade as MSVC/GCC, in which case both are set.
+#ifdef __clang__
+#define HWY_COMPILER_CLANG (__clang_major__ * 100 + __clang_minor__)
+#else
+#define HWY_COMPILER_CLANG 0
+#endif
+
+#if !HWY_COMPILER_MSVC && !HWY_COMPILER_GCC && !HWY_COMPILER_CLANG
+#error "Unsupported compiler"
+#endif
+
+#if HWY_COMPILER_MSVC
+#include <intrin.h>
+
+#define HWY_RESTRICT __restrict
+#define HWY_INLINE __forceinline
+#define HWY_NOINLINE __declspec(noinline)
+#define HWY_LIKELY(expr) expr
+#define HWY_TRAP __debugbreak
+#define HWY_TARGET_ATTR(feature_str)
+#define HWY_DIAGNOSTICS(tokens) __pragma(warning(tokens))
+#define HWY_DIAGNOSTICS_OFF(msc, gcc) HWY_DIAGNOSTICS(msc)
+
+#else
+
+#define HWY_RESTRICT __restrict__
+#define HWY_INLINE \
+  inline __attribute__((always_inline)) __attribute__((flatten))
+#define HWY_NOINLINE inline __attribute__((noinline))
+#define HWY_LIKELY(expr) __builtin_expect(!!(expr), 1)
+#define HWY_TRAP __builtin_trap
+#define HWY_TARGET_ATTR(feature_str) __attribute__((target(feature_str)))
+#define HWY_PRAGMA(tokens) _Pragma(#tokens)
+#define HWY_DIAGNOSTICS(tokens) HWY_PRAGMA(GCC diagnostic tokens)
+#define HWY_DIAGNOSTICS_OFF(msc, gcc) HWY_DIAGNOSTICS(gcc)
+
+#endif
+
+// Add to #if conditions to prevent IDE from graying out code.
+#if (defined __CDT_PARSER__) || (defined __INTELLISENSE__) || \
+    (defined Q_CREATOR_RUN)
+#define HWY_IDE 1
+#else
+#define HWY_IDE 0
+#endif
+
+// Clang 3.9 generates VINSERTF128 instead of the desired VBROADCASTF128,
+// which would free up port5. However, inline assembly isn't supported on
+// MSVC, results in incorrect output on GCC 8.3, and raises "invalid output size
+// for constraint" errors on Clang (https://gcc.godbolt.org/z/-Jt_-F), hence we
+// disable it.
+#define HWY_LOADDUP_ASM 0
+
+#endif  // HWY_COMPILER_SPECIFIC_H_
diff --git a/third_party/highway/hwy/foreach_target.h b/third_party/highway/hwy/foreach_target.h
new file mode 100644
index 000000000..3f581e1f4
--- /dev/null
+++ b/third_party/highway/hwy/foreach_target.h
@@ -0,0 +1,291 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_FOREACH_TARGET_H_
+#define HWY_FOREACH_TARGET_H_
+
+// Includes a specified file for each target in HWY_RUNTIME_TARGETS. This
+// generates a definition of Module::HWY_FUNC, called by runtime_dispatch.h.
+
+#ifndef HWY_TARGET_INCLUDE
+#error "Must set HWY_TARGET_INCLUDE before including foreach_target.h"
+#endif
+
+#include "hwy/runtime_targets.h"
+// After runtime_targets:
+#include "hwy/include_headers.h"
+
+//-----------------------------------------------------------------------------
+// SSE4
+#if HWY_RUNTIME_TARGETS & HWY_SSE4
+
+#undef HWY_NAMESPACE
+#define HWY_NAMESPACE N_SSE4
+
+#undef HWY_FUNC
+#define HWY_FUNC F_SSE4
+
+#undef HWY_ATTR
+#define HWY_ATTR HWY_ATTR_SSE4
+
+#undef HWY_BITS
+#define HWY_BITS 128
+
+#undef HWY_ALIGN
+#define HWY_ALIGN alignas(16)
+
+#undef HWY_HAS_CMP64
+#define HWY_HAS_CMP64 0
+
+#undef HWY_HAS_GATHER
+#define HWY_HAS_GATHER 0
+
+#undef HWY_HAS_VARIABLE_SHIFT
+#define HWY_HAS_VARIABLE_SHIFT 0
+
+#undef HWY_HAS_INT64
+#define HWY_HAS_INT64 1
+
+#undef HWY_HAS_DOUBLE
+#define HWY_HAS_DOUBLE 1
+
+#include HWY_TARGET_INCLUDE
+#endif
+
+//-----------------------------------------------------------------------------
+// AVX2
+#if HWY_RUNTIME_TARGETS & HWY_AVX2
+
+#undef HWY_NAMESPACE
+#define HWY_NAMESPACE N_AVX2
+
+#undef HWY_FUNC
+#define HWY_FUNC F_AVX2
+
+#undef HWY_ATTR
+#define HWY_ATTR HWY_ATTR_AVX2
+
+#undef HWY_BITS
+#define HWY_BITS 256
+
+#undef HWY_ALIGN
+#define HWY_ALIGN alignas(32)
+
+#undef HWY_HAS_CMP64
+#define HWY_HAS_CMP64 1
+
+#undef HWY_HAS_GATHER
+#define HWY_HAS_GATHER 1
+
+#undef HWY_HAS_VARIABLE_SHIFT
+#define HWY_HAS_VARIABLE_SHIFT 1
+
+#undef HWY_HAS_INT64
+#define HWY_HAS_INT64 1
+
+#undef HWY_HAS_DOUBLE
+#define HWY_HAS_DOUBLE 1
+
+#include HWY_TARGET_INCLUDE
+#endif
+
+//-----------------------------------------------------------------------------
+// AVX-512
+#if HWY_RUNTIME_TARGETS & HWY_AVX512
+
+#undef HWY_NAMESPACE
+#define HWY_NAMESPACE N_AVX512
+
+#undef HWY_FUNC
+#define HWY_FUNC F_AVX512
+
+#undef HWY_ATTR
+#define HWY_ATTR HWY_ATTR_AVX512
+
+#undef HWY_BITS
+#define HWY_BITS 512
+
+#undef HWY_ALIGN
+#define HWY_ALIGN alignas(64)
+
+#undef HWY_HAS_CMP64
+#define HWY_HAS_CMP64 1
+
+#undef HWY_HAS_GATHER
+#define HWY_HAS_GATHER 1
+
+#undef HWY_HAS_VARIABLE_SHIFT
+#define HWY_HAS_VARIABLE_SHIFT 1
+
+#undef HWY_HAS_INT64
+#define HWY_HAS_INT64 1
+
+#undef HWY_HAS_DOUBLE
+#define HWY_HAS_DOUBLE 1
+
+#include HWY_TARGET_INCLUDE
+#endif
+
+//-----------------------------------------------------------------------------
+// PPC8
+#if HWY_RUNTIME_TARGETS & HWY_PPC8
+
+#undef HWY_NAMESPACE
+#define HWY_NAMESPACE N_PPC8
+
+#undef HWY_FUNC
+#define HWY_FUNC F_PPC8
+
+#undef HWY_ATTR
+#define HWY_ATTR
+
+#undef HWY_BITS
+#define HWY_BITS 128
+
+#undef HWY_ALIGN
+#define HWY_ALIGN alignas(16)
+
+#undef HWY_HAS_CMP64
+#define HWY_HAS_CMP64 1
+
+#undef HWY_HAS_GATHER
+#define HWY_HAS_GATHER 0
+
+#undef HWY_HAS_VARIABLE_SHIFT
+#define HWY_HAS_VARIABLE_SHIFT 1
+
+#undef HWY_HAS_INT64
+#define HWY_HAS_INT64 1
+
+#undef HWY_HAS_DOUBLE
+#define HWY_HAS_DOUBLE 1
+
+#include HWY_TARGET_INCLUDE
+#endif
+
+//-----------------------------------------------------------------------------
+// ARM8
+#if HWY_RUNTIME_TARGETS & HWY_ARM8
+
+#undef HWY_NAMESPACE
+#define HWY_NAMESPACE N_ARM8
+
+#undef HWY_FUNC
+#define HWY_FUNC F_ARM8
+
+#undef HWY_ATTR
+#define HWY_ATTR HWY_ATTR_ARM8
+
+#undef HWY_BITS
+#define HWY_BITS 128
+
+#undef HWY_ALIGN
+#define HWY_ALIGN alignas(16)
+
+#undef HWY_HAS_CMP64
+#undef HWY_HAS_DOUBLE
+#ifdef __arm__
+#define HWY_HAS_CMP64 0
+#define HWY_HAS_DOUBLE 0
+#else
+#define HWY_HAS_CMP64 1
+#define HWY_HAS_DOUBLE 1
+#endif
+
+#undef HWY_HAS_INT64
+#define HWY_HAS_INT64 1
+
+#undef HWY_HAS_GATHER
+#define HWY_HAS_GATHER 0
+
+#undef HWY_HAS_VARIABLE_SHIFT
+#define HWY_HAS_VARIABLE_SHIFT 1
+
+#include HWY_TARGET_INCLUDE
+#endif
+
+//-----------------------------------------------------------------------------
+// WASM
+#if HWY_RUNTIME_TARGETS & HWY_WASM
+
+#undef HWY_NAMESPACE
+#define HWY_NAMESPACE N_WASM
+
+#undef HWY_FUNC
+#define HWY_FUNC F_WASM
+
+#undef HWY_ATTR
+#define HWY_ATTR HWY_ATTR_WASM
+
+#undef HWY_BITS
+#define HWY_BITS 128
+
+#undef HWY_ALIGN
+#define HWY_ALIGN alignas(16)
+
+#undef HWY_HAS_CMP64
+#define HWY_HAS_CMP64 0
+
+#undef HWY_HAS_GATHER
+#define HWY_HAS_GATHER 0
+
+#undef HWY_HAS_VARIABLE_SHIFT
+#define HWY_HAS_VARIABLE_SHIFT 0
+
+#undef HWY_HAS_INT64
+#define HWY_HAS_INT64 0
+
+#undef HWY_HAS_DOUBLE
+#define HWY_HAS_DOUBLE 0
+
+#include HWY_TARGET_INCLUDE
+#endif
+
+//-----------------------------------------------------------------------------
+// NONE
+
+#undef HWY_NAMESPACE
+#define HWY_NAMESPACE N_NONE
+
+#undef HWY_FUNC
+#define HWY_FUNC F_NONE
+
+#undef HWY_ATTR
+#define HWY_ATTR
+
+#undef HWY_BITS
+#define HWY_BITS 0
+
+#undef HWY_ALIGN
+#define HWY_ALIGN
+
+#undef HWY_HAS_CMP64
+#define HWY_HAS_CMP64 1
+
+#undef HWY_HAS_GATHER
+#define HWY_HAS_GATHER 1
+
+#undef HWY_HAS_VARIABLE_SHIFT
+#define HWY_HAS_VARIABLE_SHIFT 1
+
+#undef HWY_HAS_INT64
+#define HWY_HAS_INT64 1
+
+#undef HWY_HAS_DOUBLE
+#define HWY_HAS_DOUBLE 1
+
+// This is followed by the implementation part of the original CC file, so no
+// need to include again. NONE must be last because it is unconditional.
+
+#endif  // #ifndef HWY_FOREACH_TARGET_H_
diff --git a/third_party/highway/hwy/include_headers.h b/third_party/highway/hwy/include_headers.h
new file mode 100644
index 000000000..2bd0222c4
--- /dev/null
+++ b/third_party/highway/hwy/include_headers.h
@@ -0,0 +1,59 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_INCLUDE_HEADERS_H_
+#define HWY_INCLUDE_HEADERS_H_
+
+// Do not include directly.
+#if !defined(HWY_STATIC_TARGETS) && !defined(HWY_RUNTIME_TARGETS)
+#error "Must include static_targets.h or foreach_target.h"
+#endif
+
+#include <atomic>
+
+// Alternative for asm volatile("" : : : "memory"), which has no effect.
+#define HWY_FENCE std::atomic_thread_fence(std::memory_order_acq_rel)
+
+// How many lanes of type T in a full vector. NOTE: cannot use in #if because
+// this uses sizeof.
+#define HWY_LANES_OR_0(T) (HWY_BITS / (8 * sizeof(T)))
+
+#define HWY_FULL(T) ::hwy::Desc<T, HWY_LANES_OR_0(T)>
+
+// A vector of up to MAX_N lanes.
+#define HWY_CAPPED(T, MAX_N) ::hwy::Desc<T, HWY_MIN(MAX_N, HWY_LANES_OR_0(T))>
+
+#endif  // #ifndef HWY_INCLUDE_HEADERS_H_
+
+#if (HWY_RUNTIME_TARGETS | HWY_STATIC_TARGETS) & HWY_WASM
+#include "hwy/wasm.h"
+#endif
+
+#if (HWY_RUNTIME_TARGETS | HWY_STATIC_TARGETS) & HWY_ARM8
+#include "hwy/arm64_neon.h"
+#endif
+
+#if (HWY_RUNTIME_TARGETS | HWY_STATIC_TARGETS) & HWY_SSE4
+#include "hwy/x86/sse4.h"
+#endif
+
+#if (HWY_RUNTIME_TARGETS | HWY_STATIC_TARGETS) & HWY_AVX2
+#include "hwy/x86/avx2.h"
+#endif
+
+#if (HWY_RUNTIME_TARGETS | HWY_STATIC_TARGETS) & HWY_AVX512
+#include "hwy/x86/avx512.h"
+#endif
+
+#include "hwy/scalar.h"
diff --git a/third_party/highway/hwy/nanobenchmark.cc b/third_party/highway/hwy/nanobenchmark.cc
new file mode 100644
index 000000000..9ee000fe6
--- /dev/null
+++ b/third_party/highway/hwy/nanobenchmark.cc
@@ -0,0 +1,719 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "hwy/nanobenchmark.h"
+
+#include <stddef.h>
+#include <stdio.h>
+#include <stdlib.h>  // abort
+#include <string.h>  // memcpy
+#include <time.h>    // clock_gettime
+
+#include <algorithm>  // sort
+#include <atomic>
+#include <limits>
+#include <numeric>  // iota
+#include <random>
+#include <string>
+#include <vector>
+
+#include "hwy/arch.h"
+#include "hwy/compiler_specific.h"
+#if HWY_ARCH == HWY_ARCH_X86
+#if defined(_MSC_VER)
+#include <intrin.h>
+#else
+#include <cpuid.h>  // NOLINT
+#endif
+#elif HWY_ARCH == HWY_ARCH_PPC
+#include <sys/platform/ppc.h>  // NOLINT __ppc_get_timebase_freq
+#elif HWY_ARCH == HWY_ARCH_ARM
+#else
+#error "Please add support for this architecture"
+#endif
+
+namespace hwy {
+namespace platform {
+namespace {
+
+#if HWY_ARCH == HWY_ARCH_X86
+
+void Cpuid(const uint32_t level, const uint32_t count,
+           uint32_t* HWY_RESTRICT abcd) {
+#if HWY_COMPILER_MSVC
+  int regs[4];
+  __cpuidex(regs, level, count);
+  for (int i = 0; i < 4; ++i) {
+    abcd[i] = regs[i];
+  }
+#else
+  uint32_t a, b, c, d;
+  __cpuid_count(level, count, a, b, c, d);
+  abcd[0] = a;
+  abcd[1] = b;
+  abcd[2] = c;
+  abcd[3] = d;
+#endif
+}
+
+std::string BrandString() {
+  char brand_string[49];
+  uint32_t abcd[4];
+
+  // Check if brand string is supported (it is on all reasonable Intel/AMD)
+  Cpuid(0x80000000U, 0, abcd);
+  if (abcd[0] < 0x80000004U) {
+    return std::string();
+  }
+
+  for (int i = 0; i < 3; ++i) {
+    Cpuid(0x80000002U + i, 0, abcd);
+    memcpy(brand_string + i * 16, &abcd, sizeof(abcd));
+  }
+  brand_string[48] = 0;
+  return brand_string;
+}
+
+// Returns the frequency quoted inside the brand string. This does not
+// account for throttling nor Turbo Boost.
+double NominalClockRate() {
+  const std::string& brand_string = BrandString();
+  // Brand strings include the maximum configured frequency. These prefixes are
+  // defined by Intel CPUID documentation.
+  const char* prefixes[3] = {"MHz", "GHz", "THz"};
+  const double multipliers[3] = {1E6, 1E9, 1E12};
+  for (size_t i = 0; i < 3; ++i) {
+    const size_t pos_prefix = brand_string.find(prefixes[i]);
+    if (pos_prefix != std::string::npos) {
+      const size_t pos_space = brand_string.rfind(' ', pos_prefix - 1);
+      if (pos_space != std::string::npos) {
+        const std::string digits =
+            brand_string.substr(pos_space + 1, pos_prefix - pos_space - 1);
+        return std::stod(digits) * multipliers[i];
+      }
+    }
+  }
+
+  return 0.0;
+}
+
+#endif  // HWY_ARCH_X86
+
+}  // namespace
+
+// Returns tick rate. Invariant means the tick counter frequency is independent
+// of CPU throttling or sleep. May be expensive, caller should cache the result.
+double InvariantTicksPerSecond() {
+#if HWY_ARCH == HWY_ARCH_PPC
+  return __ppc_get_timebase_freq();
+#elif HWY_ARCH == HWY_ARCH_X86
+  // We assume the TSC is invariant; it is on all recent Intel/AMD CPUs.
+  return NominalClockRate();
+#else
+  // Fall back to clock_gettime nanoseconds.
+  return 1E9;
+#endif
+}
+
+}  // namespace platform
+namespace {
+
+// Prevents the compiler from eliding the computations that led to "output".
+template <class T>
+inline void PreventElision(T&& output) {
+#if HWY_COMPILER_MSVC == 0
+  // Works by indicating to the compiler that "output" is being read and
+  // modified. The +r constraint avoids unnecessary writes to memory, but only
+  // works for built-in types (typically FuncOutput).
+  asm volatile("" : "+r"(output) : : "memory");
+#else
+  // MSVC does not support inline assembly anymore (and never supported GCC's
+  // RTL constraints). Self-assignment with #pragma optimize("off") might be
+  // expected to prevent elision, but it does not with MSVC 2015. Type-punning
+  // with volatile pointers generates inefficient code on MSVC 2017.
+  static std::atomic<T> dummy(T{});
+  dummy.store(output, std::memory_order_relaxed);
+#endif
+}
+
+namespace timer {
+
+// Start/Stop return absolute timestamps and must be placed immediately before
+// and after the region to measure. We provide separate Start/Stop functions
+// because they use different fences.
+//
+// Background: RDTSC is not 'serializing'; earlier instructions may complete
+// after it, and/or later instructions may complete before it. 'Fences' ensure
+// regions' elapsed times are independent of such reordering. The only
+// documented unprivileged serializing instruction is CPUID, which acts as a
+// full fence (no reordering across it in either direction). Unfortunately
+// the latency of CPUID varies wildly (perhaps made worse by not initializing
+// its EAX input). Because it cannot reliably be deducted from the region's
+// elapsed time, it must not be included in the region to measure (i.e.
+// between the two RDTSC).
+//
+// The newer RDTSCP is sometimes described as serializing, but it actually
+// only serves as a half-fence with release semantics. Although all
+// instructions in the region will complete before the final timestamp is
+// captured, subsequent instructions may leak into the region and increase the
+// elapsed time. Inserting another fence after the final RDTSCP would prevent
+// such reordering without affecting the measured region.
+//
+// Fortunately, such a fence exists. The LFENCE instruction is only documented
+// to delay later loads until earlier loads are visible. However, Intel's
+// reference manual says it acts as a full fence (waiting until all earlier
+// instructions have completed, and delaying later instructions until it
+// completes). AMD assigns the same behavior to MFENCE.
+//
+// We need a fence before the initial RDTSC to prevent earlier instructions
+// from leaking into the region, and arguably another after RDTSC to avoid
+// region instructions from completing before the timestamp is recorded.
+// When surrounded by fences, the additional RDTSCP half-fence provides no
+// benefit, so the initial timestamp can be recorded via RDTSC, which has
+// lower overhead than RDTSCP because it does not read TSC_AUX. In summary,
+// we define Start = LFENCE/RDTSC/LFENCE; Stop = RDTSCP/LFENCE.
+//
+// Using Start+Start leads to higher variance and overhead than Stop+Stop.
+// However, Stop+Stop includes an LFENCE in the region measurements, which
+// adds a delay dependent on earlier loads. The combination of Start+Stop
+// is faster than Start+Start and more consistent than Stop+Stop because
+// the first LFENCE already delayed subsequent loads before the measured
+// region. This combination seems not to have been considered in prior work:
+// http://akaros.cs.berkeley.edu/lxr/akaros/kern/arch/x86/rdtsc_test.c
+//
+// Note: performance counters can measure 'exact' instructions-retired or
+// (unhalted) cycle counts. The RDPMC instruction is not serializing and also
+// requires fences. Unfortunately, it is not accessible on all OSes and we
+// prefer to avoid kernel-mode drivers. Performance counters are also affected
+// by several under/over-count errata, so we use the TSC instead.
+
+// Returns a 64-bit timestamp in unit of 'ticks'; to convert to seconds,
+// divide by InvariantTicksPerSecond.
+inline uint64_t Start64() {
+  uint64_t t;
+#if HWY_ARCH == HWY_ARCH_PPC
+  asm volatile("mfspr %0, %1" : "=r"(t) : "i"(268));
+#elif HWY_ARCH == HWY_ARCH_X86
+#if HWY_COMPILER_MSVC
+  _ReadWriteBarrier();
+  _mm_lfence();
+  _ReadWriteBarrier();
+  t = __rdtsc();
+  _ReadWriteBarrier();
+  _mm_lfence();
+  _ReadWriteBarrier();
+#else
+  asm volatile(
+      "lfence\n\t"
+      "rdtsc\n\t"
+      "shl $32, %%rdx\n\t"
+      "or %%rdx, %0\n\t"
+      "lfence"
+      : "=a"(t)
+      :
+      // "memory" avoids reordering. rdx = TSC >> 32.
+      // "cc" = flags modified by SHL.
+      : "rdx", "memory", "cc");
+#endif
+#else
+  // Fall back to OS - unsure how to reliably query cntvct_el0 frequency.
+  timespec ts;
+  clock_gettime(CLOCK_MONOTONIC, &ts);
+  t = ts.tv_sec * 1000000000LL + ts.tv_nsec;
+#endif
+  return t;
+}
+
+inline uint64_t Stop64() {
+  uint64_t t;
+#if HWY_ARCH == HWY_ARCH_PPC
+  asm volatile("mfspr %0, %1" : "=r"(t) : "i"(268));
+#elif HWY_ARCH == HWY_ARCH_X86
+#if HWY_COMPILER_MSVC
+  _ReadWriteBarrier();
+  unsigned aux;
+  t = __rdtscp(&aux);
+  _ReadWriteBarrier();
+  _mm_lfence();
+  _ReadWriteBarrier();
+#else
+  // Use inline asm because __rdtscp generates code to store TSC_AUX (ecx).
+  asm volatile(
+      "rdtscp\n\t"
+      "shl $32, %%rdx\n\t"
+      "or %%rdx, %0\n\t"
+      "lfence"
+      : "=a"(t)
+      :
+      // "memory" avoids reordering. rcx = TSC_AUX. rdx = TSC >> 32.
+      // "cc" = flags modified by SHL.
+      : "rcx", "rdx", "memory", "cc");
+#endif
+#else
+  t = Start64();
+#endif
+  return t;
+}
+
+// Returns a 32-bit timestamp with about 4 cycles less overhead than
+// Start64. Only suitable for measuring very short regions because the
+// timestamp overflows about once a second.
+inline uint32_t Start32() {
+  uint32_t t;
+#if HWY_ARCH == HWY_ARCH_X86
+#if HWY_COMPILER_MSVC
+  _ReadWriteBarrier();
+  _mm_lfence();
+  _ReadWriteBarrier();
+  t = static_cast<uint32_t>(__rdtsc());
+  _ReadWriteBarrier();
+  _mm_lfence();
+  _ReadWriteBarrier();
+#else
+  asm volatile(
+      "lfence\n\t"
+      "rdtsc\n\t"
+      "lfence"
+      : "=a"(t)
+      :
+      // "memory" avoids reordering. rdx = TSC >> 32.
+      : "rdx", "memory");
+#endif
+#else
+  t = static_cast<uint32_t>(Start64());
+#endif
+  return t;
+}
+
+inline uint32_t Stop32() {
+  uint32_t t;
+#if HWY_ARCH == HWY_ARCH_X86
+#if HWY_COMPILER_MSVC
+  _ReadWriteBarrier();
+  unsigned aux;
+  t = static_cast<uint32_t>(__rdtscp(&aux));
+  _ReadWriteBarrier();
+  _mm_lfence();
+  _ReadWriteBarrier();
+#else
+  // Use inline asm because __rdtscp generates code to store TSC_AUX (ecx).
+  asm volatile(
+      "rdtscp\n\t"
+      "lfence"
+      : "=a"(t)
+      :
+      // "memory" avoids reordering. rcx = TSC_AUX. rdx = TSC >> 32.
+      : "rcx", "rdx", "memory");
+#endif
+#else
+  t = static_cast<uint32_t>(Stop64());
+#endif
+  return t;
+}
+
+}  // namespace timer
+
+namespace robust_statistics {
+
+// Sorts integral values in ascending order (e.g. for Mode). About 3x faster
+// than std::sort for input distributions with very few unique values.
+template <class T>
+void CountingSort(T* values, size_t num_values) {
+  // Unique values and their frequency (similar to flat_map).
+  using Unique = std::pair<T, int>;
+  std::vector<Unique> unique;
+  for (size_t i = 0; i < num_values; ++i) {
+    const T value = values[i];
+    const auto pos =
+        std::find_if(unique.begin(), unique.end(),
+                     [value](const Unique u) { return u.first == value; });
+    if (pos == unique.end()) {
+      unique.push_back(std::make_pair(value, 1));
+    } else {
+      ++pos->second;
+    }
+  }
+
+  // Sort in ascending order of value (pair.first).
+  std::sort(unique.begin(), unique.end());
+
+  // Write that many copies of each unique value to the array.
+  T* HWY_RESTRICT p = values;
+  for (const auto& value_count : unique) {
+    std::fill(p, p + value_count.second, value_count.first);
+    p += value_count.second;
+  }
+  NANOBENCHMARK_CHECK(p == values + num_values);
+}
+
+// @return i in [idx_begin, idx_begin + half_count) that minimizes
+// sorted[i + half_count] - sorted[i].
+template <typename T>
+size_t MinRange(const T* const HWY_RESTRICT sorted, const size_t idx_begin,
+                const size_t half_count) {
+  T min_range = std::numeric_limits<T>::max();
+  size_t min_idx = 0;
+
+  for (size_t idx = idx_begin; idx < idx_begin + half_count; ++idx) {
+    NANOBENCHMARK_CHECK(sorted[idx] <= sorted[idx + half_count]);
+    const T range = sorted[idx + half_count] - sorted[idx];
+    if (range < min_range) {
+      min_range = range;
+      min_idx = idx;
+    }
+  }
+
+  return min_idx;
+}
+
+// Returns an estimate of the mode by calling MinRange on successively
+// halved intervals. "sorted" must be in ascending order. This is the
+// Half Sample Mode estimator proposed by Bickel in "On a fast, robust
+// estimator of the mode", with complexity O(N log N). The mode is less
+// affected by outliers in highly-skewed distributions than the median.
+// The averaging operation below assumes "T" is an unsigned integer type.
+template <typename T>
+T ModeOfSorted(const T* const HWY_RESTRICT sorted, const size_t num_values) {
+  size_t idx_begin = 0;
+  size_t half_count = num_values / 2;
+  while (half_count > 1) {
+    idx_begin = MinRange(sorted, idx_begin, half_count);
+    half_count >>= 1;
+  }
+
+  const T x = sorted[idx_begin + 0];
+  if (half_count == 0) {
+    return x;
+  }
+  NANOBENCHMARK_CHECK(half_count == 1);
+  const T average = (x + sorted[idx_begin + 1] + 1) / 2;
+  return average;
+}
+
+// Returns the mode. Side effect: sorts "values".
+template <typename T>
+T Mode(T* values, const size_t num_values) {
+  CountingSort(values, num_values);
+  return ModeOfSorted(values, num_values);
+}
+
+template <typename T, size_t N>
+T Mode(T (&values)[N]) {
+  return Mode(&values[0], N);
+}
+
+// Returns the median value. Side effect: sorts "values".
+template <typename T>
+T Median(T* values, const size_t num_values) {
+  NANOBENCHMARK_CHECK(!values->empty());
+  std::sort(values, values + num_values);
+  const size_t half = num_values / 2;
+  // Odd count: return middle
+  if (num_values % 2) {
+    return values[half];
+  }
+  // Even count: return average of middle two.
+  return (values[half] + values[half - 1] + 1) / 2;
+}
+
+// Returns a robust measure of variability.
+template <typename T>
+T MedianAbsoluteDeviation(const T* values, const size_t num_values,
+                          const T median) {
+  NANOBENCHMARK_CHECK(num_values != 0);
+  std::vector<T> abs_deviations;
+  abs_deviations.reserve(num_values);
+  for (size_t i = 0; i < num_values; ++i) {
+    const int64_t abs = std::abs(int64_t(values[i]) - int64_t(median));
+    abs_deviations.push_back(static_cast<T>(abs));
+  }
+  return Median(abs_deviations.data(), num_values);
+}
+
+}  // namespace robust_statistics
+
+// Ticks := platform-specific timer values (CPU cycles on x86). Must be
+// unsigned to guarantee wraparound on overflow. 32 bit timers are faster to
+// read than 64 bit.
+using Ticks = uint32_t;
+
+// Returns timer overhead / minimum measurable difference.
+Ticks TimerResolution() {
+  // Nested loop avoids exceeding stack/L1 capacity.
+  Ticks repetitions[Params::kTimerSamples];
+  for (size_t rep = 0; rep < Params::kTimerSamples; ++rep) {
+    Ticks samples[Params::kTimerSamples];
+    for (size_t i = 0; i < Params::kTimerSamples; ++i) {
+      const Ticks t0 = timer::Start32();
+      const Ticks t1 = timer::Stop32();
+      samples[i] = t1 - t0;
+    }
+    repetitions[rep] = robust_statistics::Mode(samples);
+  }
+  return robust_statistics::Mode(repetitions);
+}
+
+static const Ticks timer_resolution = TimerResolution();
+
+// Estimates the expected value of "lambda" values with a variable number of
+// samples until the variability "rel_mad" is less than "max_rel_mad".
+template <class Lambda>
+Ticks SampleUntilStable(const double max_rel_mad, double* rel_mad,
+                        const Params& p, const Lambda& lambda) {
+  // Choose initial samples_per_eval based on a single estimated duration.
+  Ticks t0 = timer::Start32();
+  lambda();
+  Ticks t1 = timer::Stop32();
+  Ticks est = t1 - t0;
+  static const double ticks_per_second = platform::InvariantTicksPerSecond();
+  const size_t ticks_per_eval =
+      static_cast<size_t>(ticks_per_second * p.seconds_per_eval);
+  size_t samples_per_eval = ticks_per_eval / est;
+  samples_per_eval = std::max(samples_per_eval, p.min_samples_per_eval);
+
+  std::vector<Ticks> samples;
+  samples.reserve(1 + samples_per_eval);
+  samples.push_back(est);
+
+  // Percentage is too strict for tiny differences, so also allow a small
+  // absolute "median absolute deviation".
+  const Ticks max_abs_mad = (timer_resolution + 99) / 100;
+  *rel_mad = 0.0;  // ensure initialized
+
+  for (size_t eval = 0; eval < p.max_evals; ++eval, samples_per_eval *= 2) {
+    samples.reserve(samples.size() + samples_per_eval);
+    for (size_t i = 0; i < samples_per_eval; ++i) {
+      t0 = timer::Start32();
+      lambda();
+      t1 = timer::Stop32();
+      samples.push_back(t1 - t0);
+    }
+
+    if (samples.size() >= p.min_mode_samples) {
+      est = robust_statistics::Mode(samples.data(), samples.size());
+    } else {
+      // For "few" (depends also on the variance) samples, Median is safer.
+      est = robust_statistics::Median(samples.data(), samples.size());
+    }
+    NANOBENCHMARK_CHECK(est != 0);
+
+    // Median absolute deviation (mad) is a robust measure of 'variability'.
+    const Ticks abs_mad = robust_statistics::MedianAbsoluteDeviation(
+        samples.data(), samples.size(), est);
+    *rel_mad = static_cast<double>(int(abs_mad)) / est;
+
+    if (*rel_mad <= max_rel_mad || abs_mad <= max_abs_mad) {
+      if (p.verbose) {
+        printf("%6zu samples => %5u (abs_mad=%4u, rel_mad=%4.2f%%)\n",
+               samples.size(), est, abs_mad, *rel_mad * 100.0);
+      }
+      return est;
+    }
+  }
+
+  if (p.verbose) {
+    printf(
+        "WARNING: rel_mad=%4.2f%% still exceeds %4.2f%% after %6zu samples.\n",
+        *rel_mad * 100.0, max_rel_mad * 100.0, samples.size());
+  }
+  return est;
+}
+
+using InputVec = std::vector<FuncInput>;
+
+// Returns vector of unique input values.
+InputVec UniqueInputs(const FuncInput* inputs, const size_t num_inputs) {
+  InputVec unique(inputs, inputs + num_inputs);
+  std::sort(unique.begin(), unique.end());
+  unique.erase(std::unique(unique.begin(), unique.end()), unique.end());
+  return unique;
+}
+
+// Returns how often we need to call func for sufficient precision, or zero
+// on failure (e.g. the elapsed time is too long for a 32-bit tick count).
+size_t NumSkip(const Func func, const uint8_t* arg, const InputVec& unique,
+               const Params& p) {
+  // Min elapsed ticks for any input.
+  Ticks min_duration = ~0u;
+
+  for (const FuncInput input : unique) {
+    // Make sure a 32-bit timer is sufficient.
+    const uint64_t t0 = timer::Start64();
+    PreventElision(func(arg, input));
+    const uint64_t t1 = timer::Stop64();
+    const uint64_t elapsed = t1 - t0;
+    if (elapsed >= (1ULL << 30)) {
+      fprintf(stderr, "Measurement failed: need 64-bit timer for input=%zu\n",
+              input);
+      return 0;
+    }
+
+    double rel_mad;
+    const Ticks total = SampleUntilStable(
+        p.target_rel_mad, &rel_mad, p,
+        [func, arg, input]() { PreventElision(func(arg, input)); });
+    min_duration = std::min(min_duration, total - timer_resolution);
+  }
+
+  // Number of repetitions required to reach the target resolution.
+  const size_t max_skip = p.precision_divisor;
+  // Number of repetitions given the estimated duration.
+  const size_t num_skip =
+      min_duration == 0 ? 0 : (max_skip + min_duration - 1) / min_duration;
+  if (p.verbose) {
+    printf("res=%u max_skip=%zu min_dur=%u num_skip=%zu\n", timer_resolution,
+           max_skip, min_duration, num_skip);
+  }
+  return num_skip;
+}
+
+// Replicates inputs until we can omit "num_skip" occurrences of an input.
+InputVec ReplicateInputs(const FuncInput* inputs, const size_t num_inputs,
+                         const size_t num_unique, const size_t num_skip,
+                         const Params& p) {
+  InputVec full;
+  if (num_unique == 1) {
+    full.assign(p.subset_ratio * num_skip, inputs[0]);
+    return full;
+  }
+
+  full.reserve(p.subset_ratio * num_skip * num_inputs);
+  for (size_t i = 0; i < p.subset_ratio * num_skip; ++i) {
+    full.insert(full.end(), inputs, inputs + num_inputs);
+  }
+  std::mt19937 rng;
+  std::shuffle(full.begin(), full.end(), rng);
+  return full;
+}
+
+// Copies the "full" to "subset" in the same order, but with "num_skip"
+// randomly selected occurrences of "input_to_skip" removed.
+void FillSubset(const InputVec& full, const FuncInput input_to_skip,
+                const size_t num_skip, InputVec* subset) {
+  const size_t count = std::count(full.begin(), full.end(), input_to_skip);
+  // Generate num_skip random indices: which occurrence to skip.
+  std::vector<uint32_t> omit(count);
+  std::iota(omit.begin(), omit.end(), 0);
+  // omit[] is the same on every call, but that's OK because they identify the
+  // Nth instance of input_to_skip, so the position within full[] differs.
+  std::mt19937 rng;
+  std::shuffle(omit.begin(), omit.end(), rng);
+  omit.resize(num_skip);
+  std::sort(omit.begin(), omit.end());
+
+  uint32_t occurrence = ~0u;  // 0 after preincrement
+  size_t idx_omit = 0;        // cursor within omit[]
+  size_t idx_subset = 0;      // cursor within *subset
+  for (const FuncInput next : full) {
+    if (next == input_to_skip) {
+      ++occurrence;
+      // Haven't removed enough already
+      if (idx_omit < num_skip) {
+        // This one is up for removal
+        if (occurrence == omit[idx_omit]) {
+          ++idx_omit;
+          continue;
+        }
+      }
+    }
+    if (idx_subset < subset->size()) {
+      (*subset)[idx_subset++] = next;
+    }
+  }
+  NANOBENCHMARK_CHECK(idx_subset == subset->size());
+  NANOBENCHMARK_CHECK(idx_omit == omit.size());
+  NANOBENCHMARK_CHECK(occurrence == count - 1);
+}
+
+// Returns total ticks elapsed for all inputs.
+Ticks TotalDuration(const Func func, const uint8_t* arg, const InputVec* inputs,
+                    const Params& p, double* max_rel_mad) {
+  double rel_mad;
+  const Ticks duration =
+      SampleUntilStable(p.target_rel_mad, &rel_mad, p, [func, arg, inputs]() {
+        for (const FuncInput input : *inputs) {
+          PreventElision(func(arg, input));
+        }
+      });
+  *max_rel_mad = std::max(*max_rel_mad, rel_mad);
+  return duration;
+}
+
+// (Nearly) empty Func for measuring timer overhead/resolution.
+HWY_NOINLINE FuncOutput EmptyFunc(const void* /*arg*/, const FuncInput input) {
+  return input;
+}
+
+// Returns overhead of accessing inputs[] and calling a function; this will
+// be deducted from future TotalDuration return values.
+Ticks Overhead(const uint8_t* arg, const InputVec* inputs, const Params& p) {
+  double rel_mad;
+  // Zero tolerance because repeatability is crucial and EmptyFunc is fast.
+  return SampleUntilStable(0.0, &rel_mad, p, [arg, inputs]() {
+    for (const FuncInput input : *inputs) {
+      PreventElision(EmptyFunc(arg, input));
+    }
+  });
+}
+
+}  // namespace
+
+size_t Measure(const Func func, const uint8_t* arg, const FuncInput* inputs,
+               const size_t num_inputs, Result* results, const Params& p) {
+  NANOBENCHMARK_CHECK(num_inputs != 0);
+  const InputVec& unique = UniqueInputs(inputs, num_inputs);
+
+  const size_t num_skip = NumSkip(func, arg, unique, p);  // never 0
+  if (num_skip == 0) return 0;  // NumSkip already printed error message
+  const float mul = 1.0f / static_cast<int>(num_skip);
+
+  const InputVec& full =
+      ReplicateInputs(inputs, num_inputs, unique.size(), num_skip, p);
+  InputVec subset(full.size() - num_skip);
+
+  const Ticks overhead = Overhead(arg, &full, p);
+  const Ticks overhead_skip = Overhead(arg, &subset, p);
+  if (overhead < overhead_skip) {
+    fprintf(stderr, "Measurement failed: overhead %u < %u\n", overhead,
+            overhead_skip);
+    return 0;
+  }
+
+  if (p.verbose) {
+    printf("#inputs=%5zu,%5zu overhead=%5u,%5u\n", full.size(), subset.size(),
+           overhead, overhead_skip);
+  }
+
+  double max_rel_mad = 0.0;
+  const Ticks total = TotalDuration(func, arg, &full, p, &max_rel_mad);
+
+  for (size_t i = 0; i < unique.size(); ++i) {
+    FillSubset(full, unique[i], num_skip, &subset);
+    const Ticks total_skip = TotalDuration(func, arg, &subset, p, &max_rel_mad);
+
+    if (total < total_skip) {
+      fprintf(stderr, "Measurement failed: total %u < %u\n", total, total_skip);
+      return 0;
+    }
+
+    const Ticks duration = (total - overhead) - (total_skip - overhead_skip);
+    results[i].input = unique[i];
+    results[i].ticks = duration * mul;
+    results[i].variability = static_cast<float>(max_rel_mad);
+  }
+
+  return unique.size();
+}
+
+}  // namespace hwy
diff --git a/third_party/highway/hwy/nanobenchmark.h b/third_party/highway/hwy/nanobenchmark.h
new file mode 100644
index 000000000..2a3109cd6
--- /dev/null
+++ b/third_party/highway/hwy/nanobenchmark.h
@@ -0,0 +1,164 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_NANOBENCHMARK_H_
+#define HWY_NANOBENCHMARK_H_
+
+// Benchmarks functions of a single integer argument with realistic branch
+// prediction hit rates. Uses a robust estimator to summarize the measurements.
+// The precision is about 0.2%.
+//
+// Examples: see nanobenchmark_test.cc.
+//
+// Background: Microbenchmarks such as http://github.com/google/benchmark
+// can measure elapsed times on the order of a microsecond. Shorter functions
+// are typically measured by repeating them thousands of times and dividing
+// the total elapsed time by this count. Unfortunately, repetition (especially
+// with the same input parameter!) influences the runtime. In time-critical
+// code, it is reasonable to expect warm instruction/data caches and TLBs,
+// but a perfect record of which branches will be taken is unrealistic.
+// Unless the application also repeatedly invokes the measured function with
+// the same parameter, the benchmark is measuring something very different -
+// a best-case result, almost as if the parameter were made a compile-time
+// constant. This may lead to erroneous conclusions about branch-heavy
+// algorithms outperforming branch-free alternatives.
+//
+// Our approach differs in three ways. Adding fences to the timer functions
+// reduces variability due to instruction reordering, improving the timer
+// resolution to about 40 CPU cycles. However, shorter functions must still
+// be invoked repeatedly. For more realistic branch prediction performance,
+// we vary the input parameter according to a user-specified distribution.
+// Thus, instead of VaryInputs(Measure(Repeat(func))), we change the
+// loop nesting to Measure(Repeat(VaryInputs(func))). We also estimate the
+// central tendency of the measurement samples with the "half sample mode",
+// which is more robust to outliers and skewed data than the mean or median.
+
+// WARNING if included from multiple translation units compiled with distinct
+// flags: this header requires textual inclusion and a predefined NB_NAMESPACE
+// macro that is unique to the current compile flags. We must also avoid
+// standard library headers such as vector and functional that define functions.
+
+#include <stddef.h>
+#include <stdint.h>
+
+// Enables sanity checks that verify correct operation at the cost of
+// longer benchmark runs.
+#ifndef NANOBENCHMARK_ENABLE_CHECKS
+#define NANOBENCHMARK_ENABLE_CHECKS 0
+#endif
+
+#define NANOBENCHMARK_CHECK_ALWAYS(condition)                             \
+  while (!(condition)) {                                                  \
+    fprintf(stderr, "Nanobenchmark check failed at line %d\n", __LINE__); \
+    abort();                                                              \
+  }
+
+#if NANOBENCHMARK_ENABLE_CHECKS
+#define NANOBENCHMARK_CHECK(condition) NANOBENCHMARK_CHECK_ALWAYS(condition)
+#else
+#define NANOBENCHMARK_CHECK(condition)
+#endif
+
+namespace hwy {
+
+namespace platform {
+
+// Returns tick rate, useful for converting measurements to seconds. Invariant
+// means the tick counter frequency is independent of CPU throttling or sleep.
+// This call may be expensive, callers should cache the result.
+double InvariantTicksPerSecond();
+
+}  // namespace platform
+
+// Input influencing the function being measured (e.g. number of bytes to copy).
+using FuncInput = size_t;
+
+// "Proof of work" returned by Func to ensure the compiler does not elide it.
+using FuncOutput = uint64_t;
+
+// Function to measure: either 1) a captureless lambda or function with two
+// arguments or 2) a lambda with capture, in which case the first argument
+// is reserved for use by MeasureClosure.
+using Func = FuncOutput (*)(const void*, FuncInput);
+
+// Internal parameters that determine precision/resolution/measuring time.
+struct Params {
+  // For measuring timer overhead/resolution. Used in a nested loop =>
+  // quadratic time, acceptable because we know timer overhead is "low".
+  // constexpr because this is used to define array bounds.
+  static constexpr size_t kTimerSamples = 256;
+
+  // Best-case precision, expressed as a divisor of the timer resolution.
+  // Larger => more calls to Func and higher precision.
+  size_t precision_divisor = 1024;
+
+  // Ratio between full and subset input distribution sizes. Cannot be less
+  // than 2; larger values increase measurement time but more faithfully
+  // model the given input distribution.
+  size_t subset_ratio = 2;
+
+  // Together with the estimated Func duration, determines how many times to
+  // call Func before checking the sample variability. Larger values increase
+  // measurement time, memory/cache use and precision.
+  double seconds_per_eval = 4E-3;
+
+  // The minimum number of samples before estimating the central tendency.
+  size_t min_samples_per_eval = 7;
+
+  // The mode is better than median for estimating the central tendency of
+  // skewed/fat-tailed distributions, but it requires sufficient samples
+  // relative to the width of half-ranges.
+  size_t min_mode_samples = 64;
+
+  // Maximum permissible variability (= median absolute deviation / center).
+  double target_rel_mad = 0.002;
+
+  // Abort after this many evals without reaching target_rel_mad. This
+  // prevents infinite loops.
+  size_t max_evals = 9;
+
+  // Whether to print additional statistics to stdout.
+  bool verbose = true;
+};
+
+// Measurement result for each unique input.
+struct Result {
+  FuncInput input;
+
+  // Robust estimate (mode or median) of duration.
+  float ticks;
+
+  // Measure of variability (median absolute deviation relative to "ticks").
+  float variability;
+};
+
+// Precisely measures the number of ticks elapsed when calling "func" with the
+// given inputs, shuffled to ensure realistic branch prediction hit rates.
+//
+// "func" returns a 'proof of work' to ensure its computations are not elided.
+// "arg" is passed to Func, or reserved for internal use by MeasureClosure.
+// "inputs" is an array of "num_inputs" (not necessarily unique) arguments to
+//   "func". The values should be chosen to maximize coverage of "func". This
+//   represents a distribution, so a value's frequency should reflect its
+//   probability in the real application. Order does not matter; for example, a
+//   uniform distribution over [0, 4) could be represented as {3,0,2,1}.
+// Returns how many Result were written to "results": one per unique input, or
+//   zero if the measurement failed (an error message goes to stderr).
+size_t Measure(const Func func, const uint8_t* arg, const FuncInput* inputs,
+               const size_t num_inputs, Result* results,
+               const Params& p = Params());
+
+}  // namespace hwy
+
+#endif  // HWY_NANOBENCHMARK_H_
diff --git a/third_party/highway/hwy/runtime_dispatch.cc b/third_party/highway/hwy/runtime_dispatch.cc
new file mode 100644
index 000000000..f65ee1812
--- /dev/null
+++ b/third_party/highway/hwy/runtime_dispatch.cc
@@ -0,0 +1,193 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "hwy/runtime_dispatch.h"
+
+#include <stdint.h>
+
+#include <atomic>
+
+#if HWY_ARCH == HWY_ARCH_X86
+#include <xmmintrin.h>
+#ifdef _MSC_VER
+#include <intrin.h>
+#else
+#include <cpuid.h>
+#endif
+#endif
+
+namespace hwy {
+
+namespace {
+
+bool IsBitSet(const uint32_t reg, const int index) {
+  return (reg & (1U << index)) != 0;
+}
+
+#if HWY_ARCH == HWY_ARCH_X86
+
+// Calls CPUID instruction with eax=level and ecx=count and returns the result
+// in abcd array where abcd = {eax, ebx, ecx, edx} (hence the name abcd).
+void Cpuid(const uint32_t level, const uint32_t count,
+           uint32_t* HWY_RESTRICT abcd) {
+#ifdef _MSC_VER
+  int regs[4];
+  __cpuidex(regs, level, count);
+  for (int i = 0; i < 4; ++i) {
+    abcd[i] = regs[i];
+  }
+#else
+  uint32_t a, b, c, d;
+  __cpuid_count(level, count, a, b, c, d);
+  abcd[0] = a;
+  abcd[1] = b;
+  abcd[2] = c;
+  abcd[3] = d;
+#endif
+}
+
+// Returns the lower 32 bits of extended control register 0.
+// Requires CPU support for "OSXSAVE" (see below).
+uint32_t ReadXCR0() {
+#ifdef _MSC_VER
+  return static_cast<uint32_t>(_xgetbv(0));
+#else
+  uint32_t xcr0, xcr0_high;
+  const uint32_t index = 0;
+  asm volatile(".byte 0x0F, 0x01, 0xD0"
+               : "=a"(xcr0), "=d"(xcr0_high)
+               : "c"(index));
+  return xcr0;
+#endif
+}
+
+#endif  // HWY_ARCH_X86
+
+// Not function-local => no compiler-generated locking.
+std::atomic<int> supported_{-1};  // Not yet initialized
+
+#if HWY_ARCH == HWY_ARCH_X86
+// Bits indicating which instruction set extensions are supported.
+constexpr uint32_t kSSE = 1 << 0;
+constexpr uint32_t kSSE2 = 1 << 1;
+constexpr uint32_t kSSE3 = 1 << 2;
+constexpr uint32_t kSSSE3 = 1 << 3;
+constexpr uint32_t kSSE41 = 1 << 4;
+constexpr uint32_t kSSE42 = 1 << 5;
+constexpr uint32_t kGroupSSE4 = kSSE | kSSE2 | kSSE3 | kSSSE3 | kSSE41 | kSSE42;
+
+constexpr uint32_t kAVX = 1u << 6;
+constexpr uint32_t kAVX2 = 1u << 7;
+constexpr uint32_t kFMA = 1u << 8;
+constexpr uint32_t kLZCNT = 1u << 9;
+constexpr uint32_t kBMI = 1u << 10;
+constexpr uint32_t kBMI2 = 1u << 11;
+constexpr uint32_t kGroupAVX2 = kAVX | kAVX2 | kFMA | kLZCNT | kBMI | kBMI2;
+
+constexpr uint32_t kAVX512F = 1u << 12;
+constexpr uint32_t kAVX512VL = 1u << 13;
+constexpr uint32_t kAVX512DQ = 1u << 14;
+constexpr uint32_t kAVX512BW = 1u << 15;
+constexpr uint32_t kGroupAVX512 = kAVX512F | kAVX512VL | kAVX512DQ | kAVX512BW;
+#endif
+
+}  // namespace
+
+TargetBitfield::TargetBitfield() {
+  bits_ = supported_.load(std::memory_order_acquire);
+  // Already initialized?
+  if (HWY_LIKELY(bits_ != -1)) {
+    return;
+  }
+
+  bits_ = HWY_NONE;
+
+#if HWY_ARCH == HWY_ARCH_X86
+  uint32_t flags = 0;
+  uint32_t abcd[4];
+
+  Cpuid(0, 0, abcd);
+  const uint32_t max_level = abcd[0];
+
+  // Standard feature flags
+  Cpuid(1, 0, abcd);
+  flags |= IsBitSet(abcd[3], 25) ? kSSE : 0;
+  flags |= IsBitSet(abcd[3], 26) ? kSSE2 : 0;
+  flags |= IsBitSet(abcd[2], 0) ? kSSE3 : 0;
+  flags |= IsBitSet(abcd[2], 9) ? kSSSE3 : 0;
+  flags |= IsBitSet(abcd[2], 19) ? kSSE41 : 0;
+  flags |= IsBitSet(abcd[2], 20) ? kSSE42 : 0;
+  flags |= IsBitSet(abcd[2], 12) ? kFMA : 0;
+  flags |= IsBitSet(abcd[2], 28) ? kAVX : 0;
+  const bool has_osxsave = IsBitSet(abcd[2], 27);
+
+  // Extended feature flags
+  Cpuid(0x80000001U, 0, abcd);
+  flags |= IsBitSet(abcd[2], 5) ? kLZCNT : 0;
+
+  // Extended features
+  if (max_level >= 7) {
+    Cpuid(7, 0, abcd);
+    flags |= IsBitSet(abcd[1], 3) ? kBMI : 0;
+    flags |= IsBitSet(abcd[1], 5) ? kAVX2 : 0;
+    flags |= IsBitSet(abcd[1], 8) ? kBMI2 : 0;
+
+    flags |= IsBitSet(abcd[1], 16) ? kAVX512F : 0;
+    flags |= IsBitSet(abcd[1], 17) ? kAVX512DQ : 0;
+    flags |= IsBitSet(abcd[1], 30) ? kAVX512BW : 0;
+    flags |= IsBitSet(abcd[1], 31) ? kAVX512VL : 0;
+  }
+
+  // Verify OS support for XSAVE, without which XMM/YMM registers are not
+  // preserved across context switches and are not safe to use.
+  if (has_osxsave) {
+    const uint32_t xcr0 = ReadXCR0();
+    // XMM
+    if (!IsBitSet(xcr0, 1)) {
+      flags = 0;
+    }
+    // YMM
+    if (!IsBitSet(xcr0, 2)) {
+      flags &= ~kGroupAVX2;
+    }
+    // ZMM + opmask
+    if ((xcr0 & 0x70) != 0x70) {
+      flags &= ~kGroupAVX512;
+    }
+  }
+
+  // Set target bit(s) if all their group's flags are all set.
+  if ((flags & kGroupAVX512) == kGroupAVX512) {
+    bits_ |= HWY_AVX512;
+  }
+  if ((flags & kGroupAVX2) == kGroupAVX2) {
+    bits_ |= HWY_AVX2;
+  }
+  if ((flags & kGroupSSE4) == kGroupSSE4) {
+    bits_ |= HWY_SSE4;
+  }
+#elif HWY_ARCH == HWY_ARCH_WASM
+  bits_ |= HWY_WASM;
+#elif HWY_ARCH == HWY_ARCH_ARM
+  bits_ |= HWY_ARM8;
+#endif
+
+  // Don't report targets that aren't enabled, otherwise foreach-target loops
+  // will not terminate.
+  bits_ &= HWY_RUNTIME_TARGETS;
+
+  supported_.store(bits_, std::memory_order_release);
+}
+
+}  // namespace hwy
diff --git a/third_party/highway/hwy/runtime_dispatch.h b/third_party/highway/hwy/runtime_dispatch.h
new file mode 100644
index 000000000..366adab4b
--- /dev/null
+++ b/third_party/highway/hwy/runtime_dispatch.h
@@ -0,0 +1,126 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_RUNTIME_DISPATCH_H_
+#define HWY_RUNTIME_DISPATCH_H_
+
+// Chooses and calls best available target instantiation. See README.md.
+
+#include <stddef.h>
+
+#include <utility>  // std::forward
+
+#include "hwy/compiler_specific.h"  // HWY_INLINE
+#include "hwy/runtime_targets.h"
+#include "hwy/targets.h"
+
+namespace hwy {
+
+// Strongly-typed enum ensures the argument to Dispatch is a single target, not
+// a bitfield. Only targets in HWY_RUNTIME_TARGETS are defined, which avoids
+// unhandled-enumerator warnings.
+enum class Target {
+#define HWY_X(target) k##target = HWY_##target,
+  HWY_FOREACH_TARGET
+#undef HWY_X
+};
+
+// Returns func.HWY_FUNC(args). Calling a member function allows stateful
+// functors. Dispatch overhead is low but prefer to call this infrequently by
+// hoisting this call.
+template <class Func, typename... Args>
+HWY_INLINE auto Dispatch(const Target target, Func&& func, Args&&... args)
+    -> decltype(std::forward<Func>(func).F_NONE(std::forward<Args>(args)...)) {
+  switch (target) {
+#define HWY_X(target)     \
+  case Target::k##target: \
+    return std::forward<Func>(func).F_##target(std::forward<Args>(args)...);
+    HWY_FOREACH_TARGET
+#undef HWY_X
+  }
+}
+
+// All targets supported by the current CPU. Cheap to construct.
+class TargetBitfield {
+ public:
+  TargetBitfield();
+
+  int Bits() const { return bits_; }
+  bool Any() const { return bits_ != 0; }
+
+  // The best target available on all supported CPUs.
+  static constexpr Target Baseline() {
+#if HWY_RUNTIME_TARGETS & HWY_WASM
+    return Target::kWASM;
+#elif HWY_RUNTIME_TARGETS & HWY_SSE4
+    return Target::kSSE4;
+#elif HWY_RUNTIME_TARGETS & HWY_ARM8
+    return Target::kARM8;
+#elif HWY_RUNTIME_TARGETS & HWY_PPC8
+    return Target::kPPC;
+#else
+    return Target::kNONE;
+#endif
+  }
+
+  // Returns 'best' (widest/most recent) target amongst those supported.
+  Target Best() const {
+#if HWY_RUNTIME_TARGETS & HWY_WASM
+    if (bits_ & HWY_WASM) return Target::kWASM;
+#endif
+#if HWY_RUNTIME_TARGETS & HWY_AVX512
+    if (bits_ & HWY_AVX512) return Target::kAVX512;
+#endif
+#if HWY_RUNTIME_TARGETS & HWY_AVX2
+    if (bits_ & HWY_AVX2) return Target::kAVX2;
+#endif
+#if HWY_RUNTIME_TARGETS & HWY_SSE4
+    if (bits_ & HWY_SSE4) return Target::kSSE4;
+#endif
+#if HWY_RUNTIME_TARGETS & HWY_PPC8
+    if (bits_ & HWY_PPC8) return Target::kPPC8;
+#endif
+#if HWY_RUNTIME_TARGETS & HWY_ARM8
+    if (bits_ & HWY_ARM8) return Target::kARM8;
+#endif
+    return Target::kNONE;
+  }
+
+  void Clear(Target target) { bits_ &= ~static_cast<int>(target); }
+
+  // Calls func.HWY_FUNC(args) for all enabled targets and returns the bitfield
+  // of enabled targets.
+  template <class Func, typename... Args>
+  HWY_INLINE int Foreach(Func&& func, Args&&... args) const {
+#define HWY_X(target)                                                   \
+  {                                                                     \
+    constexpr int kBit = HWY_##target;                                  \
+    /* if supported, OR kBit == HWY_NONE (always enabled) */            \
+    if ((bits_ & kBit) == kBit) {                                       \
+      std::forward<Func>(func).F_##target(std::forward<Args>(args)...); \
+    }                                                                   \
+  }
+    HWY_FOREACH_TARGET
+#undef HWY_X
+
+    return bits_ & HWY_RUNTIME_TARGETS;
+  }
+
+ private:
+  int bits_;
+};
+
+}  // namespace hwy
+
+#endif  // HWY_RUNTIME_DISPATCH_H_
diff --git a/third_party/highway/hwy/runtime_targets.h b/third_party/highway/hwy/runtime_targets.h
new file mode 100644
index 000000000..7f45db8c0
--- /dev/null
+++ b/third_party/highway/hwy/runtime_targets.h
@@ -0,0 +1,162 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_RUNTIME_TARGETS_H_
+#define HWY_RUNTIME_TARGETS_H_
+
+// Chooses default HWY_RUNTIME_TARGETS and provides support for generating
+// implementations (if enabled) and invocations (if supported).
+
+// HWY_RUNTIME_TARGETS is separate from HWY_STATIC_TARGETS to allow
+// static_targets.h and runtime_dispatch.h to coexist in a translation unit.
+
+#include "hwy/arch.h"
+#include "hwy/compiler_specific.h"
+#include "hwy/targets.h"
+
+//-----------------------------------------------------------------------------
+// If not already defined, choose default HWY_RUNTIME_TARGETS for this platform.
+
+#ifndef HWY_RUNTIME_TARGETS
+
+#if HWY_ARCH == HWY_ARCH_X86
+
+#if defined(HWY_DISABLE_AVX2)
+#define HWY_RUNTIME_TARGETS HWY_SSE4
+#elif (HWY_COMPILER_CLANG != 0 && HWY_COMPILER_CLANG < 700) || \
+    defined(HWY_DISABLE_AVX512)
+// Clang6 encounters backend errors when compiling AVX-512.
+#define HWY_RUNTIME_TARGETS (HWY_SSE4 | HWY_AVX2)
+#else
+#define HWY_RUNTIME_TARGETS (HWY_SSE4 | HWY_AVX2 | HWY_AVX512)
+#endif
+
+#elif HWY_ARCH == HWY_ARCH_WASM
+#define HWY_RUNTIME_TARGETS HWY_WASM
+#elif HWY_ARCH == HWY_ARCH_PPC
+#define HWY_RUNTIME_TARGETS HWY_PPC8
+#elif HWY_ARCH == HWY_ARCH_ARM
+#define HWY_RUNTIME_TARGETS HWY_ARM8
+#elif HWY_ARCH == HWY_ARCH_SCALAR
+#define HWY_RUNTIME_TARGETS HWY_NONE
+#else
+#error "Unsupported platform"
+#endif
+
+#endif  // HWY_RUNTIME_TARGETS
+
+//-----------------------------------------------------------------------------
+// X macros for generating enum/code for each enabled target.
+
+#define HWY_FOR_NONE HWY_X(NONE)
+
+#if HWY_RUNTIME_TARGETS & HWY_ARM8
+#define HWY_FOR_ARM8 HWY_X(ARM8)
+#else
+#define HWY_FOR_ARM8
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_PPC8
+#define HWY_FOR_PPC8 HWY_X(PPC8)
+#else
+#define HWY_FOR_PPC8
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_SSE4
+#define HWY_FOR_SSE4 HWY_X(SSE4)
+#else
+#define HWY_FOR_SSE4
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_AVX2
+#define HWY_FOR_AVX2 HWY_X(AVX2)
+#else
+#define HWY_FOR_AVX2
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_AVX512
+#define HWY_FOR_AVX512 HWY_X(AVX512)
+#else
+#define HWY_FOR_AVX512
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_WASM
+#define HWY_FOR_WASM HWY_X(WASM)
+#else
+#define HWY_FOR_WASM
+#endif
+
+#define HWY_FOREACH_TARGET \
+  HWY_FOR_NONE             \
+  HWY_FOR_WASM             \
+  HWY_FOR_ARM8             \
+  HWY_FOR_PPC8             \
+  HWY_FOR_SSE4             \
+  HWY_FOR_AVX2             \
+  HWY_FOR_AVX512
+
+//-----------------------------------------------------------------------------
+// Special support for declaring target-specific member functions.
+// This is easier for users than using via HWY_FOREACH_TARGET.
+
+#define HWY_DECLARE_NONE(ret, args) ret F_NONE args;
+
+#if HWY_RUNTIME_TARGETS & HWY_WASM
+#define HWY_DECLARE_WASM(ret, args) ret F_WASM args;
+#else
+#define HWY_DECLARE_WASM(ret, args)
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_ARM8
+#define HWY_DECLARE_ARM8(ret, args) ret F_ARM8 args;
+#else
+#define HWY_DECLARE_ARM8(ret, args)
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_PPC8
+#define HWY_DECLARE_PPC8(ret, args) ret F_PPC8 args;
+#else
+#define HWY_DECLARE_PPC8(ret, args)
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_SSE4
+#define HWY_DECLARE_SSE4(ret, args) ret F_SSE4 args;
+#else
+#define HWY_DECLARE_SSE4(ret, args)
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_AVX2
+#define HWY_DECLARE_AVX2(ret, args) ret F_AVX2 args;
+#else
+#define HWY_DECLARE_AVX2(ret, args)
+#endif
+
+#if HWY_RUNTIME_TARGETS & HWY_AVX512
+#define HWY_DECLARE_AVX512(ret, args) ret F_AVX512 args;
+#else
+#define HWY_DECLARE_AVX512(ret, args)
+#endif
+
+#define HWY_DECLARE(ret, args) \
+  HWY_DECLARE_NONE(ret, args)  \
+  HWY_DECLARE_WASM(ret, args)  \
+  HWY_DECLARE_ARM8(ret, args)  \
+  HWY_DECLARE_PPC8(ret, args)  \
+  HWY_DECLARE_SSE4(ret, args)  \
+  HWY_DECLARE_AVX2(ret, args)  \
+  HWY_DECLARE_AVX512(ret, args)
+
+//-----------------------------------------------------------------------------
+
+#endif  // HWY_RUNTIME_TARGETS_H_
diff --git a/third_party/highway/hwy/scalar.h b/third_party/highway/hwy/scalar.h
new file mode 100644
index 000000000..f4e9501b5
--- /dev/null
+++ b/third_party/highway/hwy/scalar.h
@@ -0,0 +1,819 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_SCALAR_H_
+#define HWY_SCALAR_H_
+
+// Single-element vectors and operations.
+
+#include <math.h>
+#include <string.h>
+
+#include "hwy/compiler_specific.h"
+#include "hwy/shared.h"
+
+namespace hwy {
+
+// Shorthand for a scalar; note that Vec0<T> is the actual data class.
+template <typename T>
+using Scalar = Desc<T, 0>;
+
+// (Wrapper class required for overloading comparison operators.)
+template <typename T>
+struct Vec0 {
+  HWY_INLINE Vec0() = default;
+  Vec0(const Vec0&) = default;
+  Vec0& operator=(const Vec0&) = default;
+  HWY_INLINE explicit Vec0(const T t) : raw(t) {}
+
+  HWY_INLINE Vec0& operator*=(const Vec0 other) {
+    return *this = (*this * other);
+  }
+  HWY_INLINE Vec0& operator/=(const Vec0 other) {
+    return *this = (*this / other);
+  }
+  HWY_INLINE Vec0& operator+=(const Vec0 other) {
+    return *this = (*this + other);
+  }
+  HWY_INLINE Vec0& operator-=(const Vec0 other) {
+    return *this = (*this - other);
+  }
+  HWY_INLINE Vec0& operator&=(const Vec0 other) {
+    return *this = (*this & other);
+  }
+  HWY_INLINE Vec0& operator|=(const Vec0 other) {
+    return *this = (*this | other);
+  }
+  HWY_INLINE Vec0& operator^=(const Vec0 other) {
+    return *this = (*this ^ other);
+  }
+
+  T raw;
+};
+
+// 0 or FF..FF, same size as Vec0.
+template <typename T>
+class Mask0 {
+  using Raw = MakeUnsigned<T>;
+
+ public:
+  static HWY_INLINE Mask0<T> FromBool(bool b) {
+    Mask0<T> mask;
+    mask.bits = b ? ~Raw(0) : 0;
+    return mask;
+  }
+
+  Raw bits;
+};
+
+// ------------------------------ Cast
+
+template <typename T, typename FromT>
+HWY_INLINE Vec0<T> BitCast(Scalar<T> /* tag */, Vec0<FromT> v) {
+  static_assert(sizeof(T) <= sizeof(FromT), "Promoting is undefined");
+  T to;
+  CopyBytes<sizeof(FromT)>(&v.raw, &to);
+  return Vec0<T>(to);
+}
+
+// ------------------------------ Set
+
+template <typename T>
+HWY_INLINE Vec0<T> Zero(Scalar<T> /* tag */) {
+  return Vec0<T>(T(0));
+}
+
+template <typename T, typename T2>
+HWY_INLINE Vec0<T> Set(Scalar<T> /* tag */, const T2 t) {
+  return Vec0<T>(t);
+}
+
+template <typename T, typename T2>
+HWY_INLINE Vec0<T> Iota(Scalar<T> /* tag */, const T2 first) {
+  return Vec0<T>(first);
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> Undefined(Scalar<T> /* tag */) {
+  return Vec0<T>(0);
+}
+
+// ================================================== SHIFTS
+
+// ------------------------------ Shift lanes by constant #bits
+
+template <int kBits, typename T>
+HWY_INLINE Vec0<T> ShiftLeft(const Vec0<T> v) {
+  static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift");
+  return Vec0<T>(static_cast<MakeUnsigned<T>>(v.raw) << kBits);
+}
+
+template <int kBits, typename T>
+HWY_INLINE Vec0<T> ShiftRight(const Vec0<T> v) {
+  static_assert(0 <= kBits && kBits < sizeof(T) * 8, "Invalid shift");
+  return Vec0<T>(v.raw >> kBits);
+}
+
+// ------------------------------ Shift lanes by independent variable #bits
+
+// Single-lane => same as above except for the argument type.
+template <typename T>
+HWY_INLINE Vec0<T> operator<<(const Vec0<T> v, const Vec0<T> bits) {
+  return Vec0<T>(static_cast<MakeUnsigned<T>>(v.raw) << bits.raw);
+}
+template <typename T>
+HWY_INLINE Vec0<T> operator>>(const Vec0<T> v, const Vec0<T> bits) {
+  return Vec0<T>(v.raw >> bits.raw);
+}
+
+// ================================================== LOGICAL
+
+template <typename Bits>
+struct BitwiseOp {
+  template <typename T, class Op>
+  Vec0<T> operator()(const Vec0<T> a, const Vec0<T> b, const Op& op) const {
+    static_assert(sizeof(T) == sizeof(Bits), "Float/int size mismatch");
+    Bits ia, ib;
+    CopyBytes<sizeof(Bits)>(&a, &ia);
+    CopyBytes<sizeof(Bits)>(&b, &ib);
+    ia = op(ia, ib);
+    T ret;
+    CopyBytes<sizeof(Bits)>(&ia, &ret);
+    return Vec0<T>(ret);
+  }
+};
+
+// ------------------------------ Bitwise AND
+
+template <typename T>
+HWY_INLINE Vec0<T> operator&(const Vec0<T> a, const Vec0<T> b) {
+  return Vec0<T>(a.raw & b.raw);
+}
+template <>
+HWY_INLINE Vec0<float> operator&(const Vec0<float> a, const Vec0<float> b) {
+  return BitwiseOp<int32_t>()(a, b, [](int32_t i, int32_t j) { return i & j; });
+}
+template <>
+HWY_INLINE Vec0<double> operator&(const Vec0<double> a, const Vec0<double> b) {
+  return BitwiseOp<int64_t>()(a, b, [](int64_t i, int64_t j) { return i & j; });
+}
+
+// ------------------------------ Bitwise AND-NOT
+
+// Returns ~a & b.
+template <typename T>
+HWY_INLINE Vec0<T> AndNot(const Vec0<T> a, const Vec0<T> b) {
+  return Vec0<T>(~a.raw & b.raw);
+}
+template <>
+HWY_INLINE Vec0<float> AndNot(const Vec0<float> a, const Vec0<float> b) {
+  return BitwiseOp<int32_t>()(a, b,
+                              [](int32_t i, int32_t j) { return ~i & j; });
+}
+template <>
+HWY_INLINE Vec0<double> AndNot(const Vec0<double> a, const Vec0<double> b) {
+  return BitwiseOp<int64_t>()(a, b,
+                              [](int64_t i, int64_t j) { return ~i & j; });
+}
+
+// ------------------------------ Bitwise OR
+
+template <typename T>
+HWY_INLINE Vec0<T> operator|(const Vec0<T> a, const Vec0<T> b) {
+  return Vec0<T>(a.raw | b.raw);
+}
+template <>
+HWY_INLINE Vec0<float> operator|(const Vec0<float> a, const Vec0<float> b) {
+  return BitwiseOp<int32_t>()(a, b, [](int32_t i, int32_t j) { return i | j; });
+}
+template <>
+HWY_INLINE Vec0<double> operator|(const Vec0<double> a, const Vec0<double> b) {
+  return BitwiseOp<int64_t>()(a, b, [](int64_t i, int64_t j) { return i | j; });
+}
+
+// ------------------------------ Bitwise XOR
+
+template <typename T>
+HWY_INLINE Vec0<T> operator^(const Vec0<T> a, const Vec0<T> b) {
+  return Vec0<T>(a.raw ^ b.raw);
+}
+template <>
+HWY_INLINE Vec0<float> operator^(const Vec0<float> a, const Vec0<float> b) {
+  return BitwiseOp<int32_t>()(a, b, [](int32_t i, int32_t j) { return i ^ j; });
+}
+template <>
+HWY_INLINE Vec0<double> operator^(const Vec0<double> a, const Vec0<double> b) {
+  return BitwiseOp<int64_t>()(a, b, [](int64_t i, int64_t j) { return i ^ j; });
+}
+
+// ------------------------------ Mask
+
+// v must be 0 or FF..FF.
+template <typename T>
+HWY_INLINE Mask0<T> MaskFromVec(const Vec0<T> v) {
+  Mask0<T> mask;
+  memcpy(&mask.bits, &v.raw, sizeof(mask.bits));
+  return mask;
+}
+
+template <typename T>
+Vec0<T> VecFromMask(const Mask0<T> mask) {
+  Vec0<T> v;
+  memcpy(&v.raw, &mask.bits, sizeof(v.raw));
+  return v;
+}
+
+// Returns mask ? yes : no.
+template <typename T>
+HWY_INLINE Vec0<T> IfThenElse(const Mask0<T> mask, const Vec0<T> yes,
+                              const Vec0<T> no) {
+  return mask.bits ? yes : no;
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> IfThenElseZero(const Mask0<T> mask, const Vec0<T> yes) {
+  return mask.bits ? yes : Vec0<T>(0);
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> IfThenZeroElse(const Mask0<T> mask, const Vec0<T> no) {
+  return mask.bits ? Vec0<T>(0) : no;
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> ZeroIfNegative(const Vec0<T> v) {
+  return v.raw < 0 ? Vec0<T>(0) : v;
+}
+
+// ================================================== ARITHMETIC
+
+template <typename T>
+HWY_INLINE Vec0<T> operator+(Vec0<T> a, Vec0<T> b) {
+  const uint64_t a64 = static_cast<int64_t>(a.raw);
+  const uint64_t b64 = static_cast<int64_t>(b.raw);
+  return Vec0<T>((a64 + b64) & ~T(0));
+}
+HWY_INLINE Vec0<float> operator+(const Vec0<float> a, const Vec0<float> b) {
+  return Vec0<float>(a.raw + b.raw);
+}
+HWY_INLINE Vec0<double> operator+(const Vec0<double> a, const Vec0<double> b) {
+  return Vec0<double>(a.raw + b.raw);
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> operator-(Vec0<T> a, Vec0<T> b) {
+  const uint64_t a64 = static_cast<int64_t>(a.raw);
+  const uint64_t b64 = static_cast<int64_t>(b.raw);
+  return Vec0<T>((a64 - b64) & ~T(0));
+}
+HWY_INLINE Vec0<float> operator-(const Vec0<float> a, const Vec0<float> b) {
+  return Vec0<float>(a.raw - b.raw);
+}
+HWY_INLINE Vec0<double> operator-(const Vec0<double> a, const Vec0<double> b) {
+  return Vec0<double>(a.raw - b.raw);
+}
+
+// ------------------------------ Saturating addition
+
+// Returns a + b clamped to the destination range.
+
+// Unsigned
+HWY_INLINE Vec0<uint8_t> SaturatedAdd(const Vec0<uint8_t> a,
+                                      const Vec0<uint8_t> b) {
+  return Vec0<uint8_t>(HWY_MIN(HWY_MAX(0, a.raw + b.raw), 255));
+}
+HWY_INLINE Vec0<uint16_t> SaturatedAdd(const Vec0<uint16_t> a,
+                                       const Vec0<uint16_t> b) {
+  return Vec0<uint16_t>(HWY_MIN(HWY_MAX(0, a.raw + b.raw), 65535));
+}
+
+// Signed
+HWY_INLINE Vec0<int8_t> SaturatedAdd(const Vec0<int8_t> a,
+                                     const Vec0<int8_t> b) {
+  return Vec0<int8_t>(HWY_MIN(HWY_MAX(-128, a.raw + b.raw), 127));
+}
+HWY_INLINE Vec0<int16_t> SaturatedAdd(const Vec0<int16_t> a,
+                                      const Vec0<int16_t> b) {
+  return Vec0<int16_t>(HWY_MIN(HWY_MAX(-32768, a.raw + b.raw), 32767));
+}
+
+// ------------------------------ Saturating subtraction
+
+// Returns a - b clamped to the destination range.
+
+// Unsigned
+HWY_INLINE Vec0<uint8_t> SaturatedSub(const Vec0<uint8_t> a,
+                                      const Vec0<uint8_t> b) {
+  return Vec0<uint8_t>(HWY_MIN(HWY_MAX(0, a.raw - b.raw), 255));
+}
+HWY_INLINE Vec0<uint16_t> SaturatedSub(const Vec0<uint16_t> a,
+                                       const Vec0<uint16_t> b) {
+  return Vec0<uint16_t>(HWY_MIN(HWY_MAX(0, a.raw - b.raw), 65535));
+}
+
+// Signed
+HWY_INLINE Vec0<int8_t> SaturatedSub(const Vec0<int8_t> a,
+                                     const Vec0<int8_t> b) {
+  return Vec0<int8_t>(HWY_MIN(HWY_MAX(-128, a.raw - b.raw), 127));
+}
+HWY_INLINE Vec0<int16_t> SaturatedSub(const Vec0<int16_t> a,
+                                      const Vec0<int16_t> b) {
+  return Vec0<int16_t>(HWY_MIN(HWY_MAX(-32768, a.raw - b.raw), 32767));
+}
+
+// ------------------------------ Average
+
+// Returns (a + b + 1) / 2
+
+HWY_INLINE Vec0<uint8_t> AverageRound(const Vec0<uint8_t> a,
+                                      const Vec0<uint8_t> b) {
+  return Vec0<uint8_t>((a.raw + b.raw + 1) / 2);
+}
+HWY_INLINE Vec0<uint16_t> AverageRound(const Vec0<uint16_t> a,
+                                       const Vec0<uint16_t> b) {
+  return Vec0<uint16_t>((a.raw + b.raw + 1) / 2);
+}
+
+// ------------------------------ Absolute value
+
+template <typename T>
+HWY_INLINE Vec0<T> Abs(const Vec0<T> a) {
+  const T i = a.raw;
+  return (i >= 0 || i == LimitsMin<T>()) ? a : Vec0<T>(-i);
+}
+
+// ------------------------------ min/max
+
+template <typename T>
+HWY_INLINE Vec0<T> Min(const Vec0<T> a, const Vec0<T> b) {
+  return Vec0<T>(HWY_MIN(a.raw, b.raw));
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> Max(const Vec0<T> a, const Vec0<T> b) {
+  return Vec0<T>(HWY_MAX(a.raw, b.raw));
+}
+
+// Returns the closest value to v within [lo, hi].
+template <typename T>
+HWY_INLINE Vec0<T> Clamp(const Vec0<T> v, const Vec0<T> lo, const Vec0<T> hi) {
+  return Min(Max(lo, v), hi);
+}
+
+// ------------------------------ Floating-point negate
+
+HWY_INLINE Vec0<float> Neg(const Vec0<float> v) {
+  const Scalar<float> df;
+  const Scalar<uint32_t> du;
+  const auto sign = BitCast(df, Set(du, 0x80000000u));
+  return v ^ sign;
+}
+
+HWY_INLINE Vec0<double> Neg(const Vec0<double> v) {
+  const Scalar<double> df;
+  const Scalar<uint64_t> du;
+  const auto sign = BitCast(df, Set(du, 0x8000000000000000ull));
+  return v ^ sign;
+}
+
+// ------------------------------ mul/div
+
+template <typename T>
+HWY_INLINE Vec0<T> operator*(const Vec0<T> a, const Vec0<T> b) {
+  if (IsFloat<T>()) {
+    return Vec0<T>(static_cast<T>(double(a.raw) * b.raw));
+  } else if (IsSigned<T>()) {
+    return Vec0<T>(static_cast<T>(int64_t(a.raw) * b.raw));
+  } else {
+    return Vec0<T>(static_cast<T>(uint64_t(a.raw) * b.raw));
+  }
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> operator/(const Vec0<T> a, const Vec0<T> b) {
+  return Vec0<T>(a.raw / b.raw);
+}
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// Returns the upper 16 bits of a * b in each lane.
+HWY_INLINE Vec0<int16_t> MulHigh(const Vec0<int16_t> a, const Vec0<int16_t> b) {
+  return Vec0<int16_t>((a.raw * b.raw) >> 16);
+}
+HWY_INLINE Vec0<uint16_t> MulHigh(const Vec0<uint16_t> a,
+                                  const Vec0<uint16_t> b) {
+  // Cast to uint32_t first to prevent overflow. Otherwise the result of
+  // uint16_t * uint16_t is in "int" which may overflow. In practice the result
+  // is the same but this way it is also defined.
+  return Vec0<uint16_t>(
+      (static_cast<uint32_t>(a.raw) * static_cast<uint32_t>(b.raw)) >> 16);
+}
+
+}  // namespace ext
+
+// Multiplies even lanes (0, 2 ..) and returns the double-wide result.
+HWY_INLINE Vec0<int64_t> MulEven(const Vec0<int32_t> a, const Vec0<int32_t> b) {
+  const int64_t a64 = a.raw;
+  return Vec0<int64_t>(a64 * b.raw);
+}
+HWY_INLINE Vec0<uint64_t> MulEven(const Vec0<uint32_t> a,
+                                  const Vec0<uint32_t> b) {
+  const uint64_t a64 = a.raw;
+  return Vec0<uint64_t>(a64 * b.raw);
+}
+
+// Approximate reciprocal
+HWY_INLINE Vec0<float> ApproximateReciprocal(const Vec0<float> v) {
+  return Vec0<float>(1.0f / v.raw);
+}
+
+namespace ext {
+// Absolute value of difference.
+HWY_INLINE Vec0<float> AbsDiff(const Vec0<float> a, const Vec0<float> b) {
+  return Vec0<float>(fabsf(a.raw - b.raw));
+}
+}  // namespace ext
+
+// ------------------------------ Floating-point multiply-add variants
+
+template <typename T>
+HWY_INLINE Vec0<T> MulAdd(const Vec0<T> mul, const Vec0<T> x,
+                          const Vec0<T> add) {
+  return mul * x + add;
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> NegMulAdd(const Vec0<T> mul, const Vec0<T> x,
+                             const Vec0<T> add) {
+  return add - mul * x;
+}
+
+// Slightly more expensive on ARM (extra negate)
+namespace ext {
+
+template <typename T>
+HWY_INLINE Vec0<T> MulSub(const Vec0<T> mul, const Vec0<T> x,
+                          const Vec0<T> sub) {
+  return mul * x - sub;
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> NegMulSub(const Vec0<T> mul, const Vec0<T> x,
+                             const Vec0<T> sub) {
+  return Neg(mul) * x - sub;
+}
+
+}  // namespace ext
+
+// ------------------------------ Floating-point square root
+
+// Approximate reciprocal square root
+HWY_INLINE Vec0<float> ApproximateReciprocalSqrt(const Vec0<float> v) {
+  float f = v.raw;
+  const float half = f * 0.5f;
+  uint32_t bits;
+  CopyBytes<4>(&f, &bits);
+  // Initial guess based on log2(f)
+  bits = 0x5F3759DF - (bits >> 1);
+  CopyBytes<4>(&bits, &f);
+  // One Newton-Raphson iteration
+  return Vec0<float>(f * (1.5f - (half * f * f)));
+}
+
+// Square root
+HWY_INLINE Vec0<float> Sqrt(const Vec0<float> v) {
+  return Vec0<float>(sqrt(v.raw));
+}
+HWY_INLINE Vec0<double> Sqrt(const Vec0<double> v) {
+  return Vec0<double>(Sqrt(Vec0<float>(v.raw)).raw);
+}
+
+// ------------------------------ Floating-point rounding
+
+// Approximation of round-to-nearest for numbers representable as integers.
+HWY_INLINE Vec0<float> Round(const Vec0<float> v) {
+  const float bias = v.raw < 0.0f ? -0.5f : 0.5f;
+  return Vec0<float>(static_cast<int32_t>(v.raw + bias));
+}
+HWY_INLINE Vec0<double> Round(const Vec0<double> v) {
+  const double bias = v.raw < 0.0 ? -0.5 : 0.5;
+  return Vec0<double>(static_cast<int64_t>(v.raw + bias));
+}
+
+HWY_INLINE Vec0<float> Trunc(const Vec0<float> v) {
+  return Vec0<float>(static_cast<int32_t>(v.raw));
+}
+HWY_INLINE Vec0<double> Trunc(const Vec0<double> v) {
+  return Vec0<double>(static_cast<int64_t>(v.raw));
+}
+
+template <typename Float, typename Bits, int kMantissaBits, int kExponentBits,
+          class V>
+V Ceiling(const V v) {
+  const Bits kExponentMask = (1ull << kExponentBits) - 1;
+  const Bits kMantissaMask = (1ull << kMantissaBits) - 1;
+  const Bits kBias = kExponentMask / 2;
+
+  Float f = v.raw;
+  const bool positive = f > 0.0f;
+
+  Bits bits;
+  CopyBytes<sizeof(Bits)>(&v, &bits);
+
+  const int exponent = ((bits >> kMantissaBits) & kExponentMask) - kBias;
+  // Already an integer.
+  if (exponent >= kMantissaBits) return v;
+  // |v| <= 1 => 0 or 1.
+  if (exponent < 0) return V(positive);
+
+  const Bits mantissa_mask = kMantissaMask >> exponent;
+  // Already an integer
+  if ((bits & mantissa_mask) == 0) return v;
+
+  // Clear fractional bits and round up
+  if (positive) bits += (kMantissaMask + 1) >> exponent;
+  bits &= ~mantissa_mask;
+
+  CopyBytes<sizeof(Bits)>(&bits, &f);
+  return V(f);
+}
+
+template <typename Float, typename Bits, int kMantissaBits, int kExponentBits,
+          class V>
+V Floor(const V v) {
+  const Bits kExponentMask = (1ull << kExponentBits) - 1;
+  const Bits kMantissaMask = (1ull << kMantissaBits) - 1;
+  const Bits kBias = kExponentMask / 2;
+
+  Float f = v.raw;
+  const bool negative = f < 0.0f;
+
+  Bits bits;
+  CopyBytes<sizeof(Bits)>(&v, &bits);
+
+  const int exponent = ((bits >> kMantissaBits) & kExponentMask) - kBias;
+  // Already an integer.
+  if (exponent >= kMantissaBits) return v;
+  // |v| <= 1 => -1 or 0.
+  if (exponent < 0) return V(negative ? -1.0 : 0.0f);
+
+  const Bits mantissa_mask = kMantissaMask >> exponent;
+  // Already an integer
+  if ((bits & mantissa_mask) == 0) return v;
+
+  // Clear fractional bits and round down
+  if (negative) bits += (kMantissaMask + 1) >> exponent;
+  bits &= ~mantissa_mask;
+
+  CopyBytes<sizeof(Bits)>(&bits, &f);
+  return V(f);
+}
+
+// Toward +infinity, aka ceiling
+HWY_INLINE Vec0<float> Ceil(const Vec0<float> v) {
+  return Ceiling<float, uint32_t, 23, 8>(v);
+}
+HWY_INLINE Vec0<double> Ceil(const Vec0<double> v) {
+  return Ceiling<double, uint64_t, 52, 11>(v);
+}
+
+// Toward -infinity, aka floor
+HWY_INLINE Vec0<float> Floor(const Vec0<float> v) {
+  return Floor<float, uint32_t, 23, 8>(v);
+}
+HWY_INLINE Vec0<double> Floor(const Vec0<double> v) {
+  return Floor<double, uint64_t, 52, 11>(v);
+}
+
+// ================================================== COMPARE
+
+template <typename T>
+HWY_INLINE Mask0<T> operator==(const Vec0<T> a, const Vec0<T> b) {
+  return Mask0<T>::FromBool(a.raw == b.raw);
+}
+
+template <typename T>
+HWY_INLINE Mask0<T> TestBit(const Vec0<T> v, const Vec0<T> bit) {
+  static_assert(!IsFloat<T>(), "Only integer vectors supported");
+  return (v & bit) == bit;
+}
+
+template <typename T>
+HWY_INLINE Mask0<T> operator<(const Vec0<T> a, const Vec0<T> b) {
+  return Mask0<T>::FromBool(a.raw < b.raw);
+}
+template <typename T>
+HWY_INLINE Mask0<T> operator>(const Vec0<T> a, const Vec0<T> b) {
+  return Mask0<T>::FromBool(a.raw > b.raw);
+}
+
+template <typename T>
+HWY_INLINE Mask0<T> operator<=(const Vec0<T> a, const Vec0<T> b) {
+  return Mask0<T>::FromBool(a.raw <= b.raw);
+}
+template <typename T>
+HWY_INLINE Mask0<T> operator>=(const Vec0<T> a, const Vec0<T> b) {
+  return Mask0<T>::FromBool(a.raw >= b.raw);
+}
+
+// ================================================== MEMORY
+
+// ------------------------------ Load
+
+template <typename T>
+HWY_INLINE Vec0<T> Load(Scalar<T> /* tag */, const T* HWY_RESTRICT aligned) {
+  T t;
+  CopyBytes<sizeof(T)>(aligned, &t);
+  return Vec0<T>(t);
+}
+
+template <typename T>
+HWY_INLINE Vec0<T> LoadU(Scalar<T> d, const T* HWY_RESTRICT p) {
+  return Load(d, p);
+}
+
+// In some use cases, "load single lane" is sufficient; otherwise avoid this.
+template <typename T>
+HWY_INLINE Vec0<T> LoadDup128(Scalar<T> d, const T* HWY_RESTRICT aligned) {
+  return Load(d, aligned);
+}
+
+// ------------------------------ Store
+
+template <typename T>
+HWY_INLINE void Store(const Vec0<T> v, Scalar<T> /* tag */,
+                      T* HWY_RESTRICT aligned) {
+  CopyBytes<sizeof(T)>(&v.raw, aligned);
+}
+
+template <typename T>
+HWY_INLINE void StoreU(const Vec0<T> v, Scalar<T> d, T* HWY_RESTRICT p) {
+  return Store(v, d, p);
+}
+
+// ------------------------------ "Non-temporal" stores
+
+template <typename T>
+HWY_INLINE void Stream(const Vec0<T> v, Scalar<T> d, T* HWY_RESTRICT aligned) {
+  return Store(v, d, aligned);
+}
+
+// ------------------------------ Gather
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+template <typename T, typename Offset>
+HWY_INLINE Vec0<T> GatherOffset(Scalar<T> d, const T* base,
+                                const Vec0<Offset> offset) {
+  static_assert(sizeof(T) == sizeof(Offset), "SVE requires same size base/ofs");
+  const uintptr_t addr = reinterpret_cast<uintptr_t>(base) + offset.raw;
+  return Load(d, reinterpret_cast<const T*>(addr));
+}
+
+template <typename T, typename Index>
+HWY_INLINE Vec0<T> GatherIndex(Scalar<T> d, const T* HWY_RESTRICT base,
+                               const Vec0<Index> index) {
+  static_assert(sizeof(T) == sizeof(Index), "SVE requires same size base/idx");
+  return Load(d, base + index.raw);
+}
+
+}  // namespace ext
+
+// ================================================== CONVERT
+
+template <typename FromT, typename ToT>
+HWY_INLINE Vec0<ToT> ConvertTo(Scalar<ToT> /* tag */, Vec0<FromT> from) {
+  return Vec0<ToT>(static_cast<ToT>(from.raw));
+}
+
+HWY_INLINE Vec0<float> ConvertTo(Scalar<float> /* tag */,
+                                 const Vec0<int32_t> v) {
+  return Vec0<float>(v.raw);
+}
+
+// Truncates (rounds toward zero).
+HWY_INLINE Vec0<int32_t> ConvertTo(Scalar<int32_t> /* tag */,
+                                   const Vec0<float> v) {
+  return Vec0<int32_t>(static_cast<int>(v.raw));
+}
+
+HWY_INLINE Vec0<uint32_t> U32FromU8(const Vec0<uint8_t> v) {
+  return ConvertTo(Scalar<uint32_t>(), v);
+}
+
+HWY_INLINE Vec0<uint8_t> U8FromU32(const Vec0<uint32_t> v) {
+  return ConvertTo(Scalar<uint8_t>(), v);
+}
+
+// Approximation of round-to-nearest for numbers representable as int32_t.
+HWY_INLINE Vec0<int32_t> NearestInt(const Vec0<float> v) {
+  const float f = v.raw;
+  const float bias = f < 0.0f ? -0.5f : 0.5f;
+  return Vec0<int32_t>(static_cast<int>(f + bias));
+}
+
+// ================================================== SWIZZLE
+
+// Unsupported: shift_*_bytes, CombineShiftRightBytes, interleave_*,
+// shuffle_*, SumsOfU8x8, SumOfLanes, upper/lower/GetHalf - these require
+// more than one lane and/or actual 128-bit vectors.
+
+template <typename T>
+HWY_INLINE T GetLane(const Vec0<T> v) {
+  return v.raw;
+}
+
+// ------------------------------ Broadcast/splat any lane
+
+template <int kLane, typename T>
+HWY_INLINE Vec0<T> Broadcast(const Vec0<T> v) {
+  static_assert(kLane == 0, "Scalar only has one lane");
+  return v;
+}
+
+// ------------------------------ Zip/unpack
+
+HWY_INLINE Vec0<uint16_t> ZipLo(const Vec0<uint8_t> a, const Vec0<uint8_t> b) {
+  return Vec0<uint16_t>((uint32_t(b.raw) << 8) + a.raw);
+}
+HWY_INLINE Vec0<uint32_t> ZipLo(const Vec0<uint16_t> a,
+                                const Vec0<uint16_t> b) {
+  return Vec0<uint32_t>((uint32_t(b.raw) << 16) + a.raw);
+}
+HWY_INLINE Vec0<uint64_t> ZipLo(const Vec0<uint32_t> a,
+                                const Vec0<uint32_t> b) {
+  return Vec0<uint64_t>((uint64_t(b.raw) << 32) + a.raw);
+}
+HWY_INLINE Vec0<int16_t> ZipLo(const Vec0<int8_t> a, const Vec0<int8_t> b) {
+  return Vec0<int16_t>((uint32_t(b.raw) << 8) + a.raw);
+}
+HWY_INLINE Vec0<int32_t> ZipLo(const Vec0<int16_t> a, const Vec0<int16_t> b) {
+  return Vec0<int32_t>((uint32_t(b.raw) << 16) + a.raw);
+}
+HWY_INLINE Vec0<int64_t> ZipLo(const Vec0<int32_t> a, const Vec0<int32_t> b) {
+  return Vec0<int64_t>((uint64_t(b.raw) << 32) + a.raw);
+}
+
+// ================================================== MISC
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// Returns a bit array of the most significant bit of each byte in "v", i.e.
+// sum_i=0..15 of (v[i] >> 7) << i; v[0] is the least-significant byte of "v".
+// This is useful for testing/branching based on comparison results.
+HWY_INLINE uint64_t movemask(const Vec0<uint8_t> v) { return v.raw >> 7; }
+
+// Returns the most significant bit of each float/double lane (see above).
+HWY_INLINE uint64_t movemask(const Vec0<float> v) {
+  // Cannot return (v < 0) because +0.0 == -0.0.
+  const auto bits = BitCast(Scalar<uint32_t>(), v);
+  return GetLane(ShiftRight<31>(bits));
+}
+HWY_INLINE uint64_t movemask(const Vec0<double> v) {
+  // Cannot return (v < 0) because +0.0 == -0.0.
+  const auto bits = BitCast(Scalar<uint64_t>(), v);
+  return GetLane(ShiftRight<63>(bits));
+}
+
+template <typename T>
+HWY_INLINE bool AllFalse(const Mask0<T> v) {
+  return v.bits == 0;
+}
+
+template <typename T>
+HWY_INLINE bool AllTrue(const Mask0<T> v) {
+  return v.bits != 0;
+}
+
+template <typename T>
+HWY_INLINE size_t CountTrue(const Mask0<T> v) {
+  return v.bits == 0 ? 0 : 1;
+}
+
+// Sum of all lanes, i.e. the only one.
+template <typename T>
+HWY_INLINE Vec0<T> SumOfLanes(const Vec0<T> v0) {
+  return v0;
+}
+
+}  // namespace ext
+}  // namespace hwy
+
+#endif  // HWY_SCALAR_H_
diff --git a/third_party/highway/hwy/shared.h b/third_party/highway/hwy/shared.h
new file mode 100644
index 000000000..1b61b0b0b
--- /dev/null
+++ b/third_party/highway/hwy/shared.h
@@ -0,0 +1,97 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_SHARED_H_
+#define HWY_SHARED_H_
+
+// Definitions shared between target-specific headers and possibly also users.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "hwy/compiler_specific.h"
+#include "hwy/type_traits.h"
+
+// 4 instances of a given literal value, useful as input to LoadDup128.
+#define HWY_REP4(literal) literal, literal, literal, literal
+
+#define HWY_MIN(a, b) ((a) < (b) ? (a) : (b))
+#define HWY_MAX(a, b) ((a) < (b) ? (b) : (a))
+
+namespace hwy {
+
+// SIMD operations are implemented as overloaded functions selected using a
+// "descriptor" D := Desc<T, N>. T is the lane type, N the number of lanes.
+// kLanesOr0 is 0 for scalar (some SIMD users need kLanes=1).
+template <typename Lane, size_t kLanesOr0>
+struct Desc {
+  constexpr Desc() = default;
+
+  using T = Lane;
+
+  // How far to advance loop counters (>= 1).
+  static constexpr size_t N = HWY_MAX(kLanesOr0, 1);
+  static_assert((N & (N - 1)) == 0, "N must be a power of two");
+
+  // For creating a second Desc matching the first, use the original kLanesOr0
+  // multiplied/divided by ratio of sizeof(Lane).
+  static constexpr size_t LanesOr0() { return kLanesOr0; }
+};
+
+// Avoids linker errors in pre-C++17 debug builds.
+template <typename Lane, size_t kLanesOr0>
+constexpr size_t Desc<Lane, kLanesOr0>::N;
+
+// Alias for the actual vector data, e.g. Vec0<float> for Desc<float, 0>,
+// To avoid inadvertent conversions between vectors of different lengths, they
+// have distinct types (Vec128<T, N>) on x86. On PPC, we do not use wrapper
+// class templates due to poor code generation.
+template <class D>
+using VT = decltype(Zero(D()));
+
+// Type tags for GetHalf(v).
+struct Upper {};
+struct Lower {};
+
+// Unfortunately the GCC/Clang intrinsics do not accept int64_t*.
+using GatherIndex64 = long long int;  // NOLINT(google-runtime-int)
+static_assert(sizeof(GatherIndex64) == 8, "Must be 64-bit type");
+
+// The source/destination must not overlap/alias.
+template <size_t kBytes, typename From, typename To>
+HWY_INLINE void CopyBytes(const From* from, To* to) {
+#if HWY_COMPILER_MSVC
+  const uint8_t* HWY_RESTRICT from_bytes =
+      reinterpret_cast<const uint8_t*>(from);
+  uint8_t* HWY_RESTRICT to_bytes = reinterpret_cast<uint8_t*>(to);
+  for (size_t i = 0; i < kBytes; ++i) {
+    to_bytes[i] = from_bytes[i];
+  }
+#else
+  // Avoids horrible codegen on Clang (series of PINSRB)
+  __builtin_memcpy(to, from, kBytes);
+#endif
+}
+
+static HWY_INLINE size_t PopCount(const uint64_t x) {
+#ifdef _MSC_VER
+  return _mm_popcnt_u64(x);
+#else
+  return static_cast<size_t>(__builtin_popcountll(x));
+#endif
+}
+
+}  // namespace hwy
+
+#endif  // HWY_SHARED_H_
diff --git a/third_party/highway/hwy/static_targets.h b/third_party/highway/hwy/static_targets.h
new file mode 100644
index 000000000..7c2e367bd
--- /dev/null
+++ b/third_party/highway/hwy/static_targets.h
@@ -0,0 +1,136 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_STATIC_TARGETS_H_
+#define HWY_STATIC_TARGETS_H_
+
+// Defines HWY_STATIC_TARGETS and chooses the best available.
+
+#include "hwy/arch.h"
+#include "hwy/targets.h"
+
+#ifndef HWY_STATIC_TARGETS
+
+// Any targets in HWY_STATIC_TARGETS are eligible for use without runtime
+// dispatch, so they must be supported by both the compiler and CPU.
+#if HWY_ARCH == HWY_ARCH_X86
+#if defined(HWY_DISABLE_AVX2)
+// Clang6 encounters backend errors when compiling AVX2.
+#define HWY_STATIC_TARGETS (HWY_SSE4)
+#else
+#define HWY_STATIC_TARGETS (HWY_SSE4 | HWY_AVX2)
+#endif
+#elif HWY_ARCH == HWY_ARCH_PPC
+#define HWY_STATIC_TARGETS HWY_PPC8
+#elif HWY_ARCH == HWY_ARCH_ARM
+#define HWY_STATIC_TARGETS HWY_ARM8
+#elif HWY_ARCH == HWY_ARCH_WASM
+#define HWY_STATIC_TARGETS HWY_WASM
+#elif HWY_ARCH == HWY_ARCH_SCALAR
+#define HWY_STATIC_TARGETS HWY_NONE
+#else
+#error "Unsupported platform"
+#endif
+
+#endif  // HWY_STATIC_TARGETS
+
+// After HWY_STATIC_TARGETS:
+#include "hwy/include_headers.h"
+
+// It is permissible to include static_targets.h BEFORE foreach_target.h,
+// but not the reverse because that would override the intended runtime target.
+#ifdef HWY_NAMESPACE
+#error "Do not include static_targets.h after foreach_target.h"
+#endif
+
+// Choose best available:
+
+//-----------------------------------------------------------------------------
+#if HWY_STATIC_TARGETS & HWY_AVX512
+#define HWY_BITS 512
+#define HWY_ALIGN alignas(64)
+#define HWY_HAS_CMP64 1
+#define HWY_HAS_GATHER 1
+#define HWY_HAS_VARIABLE_SHIFT 1
+#define HWY_HAS_INT64 1
+#define HWY_HAS_DOUBLE 1
+#define HWY_ATTR HWY_ATTR_AVX512
+//-----------------------------------------------------------------------------
+#elif HWY_STATIC_TARGETS & HWY_AVX2
+#define HWY_BITS 256
+#define HWY_ALIGN alignas(32)
+#define HWY_HAS_CMP64 1
+#define HWY_HAS_GATHER 1
+#define HWY_HAS_VARIABLE_SHIFT 1
+#define HWY_HAS_INT64 1
+#define HWY_HAS_DOUBLE 1
+#define HWY_ATTR HWY_ATTR_AVX2
+//-----------------------------------------------------------------------------
+#elif HWY_STATIC_TARGETS & HWY_SSE4
+#define HWY_BITS 128
+#define HWY_ALIGN alignas(16)
+#define HWY_HAS_CMP64 0
+#define HWY_HAS_GATHER 0
+#define HWY_HAS_VARIABLE_SHIFT 0
+#define HWY_HAS_INT64 1
+#define HWY_HAS_DOUBLE 1
+#define HWY_ATTR HWY_ATTR_SSE4
+//-----------------------------------------------------------------------------
+#elif HWY_STATIC_TARGETS & HWY_PPC8
+#define HWY_BITS 128
+#define HWY_ALIGN alignas(16)
+#define HWY_HAS_CMP64 1
+#define HWY_HAS_GATHER 0
+#define HWY_HAS_VARIABLE_SHIFT 1
+#define HWY_HAS_INT64 1
+#define HWY_HAS_DOUBLE 1
+#define HWY_ATTR
+//-----------------------------------------------------------------------------
+#elif HWY_STATIC_TARGETS & HWY_WASM
+#define HWY_BITS 128
+#define HWY_ALIGN alignas(16)
+#define HWY_HAS_CMP64 0
+#define HWY_HAS_GATHER 0
+#define HWY_HAS_VARIABLE_SHIFT 0
+#define HWY_HAS_INT64 0
+#define HWY_HAS_DOUBLE 0
+#define HWY_ATTR HWY_ATTR_WASM
+//-----------------------------------------------------------------------------
+#elif HWY_STATIC_TARGETS & HWY_ARM8
+#define HWY_BITS 128
+#define HWY_ALIGN alignas(16)
+#ifdef __arm__
+#define HWY_HAS_CMP64 0
+#define HWY_HAS_DOUBLE 0
+#else
+#define HWY_HAS_CMP64 1
+#define HWY_HAS_DOUBLE 1
+#endif
+#define HWY_HAS_INT64 1
+#define HWY_HAS_GATHER 0
+#define HWY_HAS_VARIABLE_SHIFT 1
+#define HWY_ATTR HWY_ATTR_ARM8
+//-----------------------------------------------------------------------------
+#else  // NONE
+#define HWY_BITS 0
+#define HWY_ALIGN
+#define HWY_HAS_CMP64 0
+#define HWY_HAS_GATHER 1
+#define HWY_HAS_VARIABLE_SHIFT 1
+#define HWY_HAS_INT64 1
+#define HWY_HAS_DOUBLE 1
+#define HWY_ATTR
+#endif
+
+#endif  // HWY_STATIC_TARGETS_H_
diff --git a/third_party/highway/hwy/targets.h b/third_party/highway/hwy/targets.h
new file mode 100644
index 000000000..893cebc29
--- /dev/null
+++ b/third_party/highway/hwy/targets.h
@@ -0,0 +1,27 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TARGETS_H_
+#define HWY_TARGETS_H_
+
+// Unique bit value for each target. Used in static/runtime_targets.h.
+#define HWY_NONE 0
+#define HWY_PPC8 1  // v2.07 or 3
+#define HWY_AVX2 2
+#define HWY_SSE4 4
+#define HWY_ARM8 8
+#define HWY_AVX512 16
+#define HWY_WASM 32
+
+#endif  // HWY_TARGETS_H_
diff --git a/third_party/highway/hwy/tests/arithmetic_test.cc b/third_party/highway/hwy/tests/arithmetic_test.cc
new file mode 100644
index 000000000..da78ceee3
--- /dev/null
+++ b/third_party/highway/hwy/tests/arithmetic_test.cc
@@ -0,0 +1,934 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/arithmetic_test.cc"
+
+#include "hwy/tests/test_util.h"
+struct ArithmeticTest {
+  HWY_DECLARE(void, ())
+};
+TEST(HwyArithmeticTest, Run) { hwy::RunTests<ArithmeticTest>(); }
+
+#endif  // HWY_TARGET_INCLUDE
+#include "hwy/tests/test_target_util.h"
+
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+constexpr HWY_FULL(uint8_t) du8;
+constexpr HWY_FULL(uint16_t) du16;
+constexpr HWY_FULL(uint32_t) du32;
+constexpr HWY_FULL(uint64_t) du64;
+constexpr HWY_FULL(int8_t) di8;
+constexpr HWY_FULL(int16_t) di16;
+constexpr HWY_FULL(int32_t) di32;
+constexpr HWY_FULL(int64_t) di64;
+constexpr HWY_FULL(float) df;
+constexpr HWY_FULL(double) dd;
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestPlusMinus(D d) {
+  using T = typename D::T;
+  const auto v2 = Iota(D(), 2);
+  const auto v3 = Iota(D(), 3);
+  const auto v4 = Iota(D(), 4);
+
+  T lanes[d.N] = {};  // Initialized for static analysis.
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (2 + i) + (3 + i);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, v2 + v3);
+  HWY_ASSERT_VEC_EQ(d, v3, (v2 + v3) - v2);
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (2 + i) + (4 + i);
+  }
+  auto sum = v2;
+  sum += v4;  // sum == 6,8..
+  HWY_ASSERT_VEC_EQ(d, lanes, sum);
+
+  sum -= v4;
+  HWY_ASSERT_VEC_EQ(d, v2, sum);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestUnsignedSaturatingArithmetic(D d) {
+  using T = typename D::T;
+  const auto v0 = Zero(d);
+  const auto vi = Iota(d, 1);
+  const auto vm = Set(d, LimitsMax<T>());
+
+  HWY_ASSERT_VEC_EQ(d, v0 + v0, SaturatedAdd(v0, v0));
+  HWY_ASSERT_VEC_EQ(d, v0 + vi, SaturatedAdd(v0, vi));
+  HWY_ASSERT_VEC_EQ(d, v0 + vm, SaturatedAdd(v0, vm));
+  HWY_ASSERT_VEC_EQ(d, vm, SaturatedAdd(vi, vm));
+  HWY_ASSERT_VEC_EQ(d, vm, SaturatedAdd(vm, vm));
+
+  HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(v0, v0));
+  HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(v0, vi));
+  HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(vi, vi));
+  HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(vi, vm));
+  HWY_ASSERT_VEC_EQ(d, vm - vi, SaturatedSub(vm, vi));
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSignedSaturatingArithmetic(D d) {
+  using T = typename D::T;
+  const auto v0 = Zero(d);
+  const auto vi = Iota(d, 1);
+  const auto vpm = Set(d, LimitsMax<T>());
+  const auto vn = Iota(d, -T(d.N));
+  const auto vnm = Set(d, LimitsMin<T>());
+
+  HWY_ASSERT_VEC_EQ(d, v0, SaturatedAdd(v0, v0));
+  HWY_ASSERT_VEC_EQ(d, vi, SaturatedAdd(v0, vi));
+  HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAdd(v0, vpm));
+  HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAdd(vi, vpm));
+  HWY_ASSERT_VEC_EQ(d, vpm, SaturatedAdd(vpm, vpm));
+
+  HWY_ASSERT_VEC_EQ(d, v0, SaturatedSub(v0, v0));
+  HWY_ASSERT_VEC_EQ(d, v0 - vi, SaturatedSub(v0, vi));
+  HWY_ASSERT_VEC_EQ(d, vn, SaturatedSub(vn, v0));
+  HWY_ASSERT_VEC_EQ(d, vnm, SaturatedSub(vnm, vi));
+  HWY_ASSERT_VEC_EQ(d, vnm, SaturatedSub(vnm, vpm));
+}
+
+HWY_NOINLINE HWY_ATTR void TestSaturatingArithmetic() {
+  TestUnsignedSaturatingArithmetic(du8);
+  TestUnsignedSaturatingArithmetic(du16);
+  TestSignedSaturatingArithmetic(di8);
+  TestSignedSaturatingArithmetic(di16);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestAverageT(D d) {
+  using T = typename D::T;
+  const auto v0 = Zero(d);
+  const auto v1 = Set(d, T(1));
+  const auto v2 = Set(d, T(2));
+
+  HWY_ASSERT_VEC_EQ(d, v0, AverageRound(v0, v0));
+  HWY_ASSERT_VEC_EQ(d, v1, AverageRound(v0, v1));
+  HWY_ASSERT_VEC_EQ(d, v1, AverageRound(v1, v1));
+  HWY_ASSERT_VEC_EQ(d, v2, AverageRound(v1, v2));
+  HWY_ASSERT_VEC_EQ(d, v2, AverageRound(v2, v2));
+}
+
+HWY_NOINLINE HWY_ATTR void TestAverage() {
+  TestAverageT(du8);
+  TestAverageT(du16);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestAbsT(D d) {
+  using T = typename D::T;
+  const auto v0 = Zero(d);
+  const auto vp1 = Set(d, T(1));
+  const auto vn1 = Set(d, T(-1));
+  const auto vpm = Set(d, LimitsMax<T>());
+  const auto vnm = Set(d, LimitsMin<T>());
+
+  HWY_ASSERT_VEC_EQ(d, v0, Abs(v0));
+  HWY_ASSERT_VEC_EQ(d, vp1, Abs(vp1));
+  HWY_ASSERT_VEC_EQ(d, vp1, Abs(vn1));
+  HWY_ASSERT_VEC_EQ(d, vpm, Abs(vpm));
+  HWY_ASSERT_VEC_EQ(d, vnm, Abs(vnm));
+}
+
+HWY_NOINLINE HWY_ATTR void TestAbs() {
+  TestAbsT(di8);
+  TestAbsT(di16);
+  TestAbsT(di32);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestUnsignedShifts(D d) {
+  using T = typename D::T;
+  constexpr int kSign = (sizeof(T) * 8) - 1;
+  const auto v0 = Zero(d);
+  const auto vi = Iota(d, 0);
+  HWY_ALIGN T expected[d.N] = {};  // Initialized for static analysis.
+
+  // Shifting out of right side => zero
+  HWY_ASSERT_VEC_EQ(d, v0, ShiftRight<7>(vi));
+#if !HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  HWY_ASSERT_VEC_EQ(d, v0, ShiftRightSame(vi, 7));
+#endif
+
+  // Simple left shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(i << 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, hwy::ShiftLeft<1>(vi));
+#if !HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftLeftSame(vi, 1));
+#endif
+
+  // Simple right shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(i >> 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftRight<1>(vi));
+#if !HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftRightSame(vi, 1));
+#endif
+
+  // Verify truncation for left-shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = (T(i) << kSign) & ~T(0);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, hwy::ShiftLeft<kSign>(vi));
+#if !HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftLeftSame(vi, kSign));
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSignedShifts(D d) {
+  using T = typename D::T;
+  const auto v0 = Zero(d);
+  const auto vi = Iota(d, 0);
+  HWY_ALIGN T expected[d.N] = {};  // Initialized for static analysis.
+
+  // Shifting out of right side => zero
+  HWY_ASSERT_VEC_EQ(d, v0, ShiftRight<7>(vi));
+#if !HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  HWY_ASSERT_VEC_EQ(d, v0, ShiftRightSame(vi, 7));
+#endif
+
+  // Simple left shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(i << 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, hwy::ShiftLeft<1>(vi));
+#if !HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftLeftSame(vi, 1));
+#endif
+
+  // Simple right shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(i >> 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftRight<1>(vi));
+#if !HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftRightSame(vi, 1));
+#endif
+
+  // Sign extension
+  constexpr T min = LimitsMin<T>();
+  const auto vn = Iota(d, min);
+  for (size_t i = 0; i < d.N; ++i) {
+    // We want a right-shift here, which is undefined behavior for negative
+    // numbers. Since we want (-1)>>1 to be -1, we need to adjust rounding if
+    // minT is odd and negative.
+    T minT = min + i;
+    expected[i] = T(minT / 2 + (minT < 0 ? minT % 2 : 0));
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftRight<1>(vn));
+#if !HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftRightSame(vn, 1));
+#endif
+
+  // Shifting negative left
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T((min + i) << 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, hwy::ShiftLeft<1>(vn));
+#if !HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  HWY_ASSERT_VEC_EQ(d, expected, ShiftLeftSame(vn, 1));
+#endif
+}
+
+#if HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestUnsignedVarShifts(D d) {
+  using T = typename D::T;
+  constexpr int kSign = (sizeof(T) * 8) - 1;
+  const auto v0 = Zero(d);
+  const auto v1 = Set(d, 1);
+  const auto vi = Iota(d, 0);
+  HWY_ALIGN T expected[d.N];
+
+  // Shifting out of right side => zero
+  HWY_ASSERT_VEC_EQ(d, v0, vi >> Set(d, 7));
+
+  // Simple left shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(i << 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, vi << Set(d, 1));
+
+  // Simple right shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(i >> 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, vi >> Set(d, 1));
+
+  // Verify truncation for left-shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = (T(i) << kSign) & ~T(0);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, vi << Set(d, kSign));
+
+  // Verify variable left shift (assumes < 32 lanes)
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(1) << i;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, v1 << vi);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSignedVarLeftShifts(D d) {
+  using T = typename D::T;
+  const auto v1 = Set(d, 1);
+  const auto vi = Iota(d, 0);
+
+  HWY_ALIGN T expected[d.N];
+
+  // Simple left shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(i << 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, vi << v1);
+
+  // Shifting negative numbers left
+  constexpr T min = LimitsMin<T>();
+  const auto vn = Iota(d, min);
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T((min + i) << 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, vn << v1);
+
+  // Differing shift counts (assumes < 32 lanes)
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(1) << i;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, v1 << vi);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSignedVarRightShifts(D d) {
+  using T = typename D::T;
+  const auto v0 = Zero(d);
+  const auto vi = Iota(d, 0);
+  const auto vmax = Set(d, LimitsMax<T>());
+  HWY_ALIGN T expected[d.N];
+
+  // Shifting out of right side => zero
+  HWY_ASSERT_VEC_EQ(d, v0, vi >> Set(d, 7));
+
+  // Simple right shift
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = T(i >> 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, vi >> Set(d, 1));
+
+  // Sign extension
+  constexpr T min = LimitsMin<T>();
+  const auto vn = Iota(d, min);
+  for (size_t i = 0; i < d.N; ++i) {
+    // We want a right-shift here, which is undefined behavior for negative
+    // numbers. Since we want (-1)>>1 to be -1, we need to adjust rounding if
+    // minT is odd and negative.
+    T minT = min + i;
+    expected[i] = T(minT / 2 + (minT < 0 ? minT % 2 : 0));
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, vn >> Set(d, 1));
+
+  // Differing shift counts (assumes < 32 lanes)
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = LimitsMax<T>() >> i;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, vmax >> vi);
+}
+
+#endif
+
+HWY_NOINLINE HWY_ATTR void TestShifts() {
+  // No u8.
+  TestUnsignedShifts(du16);
+  TestUnsignedShifts(du32);
+#if HWY_HAS_INT64
+  TestUnsignedShifts(du64);
+#endif
+  // No i8.
+  TestSignedShifts(di16);
+  TestSignedShifts(di32);
+  // No i64/f32/f64.
+
+#if HWY_HAS_VARIABLE_SHIFT || HWY_IDE
+  TestUnsignedVarShifts(du32);
+  TestUnsignedVarShifts(du64);
+  TestSignedVarLeftShifts(di32);
+  TestSignedVarRightShifts(di32);
+  TestSignedVarLeftShifts(di64);
+// No i64 (right-shift).
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestUnsignedMinMax(D d) {
+  using T = typename D::T;
+  const auto v0 = Zero(d);
+  const auto v1 = Iota(d, 1);
+  const auto v2 = Iota(d, 2);
+  const auto v_max = Iota(d, LimitsMax<T>() - d.N + 1);
+  HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v2));
+  HWY_ASSERT_VEC_EQ(d, v2, Max(v1, v2));
+  HWY_ASSERT_VEC_EQ(d, v0, Min(v1, v0));
+  HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v0));
+  HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v_max));
+  HWY_ASSERT_VEC_EQ(d, v_max, Max(v1, v_max));
+  HWY_ASSERT_VEC_EQ(d, v0, Min(v0, v_max));
+  HWY_ASSERT_VEC_EQ(d, v_max, Max(v0, v_max));
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSignedMinMax(D d) {
+  using T = typename D::T;
+  const auto v1 = Iota(d, 1);
+  const auto v2 = Iota(d, 2);
+  const auto v_neg = Iota(d, -T(d.N));
+  const auto v_neg_max = Iota(d, LimitsMin<T>());
+  HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v2));
+  HWY_ASSERT_VEC_EQ(d, v2, Max(v1, v2));
+  HWY_ASSERT_VEC_EQ(d, v_neg, Min(v1, v_neg));
+  HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v_neg));
+  HWY_ASSERT_VEC_EQ(d, v_neg_max, Min(v1, v_neg_max));
+  HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v_neg_max));
+  HWY_ASSERT_VEC_EQ(d, v_neg_max, Min(v_neg, v_neg_max));
+  HWY_ASSERT_VEC_EQ(d, v_neg, Max(v_neg, v_neg_max));
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestFloatMinMax(D d) {
+  using T = typename D::T;
+  const auto v1 = Iota(d, 1);
+  const auto v2 = Iota(d, 2);
+  const auto v_neg = Iota(d, -T(d.N));
+  HWY_ASSERT_VEC_EQ(d, v1, Min(v1, v2));
+  HWY_ASSERT_VEC_EQ(d, v2, Max(v1, v2));
+  HWY_ASSERT_VEC_EQ(d, v_neg, Min(v1, v_neg));
+  HWY_ASSERT_VEC_EQ(d, v1, Max(v1, v_neg));
+}
+
+HWY_NOINLINE HWY_ATTR void TestMinMax() {
+  TestUnsignedMinMax(du8);
+  TestUnsignedMinMax(du16);
+  TestUnsignedMinMax(du32);
+  // No u64.
+  TestSignedMinMax(di8);
+  TestSignedMinMax(di16);
+  TestSignedMinMax(di32);
+  // No i64.
+  HWY_FOREACH_F(TestFloatMinMax);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestUnsignedMul(D d) {
+  using T = typename D::T;
+  const auto v0 = Zero(d);
+  const auto v1 = Set(d, T(1));
+  const auto vi = Iota(d, 1);
+  const auto vj = Iota(d, 3);
+  T lanes[d.N] = {};  // Initialized for static analysis.
+  HWY_ASSERT_VEC_EQ(d, v0, v0 * v0);
+  HWY_ASSERT_VEC_EQ(d, v1, v1 * v1);
+  HWY_ASSERT_VEC_EQ(d, vi, v1 * vi);
+  HWY_ASSERT_VEC_EQ(d, vi, vi * v1);
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (1 + i) * (1 + i);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, vi * vi);
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (1 + i) * (3 + i);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, vi * vj);
+
+  const T max = LimitsMax<T>();
+  const auto vmax = Set(d, max);
+  HWY_ASSERT_VEC_EQ(d, vmax, vmax * v1);
+  HWY_ASSERT_VEC_EQ(d, vmax, v1 * vmax);
+
+  const size_t bits = sizeof(T) * 8;
+  const uint64_t mask = (1ull << bits) - 1;
+  const T max2 = (uint64_t(max) * max) & mask;
+  HWY_ASSERT_VEC_EQ(d, Set(d, max2), vmax * vmax);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSignedMul(D d) {
+  using T = typename D::T;
+  const auto v0 = Zero(d);
+  const auto v1 = Set(d, T(1));
+  const auto vi = Iota(d, 1);
+  const auto vn = Iota(d, -T(d.N));
+  T lanes[d.N] = {};  // Initialized for static analysis.
+  HWY_ASSERT_VEC_EQ(d, v0, v0 * v0);
+  HWY_ASSERT_VEC_EQ(d, v1, v1 * v1);
+  HWY_ASSERT_VEC_EQ(d, vi, v1 * vi);
+  HWY_ASSERT_VEC_EQ(d, vi, vi * v1);
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (1 + i) * (1 + i);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, vi * vi);
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (-T(d.N) + i) * (1 + i);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, vn * vi);
+  HWY_ASSERT_VEC_EQ(d, lanes, vi * vn);
+}
+
+HWY_NOINLINE HWY_ATTR void TestMul() {
+  // No u8.
+  TestUnsignedMul(du16);
+  TestUnsignedMul(du32);
+  // No u64,i8.
+  TestSignedMul(di16);
+  TestSignedMul(di32);
+  // No i64.
+}
+
+template <class D, class D2>
+HWY_NOINLINE HWY_ATTR void TestMulHighT(D d, D2 /*d2*/) {
+  using T = typename D::T;
+  using Wide = typename D2::T;
+  HWY_ALIGN T in_lanes[d.N] = {};        // Initialized for static analysis.
+  HWY_ALIGN T expected_lanes[d.N] = {};  // Initialized for static analysis.
+  const auto vi = Iota(d, 1);
+  const auto vni = Iota(d, -T(d.N));
+
+  const auto v0 = Zero(d);
+  HWY_ASSERT_VEC_EQ(d, v0, ext::MulHigh(v0, v0));
+  HWY_ASSERT_VEC_EQ(d, v0, ext::MulHigh(v0, vi));
+  HWY_ASSERT_VEC_EQ(d, v0, ext::MulHigh(vi, v0));
+
+  // Large positive squared
+  for (size_t i = 0; i < d.N; ++i) {
+    in_lanes[i] = LimitsMax<T>() >> i;
+    expected_lanes[i] = (Wide(in_lanes[i]) * in_lanes[i]) >> 16;
+  }
+  auto v = Load(d, in_lanes);
+  HWY_ASSERT_VEC_EQ(d, expected_lanes, ext::MulHigh(v, v));
+
+  // Large positive * small positive
+  for (size_t i = 0; i < d.N; ++i) {
+    expected_lanes[i] = (Wide(in_lanes[i]) * (1 + i)) >> 16;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected_lanes, ext::MulHigh(v, vi));
+  HWY_ASSERT_VEC_EQ(d, expected_lanes, ext::MulHigh(vi, v));
+
+  // Large positive * small negative
+  for (size_t i = 0; i < d.N; ++i) {
+    expected_lanes[i] = (Wide(in_lanes[i]) * T(i - d.N)) >> 16;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected_lanes, ext::MulHigh(v, vni));
+  HWY_ASSERT_VEC_EQ(d, expected_lanes, ext::MulHigh(vni, v));
+}
+
+HWY_NOINLINE HWY_ATTR void TestMulHigh() {
+  TestMulHighT(di16, di32);
+  TestMulHighT(du16, du32);
+}
+
+template <class D, class D2>
+HWY_NOINLINE HWY_ATTR void TestMulEvenT(D d, D2 d2) {
+  using T = typename D::T;
+  using Wide = typename D2::T;
+
+  const auto v0 = Zero(d);
+  HWY_ASSERT_VEC_EQ(d2, Zero(d2), MulEven(v0, v0));
+
+  // scalar has N=1 and we write to "lane 1" below, though it isn't used by
+  // the actual MulEven.
+  HWY_ALIGN T in_lanes[HWY_MAX(d.N, 2)];
+  HWY_ALIGN Wide expected[d2.N];
+  for (size_t i = 0; i < d.N; i += 2) {
+    in_lanes[i + 0] = LimitsMax<T>() >> i;
+    in_lanes[i + 1] = 1;  // will be overwritten with upper half of result
+    expected[i / 2] = Wide(in_lanes[i + 0]) * in_lanes[i + 0];
+  }
+
+  const auto v = Load(d, in_lanes);
+  HWY_ASSERT_VEC_EQ(d2, expected, MulEven(v, v));
+}
+
+HWY_NOINLINE HWY_ATTR void TestMulEven() {
+  TestMulEvenT(di32, di64);
+  TestMulEvenT(du32, du64);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestMulAdd(D d) {
+  using T = typename D::T;
+  const auto k0 = Zero(d);
+  const auto v1 = Iota(d, 1);
+  const auto v2 = Iota(d, 2);
+  T lanes[d.N];
+  HWY_ASSERT_VEC_EQ(d, k0, MulAdd(k0, k0, k0));
+  HWY_ASSERT_VEC_EQ(d, v2, MulAdd(k0, v1, v2));
+  HWY_ASSERT_VEC_EQ(d, v2, MulAdd(v1, k0, v2));
+  HWY_ASSERT_VEC_EQ(d, k0, NegMulAdd(k0, k0, k0));
+  HWY_ASSERT_VEC_EQ(d, v2, NegMulAdd(k0, v1, v2));
+  HWY_ASSERT_VEC_EQ(d, v2, NegMulAdd(v1, k0, v2));
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (i + 1) * (i + 2);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, MulAdd(v2, v1, k0));
+  HWY_ASSERT_VEC_EQ(d, lanes, MulAdd(v1, v2, k0));
+  HWY_ASSERT_VEC_EQ(d, lanes, NegMulAdd(Neg(v2), v1, k0));
+  HWY_ASSERT_VEC_EQ(d, lanes, NegMulAdd(v1, Neg(v2), k0));
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (i + 2) * (i + 2) + (i + 1);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, MulAdd(v2, v2, v1));
+  HWY_ASSERT_VEC_EQ(d, lanes, NegMulAdd(Neg(v2), v2, v1));
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = -T(i + 2) * (i + 2) + (1 + i);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, NegMulAdd(v2, v2, v1));
+
+  HWY_ASSERT_VEC_EQ(d, k0, ext::MulSub(k0, k0, k0));
+  HWY_ASSERT_VEC_EQ(d, k0, ext::NegMulSub(k0, k0, k0));
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = -T(i + 2);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::MulSub(k0, v1, v2));
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::MulSub(v1, k0, v2));
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::NegMulSub(Neg(k0), v1, v2));
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::NegMulSub(v1, Neg(k0), v2));
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (i + 1) * (i + 2);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::MulSub(v1, v2, k0));
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::MulSub(v2, v1, k0));
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::NegMulSub(Neg(v1), v2, k0));
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::NegMulSub(v2, Neg(v1), k0));
+
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = (i + 2) * (i + 2) - (1 + i);
+  }
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::MulSub(v2, v2, v1));
+  HWY_ASSERT_VEC_EQ(d, lanes, ext::NegMulSub(Neg(v2), v2, v1));
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSquareRoot(D d) {
+  const auto vi = Iota(d, 0);
+  HWY_ASSERT_VEC_EQ(d, vi, Sqrt(vi * vi));
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestReciprocalSquareRoot(D d) {
+  const auto v = Set(d, 123.0f);
+  HWY_ALIGN float lanes[d.N];
+  Store(ApproximateReciprocalSqrt(v), d, lanes);
+  for (size_t i = 0; i < d.N; ++i) {
+    float err = lanes[i] - 0.090166f;
+    if (err < 0.0f) err = -err;
+    HWY_ASSERT_EQ(true, err < 1E-4f);
+  }
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestRound(D d) {
+  using T = typename D::T;
+  // Numbers close to 0
+  {
+    const auto v = Set(d, 0.4);
+    const auto zero = Set(d, 0);
+    const auto one = Set(d, 1);
+    HWY_ASSERT_VEC_EQ(d, one, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, zero, Floor(v));
+    HWY_ASSERT_VEC_EQ(d, zero, Round(v));
+    HWY_ASSERT_VEC_EQ(d, zero, Trunc(v));
+  }
+  {
+    const auto v = Set(d, -0.4);
+    const auto zero = Set(d, 0);
+    const auto one = Set(d, -1);
+    HWY_ASSERT_VEC_EQ(d, zero, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, one, Floor(v));
+    HWY_ASSERT_VEC_EQ(d, zero, Round(v));
+    HWY_ASSERT_VEC_EQ(d, zero, Trunc(v));
+  }
+  // Integer positive
+  {
+    const auto v = Iota(d, 4.0);
+    HWY_ASSERT_VEC_EQ(d, v, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, v, Floor(v));
+    HWY_ASSERT_VEC_EQ(d, v, Round(v));
+    HWY_ASSERT_VEC_EQ(d, v, Trunc(v));
+  }
+
+  // Integer negative
+  {
+    const auto v = Iota(d, T(-32.0));
+    HWY_ASSERT_VEC_EQ(d, v, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, v, Floor(v));
+    HWY_ASSERT_VEC_EQ(d, v, Round(v));
+    HWY_ASSERT_VEC_EQ(d, v, Trunc(v));
+  }
+
+  // Huge positive
+  {
+    const auto v = Set(d, T(1E15));
+    HWY_ASSERT_VEC_EQ(d, v, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, v, Floor(v));
+  }
+
+  // Huge negative
+  {
+    const auto v = Set(d, T(-1E15));
+    HWY_ASSERT_VEC_EQ(d, v, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, v, Floor(v));
+  }
+
+  // Above positive
+  {
+    const auto v = Iota(d, T(2.0001));
+    const auto v3 = Iota(d, T(3));
+    const auto v2 = Iota(d, T(2));
+    HWY_ASSERT_VEC_EQ(d, v3, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, v2, Floor(v));
+    HWY_ASSERT_VEC_EQ(d, v2, Round(v));
+    HWY_ASSERT_VEC_EQ(d, v2, Trunc(v));
+  }
+
+  // Below positive
+  {
+    const auto v = Iota(d, T(3.9999));
+    const auto v4 = Iota(d, T(4));
+    const auto v3 = Iota(d, T(3));
+    HWY_ASSERT_VEC_EQ(d, v4, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, v3, Floor(v));
+    HWY_ASSERT_VEC_EQ(d, v4, Round(v));
+    HWY_ASSERT_VEC_EQ(d, v3, Trunc(v));
+  }
+
+  // Above negative
+  {
+    // WARNING: using iota => ensure negative value is low enough that
+    // even 16 lanes remain negative, otherwise trunc will behave differently
+    // for positive/negative values.
+    const auto v = Iota(d, T(-19.9999));
+    const auto v3 = Iota(d, T(-19));
+    const auto v4 = Iota(d, T(-20));
+    HWY_ASSERT_VEC_EQ(d, v3, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, v4, Floor(v));
+    HWY_ASSERT_VEC_EQ(d, v4, Round(v));
+    HWY_ASSERT_VEC_EQ(d, v3, Trunc(v));
+  }
+
+  // Below negative
+  {
+    const auto v = Iota(d, T(-18.0001));
+    const auto v2 = Iota(d, T(-18));
+    const auto v3 = Iota(d, T(-19));
+    HWY_ASSERT_VEC_EQ(d, v2, Ceil(v));
+    HWY_ASSERT_VEC_EQ(d, v3, Floor(v));
+    HWY_ASSERT_VEC_EQ(d, v2, Round(v));
+    HWY_ASSERT_VEC_EQ(d, v2, Trunc(v));
+  }
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestIntFromFloat(D d) {
+  // Integer positive
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 4), ConvertTo(d, Iota(df, 4.0f)));
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 4), NearestInt(Iota(df, 4.0f)));
+
+  // Integer negative
+  HWY_ASSERT_VEC_EQ(d, Iota(d, -32), ConvertTo(d, Iota(df, -32.0f)));
+  HWY_ASSERT_VEC_EQ(d, Iota(d, -32), NearestInt(Iota(df, -32.0f)));
+
+  // Above positive
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 2), ConvertTo(d, Iota(df, 2.001f)));
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 2), NearestInt(Iota(df, 2.001f)));
+
+  // Below positive
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 3), ConvertTo(d, Iota(df, 3.9999f)));
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 4), NearestInt(Iota(df, 3.9999f)));
+
+  // Above negative
+  HWY_ASSERT_VEC_EQ(d, Iota(d, -23), ConvertTo(d, Iota(df, -23.9999f)));
+  HWY_ASSERT_VEC_EQ(d, Iota(d, -24), NearestInt(Iota(df, -23.9999f)));
+
+  // Below negative
+  HWY_ASSERT_VEC_EQ(d, Iota(d, -24), ConvertTo(d, Iota(df, -24.001f)));
+  HWY_ASSERT_VEC_EQ(d, Iota(d, -24), NearestInt(Iota(df, -24.001f)));
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestFloatFromInt(D d) {
+  // Integer positive
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 4.0f), ConvertTo(d, Iota(di32, 4)));
+
+  // Integer negative
+  HWY_ASSERT_VEC_EQ(d, Iota(d, -32.0f), ConvertTo(d, Iota(di32, -32)));
+
+  // Above positive
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 2.0f), ConvertTo(d, Iota(di32, 2)));
+
+  // Below positive
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 4.0f), ConvertTo(d, Iota(di32, 4)));
+
+  // Above negative
+  HWY_ASSERT_VEC_EQ(d, Iota(d, -4.0f), ConvertTo(d, Iota(di32, -4)));
+
+  // Below negative
+  HWY_ASSERT_VEC_EQ(d, Iota(d, -2.0f), ConvertTo(d, Iota(di32, -2)));
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSumsOfU8(D d) {
+#if HWY_BITS != 0 || HWY_IDE
+  HWY_ALIGN uint8_t in_bytes[du8.N];
+  uint64_t sums[d.N] = {0};
+  for (size_t i = 0; i < du8.N; ++i) {
+    const size_t group = i / 8;
+    in_bytes[i] = static_cast<uint8_t>(2 * i + 1);
+    sums[group] += in_bytes[i];
+  }
+  const auto v = Load(du8, in_bytes);
+  HWY_ASSERT_VEC_EQ(d, sums, ext::SumsOfU8x8(v));
+#else
+  (void)d;
+#endif
+}
+
+HWY_NOINLINE HWY_ATTR void TestMinPos() {
+#if HWY_BITS == 128 || HWY_IDE
+  const HWY_CAPPED(uint16_t, 8) du16x8;
+  HWY_ALIGN uint16_t in_bytes[du16x8.N];
+
+  // Check for the minimum value in each position.
+  for (uint16_t ret_pos = 0; ret_pos < du16x8.N; ret_pos++) {
+    for (size_t i = 0; i < du16x8.N; ++i) {
+      // The minimum value is when i == 0, since i < d16.N and 3 is coprime
+      // with d16.N, therefore no other value of i has i * 3 % d16.N == 0.
+      in_bytes[(ret_pos + i) % du16x8.N] = 777U + i * 3 % du16x8.N;
+    }
+
+    const auto v = Load(du16x8, in_bytes);
+    auto ext_minpos_v = ext::minpos(v);
+
+    HWY_ASSERT_EQ(uint16_t(777), GetLane(ext_minpos_v));
+    ext_minpos_v = ShiftRightLanes<1>(ext_minpos_v);
+    HWY_ASSERT_EQ(ret_pos, GetLane(ext_minpos_v));
+  }
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestHorzSumT(D d) {
+  using T = typename D::T;
+
+  HWY_ALIGN T in_lanes[d.N];
+  double sum = 0.0;
+  for (size_t i = 0; i < d.N; ++i) {
+    in_lanes[i] = 1u << i;
+    sum += in_lanes[i];
+  }
+  const auto v = Load(d, in_lanes);
+  const auto expected = Set(d, T(sum));
+  HWY_ASSERT_VEC_EQ(d, expected, ext::SumOfLanes(v));
+}
+
+HWY_NOINLINE HWY_ATTR void TestHorzSum() {
+  // No u16.
+  TestHorzSumT(du32);
+#if HWY_HAS_INT64
+  TestHorzSumT(du64);
+#endif
+
+  // No i8/i16.
+  TestHorzSumT(di32);
+#if HWY_HAS_INT64
+  TestHorzSumT(di64);
+#endif
+
+  HWY_FOREACH_F(TestHorzSumT);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestAbsDiffT(D d) {
+  using T = typename D::T;
+
+  HWY_ALIGN T in_lanes_a[d.N];
+  HWY_ALIGN T in_lanes_b[d.N];
+  HWY_ALIGN T out_lanes[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    in_lanes_a[i] = (i ^ 1u) << i;
+    in_lanes_b[i] = i << i;
+    out_lanes[i] = fabsf(in_lanes_a[i] - in_lanes_b[i]);
+  }
+  const auto a = Load(d, in_lanes_a);
+  const auto b = Load(d, in_lanes_b);
+  const auto expected = Load(d, out_lanes);
+  HWY_ASSERT_VEC_EQ(d, expected, ext::AbsDiff(a, b));
+  HWY_ASSERT_VEC_EQ(d, expected, ext::AbsDiff(b, a));
+}
+
+HWY_NOINLINE HWY_ATTR void TestAbsDiff() { TestAbsDiffT(df); }
+
+HWY_NOINLINE HWY_ATTR void TestArithmetic() {
+  (void)dd;
+  (void)di64;
+  (void)du64;
+
+  HWY_FOREACH_UIF(TestPlusMinus);
+  TestSaturatingArithmetic();
+
+  TestShifts();
+  TestMinMax();
+  TestAverage();
+  TestAbs();
+  TestMul();
+  TestMulHigh();
+  TestMulEven();
+
+  HWY_FOREACH_F(TestMulAdd);
+  HWY_FOREACH_F(TestSquareRoot);
+  TestReciprocalSquareRoot(df);
+  HWY_FOREACH_F(TestRound);
+  TestIntFromFloat(di32);
+  TestFloatFromInt(df);
+
+  TestSumsOfU8(du64);
+  TestMinPos();
+  TestHorzSum();
+
+  TestAbsDiff();
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+
+// Instantiate for the current target.
+void ArithmeticTest::HWY_FUNC() { hwy::HWY_NAMESPACE::TestArithmetic(); }
diff --git a/third_party/highway/hwy/tests/compare_test.cc b/third_party/highway/hwy/tests/compare_test.cc
new file mode 100644
index 000000000..a1b48aec4
--- /dev/null
+++ b/third_party/highway/hwy/tests/compare_test.cc
@@ -0,0 +1,167 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/compare_test.cc"
+
+#include "hwy/tests/test_util.h"
+struct CompareTest {
+  HWY_DECLARE(void, ())
+};
+TEST(HwyCompareTest, Run) { hwy::RunTests<CompareTest>(); }
+
+#endif  // HWY_TARGET_INCLUDE
+#include "hwy/tests/test_target_util.h"
+
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+constexpr HWY_FULL(uint8_t) du8;
+constexpr HWY_FULL(uint16_t) du16;
+constexpr HWY_FULL(uint32_t) du32;
+constexpr HWY_FULL(uint64_t) du64;
+constexpr HWY_FULL(int8_t) di8;
+constexpr HWY_FULL(int16_t) di16;
+constexpr HWY_FULL(int32_t) di32;
+constexpr HWY_FULL(int64_t) di64;
+constexpr HWY_FULL(float) df;
+constexpr HWY_FULL(double) dd;
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestEquality(D d) {
+  const auto v2 = Iota(d, 2);
+  const auto v2b = Iota(d, 2);
+  const auto v3 = Iota(d, 3);
+
+  HWY_ASSERT_EQ(false, ext::AllTrue(v2 == v3));
+  HWY_ASSERT_EQ(true, ext::AllFalse(v2 == v3));
+  HWY_ASSERT_EQ(true, ext::AllTrue(v2 == v2b));
+  HWY_ASSERT_EQ(false, ext::AllFalse(v2 == v2b));
+}
+
+// Integer and floating-point.
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestInequality(D d) {
+  using T = typename D::T;
+  const auto v2 = Iota(d, 2);
+  const auto vn = Iota(d, -T(d.N));
+
+  HWY_ASSERT_EQ(true, ext::AllTrue(v2 > vn));
+  HWY_ASSERT_EQ(true, ext::AllTrue(vn < v2));
+  HWY_ASSERT_EQ(true, ext::AllFalse(v2 < vn));
+  HWY_ASSERT_EQ(true, ext::AllFalse(vn > v2));
+}
+
+// Returns "bits" after zeroing any upper bits that wouldn't be returned by
+// movemask for the given vector "D".
+template <class D>
+uint64_t ValidBits(D d, const uint64_t bits) {
+  constexpr size_t shift = 64 - d.N;  // 0..63 - avoids UB for d.N == 64.
+  static_assert(shift < 64, "d.N out of range");  // Silences clang-tidy.
+  // NOLINTNEXTLINE(clang-analyzer-core.UndefinedBinaryOperatorResult)
+  return (bits << shift) >> shift;
+}
+
+HWY_NOINLINE HWY_ATTR void TestMovemask() {
+  HWY_ALIGN constexpr uint8_t kBytes[kTestMaxVectorSize] = {
+      0x80, 0xFF, 0x7F, 0x00, 0x01, 0x10, 0x20, 0x40, 0x80, 0x02, 0x04,
+      0x08, 0xC0, 0xC1, 0xFE, 0x0F, 0x0F, 0xFE, 0xC1, 0xC0, 0x08, 0x04,
+      0x02, 0x80, 0x40, 0x20, 0x10, 0x01, 0x00, 0x7F, 0xFF, 0x80, 0x0F,
+      0xFE, 0xC1, 0xC0, 0x08, 0x04, 0x02, 0x80, 0x40, 0x20, 0x10, 0x01,
+      0x00, 0x7F, 0xFF, 0x80, 0x80, 0xFF, 0x7F, 0x00, 0x01, 0x10, 0x20,
+      0x40, 0x80, 0x02, 0x04, 0x08, 0xC0, 0xC1, 0xFE, 0x0F};
+  const auto bytes = Load(du8, kBytes);
+  HWY_ASSERT_EQ(ValidBits(du8, 0x7103C08EC08E7103ull), ext::movemask(bytes));
+
+  HWY_ALIGN const float kLanesF[kTestMaxVectorSize / sizeof(float)] = {
+      -1.0f,  1E30f, -0.0f, 1E-30f, 0.0f,  -0.0f, 1E30f, -1.0f,
+      1E-30f, -0.0f, 1E30f, -1.0f,  -1.0f, 1E30f, -0.0f, 1E-30f};
+  const auto vf = Load(df, kLanesF);
+  HWY_ASSERT_EQ(ValidBits(df, 0x5aa5), ext::movemask(vf));
+
+#if HWY_HAS_DOUBLE
+  HWY_ALIGN const double kLanesD[kTestMaxVectorSize / sizeof(double)] = {
+      1E300, -1E-300, -0.0, 1E-10, -0.0, 0.0, 1E300, -1E-10};
+  const auto vd = Load(dd, kLanesD);
+  HWY_ASSERT_EQ(ValidBits(dd, 0x96), ext::movemask(vd));
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestAllTrueFalse(D d) {
+  using T = typename D::T;
+  const auto zero = Zero(d);
+  const T max = LimitsMax<T>();
+  const T min_nonzero = LimitsMin<T>() + 1;
+
+  auto v = zero;
+  HWY_ALIGN T lanes[d.N] = {};  // Initialized for clang-analyzer.
+  Store(v, d, lanes);
+  HWY_ASSERT_EQ(true, ext::AllTrue(v == zero));
+  HWY_ASSERT_EQ(false, ext::AllFalse(v == zero));
+
+  // Set each lane to nonzero and back to zero
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = max;
+    v = Load(d, lanes);
+    HWY_ASSERT_EQ(false, ext::AllTrue(v == zero));
+#if HWY_BITS != 0
+    HWY_ASSERT_EQ(false, ext::AllFalse(v == zero));
+#endif
+
+    lanes[i] = min_nonzero;
+    v = Load(d, lanes);
+    HWY_ASSERT_EQ(false, ext::AllTrue(v == zero));
+#if HWY_BITS != 0
+    HWY_ASSERT_EQ(false, ext::AllFalse(v == zero));
+#endif
+
+    // Reset to all zero
+    lanes[i] = T(0);
+    v = Load(d, lanes);
+    HWY_ASSERT_EQ(true, ext::AllTrue(v == zero));
+    HWY_ASSERT_EQ(false, ext::AllFalse(v == zero));
+  }
+}
+
+HWY_NOINLINE HWY_ATTR void TestCompare() {
+  (void)dd;
+  (void)di64;
+  (void)du64;
+
+  HWY_FOREACH_UIF(TestEquality);
+
+  HWY_FOREACH_F(TestInequality);
+  TestInequality(di8);
+  TestInequality(di16);
+  TestInequality(di32);
+#if HWY_HAS_CMP64
+  TestInequality(di64);
+#endif
+
+  TestMovemask();
+
+  HWY_FOREACH_U(TestAllTrueFalse);
+  HWY_FOREACH_I(TestAllTrueFalse);
+  // No float.
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+
+// Instantiate for the current target.
+void CompareTest::HWY_FUNC() { hwy::HWY_NAMESPACE::TestCompare(); }
diff --git a/third_party/highway/hwy/tests/convert_test.cc b/third_party/highway/hwy/tests/convert_test.cc
new file mode 100644
index 000000000..6a63de76a
--- /dev/null
+++ b/third_party/highway/hwy/tests/convert_test.cc
@@ -0,0 +1,238 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/convert_test.cc"
+
+#include "hwy/tests/test_util.h"
+struct ConvertTest {
+  HWY_DECLARE(void, ())
+};
+TEST(HwyConvertTest, Run) { hwy::RunTests<ConvertTest>(); }
+
+#endif  // HWY_TARGET_INCLUDE
+#include "hwy/tests/test_target_util.h"
+
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+constexpr HWY_FULL(uint8_t) du8;
+constexpr HWY_FULL(uint32_t) du32;
+
+#if HWY_BITS != 0 || HWY_IDE
+constexpr HWY_FULL(uint16_t) du16;
+constexpr HWY_FULL(uint64_t) du64;
+constexpr HWY_FULL(int8_t) di8;
+constexpr HWY_FULL(int16_t) di16;
+constexpr HWY_FULL(int32_t) di32;
+constexpr HWY_FULL(int64_t) di64;
+constexpr HWY_FULL(float) df;
+#endif
+
+template <typename FromT, typename ToT>
+HWY_NOINLINE HWY_ATTR void TestCastFromTo() {
+  const HWY_FULL(FromT) dfrom;
+  const HWY_FULL(ToT) dto;
+  const auto vf = Iota(dfrom, 1);
+  const auto vt = BitCast(dto, vf);
+  static_assert(sizeof(vf) == sizeof(vt), "Cast must return same size");
+  // Must return the same bits
+  HWY_ALIGN FromT from_lanes[dfrom.N];
+  HWY_ALIGN ToT to_lanes[dto.N];
+  Store(vf, dfrom, from_lanes);
+  Store(vt, dto, to_lanes);
+  HWY_ASSERT_EQ(true, BytesEqual(from_lanes, to_lanes, sizeof(vf)));
+}
+
+// From D to all types.
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestCastFrom(D /*d*/) {
+  using FromT = typename D::T;
+  TestCastFromTo<FromT, uint8_t>();
+  TestCastFromTo<FromT, uint16_t>();
+  TestCastFromTo<FromT, uint32_t>();
+#if HWY_HAS_INT64
+  TestCastFromTo<FromT, uint64_t>();
+#endif
+  TestCastFromTo<FromT, int8_t>();
+  TestCastFromTo<FromT, int16_t>();
+  TestCastFromTo<FromT, int32_t>();
+#if HWY_HAS_INT64
+  TestCastFromTo<FromT, int64_t>();
+#endif
+  TestCastFromTo<FromT, float>();
+#if HWY_HAS_DOUBLE
+  TestCastFromTo<FromT, double>();
+#endif
+}
+
+HWY_NOINLINE HWY_ATTR void TestCast() {
+#if HWY_BITS == 0 || HWY_IDE
+  // Promotion is undefined => only test same-sized types.
+  TestCastFromTo<uint8_t, uint8_t>();
+  TestCastFromTo<int8_t, int8_t>();
+  TestCastFromTo<uint8_t, int8_t>();
+  TestCastFromTo<int8_t, uint8_t>();
+
+  TestCastFromTo<uint16_t, uint16_t>();
+  TestCastFromTo<int16_t, int16_t>();
+  TestCastFromTo<uint16_t, int16_t>();
+  TestCastFromTo<int16_t, uint16_t>();
+
+  TestCastFromTo<uint32_t, uint32_t>();
+  TestCastFromTo<int32_t, int32_t>();
+  TestCastFromTo<uint32_t, int32_t>();
+  TestCastFromTo<int32_t, uint32_t>();
+  TestCastFromTo<uint32_t, float>();
+  TestCastFromTo<int32_t, float>();
+  TestCastFromTo<float, float>();
+  TestCastFromTo<float, uint32_t>();
+  TestCastFromTo<float, int32_t>();
+
+#if HWY_HAS_INT64
+  TestCastFromTo<uint64_t, uint64_t>();
+  TestCastFromTo<int64_t, int64_t>();
+  TestCastFromTo<uint64_t, int64_t>();
+  TestCastFromTo<int64_t, uint64_t>();
+#endif
+#if HWY_HAS_DOUBLE
+  TestCastFromTo<uint64_t, double>();
+  TestCastFromTo<int64_t, double>();
+  TestCastFromTo<double, double>();
+  TestCastFromTo<double, uint64_t>();
+  TestCastFromTo<double, int64_t>();
+#endif
+#else
+  HWY_FOREACH_U(TestCastFrom);
+  HWY_FOREACH_I(TestCastFrom);
+#endif
+
+  // Float <-> u/i32
+  TestCastFromTo<uint32_t, float>();
+  TestCastFromTo<int32_t, float>();
+  TestCastFromTo<float, uint32_t>();
+  TestCastFromTo<float, int32_t>();
+}
+
+template <typename FromT, typename ToT>
+HWY_NOINLINE HWY_ATTR void TestPromoteT() {
+  constexpr size_t N = HWY_LANES_OR_0(ToT);
+  const Desc<FromT, N> from_d;
+  const Desc<ToT, N> to_d;
+
+  const auto from = Iota(from_d, 1);
+  const auto from_n1 = Set(from_d, FromT(-1));
+  const auto from_min = Set(from_d, LimitsMin<FromT>());
+  const auto from_max = Set(from_d, LimitsMax<FromT>());
+  const auto to = Iota(to_d, 1);
+  const auto to_n1 = Set(to_d, ToT(FromT(-1)));
+  const auto to_min = Set(to_d, ToT(LimitsMin<FromT>()));
+  const auto to_max = Set(to_d, ToT(LimitsMax<FromT>()));
+  HWY_ASSERT_VEC_EQ(to_d, to, ConvertTo(to_d, from));
+  HWY_ASSERT_VEC_EQ(to_d, to_n1, ConvertTo(to_d, from_n1));
+  HWY_ASSERT_VEC_EQ(to_d, to_min, ConvertTo(to_d, from_min));
+  HWY_ASSERT_VEC_EQ(to_d, to_max, ConvertTo(to_d, from_max));
+}
+
+template <typename FromT, typename ToT>
+HWY_NOINLINE HWY_ATTR void TestDemoteT() {
+  constexpr size_t N = HWY_LANES_OR_0(FromT);
+  const Desc<FromT, N> from_d;
+  const Desc<ToT, N> to_d;
+
+  const auto from = Iota(from_d, 1);
+  const auto from_n1 = Set(from_d, FromT(ToT(-1)));
+  const auto from_min = Set(from_d, FromT(LimitsMin<ToT>()));
+  const auto from_max = Set(from_d, FromT(LimitsMax<ToT>()));
+  const auto to = Iota(to_d, 1);
+  const auto to_n1 = Set(to_d, ToT(-1));
+  const auto to_min = Set(to_d, LimitsMin<ToT>());
+  const auto to_max = Set(to_d, LimitsMax<ToT>());
+  HWY_ASSERT_VEC_EQ(to_d, to, ConvertTo(to_d, from));
+  HWY_ASSERT_VEC_EQ(to_d, to_n1, ConvertTo(to_d, from_n1));
+  HWY_ASSERT_VEC_EQ(to_d, to_min, ConvertTo(to_d, from_min));
+  HWY_ASSERT_VEC_EQ(to_d, to_max, ConvertTo(to_d, from_max));
+}
+
+template <typename FromT, typename ToT>
+HWY_NOINLINE HWY_ATTR void TestDupPromoteT() {
+  constexpr size_t N = HWY_LANES_OR_0(ToT);
+  const Desc<FromT, N> from_d;
+  const Desc<ToT, N> to_d;
+
+  const auto from = Iota(from_d, 1);
+  const auto from_n1 = Set(from_d, FromT(-1));
+  const auto from_min = Set(from_d, LimitsMin<FromT>());
+  const auto from_max = Set(from_d, LimitsMax<FromT>());
+  const auto to = Iota(to_d, 1);
+  const auto to_n1 = Set(to_d, ToT(FromT(-1)));
+  const auto to_min = Set(to_d, ToT(LimitsMin<FromT>()));
+  const auto to_max = Set(to_d, ToT(LimitsMax<FromT>()));
+  HWY_ASSERT_VEC_EQ(to_d, to, ConvertTo(to_d, from));
+  HWY_ASSERT_VEC_EQ(to_d, to_n1, ConvertTo(to_d, from_n1));
+  HWY_ASSERT_VEC_EQ(to_d, to_min, ConvertTo(to_d, from_min));
+  HWY_ASSERT_VEC_EQ(to_d, to_max, ConvertTo(to_d, from_max));
+}
+
+HWY_NOINLINE HWY_ATTR void TestConvert() {
+#if HWY_BITS != 0 || HWY_IDE
+  (void)di64;
+  (void)du64;
+#endif
+
+  TestCast();
+
+  HWY_ALIGN uint8_t lanes8[du8.N];
+  Store(Iota(du8, 0), du8, lanes8);
+  HWY_ASSERT_VEC_EQ(du32, Iota(du32, 0), U32FromU8(LoadDup128(du8, lanes8)));
+  Store(Iota(du8, 0x7F), du8, lanes8);
+  HWY_ASSERT_VEC_EQ(du32, Iota(du32, 0x7F), U32FromU8(LoadDup128(du8, lanes8)));
+  const HWY_CAPPED(uint8_t, du32.N) p8;
+  HWY_ASSERT_VEC_EQ(p8, Iota(p8, 0), U8FromU32(Iota(du32, 0)));
+  HWY_ASSERT_VEC_EQ(p8, Iota(p8, 0x7F), U8FromU32(Iota(du32, 0x7F)));
+
+  // Promote: no u64,i64
+#if HWY_HAS_DOUBLE
+  TestPromoteT<float, double>();
+#endif
+  TestPromoteT<uint8_t, int16_t>();
+  TestPromoteT<uint8_t, int32_t>();
+  TestPromoteT<uint16_t, int32_t>();
+  TestPromoteT<int8_t, int16_t>();
+  TestPromoteT<int8_t, int32_t>();
+  TestPromoteT<int16_t, int32_t>();
+#if HWY_HAS_INT64
+  TestPromoteT<uint32_t, uint64_t>();
+  TestPromoteT<int32_t, int64_t>();
+#endif
+
+  // Demote
+  TestDemoteT<int16_t, int8_t>();
+  TestDemoteT<int32_t, int8_t>();
+  TestDemoteT<int32_t, int16_t>();
+  TestDemoteT<int16_t, uint8_t>();
+  TestDemoteT<int32_t, uint8_t>();
+  TestDemoteT<int32_t, uint16_t>();
+
+  TestDupPromoteT<uint8_t, uint32_t>();
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+
+// Instantiate for the current target.
+void ConvertTest::HWY_FUNC() { hwy::HWY_NAMESPACE::TestConvert(); }
diff --git a/third_party/highway/hwy/tests/hwy_test.cc b/third_party/highway/hwy/tests/hwy_test.cc
new file mode 100644
index 000000000..69009f99f
--- /dev/null
+++ b/third_party/highway/hwy/tests/hwy_test.cc
@@ -0,0 +1,455 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/hwy_test.cc"
+
+#include "hwy/tests/test_util.h"
+struct HwyTest {
+  HWY_DECLARE(void, ())
+};
+TEST(HwyTest, Run) { hwy::RunTests<HwyTest>(); }
+
+#endif  // HWY_TARGET_INCLUDE
+#include "hwy/tests/test_target_util.h"
+
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+constexpr HWY_FULL(uint8_t) du8;
+constexpr HWY_FULL(uint16_t) du16;
+constexpr HWY_FULL(uint32_t) du32;
+constexpr HWY_FULL(uint64_t) du64;
+constexpr HWY_FULL(int8_t) di8;
+constexpr HWY_FULL(int16_t) di16;
+constexpr HWY_FULL(int32_t) di32;
+constexpr HWY_FULL(int64_t) di64;
+constexpr HWY_FULL(float) df;
+constexpr HWY_FULL(double) dd;
+
+namespace examples {
+
+namespace {
+HWY_NOINLINE HWY_ATTR void FloorLog2(const uint8_t* HWY_RESTRICT values,
+                                     uint8_t* HWY_RESTRICT log2) {
+  // Descriptors for all required data types:
+  const HWY_FULL(int32_t) d32;
+  const HWY_FULL(float) df;
+  const HWY_CAPPED(uint8_t, d32.N) d8;
+
+  const auto u8 = Load(d8, values);
+  const auto bits = BitCast(d32, ConvertTo(df, ConvertTo(d32, u8)));
+  const auto exponent = ShiftRight<23>(bits) - Set(d32, 127);
+  Store(ConvertTo(d8, exponent), d8, log2);
+}
+}  // namespace
+
+HWY_NOINLINE HWY_ATTR void TestFloorLog2() {
+  const size_t kStep = HWY_FULL(int32_t)::N;
+  const size_t kBytes = 32;
+  static_assert(kBytes % kStep == 0, "Must be a multiple of kStep");
+
+  uint8_t in[kBytes];
+  uint8_t expected[kBytes];
+  RandomState rng{1234};
+  for (size_t i = 0; i < kBytes; ++i) {
+    expected[i] = Random32(&rng) & 7;
+    in[i] = 1u << expected[i];
+  }
+  uint8_t out[32];
+  for (size_t i = 0; i < kBytes; i += kStep) {
+    FloorLog2(in + i, out + i);
+  }
+  int sum = 0;
+  for (size_t i = 0; i < kBytes; ++i) {
+    HWY_ASSERT_EQ(expected[i], out[i]);
+    sum += out[i];
+  }
+  PreventElision(sum);
+}
+
+HWY_NOINLINE HWY_ATTR void Copy(const uint8_t* HWY_RESTRICT from,
+                                const size_t size, uint8_t* HWY_RESTRICT to) {
+  // Width-agnostic (library-specified N)
+  const HWY_FULL(uint8_t) d;
+  const Scalar<uint8_t> ds;
+  size_t i = 0;
+  for (; i + d.N <= size; i += d.N) {
+    const auto bytes = Load(d, from + i);
+    Store(bytes, d, to + i);
+  }
+
+  for (; i < size; ++i) {
+    // (Same loop body as above, could factor into a shared template)
+    const auto bytes = Load(ds, from + i);
+    Store(bytes, ds, to + i);
+  }
+}
+
+HWY_NOINLINE HWY_ATTR void TestCopy() {
+  RandomState rng{1234};
+  const size_t kSize = 34;
+  HWY_ALIGN uint8_t from[kSize];
+  for (unsigned char& i : from) {
+    i = Random32(&rng) & 0xFF;
+  }
+  HWY_ALIGN uint8_t to[kSize];
+  Copy(from, kSize, to);
+  for (size_t i = 0; i < kSize; ++i) {
+    HWY_ASSERT_EQ(from[i], to[i]);
+  }
+}
+
+template <typename T>
+HWY_NOINLINE HWY_ATTR void MulAdd(const T* HWY_RESTRICT mul_array,
+                                  const T* HWY_RESTRICT add_array,
+                                  const size_t size, T* HWY_RESTRICT x_array) {
+  // Type-agnostic (caller-specified lane type) and width-agnostic (uses
+  // best available instruction set).
+  const HWY_FULL(T) d;
+  for (size_t i = 0; i < size; i += d.N) {
+    const auto mul = Load(d, mul_array + i);
+    const auto add = Load(d, add_array + i);
+    auto x = Load(d, x_array + i);
+    x = MulAdd(mul, x, add);
+    Store(x, d, x_array + i);
+  }
+}
+
+template <typename T>
+HWY_NOINLINE HWY_ATTR T SumMulAdd() {
+  RandomState rng{1234};
+  const size_t kSize = 64;
+  HWY_ALIGN T mul[kSize];
+  HWY_ALIGN T x[kSize];
+  HWY_ALIGN T add[kSize];
+  for (size_t i = 0; i < kSize; ++i) {
+    mul[i] = Random32(&rng) & 0xF;
+    x[i] = Random32(&rng) & 0xFF;
+    add[i] = Random32(&rng) & 0xFF;
+  }
+  MulAdd(mul, add, kSize, x);
+  double sum = 0.0;
+  for (auto xi : x) {
+    sum += xi;
+  }
+  return sum;
+}
+
+HWY_NOINLINE HWY_ATTR void TestExamples() {
+  TestFloorLog2();
+  TestCopy();
+
+  HWY_ASSERT_EQ(78944.0f, SumMulAdd<float>());
+#if HWY_HAS_DOUBLE
+  HWY_ASSERT_EQ(78944.0, SumMulAdd<double>());
+#endif
+}
+
+}  // namespace examples
+
+namespace basic {
+
+// util.h
+
+HWY_NOINLINE HWY_ATTR void TestLimits() {
+  HWY_ASSERT_EQ(uint8_t(0), LimitsMin<uint8_t>());
+  HWY_ASSERT_EQ(uint16_t(0), LimitsMin<uint16_t>());
+  HWY_ASSERT_EQ(uint32_t(0), LimitsMin<uint32_t>());
+  HWY_ASSERT_EQ(uint64_t(0), LimitsMin<uint64_t>());
+
+  HWY_ASSERT_EQ(int8_t(-128), LimitsMin<int8_t>());
+  HWY_ASSERT_EQ(int16_t(-32768), LimitsMin<int16_t>());
+  HWY_ASSERT_EQ(int32_t(0x80000000u), LimitsMin<int32_t>());
+  HWY_ASSERT_EQ(int64_t(0x8000000000000000ull), LimitsMin<int64_t>());
+
+  HWY_ASSERT_EQ(uint8_t(0xFF), LimitsMax<uint8_t>());
+  HWY_ASSERT_EQ(uint16_t(0xFFFF), LimitsMax<uint16_t>());
+  HWY_ASSERT_EQ(uint32_t(0xFFFFFFFFu), LimitsMax<uint32_t>());
+  HWY_ASSERT_EQ(uint64_t(0xFFFFFFFFFFFFFFFFull), LimitsMax<uint64_t>());
+
+  HWY_ASSERT_EQ(int8_t(0x7F), LimitsMax<int8_t>());
+  HWY_ASSERT_EQ(int16_t(0x7FFF), LimitsMax<int16_t>());
+  HWY_ASSERT_EQ(int32_t(0x7FFFFFFFu), LimitsMax<int32_t>());
+  HWY_ASSERT_EQ(int64_t(0x7FFFFFFFFFFFFFFFull), LimitsMax<int64_t>());
+}
+
+// Test the ToString used to output test failures
+
+HWY_NOINLINE HWY_ATTR void TestToString() {
+  char buf[32];
+  const char* end;
+
+  end = ToString(int64_t(0), buf);
+  HWY_ASSERT_EQ('0', end[-1]);
+  HWY_ASSERT_EQ('\0', end[0]);
+
+  end = ToString(int64_t(3), buf);
+  HWY_ASSERT_EQ('3', end[-1]);
+  HWY_ASSERT_EQ('\0', end[0]);
+
+  end = ToString(int64_t(-1), buf);
+  HWY_ASSERT_EQ('-', end[-2]);
+  HWY_ASSERT_EQ('1', end[-1]);
+  HWY_ASSERT_EQ('\0', end[0]);
+
+  ToString(0x7FFFFFFFFFFFFFFFLL, buf);
+  HWY_ASSERT_EQ(true, StringsEqual("9223372036854775807", buf));
+
+  ToString(int64_t(0x8000000000000000ULL), buf);
+  HWY_ASSERT_EQ(true, StringsEqual("-9223372036854775808", buf));
+
+  ToString(0.0, buf);
+  HWY_ASSERT_EQ(true, StringsEqual("0.0", buf));
+  ToString(4.0, buf);
+  HWY_ASSERT_EQ(true, StringsEqual("4.0", buf));
+  ToString(-1.0, buf);
+  HWY_ASSERT_EQ(true, StringsEqual("-1.0", buf));
+  ToString(-1.25, buf);
+  HWY_ASSERT_STRING_EQ("-1.2500000000000000", const_cast<const char*>(buf));
+  ToString(2.125f, buf);
+  HWY_ASSERT_STRING_EQ("2.12500000", const_cast<const char*>(buf));
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestIsUnsigned(D /*d*/) {
+  using T = typename D::T;
+  static_assert(!IsFloat<T>(), "Expected !IsFloat");
+  static_assert(!IsSigned<T>(), "Expected !IsSigned");
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestIsSigned(D /*d*/) {
+  using T = typename D::T;
+  static_assert(!IsFloat<T>(), "Expected !IsFloat");
+  static_assert(IsSigned<T>(), "Expected IsSigned");
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestIsFloat(D /*d*/) {
+  using T = typename D::T;
+  static_assert(IsFloat<T>(), "Expected IsFloat");
+  static_assert(IsSigned<T>(), "Floats are also considered signed");
+}
+
+HWY_NOINLINE HWY_ATTR void TestType() {
+  HWY_FOREACH_U(TestIsUnsigned);
+  HWY_FOREACH_I(TestIsSigned);
+  HWY_FOREACH_F(TestIsFloat);
+}
+
+// Ensures wraparound (mod 2^bits)
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestOverflowT(D d) {
+  using T = typename D::T;
+  const auto v1 = Set(d, T(1));
+  const auto vmax = Set(d, LimitsMax<T>());
+  const auto vmin = Set(d, LimitsMin<T>());
+  // Unsigned underflow / negative -> positive
+  HWY_ASSERT_VEC_EQ(d, vmax, vmin - v1);
+  // Unsigned overflow / positive -> negative
+  HWY_ASSERT_VEC_EQ(d, vmin, vmax + v1);
+}
+
+HWY_NOINLINE HWY_ATTR void TestOverflow() {
+  HWY_FOREACH_U(TestOverflowT);
+  HWY_FOREACH_I(TestOverflowT);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestName(D d) {
+  using T = typename D::T;
+  char expected[7] = {IsFloat<T>() ? 'f' : (IsSigned<T>() ? 'i' : 'u')};
+  char* end = ToString(sizeof(T) * 8, expected + 1);
+  if (D::N != 1) {
+    *end++ = 'x';
+    end = ToString(d.N, end);
+  }
+  if (!StringsEqual(expected, TypeName<T, D::N>())) {
+    NotifyFailure(__FILE__, __LINE__, HWY_BITS, expected, -1, expected,
+                  TypeName<T, D::N>());
+  }
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSet(D d) {
+  using T = typename D::T;
+
+  // Zero
+  const auto v0 = Zero(d);
+  HWY_ALIGN T expected[d.N] = {};  // zero-initialized.
+  HWY_ASSERT_VEC_EQ(d, expected, v0);
+
+  // Set
+  const auto v2 = Set(d, T(2));
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = 2;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, v2);
+
+  // iota
+  const auto vi = Iota(d, T(5));
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = 5 + i;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, vi);
+
+  // undefined
+  const auto vu = Undefined(d);
+  Store(vu, d, expected);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestCopyAndAssign(D d) {
+  using V = VT<D>;
+
+  // copy V
+  const auto v3 = Iota(d, 3);
+  V v3b(v3);
+  HWY_ASSERT_VEC_EQ(d, v3, v3b);
+
+  // assign V
+  V v3c;
+  v3c = v3;
+  HWY_ASSERT_VEC_EQ(d, v3, v3c);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestHalf(D d) {
+#if HWY_BITS != 0 || HWY_IDE
+  using T = typename D::T;
+  size_t i;
+  constexpr size_t N2 = (d.N + 1) / 2;
+  const Desc<T, N2> d2;
+
+  const auto v = Iota(d, 1);
+  HWY_ALIGN T lanes[d.N] = {0};
+
+  Store(LowerHalf(v), d2, lanes);
+  i = 0;
+  for (; i < N2; ++i) {
+    HWY_ASSERT_EQ(T(1 + i), lanes[i]);
+  }
+  // Other half remains unchanged
+  for (; i < d.N; ++i) {
+    HWY_ASSERT_EQ(T(0), lanes[i]);
+  }
+  Store(LowerHalf(v), d2, lanes);  // Also test the wrapper
+  i = 0;
+  for (; i < N2; ++i) {
+    HWY_ASSERT_EQ(T(1 + i), lanes[i]);
+  }
+  // Other half remains unchanged
+  for (; i < d.N; ++i) {
+    HWY_ASSERT_EQ(T(0), lanes[i]);
+  }
+
+  Store(UpperHalf(v), d2, lanes);
+  i = 0;
+  for (; i < N2; ++i) {
+    HWY_ASSERT_EQ(T(N2 + 1 + i), lanes[i]);
+  }
+  // Other half remains unchanged
+  for (; i < d.N; ++i) {
+    HWY_ASSERT_EQ(T(0), lanes[i]);
+  }
+  Store(UpperHalf(v), d2, lanes);  // Also test the wrapper
+  i = 0;
+  for (; i < N2; ++i) {
+    HWY_ASSERT_EQ(T(N2 + 1 + i), lanes[i]);
+  }
+  // Other half remains unchanged
+  for (; i < d.N; ++i) {
+    HWY_ASSERT_EQ(T(0), lanes[i]);
+  }
+
+  // Ensure lanes are contiguous
+  const auto vi = Iota(d2, 1);
+  Store(vi, d2, lanes);
+  for (size_t i = 1; i < N2; ++i) {
+    HWY_ASSERT_EQ(T(lanes[i - 1] + 1), lanes[i]);
+  }
+#else
+  (void)d;
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestQuarterT(D d) {
+#if HWY_BITS != 0 || HWY_IDE
+  using T = typename D::T;
+  constexpr size_t N4 = (d.N + 3) / 4;
+  const HWY_CAPPED(T, N4) d4;
+
+  const auto v = Iota(d, 1);
+  HWY_ALIGN T lanes[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = 123;
+  }
+  const auto lo = LowerHalf(LowerHalf(v));
+  Store(lo, d4, lanes);
+  size_t i = 0;
+  for (; i < N4; ++i) {
+    HWY_ASSERT_EQ(T(i + 1), lanes[i]);
+  }
+  // Other lanes remain unchanged
+  for (; i < d.N; ++i) {
+    HWY_ASSERT_EQ(T(123), lanes[i]);
+  }
+#else
+  (void)d;
+#endif
+}
+
+HWY_NOINLINE HWY_ATTR void TestQuarter() {
+  TestQuarterT(du8);
+  TestQuarterT(du16);
+  TestQuarterT(du32);
+  TestQuarterT(di8);
+  TestQuarterT(di16);
+  TestQuarterT(di32);
+  TestQuarterT(df);
+}
+
+HWY_NOINLINE HWY_ATTR void TestBasic() {
+  (void)dd;
+  (void)di64;
+  (void)du64;
+
+  TestLimits();
+  TestToString();
+  TestType();
+  HWY_FOREACH_UIF(TestName);
+  TestOverflow();
+  HWY_FOREACH_UIF(TestSet);
+  HWY_FOREACH_UIF(TestCopyAndAssign);
+  HWY_FOREACH_UIF(TestHalf);
+  TestQuarter();
+}
+
+}  // namespace basic
+
+HWY_NOINLINE HWY_ATTR void TestHwy() {
+  examples::TestExamples();
+  basic::TestBasic();
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+
+// Instantiate for the current target.
+void HwyTest::HWY_FUNC() { hwy::HWY_NAMESPACE::TestHwy(); }
diff --git a/third_party/highway/hwy/tests/logical_test.cc b/third_party/highway/hwy/tests/logical_test.cc
new file mode 100644
index 000000000..953872c69
--- /dev/null
+++ b/third_party/highway/hwy/tests/logical_test.cc
@@ -0,0 +1,187 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/logical_test.cc"
+
+#include "hwy/tests/test_util.h"
+struct LogicalTest {
+  HWY_DECLARE(void, ())
+};
+TEST(HwyLogicalTest, Run) { hwy::RunTests<LogicalTest>(); }
+
+#endif  // HWY_TARGET_INCLUDE
+#include "hwy/tests/test_target_util.h"
+
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+constexpr HWY_FULL(uint8_t) du8;
+constexpr HWY_FULL(uint16_t) du16;
+constexpr HWY_FULL(uint32_t) du32;
+constexpr HWY_FULL(uint64_t) du64;
+constexpr HWY_FULL(int8_t) di8;
+constexpr HWY_FULL(int16_t) di16;
+constexpr HWY_FULL(int32_t) di32;
+constexpr HWY_FULL(int64_t) di64;
+constexpr HWY_FULL(float) df;
+constexpr HWY_FULL(double) dd;
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestLogicalT(D d) {
+  const auto v0 = Zero(d);
+  const auto vi = Iota(d, 0);
+
+  HWY_ASSERT_VEC_EQ(d, v0, v0 & vi);
+  HWY_ASSERT_VEC_EQ(d, v0, vi & v0);
+  HWY_ASSERT_VEC_EQ(d, vi, vi & vi);
+
+  HWY_ASSERT_VEC_EQ(d, vi, v0 | vi);
+  HWY_ASSERT_VEC_EQ(d, vi, vi | v0);
+  HWY_ASSERT_VEC_EQ(d, vi, vi | vi);
+
+  HWY_ASSERT_VEC_EQ(d, vi, v0 ^ vi);
+  HWY_ASSERT_VEC_EQ(d, vi, vi ^ v0);
+  HWY_ASSERT_VEC_EQ(d, v0, vi ^ vi);
+
+  HWY_ASSERT_VEC_EQ(d, vi, AndNot(v0, vi));
+  HWY_ASSERT_VEC_EQ(d, v0, AndNot(vi, v0));
+  HWY_ASSERT_VEC_EQ(d, v0, AndNot(vi, vi));
+
+  auto v = vi;
+  v &= vi;
+  HWY_ASSERT_VEC_EQ(d, vi, v);
+  v &= v0;
+  HWY_ASSERT_VEC_EQ(d, v0, v);
+
+  v |= vi;
+  HWY_ASSERT_VEC_EQ(d, vi, v);
+  v |= v0;
+  HWY_ASSERT_VEC_EQ(d, vi, v);
+
+  v ^= vi;
+  HWY_ASSERT_VEC_EQ(d, v0, v);
+  v ^= v0;
+  HWY_ASSERT_VEC_EQ(d, v0, v);
+}
+
+// Vec <-> Mask, IfThen*
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestIfThenElse(D d) {
+  using T = typename D::T;
+  RandomState rng{1234};
+  const T no(0);
+  T yes;
+  memset(&yes, 0xFF, sizeof(yes));
+
+  HWY_ALIGN T in1[d.N] = {};         // Initialized for clang-analyzer.
+  HWY_ALIGN T in2[d.N] = {};         // Initialized for clang-analyzer.
+  HWY_ALIGN T mask_lanes[d.N] = {};  // Initialized for clang-analyzer.
+  for (size_t i = 0; i < d.N; ++i) {
+    in1[i] = int32_t(Random32(&rng));
+    in2[i] = int32_t(Random32(&rng));
+    mask_lanes[i] = (Random32(&rng) & 1024) ? no : yes;
+  }
+
+  const auto vec = Load(d, mask_lanes);
+  const auto mask = MaskFromVec(vec);
+  HWY_ASSERT_VEC_EQ(d, vec, VecFromMask(mask));
+
+  HWY_ALIGN T out_lanes1[d.N];
+  HWY_ALIGN T out_lanes2[d.N];
+  HWY_ALIGN T out_lanes3[d.N];
+  Store(IfThenElse(mask, Load(d, in1), Load(d, in2)), d, out_lanes1);
+  Store(IfThenElseZero(mask, Load(d, in1)), d, out_lanes2);
+  Store(IfThenZeroElse(mask, Load(d, in2)), d, out_lanes3);
+  for (size_t i = 0; i < d.N; ++i) {
+    // Cannot reliably compare against yes (NaN).
+    HWY_ASSERT_EQ((mask_lanes[i] == no) ? in2[i] : in1[i], out_lanes1[i]);
+    HWY_ASSERT_EQ((mask_lanes[i] == no) ? no : in1[i], out_lanes2[i]);
+    HWY_ASSERT_EQ((mask_lanes[i] == no) ? in2[i] : no, out_lanes3[i]);
+  }
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestTestBit(D d) {
+  using T = typename D::T;
+  const size_t kNumBits = sizeof(T) * 8;
+  for (size_t i = 0; i < kNumBits; ++i) {
+    const auto bit1 = Set(d, 1ull << i);
+    const auto bit2 = Set(d, 1ull << ((i + 1) % kNumBits));
+    const auto bit3 = Set(d, 1ull << ((i + 2) % kNumBits));
+    const auto bits12 = bit1 | bit2;
+    const auto bits23 = bit2 | bit3;
+    HWY_ASSERT_EQ(true, ext::AllTrue(TestBit(bit1, bit1)));
+    HWY_ASSERT_EQ(true, ext::AllTrue(TestBit(bits12, bit1)));
+    HWY_ASSERT_EQ(true, ext::AllTrue(TestBit(bits12, bit2)));
+
+    HWY_ASSERT_EQ(true, ext::AllFalse(TestBit(bits12, bit3)));
+    HWY_ASSERT_EQ(true, ext::AllFalse(TestBit(bits23, bit1)));
+    HWY_ASSERT_EQ(true, ext::AllFalse(TestBit(bit1, bit2)));
+    HWY_ASSERT_EQ(true, ext::AllFalse(TestBit(bit2, bit1)));
+    HWY_ASSERT_EQ(true, ext::AllFalse(TestBit(bit1, bit3)));
+    HWY_ASSERT_EQ(true, ext::AllFalse(TestBit(bit3, bit1)));
+    HWY_ASSERT_EQ(true, ext::AllFalse(TestBit(bit2, bit3)));
+    HWY_ASSERT_EQ(true, ext::AllFalse(TestBit(bit3, bit2)));
+  }
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestCountTrue(D d) {
+  using T = typename D::T;
+
+  // For all combinations of zero/nonzero state of subset of lanes:
+  const size_t max_lanes = std::min(d.N, size_t(10));
+
+  HWY_ALIGN T lanes[d.N];
+  std::fill(lanes, lanes + d.N, T(1));
+
+  for (size_t code = 0; code < (1ull << max_lanes); ++code) {
+    // Number of zeros written (to also verify PopCount).
+    uint64_t expected = 0;
+    for (size_t i = 0; i < max_lanes; ++i) {
+      lanes[i] = T(1);
+      if (code & (1ull << i)) {
+        ++expected;
+        lanes[i] = T(0);
+      }
+    }
+
+    const uint64_t actual = ext::CountTrue(Load(d, lanes) == Zero(d));
+    HWY_ASSERT_EQ(expected, actual);
+  }
+}
+
+HWY_NOINLINE HWY_ATTR void TestLogical() {
+  (void)dd;
+  (void)di64;
+  (void)du64;
+
+  HWY_FOREACH_UIF(TestLogicalT);
+  HWY_FOREACH_UIF(TestIfThenElse);
+
+  HWY_FOREACH_U(TestTestBit);
+  HWY_FOREACH_I(TestTestBit);
+
+  HWY_FOREACH_UIF(TestCountTrue);
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+
+// Instantiate for the current target.
+void LogicalTest::HWY_FUNC() { hwy::HWY_NAMESPACE::TestLogical(); }
diff --git a/third_party/highway/hwy/tests/memory_test.cc b/third_party/highway/hwy/tests/memory_test.cc
new file mode 100644
index 000000000..ab5a00f47
--- /dev/null
+++ b/third_party/highway/hwy/tests/memory_test.cc
@@ -0,0 +1,243 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/memory_test.cc"
+
+#include "hwy/cache_control.h"
+#include "hwy/tests/test_util.h"
+struct MemoryTest {
+  HWY_DECLARE(void, ())
+};
+TEST(HwyMemoryTest, Run) { hwy::RunTests<MemoryTest>(); }
+
+#endif  // HWY_TARGET_INCLUDE
+#include "hwy/tests/test_target_util.h"
+
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+constexpr HWY_FULL(uint8_t) du8;
+constexpr HWY_FULL(uint16_t) du16;
+constexpr HWY_FULL(uint32_t) du32;
+constexpr HWY_FULL(uint64_t) du64;
+constexpr HWY_FULL(int8_t) di8;
+constexpr HWY_FULL(int16_t) di16;
+constexpr HWY_FULL(int32_t) di32;
+constexpr HWY_FULL(int64_t) di64;
+constexpr HWY_FULL(float) df;
+constexpr HWY_FULL(double) dd;
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestLoadStore(D d) {
+  using T = typename D::T;
+  const auto hi = Iota(d, 1 + d.N);
+  const auto lo = Iota(d, 1);
+  HWY_ALIGN T lanes[2 * d.N];
+  Store(hi, d, lanes + d.N);
+  Store(lo, d, lanes);
+
+  // Aligned load
+  const auto lo2 = Load(d, lanes);
+  HWY_ASSERT_VEC_EQ(d, lo2, lo);
+
+  // Aligned store
+  HWY_ALIGN T lanes2[2 * d.N];
+  Store(lo2, d, lanes2);
+  Store(hi, d, lanes2 + d.N);
+  for (size_t i = 0; i < 2 * d.N; ++i) {
+    HWY_ASSERT_EQ(lanes[i], lanes2[i]);
+  }
+
+  // Unaligned load
+  const auto vu = LoadU(d, lanes + 1);
+  HWY_ALIGN T lanes3[d.N];
+  Store(vu, d, lanes3);
+  for (size_t i = 0; i < d.N; ++i) {
+    HWY_ASSERT_EQ(T(i + 2), lanes3[i]);
+  }
+
+  // Unaligned store
+  StoreU(lo2, d, lanes2 + d.N / 2);
+  size_t i = 0;
+  for (; i < d.N / 2; ++i) {
+    HWY_ASSERT_EQ(lanes[i], lanes2[i]);
+  }
+  for (; i < 3 * d.N / 2; ++i) {
+    HWY_ASSERT_EQ(T(i - d.N / 2 + 1), lanes2[i]);
+  }
+  // Subsequent values remain unchanged.
+  for (; i < 2 * d.N; ++i) {
+    HWY_ASSERT_EQ(T(i + 1), lanes2[i]);
+  }
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestLoadDup128(D d) {
+#if HWY_BITS != 0 || HWY_IDE
+  using T = typename D::T;
+  constexpr size_t N128 = 16 / sizeof(T);
+  alignas(16) T lanes[N128];
+  for (size_t i = 0; i < N128; ++i) {
+    lanes[i] = static_cast<T>(1 + i);
+  }
+  const auto v = LoadDup128(d, lanes);
+  HWY_ALIGN T out[d.N];
+  Store(v, d, out);
+  for (size_t i = 0; i < d.N; ++i) {
+    HWY_ASSERT_EQ(T(i % N128 + 1), out[i]);
+  }
+#else
+  (void)d;
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestStreamT(D d) {
+  using T = typename D::T;
+  const auto v = Iota(d, 0);
+  HWY_ALIGN T out[d.N];
+  Stream(v, d, out);
+  StoreFence();
+  for (size_t i = 0; i < d.N; ++i) {
+    HWY_ASSERT_EQ(T(i), out[i]);
+  }
+}
+
+#if HWY_HAS_GATHER || HWY_IDE
+
+// kShift must be log2(sizeof(T)).
+template <typename Offset, int kShift, class D>
+HWY_NOINLINE HWY_ATTR void TestGatherT(D d) {
+  using T = typename D::T;
+  static_assert(sizeof(T) == (1 << kShift), "Incorrect kShift");
+
+  // Base points to middle; |max_offset| + sizeof(T) <= kNumBytes / 2.
+  constexpr size_t kNumBytes = kMaxVectorSize * 2;
+  uint8_t bytes[kNumBytes];
+  for (size_t i = 0; i < kNumBytes; ++i) {
+    bytes[i] = i + 1;
+  }
+  const uint8_t* middle = bytes + kNumBytes / 2;
+
+  // Offsets: combinations of aligned, repeated, negative.
+  HWY_ALIGN Offset offset_lanes[HWY_MAX(d.N, 16)] = {
+      2, 12, 4, 4, -16, -16, -21, -20, 8, 8, 8, -13, -13, -20, 20, 3};
+
+  HWY_ALIGN T expected[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    CopyBytes<sizeof(T)>(middle + offset_lanes[i], &expected[i]);
+  }
+
+  const auto offsets = Load(HWY_FULL(Offset)(), offset_lanes);
+  auto actual =
+      ext::GatherOffset(d, reinterpret_cast<const T*>(middle), offsets);
+  HWY_ASSERT_VEC_EQ(d, expected, actual);
+
+  // Indices
+  HWY_ALIGN const Offset index_lanes[HWY_MAX(d.N, 16)] = {
+      1, -2, 0, 1, 3, -2, -1, 2, 4, -3, 5, -5, 0, 2, -4, 0};
+  for (size_t i = 0; i < d.N; ++i) {
+    CopyBytes<sizeof(T)>(
+        middle + index_lanes[i] * static_cast<Offset>(sizeof(T)), &expected[i]);
+  }
+  const auto indices = Load(HWY_FULL(Offset)(), index_lanes);
+  actual = ext::GatherIndex(d, reinterpret_cast<const T*>(middle), indices);
+  HWY_ASSERT_VEC_EQ(d, expected, actual);
+}
+
+template <typename Offset, int kShift, class D>
+HWY_NOINLINE HWY_ATTR void TestFloatGatherT(D d) {
+  using T = typename D::T;
+  static_assert(sizeof(T) == (1 << kShift), "Incorrect kShift");
+
+  constexpr size_t kNumValues = 16;
+  // Base points to middle; |max_index| < kNumValues / 2.
+  HWY_ALIGN const T values[HWY_MAX(d.N, kNumValues)] = {
+      T(100.0), T(110.0), T(111.0), T(128.0), T(1024.0), T(-1.0),
+      T(-2.0),  T(-3.0),  T(0.25),  T(0.5),   T(0.75),   T(1.25),
+      T(1.5),   T(1.75),  T(-0.25), T(-0.5)};
+  const T* middle = values + kNumValues / 2;
+
+  // Indices: combinations of aligned, repeated, negative.
+  HWY_ALIGN const Offset index_lanes[HWY_MAX(d.N, 16)] = {1, -6, 0,  1,
+                                                          3, -6, -1, 7};
+  HWY_ALIGN T expected[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    CopyBytes<sizeof(T)>(middle + index_lanes[i], &expected[i]);
+  }
+  const auto indices = Load(HWY_FULL(Offset)(), index_lanes);
+  auto actual = ext::GatherIndex(d, middle, indices);
+  HWY_ASSERT_VEC_EQ(d, expected, actual);
+
+  // Offsets: same as index * sizeof(T).
+  const auto offsets = hwy::ShiftLeft<kShift>(indices);
+  actual = ext::GatherOffset(d, middle, offsets);
+  HWY_ASSERT_VEC_EQ(d, expected, actual);
+}
+
+#endif
+
+HWY_NOINLINE HWY_ATTR void TestStream() {
+  // No u8,u16.
+  TestStreamT(du32);
+  TestStreamT(du64);
+  // No i8,i16.
+  TestStreamT(di32);
+  TestStreamT(di64);
+  HWY_FOREACH_F(TestStreamT);
+}
+
+HWY_NOINLINE HWY_ATTR void TestGather() {
+#if HWY_HAS_GATHER || HWY_IDE
+  // No u8,u16.
+  TestGatherT<int32_t, 2>(du32);
+  TestGatherT<int64_t, 3>(du64);
+  // No i8,i16.
+  TestGatherT<int32_t, 2>(di32);
+  TestGatherT<int64_t, 3>(di64);
+
+  // Can't use HWY_FOREACH because kShift depends on type.
+  TestFloatGatherT<int32_t, 2>(df);
+#if HWY_HAS_DOUBLE
+  TestFloatGatherT<int64_t, 3>(dd);
+#endif
+#endif
+}
+
+HWY_NOINLINE HWY_ATTR void TestMemory() {
+  (void)dd;
+  (void)di64;
+  (void)du64;
+
+  HWY_FOREACH_UIF(TestLoadStore);
+  HWY_FOREACH_UIF(TestLoadDup128);
+  TestStream();
+  TestGather();
+  // Test that these functions compile.
+  LoadFence();
+  StoreFence();
+  int test = 0;
+  Prefetch(&test);
+  FlushCacheline(&test);
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+
+// Instantiate for the current target.
+void MemoryTest::HWY_FUNC() { hwy::HWY_NAMESPACE::TestMemory(); }
diff --git a/third_party/highway/hwy/tests/nanobenchmark_test.cc b/third_party/highway/hwy/tests/nanobenchmark_test.cc
new file mode 100644
index 000000000..09f3d23cb
--- /dev/null
+++ b/third_party/highway/hwy/tests/nanobenchmark_test.cc
@@ -0,0 +1,105 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "hwy/nanobenchmark.h"
+
+#include <stdio.h>
+#include <stdlib.h>  // strtol
+#include <unistd.h>  // sleep
+
+#include <random>
+
+namespace hwy {
+namespace {
+
+uint64_t Div(const void*, FuncInput in) {
+  // Here we're measuring the throughput because benchmark invocations are
+  // independent.
+  const int64_t d1 = 0xFFFFFFFFFFll / int64_t(in);  // IDIV
+  return d1;
+}
+
+template <size_t N>
+void MeasureDiv(const FuncInput (&inputs)[N]) {
+  Result results[N];
+  Params params;
+  params.max_evals = 4;  // avoid test timeout
+  const size_t num_results = Measure(&Div, nullptr, inputs, N, results, params);
+  for (size_t i = 0; i < num_results; ++i) {
+    printf("%5zu: %6.2f ticks; MAD=%4.2f%%\n", results[i].input,
+           results[i].ticks, results[i].variability * 100.0);
+  }
+}
+
+std::mt19937 rng;
+
+// A function whose runtime depends on rng.
+uint64_t Random(const void* /*arg*/, FuncInput in) {
+  const size_t r = rng() & 0xF;
+  uint32_t ret = in;
+  for (size_t i = 0; i < r; ++i) {
+    ret /= ((rng() & 1) + 2);
+  }
+  return ret;
+}
+
+// Ensure the measured variability is high.
+template <size_t N>
+void MeasureRandom(const FuncInput (&inputs)[N]) {
+  Result results[N];
+  Params p;
+  p.max_evals = 4;  // avoid test timeout
+  p.verbose = false;
+  const size_t num_results = Measure(&Random, nullptr, inputs, N, results, p);
+  for (size_t i = 0; i < num_results; ++i) {
+    NANOBENCHMARK_CHECK(results[i].variability > 1E-3);
+  }
+}
+
+template <size_t N>
+void EnsureLongMeasurementFails(const FuncInput (&inputs)[N]) {
+  printf("Expect a 'measurement failed' below:\n");
+  Result results[N];
+
+  const size_t num_results = Measure(
+      [](const void*, const FuncInput input) {
+        // Loop until the sleep succeeds (not interrupted by signal). We assume
+        // >= 512 MHz, so 2 seconds will exceed the 1 << 30 tick safety limit.
+        while (sleep(2) != 0) {
+        }
+        return input;
+      },
+      nullptr, inputs, N, results);
+  NANOBENCHMARK_CHECK(num_results == 0);
+  (void)num_results;
+}
+
+void RunAll(const int argc, char** /*argv*/) {
+  // unpredictable == 1 but the compiler doesn't know that.
+  const int unpredictable = argc != 999;
+  static const FuncInput inputs[] = {static_cast<FuncInput>(unpredictable) + 2,
+                                     static_cast<FuncInput>(unpredictable + 9)};
+
+  MeasureDiv(inputs);
+  MeasureRandom(inputs);
+  EnsureLongMeasurementFails(inputs);
+}
+
+}  // namespace
+}  // namespace hwy
+
+int main(int argc, char* argv[]) {
+  hwy::RunAll(argc, argv);
+  return 0;
+}
diff --git a/third_party/highway/hwy/tests/swizzle_test.cc b/third_party/highway/hwy/tests/swizzle_test.cc
new file mode 100644
index 000000000..4b03b8208
--- /dev/null
+++ b/third_party/highway/hwy/tests/swizzle_test.cc
@@ -0,0 +1,555 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TARGET_INCLUDE
+#define HWY_TARGET_INCLUDE "tests/swizzle_test.cc"
+
+#include "hwy/tests/test_util.h"
+struct SwizzleTest {
+  HWY_DECLARE(void, ())
+};
+TEST(HwySwizzleTest, Run) { hwy::RunTests<SwizzleTest>(); }
+
+#endif  // HWY_TARGET_INCLUDE
+#include "hwy/tests/test_target_util.h"
+
+namespace hwy {
+namespace HWY_NAMESPACE {
+namespace {
+
+constexpr HWY_FULL(uint8_t) du8;
+constexpr HWY_FULL(uint16_t) du16;
+constexpr HWY_FULL(uint32_t) du32;
+constexpr HWY_FULL(uint64_t) du64;
+constexpr HWY_FULL(int8_t) di8;
+constexpr HWY_FULL(int16_t) di16;
+constexpr HWY_FULL(int32_t) di32;
+constexpr HWY_FULL(int64_t) di64;
+constexpr HWY_FULL(float) df;
+constexpr HWY_FULL(double) dd;
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestShiftBytesT(D d) {
+#if HWY_BITS != 0 || HWY_IDE
+  using T = typename D::T;
+  // Zero remains zero
+  const auto v0 = Zero(d);
+  HWY_ASSERT_VEC_EQ(d, v0, ShiftLeftBytes<1>(v0));
+  HWY_ASSERT_VEC_EQ(d, v0, ShiftRightBytes<1>(v0));
+
+  // Zero after shifting out the high/low byte
+  HWY_ALIGN uint8_t bytes[du8.N] = {0};
+  bytes[du8.N - 1] = 0x7F;
+  const auto vhi = BitCast(d, Load(du8, bytes));
+  bytes[du8.N - 1] = 0;
+  bytes[0] = 0x7F;
+  const auto vlo = BitCast(d, Load(du8, bytes));
+  HWY_ASSERT_EQ(true, ext::AllTrue(ShiftLeftBytes<1>(vhi) == v0));
+  HWY_ASSERT_EQ(true, ext::AllTrue(ShiftRightBytes<1>(vlo) == v0));
+
+  HWY_ALIGN T in[d.N];
+  const uint8_t* in_bytes = reinterpret_cast<const uint8_t*>(in);
+  const auto v = BitCast(d, Iota(du8, 1));
+  Store(v, d, in);
+
+  // Shifting by one lane is the same as shifting #bytes
+  HWY_ASSERT_VEC_EQ(d, ShiftLeftLanes<1>(v), ShiftLeftBytes<sizeof(T)>(v));
+  HWY_ASSERT_VEC_EQ(d, ShiftRightLanes<1>(v), ShiftRightBytes<sizeof(T)>(v));
+  // Two lanes, we can only try to shift lanes if there are at least two
+  // lanes.
+  if (D::N > 2) {
+    // If there are only two lanes we just define the one lane version to
+    // avoid running into static_asserts.
+    const size_t shift_bytes = D::N > 2 ? 2 * sizeof(T) : sizeof(T);
+    const size_t shift_lanes = D::N > 2 ? 2 : 1;
+    HWY_ASSERT_VEC_EQ(d, ShiftLeftLanes<shift_lanes>(v),
+                      ShiftLeftBytes<shift_bytes>(v));
+    HWY_ASSERT_VEC_EQ(d, ShiftRightLanes<shift_lanes>(v),
+                      ShiftRightBytes<shift_bytes>(v));
+  }
+
+  HWY_ALIGN T shifted[d.N];
+  const uint8_t* shifted_bytes = reinterpret_cast<const uint8_t*>(shifted);
+
+  const size_t kBlockSize = HWY_MIN(du8.N, 16);
+  Store(ShiftLeftBytes<1>(v), d, shifted);
+  for (size_t block = 0; block < du8.N; block += kBlockSize) {
+    HWY_ASSERT_EQ(uint8_t(0), shifted_bytes[block]);
+    HWY_ASSERT_EQ(true, BytesEqual(in_bytes + block, shifted_bytes + block + 1,
+                                   kBlockSize - 1));
+  }
+
+  Store(ShiftRightBytes<1>(v), d, shifted);
+  for (size_t block = 0; block < du8.N; block += kBlockSize) {
+    HWY_ASSERT_EQ(uint8_t(0), shifted_bytes[block + kBlockSize - 1]);
+    HWY_ASSERT_EQ(true, BytesEqual(in_bytes + block + 1, shifted_bytes + block,
+                                   kBlockSize - 1));
+  }
+#else
+  (void)d;
+#endif
+}
+
+template <typename D, int kLane>
+struct TestBroadcastR {
+  HWY_NOINLINE HWY_ATTR void operator()() const {
+    using T = typename D::T;
+    const D d;
+    HWY_ALIGN T in_lanes[d.N] = {};
+    constexpr size_t kBlockN = HWY_MIN(d.N * sizeof(T), 16) / sizeof(T);
+    // Need to set within each 128-bit block
+    for (size_t block = 0; block < d.N; block += kBlockN) {
+      in_lanes[block + kLane] = block + 1;
+    }
+    const auto in = Load(d, in_lanes);
+    HWY_ALIGN T out_lanes[d.N];
+    Store(Broadcast<kLane>(in), d, out_lanes);
+    for (size_t block = 0; block < d.N; block += kBlockN) {
+      for (size_t i = 0; i < kBlockN; ++i) {
+        HWY_ASSERT_EQ(T(block + 1), out_lanes[block + i]);
+      }
+    }
+
+    TestBroadcastR<D, kLane - 1>()();
+  }
+};
+
+template <class D>
+struct TestBroadcastR<D, -1> {
+  void operator()() const {}
+};
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestBroadcastT(D d) {
+  using T = typename D::T;
+  TestBroadcastR<D, HWY_MIN(d.N, 16 / sizeof(T)) - 1>()();
+}
+
+HWY_NOINLINE HWY_ATTR void TestBroadcast() {
+  // No u8.
+  TestBroadcastT(du16);
+  TestBroadcastT(du32);
+#if HWY_HAS_INT64
+  TestBroadcastT(du64);
+#endif
+  // No i8.
+  TestBroadcastT(di16);
+#if HWY_HAS_INT64
+  TestBroadcastT(di64);
+#endif
+  HWY_FOREACH_F(TestBroadcastT);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestPermuteT(D d) {
+#if HWY_BITS >= 256 || HWY_IDE
+  {
+    using T = typename D::T;
+    // Too many permutations to test exhaustively; choose one with repeated
+    // and cross-block indices and ensure indices do not exceed #lanes.
+    HWY_ALIGN int32_t idx[kTestMaxVectorSize / sizeof(int32_t)] = {
+        1,        3,        2,        2,        8 % d.N, 1,       7,       6,
+        15 % d.N, 14 % d.N, 14 % d.N, 15 % d.N, 4,       9 % d.N, 8 % d.N, 5};
+    const auto v = Iota(d, 1);
+    HWY_ALIGN T expected_lanes[d.N];
+    for (size_t i = 0; i < d.N; ++i) {
+      expected_lanes[i] = idx[i] + 1;  // == v[idx[i]]
+    }
+
+    const auto opaque = SetTableIndices(d, idx);
+    const auto actual = TableLookupLanes(v, opaque);
+    HWY_ASSERT_VEC_EQ(d, expected_lanes, actual);
+  }
+#endif
+#if HWY_BITS == 128 || HWY_IDE
+  using T = typename D::T;
+  // Test all possible permutations.
+  HWY_ALIGN int32_t idx[d.N];
+  const auto v = Iota(d, 1);
+  HWY_ALIGN T expected_lanes[d.N];
+
+  const int32_t N = static_cast<int32_t>(d.N);
+  for (int32_t i0 = 0; i0 < N; ++i0) {
+    idx[0] = i0;
+    for (int32_t i1 = 0; i1 < N; ++i1) {
+      idx[1] = i1;
+      for (int32_t i2 = 0; i2 < N; ++i2) {
+        idx[2] = i2;
+        for (int32_t i3 = 0; i3 < N; ++i3) {
+          idx[3] = i3;
+
+          for (size_t i = 0; i < d.N; ++i) {
+            expected_lanes[i] = idx[i] + 1;  // == v[idx[i]]
+          }
+
+          const auto opaque = SetTableIndices(d, idx);
+          const auto actual = TableLookupLanes(v, opaque);
+          HWY_ASSERT_VEC_EQ(d, expected_lanes, actual);
+        }
+      }
+    }
+  }
+#endif
+#if HWY_BITS == 0
+  (void)d;
+#endif
+}
+
+HWY_NOINLINE HWY_ATTR void TestPermute() {
+  // Only uif32.
+  TestPermuteT(du32);
+  TestPermuteT(di32);
+  TestPermuteT(df);
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestInterleave(D d) {
+// Not supported by scalar.h: zip(f32, f32) would need to return f32x2.
+#if HWY_BITS != 0 || HWY_IDE
+  using T = typename D::T;
+  HWY_ALIGN T even_lanes[d.N];
+  HWY_ALIGN T odd_lanes[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    even_lanes[i] = 2 * i + 0;
+    odd_lanes[i] = 2 * i + 1;
+  }
+  const auto even = Load(d, even_lanes);
+  const auto odd = Load(d, odd_lanes);
+
+  HWY_ALIGN T lo_lanes[d.N];
+  HWY_ALIGN T hi_lanes[d.N];
+  Store(InterleaveLo(even, odd), d, lo_lanes);
+  Store(InterleaveHi(even, odd), d, hi_lanes);
+
+  constexpr size_t kBlockN = 16 / sizeof(T);
+  for (size_t i = 0; i < d.N; ++i) {
+    const size_t block = i / kBlockN;
+    const size_t lo = (i % kBlockN) + block * 2 * kBlockN;
+    HWY_ASSERT_EQ(T(lo), lo_lanes[i]);
+    HWY_ASSERT_EQ(T(lo + kBlockN), hi_lanes[i]);
+  }
+#else
+  (void)d;
+#endif
+}
+
+template <typename T, typename WideT>
+HWY_NOINLINE HWY_ATTR void TestZipT() {
+  static_assert(sizeof(T) * 2 == sizeof(WideT), "Expected 2x size");
+  static_assert(IsSigned<T>() == IsSigned<WideT>(), "Should have same sign");
+  const HWY_FULL(T) d;
+  const HWY_FULL(WideT) dw;
+  HWY_ALIGN T even_lanes[d.N];
+  HWY_ALIGN T odd_lanes[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    even_lanes[i] = static_cast<T>(2 * i + 0);
+    odd_lanes[i] = static_cast<T>(2 * i + 1);
+  }
+  const auto even = Load(d, even_lanes);
+  const auto odd = Load(d, odd_lanes);
+
+  HWY_ALIGN WideT lo_lanes[dw.N];
+  Store(ZipLo(even, odd), dw, lo_lanes);
+#if HWY_BITS != 0
+  HWY_ALIGN WideT hi_lanes[dw.N];
+  Store(ZipHi(even, odd), dw, hi_lanes);
+#endif
+
+  constexpr WideT kBlockN = static_cast<WideT>(16 / sizeof(WideT));
+  for (size_t i = 0; i < dw.N; ++i) {
+    const size_t block = i / kBlockN;
+    // Value of least-significant lane in lo-vector.
+    const WideT lo =
+        static_cast<WideT>(2 * (i % kBlockN) + 4 * block * kBlockN);
+    const WideT kBits = static_cast<WideT>(sizeof(T) * 8);
+    const WideT expected_lo =
+        static_cast<WideT>((static_cast<WideT>(lo + 1) << kBits) + lo);
+
+    HWY_ASSERT_EQ(expected_lo, lo_lanes[i]);
+#if HWY_BITS != 0
+    const WideT expected_hi = static_cast<WideT>(
+        (static_cast<WideT>(lo + 2 * kBlockN + 1) << kBits) + lo + 2 * kBlockN);
+    HWY_ASSERT_EQ(expected_hi, hi_lanes[i]);
+#endif
+  }
+}
+
+HWY_NOINLINE HWY_ATTR void TestZip() {
+  TestZipT<uint8_t, uint16_t>();
+  TestZipT<uint16_t, uint32_t>();
+#if HWY_HAS_INT64
+  TestZipT<uint32_t, uint64_t>();
+#endif
+  // No 64-bit nor float.
+  TestZipT<int8_t, int16_t>();
+  TestZipT<int16_t, int32_t>();
+#if HWY_HAS_INT64
+  TestZipT<int32_t, int64_t>();
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestShuffleT(D d) {
+// Not supported by scalar.h (its vector size is always less than 16 bytes)
+#if HWY_BITS != 0 || HWY_IDE
+  using T = typename D::T;
+  RandomState rng{1234};
+  constexpr size_t N8 = HWY_FULL(uint8_t)::N;
+  HWY_ALIGN uint8_t in_bytes[N8];
+  for (size_t i = 0; i < N8; ++i) {
+    in_bytes[i] = Random32(&rng) & 0xFF;
+  }
+  const auto in = Load(du8, in_bytes);
+  HWY_ALIGN const uint8_t index_bytes[kTestMaxVectorSize] = {
+      // Same index as source, multiple outputs from same input,
+      // unused input (9), ascending/descending and nonconsecutive neighbors.
+      0,  2,  1, 2, 15, 12, 13, 14, 6,  7,  8,  5,  4,  3,  10, 11,
+      11, 10, 3, 4, 5,  8,  7,  6,  14, 13, 12, 15, 2,  1,  2,  0,
+      4,  3,  2, 2, 5,  6,  7,  7,  15, 15, 15, 15, 15, 15, 0,  1};
+  const auto indices = Load(du8, index_bytes);
+  HWY_ALIGN T out_lanes[d.N];
+  Store(TableLookupBytes(BitCast(d, in), indices), d, out_lanes);
+  const uint8_t* out_bytes = reinterpret_cast<const uint8_t*>(out_lanes);
+
+  for (size_t block = 0; block < N8; block += 16) {
+    for (size_t i = 0; i < 16; ++i) {
+      const uint8_t expected = in_bytes[block + index_bytes[block + i]];
+      HWY_ASSERT_EQ(expected, out_bytes[block + i]);
+    }
+  }
+#else
+  (void)d;
+#endif
+}
+
+template <class D, int kBytes>
+struct TestCombineShiftRightR {
+  HWY_NOINLINE HWY_ATTR void operator()() const {
+#if HWY_BITS != 0 || HWY_IDE
+    using T = typename D::T;
+    const D d;
+    const HWY_FULL(uint8_t) d8;
+    const auto lo = BitCast(d, Iota(d8, 1));
+    const auto hi = BitCast(d, Iota(d8, 1 + d8.N));
+
+    HWY_ALIGN T lanes[D::N];
+    Store(CombineShiftRightBytes<kBytes>(hi, lo), d, lanes);
+    const uint8_t* bytes = reinterpret_cast<const uint8_t*>(lanes);
+
+    const size_t kBlockSize = 16;
+    for (size_t i = 0; i < d8.N; ++i) {
+      const size_t block = i / kBlockSize;
+      const size_t lane = i % kBlockSize;
+      const size_t first_lo = block * kBlockSize;
+      const size_t idx = lane + kBytes;
+      const size_t offset = (idx < kBlockSize) ? 0 : d8.N - kBlockSize;
+      const bool at_end = idx >= 2 * kBlockSize;
+      const uint8_t expected = at_end ? 0 : (first_lo + idx + 1 + offset);
+      HWY_ASSERT_EQ(expected, bytes[i]);
+    }
+
+    TestCombineShiftRightR<D, kBytes - 1>()();
+#endif
+  }
+};
+
+template <class D>
+struct TestCombineShiftRightR<D, 0> {
+  HWY_ATTR void operator()() const {}
+};
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestCombineShiftRightT(D /*d*/) {
+  TestCombineShiftRightR<D, 15>()();
+}
+
+#if HWY_BITS != 0 || HWY_IDE
+
+template <class D, class V>
+HWY_NOINLINE HWY_ATTR void VerifyLanes32(D d, V v, const int i3, const int i2,
+                                         const int i1, const int i0,
+                                         const char* filename, const int line) {
+  using T = typename D::T;
+  HWY_ALIGN T lanes[d.N];
+  Store(v, d, lanes);
+  constexpr size_t kBlockN = 16 / sizeof(T);
+  for (size_t block = 0; block < d.N; block += kBlockN) {
+    AssertEqual(T(block + i3), lanes[block + 3], filename, line);
+    AssertEqual(T(block + i2), lanes[block + 2], filename, line);
+    AssertEqual(T(block + i1), lanes[block + 1], filename, line);
+    AssertEqual(T(block + i0), lanes[block + 0], filename, line);
+  }
+}
+
+template <class D, class V>
+HWY_NOINLINE HWY_ATTR void VerifyLanes64(D d, V v, const int i1, const int i0,
+                                         const char* filename, const int line) {
+  using T = typename D::T;
+  HWY_ALIGN T lanes[d.N];
+  Store(v, d, lanes);
+  constexpr size_t kBlockN = 16 / sizeof(T);
+  for (size_t block = 0; block < d.N; block += kBlockN) {
+    AssertEqual(T(block + i1), lanes[block + 1], filename, line);
+    AssertEqual(T(block + i0), lanes[block + 0], filename, line);
+  }
+}
+
+#define VERIFY_LANES_32(d, v, i3, i2, i1, i0) \
+  VerifyLanes32((d), (v), (i3), (i2), (i1), (i0), __FILE__, __LINE__)
+
+#define VERIFY_LANES_64(d, v, i1, i0) \
+  VerifyLanes64((d), (v), (i1), (i0), __FILE__, __LINE__)
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSpecialShuffle32(D d) {
+  const auto v = Iota(d, 0);
+  VERIFY_LANES_32(d, Shuffle2301(v), 2, 3, 0, 1);
+  VERIFY_LANES_32(d, Shuffle1032(v), 1, 0, 3, 2);
+  VERIFY_LANES_32(d, Shuffle0321(v), 0, 3, 2, 1);
+  VERIFY_LANES_32(d, Shuffle2103(v), 2, 1, 0, 3);
+  VERIFY_LANES_32(d, Shuffle0123(v), 0, 1, 2, 3);
+}
+
+#if HWY_HAS_INT64
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestSpecialShuffle64(D d) {
+  const auto v = Iota(d, 0);
+  VERIFY_LANES_64(d, Shuffle01(v), 0, 1);
+}
+#endif
+
+#endif
+
+HWY_NOINLINE HWY_ATTR void TestSpecialShuffles() {
+#if HWY_BITS != 0 || HWY_IDE
+  TestSpecialShuffle32(di32);
+#if HWY_HAS_INT64
+  TestSpecialShuffle64(di64);
+#endif
+  // Can't use HWY_FOREACH_F, function depends on lane type
+  TestSpecialShuffle32(df);
+#if HWY_HAS_DOUBLE
+  TestSpecialShuffle64(dd);
+#endif
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestConcatHalves(D d) {
+#if HWY_BITS != 0 || HWY_IDE
+  using T = typename D::T;
+  // Construct inputs such that interleaved halves == iota.
+  const auto expected = Iota(d, 1);
+
+  HWY_ALIGN T lo[d.N];
+  HWY_ALIGN T hi[d.N];
+  size_t i;
+  for (i = 0; i < d.N / 2; ++i) {
+    lo[i] = 1 + i;
+    hi[i] = lo[i] + d.N / 2;
+  }
+  for (; i < d.N; ++i) {
+    lo[i] = hi[i] = 0;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, ConcatLoLo(Load(d, hi), Load(d, lo)));
+
+  // Same for high blocks.
+  for (i = 0; i < d.N / 2; ++i) {
+    lo[i] = hi[i] = 0;
+  }
+  for (; i < d.N; ++i) {
+    lo[i] = 1 + i - d.N / 2;
+    hi[i] = lo[i] + d.N / 2;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, ConcatHiHi(Load(d, hi), Load(d, lo)));
+#else
+  (void)d;
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestConcatLoHi(D d) {
+#if HWY_BITS != 0 || HWY_IDE
+  // Middle part of Iota(1) == Iota(1 + d.N / 2).
+  const auto lo = Iota(d, 1);
+  const auto hi = Iota(d, 1 + d.N);
+  HWY_ASSERT_VEC_EQ(d, Iota(d, 1 + d.N / 2), ConcatLoHi(hi, lo));
+#else
+  (void)d;
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestConcatHiLo(D d) {
+#if HWY_BITS != 0 || HWY_IDE
+  using T = typename D::T;
+  const auto lo = Iota(d, 1);
+  const auto hi = Iota(d, 1 + d.N);
+  T expected[d.N];
+  size_t i = 0;
+  for (; i < d.N / 2; ++i) {
+    expected[i] = 1 + i;
+  }
+  for (; i < d.N; ++i) {
+    expected[i] = 1 + i + d.N;
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, ConcatHiLo(hi, lo));
+#else
+  (void)d;
+#endif
+}
+
+template <class D>
+HWY_NOINLINE HWY_ATTR void TestOddEven(D d) {
+#if HWY_BITS != 0 || HWY_IDE
+  using T = typename D::T;
+  const auto even = Iota(d, 1);
+  const auto odd = Iota(d, 1 + d.N);
+  T expected[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    expected[i] = 1 + i + ((i & 1) ? d.N : 0);
+  }
+  HWY_ASSERT_VEC_EQ(d, expected, OddEven(odd, even));
+#else
+  (void)d;
+#endif
+}
+
+HWY_NOINLINE HWY_ATTR void TestSwizzle() {
+  (void)dd;
+  (void)di64;
+  (void)du64;
+
+  HWY_FOREACH_UI(TestShiftBytesT);
+  TestBroadcast();
+  HWY_FOREACH_UIF(TestInterleave);
+  TestPermute();
+  TestZip();
+  HWY_FOREACH_UI(TestShuffleT);
+
+  HWY_FOREACH_UI(TestCombineShiftRightT);
+  TestSpecialShuffles();
+  HWY_FOREACH_UIF(TestConcatHalves);
+  HWY_FOREACH_UIF(TestConcatLoHi);
+  HWY_FOREACH_UIF(TestConcatHiLo);
+  HWY_FOREACH_UIF(TestOddEven);
+}
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
+
+// Instantiate for the current target.
+void SwizzleTest::HWY_FUNC() { hwy::HWY_NAMESPACE::TestSwizzle(); }
diff --git a/third_party/highway/hwy/tests/test_target_util.h b/third_party/highway/hwy/tests/test_target_util.h
new file mode 100644
index 000000000..6be876bb2
--- /dev/null
+++ b/third_party/highway/hwy/tests/test_target_util.h
@@ -0,0 +1,148 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Empty include guard to avoid Copybara warning
+#ifndef HWY_TESTS_TEST_TARGET_UTIL_H
+#define HWY_TESTS_TEST_TARGET_UTIL_H
+#endif
+
+#include <stddef.h>
+
+// Tests already include test_util (for TEST()). Including it "again" makes its
+// definitions visible to the IDE.
+#include "hwy/foreach_target.h"
+#include "hwy/tests/test_util.h"
+
+#ifndef HWY_NAMESPACE
+#error "foreach_target.h should have set HWY_NAMESPACE."
+#endif
+
+namespace hwy {
+namespace HWY_NAMESPACE {
+// NOLINTNEXTLINE(google-build-namespaces)
+namespace {
+
+// Compare non-vector, non-string T.
+template <typename T>
+void AssertEqual(const T expected, const T actual, const char* filename = "",
+                 const int line = -1, const size_t lane = 0,
+                 const char* name = nullptr) {
+  if (name == nullptr) name = TypeName<T, 1>();
+  char expected_buf[100];
+  char actual_buf[100];
+  ToString(expected, expected_buf);
+  ToString(actual, actual_buf);
+  // Rely on string comparison to ensure similar floats are "equal".
+  if (!StringsEqual(expected_buf, actual_buf)) {
+    NotifyFailure(filename, line, HWY_BITS, name, lane, expected_buf,
+                  actual_buf);
+  }
+}
+
+HWY_ATTR void AssertStringEqual(const char* expected, const char* actual,
+                                const char* filename = "", const int line = -1,
+                                const size_t lane = 0) {
+  if (!StringsEqual(expected, actual)) {
+    NotifyFailure(filename, line, HWY_BITS, "string", lane, expected, actual);
+  }
+}
+
+// Compare expected vector to vector.
+template <class D, class V>
+HWY_ATTR void AssertVecEqual(D d, const V expected, const V actual,
+                             const char* filename, const int line) {
+  HWY_ALIGN typename D::T expected_lanes[d.N];
+  HWY_ALIGN typename D::T actual_lanes[d.N];
+  Store(expected, d, expected_lanes);
+  Store(actual, d, actual_lanes);
+  for (size_t i = 0; i < d.N; ++i) {
+    AssertEqual(expected_lanes[i], actual_lanes[i], filename, line, i,
+                TypeName<typename D::T, D::N>());
+  }
+}
+
+// Compare expected lanes to vector.
+template <class D, class V>
+HWY_ATTR void AssertVecEqual(D d, const typename D::T (&expected)[D::N],
+                             V actual, const char* filename, int line) {
+  AssertVecEqual(d, LoadU(d, expected), actual, filename, line);
+}
+
+// Only define macros once to avoid warnings/errors.
+#ifndef HWY_ASSERT_EQ
+
+#define HWY_ASSERT_EQ(expected, actual) \
+  AssertEqual(expected, actual, __FILE__, __LINE__)
+
+#define HWY_ASSERT_STRING_EQ(expected, actual) \
+  AssertStringEqual(expected, actual, __FILE__, __LINE__)
+
+#define HWY_ASSERT_VEC_EQ(d, expected, actual) \
+  AssertVecEqual(d, expected, actual, __FILE__, __LINE__)
+
+// Type lists: call func for Unsigned/Signed lane types.
+#if HWY_HAS_INT64
+
+#define HWY_FOREACH_U(func) \
+  func(du8);                \
+  func(du16);               \
+  func(du32);               \
+  func(du64);
+
+#define HWY_FOREACH_I(func) \
+  func(di8);                \
+  func(di16);               \
+  func(di32);               \
+  func(di64);
+
+#else
+
+#define HWY_FOREACH_U(func) \
+  func(du8);                \
+  func(du16);               \
+  func(du32);
+
+#define HWY_FOREACH_I(func) \
+  func(di8);                \
+  func(di16);               \
+  func(di32);
+
+#endif
+
+#define HWY_FOREACH_UI(func) \
+  HWY_FOREACH_U(func);       \
+  HWY_FOREACH_I(func);
+
+#endif  // HWY_ASSERT_EQ
+
+// Exception: these change according to HWY_HAS_DOUBLE, so always redefine.
+#undef HWY_FOREACH_F
+#undef HWY_FOREACH_UIF
+
+#if HWY_HAS_DOUBLE
+#define HWY_FOREACH_F(func) \
+  func(df);                 \
+  func(dd);
+#else
+#define HWY_FOREACH_F(func) func(df);
+#endif
+
+#define HWY_FOREACH_UIF(func) \
+  HWY_FOREACH_UI(func);       \
+  HWY_FOREACH_F(func);
+
+}  // namespace
+// NOLINTNEXTLINE(google-readability-namespace-comments)
+}  // namespace HWY_NAMESPACE
+}  // namespace hwy
diff --git a/third_party/highway/hwy/tests/test_util.h b/third_party/highway/hwy/tests/test_util.h
new file mode 100644
index 000000000..b493dd050
--- /dev/null
+++ b/third_party/highway/hwy/tests/test_util.h
@@ -0,0 +1,192 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TESTS_TEST_UTIL_H_
+#define HWY_TESTS_TEST_UTIL_H_
+
+// SIMD-independent helper functions for use by *_test.cc.
+
+#include <stdio.h>
+#include <string.h>
+
+#include <random>
+
+#include "hwy/arch.h"  // kMaxVectorSize
+#include "hwy/runtime_dispatch.h"
+#include "hwy/type_traits.h"
+
+// Caller wants to use gtest.
+#ifdef HWY_USE_GTEST
+#include "gtest/gtest.h"
+#else
+// The tests are gtest-compatible and we only need to define this. Note that
+// cmake scans for a TEST() marker in the source code. This macro is followed by
+// the body of the test, and we rely on C99/C++ guarantees that reaching the end
+// of main is equivalent to return 0. Note that this means we can only have one
+// test case per file, which also minimizes runtime-dispatch boilerplate.
+#define TEST(test_case, test_name) int main()
+#endif  // HWY_USE_GTEST
+
+namespace hwy {
+// NOLINTNEXTLINE(google-build-namespaces)
+namespace {
+
+// The maximum vector size used in tests when defining test data. This is at
+// least the kMaxVectorSize but it can be bigger. If you increased
+// kMaxVectorSize, you also need to increase this constant and update all the
+// tests that use it to define bigger arrays of test data.
+constexpr size_t kTestMaxVectorSize = 64;
+static_assert(kTestMaxVectorSize >= kMaxVectorSize,
+              "All kTestMaxVectorSize test arrays need to be updated");
+
+// Calls Test()() for each instruction set via TargetBitfield.
+template <class Test>
+int RunTests() {
+  setvbuf(stdin, nullptr, _IONBF, 0);
+
+  const int targets = TargetBitfield().Foreach(Test());
+  printf("Successfully tested instruction sets: 0x%x.\n", targets);
+  return 0;
+}
+
+[[noreturn]] void NotifyFailure(const char* filename, const int line,
+                                const size_t bits, const char* type_name,
+                                const size_t lane, const char* expected,
+                                const char* actual) {
+  fprintf(stderr,
+          "%s:%d: bits %zu, %s lane %zu mismatch: expected '%s', got '%s'.\n",
+          filename, line, bits, type_name, lane, expected, actual);
+  HWY_TRAP();
+}
+
+// Random numbers
+typedef std::mt19937 RandomState;
+HWY_INLINE uint32_t Random32(RandomState* rng) {
+  return static_cast<uint32_t>((*rng)());
+}
+
+// Prevents the compiler from eliding the computations that led to "output".
+// Works by indicating to the compiler that "output" is being read and modified.
+// The +r constraint avoids unnecessary writes to memory, but only works for
+// built-in types.
+template <class T>
+inline void PreventElision(T&& output) {
+#ifndef _MSC_VER
+  asm volatile("" : "+r"(output) : : "memory");
+#endif
+}
+
+// Returns a name for the vector/part/scalar. The type prefix is u/i/f for
+// unsigned/signed/floating point, followed by the number of bits per lane;
+// then 'x' followed by the number of lanes. Example: u8x16. This is useful for
+// understanding which instantiation of a generic test failed.
+template <typename T, size_t N>
+const char* TypeName() {
+  constexpr char prefix = IsFloat<T>() ? 'f' : (IsSigned<T>() ? 'i' : 'u');
+
+  constexpr size_t bits = sizeof(T) * 8;
+  constexpr char bits10 = '0' + (bits / 10);
+  constexpr char bits1 = '0' + (bits % 10);
+
+  // Scalars: omit the xN suffix.
+  if (N == 1) {
+    static constexpr char name1[8] = {prefix, bits1};
+    static constexpr char name2[8] = {prefix, bits10, bits1};
+    return sizeof(T) == 1 ? name1 : name2;
+  }
+
+  constexpr char N1 = (N < 10) ? '\0' : '0' + (N % 10);
+  constexpr char N10 = (N < 10) ? '0' + (N % 10) : '0' + (N / 10);
+
+  static constexpr char name1[8] = {prefix, bits1, 'x', N10, N1};
+  static constexpr char name2[8] = {prefix, bits10, bits1, 'x', N10, N1};
+  return sizeof(T) == 1 ? name1 : name2;
+}
+
+// Value to string
+
+// Returns end of string (position of '\0').
+template <typename T>
+inline char* ToString(T value, char* to) {
+  char reversed[100];
+  char* pos = reversed;
+  int64_t before;
+  do {
+    before = value;
+    value /= 10;
+    const int64_t mod = before - value * 10;
+    *pos++ = "9876543210123456789"[9 + mod];
+  } while (value != 0);
+  if (before < 0) *pos++ = '-';
+
+  // Reverse the string
+  const int num_chars = pos - reversed;
+  for (int i = 0; i < num_chars; ++i) {
+    to[i] = pos[-1 - i];
+  }
+  to[num_chars] = '\0';
+  return to + num_chars;
+}
+
+template <>
+inline char* ToString<float>(const float value, char* to) {
+  if (!std::isfinite(value)) {
+    strncpy(to, "<not-finite>", 13);
+    return to + strlen(to);
+  }
+  const int64_t truncated = static_cast<int64_t>(value);
+  char* end = ToString(truncated, to);
+  *end++ = '.';
+  int64_t frac = static_cast<int64_t>((value - truncated) * 1E8);
+  if (frac < 0) frac = -frac;
+  return ToString(frac, end);
+}
+
+template <>
+inline char* ToString<double>(const double value, char* to) {
+  if (!std::isfinite(value)) {
+    strncpy(to, "<not-finite>", 13);
+    return to + strlen(to);
+  }
+  const int64_t truncated = static_cast<int64_t>(value);
+  char* end = ToString(truncated, to);
+  *end++ = '.';
+  int64_t frac = static_cast<int64_t>((value - truncated) * 1E16);
+  if (frac < 0) frac = -frac;
+  return ToString(frac, end);
+}
+
+// String comparison
+
+template <typename T1, typename T2>
+inline bool BytesEqual(const T1* p1, const T2* p2, const size_t size) {
+  const uint8_t* bytes1 = reinterpret_cast<const uint8_t*>(p1);
+  const uint8_t* bytes2 = reinterpret_cast<const uint8_t*>(p2);
+  for (size_t i = 0; i < size; ++i) {
+    if (bytes1[i] != bytes2[i]) return false;
+  }
+  return true;
+}
+
+inline bool StringsEqual(const char* s1, const char* s2) {
+  while (*s1 == *s2++) {
+    if (*s1++ == '\0') return true;
+  }
+  return false;
+}
+
+}  // namespace
+}  // namespace hwy
+
+#endif  // HWY_TESTS_TEST_UTIL_H_
diff --git a/third_party/highway/hwy/type_traits.h b/third_party/highway/hwy/type_traits.h
new file mode 100644
index 000000000..53805dfc2
--- /dev/null
+++ b/third_party/highway/hwy/type_traits.h
@@ -0,0 +1,91 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_TYPE_TRAITS_H_
+#define HWY_TYPE_TRAITS_H_
+
+// Subset of type_traits / numeric_limits for faster compilation.
+
+#include <stddef.h>
+#include <stdint.h>
+
+namespace hwy {
+
+template <bool Condition, class T>
+struct EnableIfT {};
+template <class T>
+struct EnableIfT<true, T> {
+  using type = T;
+};
+
+template <bool Condition, class T = void>
+using EnableIf = typename EnableIfT<Condition, T>::type;
+
+template <typename T>
+constexpr bool IsFloat() {
+  return T(1.25) != T(1);
+}
+
+template <typename T>
+constexpr bool IsSigned() {
+  return T(0) > T(-1);
+}
+
+// Insert into template/function arguments to enable this overload only for
+// <= 128-bit vectors (excluding NONE).
+#define HWY_IF128(T, N) EnableIf<N != 0 && (N * sizeof(T) <= 16)>* = nullptr
+
+#define HWY_IF_FLOAT(T) EnableIf<IsFloat<T>()>* = nullptr
+
+// Largest/smallest representable integer values.
+template <typename T>
+constexpr T LimitsMax() {
+  return IsSigned<T>() ? T((1ULL << (sizeof(T) * 8 - 1)) - 1)
+                       : static_cast<T>(~0ull);
+}
+template <typename T>
+constexpr T LimitsMin() {
+  return IsSigned<T>() ? T(-1) - LimitsMax<T>() : T(0);
+}
+
+// Empty struct used as a size tag type.
+template <size_t N>
+struct SizeTag {};
+
+// The unsigned integer type whose size is kSize bytes.
+template <size_t kSize>
+struct MakeUnsignedT;
+template <>
+struct MakeUnsignedT<1> {
+  using type = uint8_t;
+};
+template <>
+struct MakeUnsignedT<2> {
+  using type = uint16_t;
+};
+template <>
+struct MakeUnsignedT<4> {
+  using type = uint32_t;
+};
+template <>
+struct MakeUnsignedT<8> {
+  using type = uint64_t;
+};
+
+template <typename T>
+using MakeUnsigned = typename MakeUnsignedT<sizeof(T)>::type;
+
+}  // namespace hwy
+
+#endif  // HWY_TYPE_TRAITS_H_
diff --git a/third_party/highway/hwy/wasm.h b/third_party/highway/hwy/wasm.h
new file mode 100644
index 000000000..0821f38cb
--- /dev/null
+++ b/third_party/highway/hwy/wasm.h
@@ -0,0 +1,2287 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_WASM_H_
+#define HWY_WASM_H_
+
+// 128-bit WASM vectors and operations.
+
+#include <wasm_simd128.h>
+
+#include <algorithm>
+#include <cmath>
+#include <cstdint>
+#include <cstdlib>
+
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+#include <stdio.h>
+#endif
+
+#include "hwy/shared.h"
+
+#define HWY_ATTR_WASM HWY_TARGET_ATTR("simd128")
+namespace hwy {
+
+template <typename T>
+struct Raw128 {
+  using type = __v128_u;
+};
+template <>
+struct Raw128<float> {
+  using type = __f32x4;
+};
+
+template <typename T>
+using Full128 = Desc<T, 16 / sizeof(T)>;
+
+template <typename T, size_t N = 16 / sizeof(T)>
+class Vec128 {
+  using Raw = typename Raw128<T>::type;
+
+ public:
+  // Compound assignment. Only usable if there is a corresponding non-member
+  // binary operator overload. For example, only f32 and f64 support division.
+  HWY_ATTR_WASM HWY_INLINE Vec128& operator*=(const Vec128 other) {
+    return *this = (*this * other);
+  }
+  HWY_ATTR_WASM HWY_INLINE Vec128& operator/=(const Vec128 other) {
+    return *this = (*this / other);
+  }
+  HWY_ATTR_WASM HWY_INLINE Vec128& operator+=(const Vec128 other) {
+    return *this = (*this + other);
+  }
+  HWY_ATTR_WASM HWY_INLINE Vec128& operator-=(const Vec128 other) {
+    return *this = (*this - other);
+  }
+  HWY_ATTR_WASM HWY_INLINE Vec128& operator&=(const Vec128 other) {
+    return *this = (*this & other);
+  }
+  HWY_ATTR_WASM HWY_INLINE Vec128& operator|=(const Vec128 other) {
+    return *this = (*this | other);
+  }
+  HWY_ATTR_WASM HWY_INLINE Vec128& operator^=(const Vec128 other) {
+    return *this = (*this ^ other);
+  }
+
+  Raw raw;
+};
+
+// Integer: FF..FF or 0. Float: MSB, all other bits undefined - see README.
+template <typename T, size_t N = 16 / sizeof(T)>
+class Mask128 {
+  using Raw = typename Raw128<T>::type;
+
+ public:
+  Raw raw;
+};
+
+// ------------------------------ Cast
+
+HWY_ATTR_WASM HWY_INLINE __v128_u BitCastToInteger(__v128_u v) { return v; }
+HWY_ATTR_WASM HWY_INLINE __v128_u BitCastToInteger(__f32x4 v) {
+  return static_cast<__v128_u>(v);
+}
+HWY_ATTR_WASM HWY_INLINE __v128_u BitCastToInteger(__f64x2 v) {
+  return static_cast<__v128_u>(v);
+}
+
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N * sizeof(T)> cast_to_u8(
+    Vec128<T, N> v) {
+  return Vec128<uint8_t, N * sizeof(T)>{BitCastToInteger(v.raw)};
+}
+
+// Cannot rely on function overloading because return types differ.
+template <typename T>
+struct BitCastFromInteger128 {
+  HWY_ATTR_WASM HWY_INLINE __v128_u operator()(__v128_u v) { return v; }
+};
+template <>
+struct BitCastFromInteger128<float> {
+  HWY_ATTR_WASM HWY_INLINE __f32x4 operator()(__v128_u v) {
+    return static_cast<__f32x4>(v);
+  }
+};
+
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> cast_u8_to(
+    Desc<T, N> /* tag */, Vec128<uint8_t, N * sizeof(T)> v) {
+  return Vec128<T, N>{BitCastFromInteger128<T>()(v.raw)};
+}
+
+template <typename T, size_t N, typename FromT>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> BitCast(
+    Desc<T, N> d, Vec128<FromT, N * sizeof(T) / sizeof(FromT)> v) {
+  return cast_u8_to(d, cast_to_u8(v));
+}
+
+// ------------------------------ Set
+
+// Returns an all-zero vector/part.
+template <typename T, size_t N, HWY_IF128(T, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> Zero(Desc<T, N> /* tag */) {
+  return Vec128<T, N>{wasm_i32x4_splat(0)};
+}
+template <size_t N, HWY_IF128(float, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Zero(Desc<float, N> /* tag */) {
+  return Vec128<float, N>{wasm_f32x4_splat(0.0f)};
+}
+
+// Returns a vector/part with all lanes set to "t".
+template <size_t N, HWY_IF128(uint8_t, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> Set(Desc<uint8_t, N> /* tag */,
+                                                const uint8_t t) {
+  return Vec128<uint8_t, N>{wasm_i8x16_splat(t)};
+}
+template <size_t N, HWY_IF128(uint16_t, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> Set(Desc<uint16_t, N> /* tag */,
+                                                 const uint16_t t) {
+  return Vec128<uint16_t, N>{wasm_i16x8_splat(t)};
+}
+template <size_t N, HWY_IF128(uint32_t, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> Set(Desc<uint32_t, N> /* tag */,
+                                                 const uint32_t t) {
+  return Vec128<uint32_t, N>{wasm_i32x4_splat(t)};
+}
+
+template <size_t N, HWY_IF128(int8_t, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> Set(Desc<int8_t, N> /* tag */,
+                                               const int8_t t) {
+  return Vec128<int8_t, N>{wasm_i8x16_splat(t)};
+}
+template <size_t N, HWY_IF128(int16_t, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> Set(Desc<int16_t, N> /* tag */,
+                                                const int16_t t) {
+  return Vec128<int16_t, N>{wasm_i16x8_splat(t)};
+}
+template <size_t N, HWY_IF128(int32_t, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> Set(Desc<int32_t, N> /* tag */,
+                                                const int32_t t) {
+  return Vec128<int32_t, N>{wasm_i32x4_splat(t)};
+}
+
+template <size_t N, HWY_IF128(float, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Set(Desc<float, N> /* tag */,
+                                              const float t) {
+  return Vec128<float, N>{wasm_f32x4_splat(t)};
+}
+
+// Returns a vector with lane i=[0, N) set to "first" + i. Unique per-lane
+// values are required to detect lane-crossing bugs.
+template <typename T, size_t N, typename T2, HWY_IF128(T, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> Iota(Desc<T, N> d, const T2 first) {
+  alignas(16) T lanes[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = first + i;
+  }
+  return Load(d, lanes);
+}
+
+HWY_DIAGNOSTICS(push)
+HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized")
+
+// Returns a vector with uninitialized elements.
+template <typename T, size_t N, HWY_IF128(T, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> Undefined(Desc<T, N> /* tag */) {
+  __v128_u raw;
+  return Vec128<T, N>{raw};
+}
+template <size_t N, HWY_IF128(float, N)>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Undefined(Desc<float, N> /* tag */) {
+  __f32x4 raw;
+  return Vec128<float, N>{raw};
+}
+
+HWY_DIAGNOSTICS(pop)
+
+// ================================================== ARITHMETIC
+
+// ------------------------------ Addition
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> operator+(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{wasm_i8x16_add(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> operator+(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{wasm_i16x8_add(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> operator+(
+    const Vec128<uint32_t, N> a, const Vec128<uint32_t, N> b) {
+  return Vec128<uint32_t, N>{wasm_i32x4_add(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> operator+(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{wasm_i8x16_add(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> operator+(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{wasm_i16x8_add(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> operator+(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Vec128<int32_t, N>{wasm_i32x4_add(a.raw, b.raw)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> operator+(const Vec128<float, N> a,
+                                                    const Vec128<float, N> b) {
+  return Vec128<float, N>{wasm_f32x4_add(a.raw, b.raw)};
+}
+
+// ------------------------------ Subtraction
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> operator-(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{wasm_i8x16_sub(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> operator-(Vec128<uint16_t, N> a,
+                                                       Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{wasm_i16x8_sub(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> operator-(
+    const Vec128<uint32_t, N> a, const Vec128<uint32_t, N> b) {
+  return Vec128<uint32_t, N>{wasm_i32x4_sub(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> operator-(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{wasm_i8x16_sub(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> operator-(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{wasm_i16x8_sub(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> operator-(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Vec128<int32_t, N>{wasm_i32x4_sub(a.raw, b.raw)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> operator-(const Vec128<float, N> a,
+                                                    const Vec128<float, N> b) {
+  return Vec128<float, N>{wasm_f32x4_sub(a.raw, b.raw)};
+}
+
+// ------------------------------ Saturating addition
+
+// Returns a + b clamped to the destination range.
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> SaturatedAdd(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{wasm_u8x16_add_saturate(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> SaturatedAdd(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{wasm_u16x8_add_saturate(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> SaturatedAdd(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{wasm_i8x16_add_saturate(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> SaturatedAdd(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{wasm_i16x8_add_saturate(a.raw, b.raw)};
+}
+
+// ------------------------------ Saturating subtraction
+
+// Returns a - b clamped to the destination range.
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> SaturatedSub(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{wasm_u8x16_sub_saturate(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> SaturatedSub(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{wasm_u16x8_sub_saturate(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> SaturatedSub(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{wasm_i8x16_sub_saturate(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> SaturatedSub(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{wasm_i16x8_sub_saturate(a.raw, b.raw)};
+}
+
+// ------------------------------ Average
+
+// Returns (a + b + 1) / 2
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> AverageRound(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  // TODO(eustas): declared in JS spec, but not in include file.
+  alignas(16) uint8_t a_lanes[16];
+  alignas(16) uint8_t b_lanes[16];
+  alignas(16) uint8_t output[16];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 16; ++i) {
+    output[i] = (a_lanes[i] + b_lanes[i] + 1) >> 1;
+  }
+  return Vec128<uint8_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> AverageRound(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  // TODO(eustas): declared in JS spec, but not in include file.
+  alignas(16) uint16_t a_lanes[8];
+  alignas(16) uint16_t b_lanes[8];
+  alignas(16) uint16_t output[8];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 8; ++i) {
+    output[i] = (a_lanes[i] + b_lanes[i] + 1) >> 1;
+  }
+  return Vec128<uint16_t, N>{wasm_v128_load(output)};
+}
+
+// ------------------------------ Absolute value
+
+// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1.
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) {
+  // TODO(eustas): return unsigned_min(v, ~v + 1)?
+  alignas(16) int8_t input[16];
+  alignas(16) int8_t output[16];
+  wasm_v128_store(input, v.raw);
+  for (size_t i = 0; i < 16; ++i) {
+    output[i] = std::abs(input[i]);
+  }
+  // what should happen to -128?
+  return Vec128<int8_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) {
+  // TODO(eustas): return unsigned_min(v, ~v + 1)?
+  alignas(16) int16_t input[8];
+  alignas(16) int16_t output[8];
+  wasm_v128_store(input, v.raw);
+  for (size_t i = 0; i < 8; ++i) {
+    output[i] = std::abs(input[i]);
+  }
+  return Vec128<int16_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) {
+  // TODO(eustas): return unsigned_min(v, ~v + 1)?
+  alignas(16) int32_t input[4];
+  alignas(16) int32_t output[4];
+  wasm_v128_store(input, v.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::abs(input[i]);
+  }
+  // what should happen to -128?
+  return Vec128<int32_t, N>{wasm_v128_load(output)};
+}
+
+// ------------------------------ Shift lanes by constant #bits
+
+// Unsigned
+template <int kBits, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> ShiftLeft(
+    const Vec128<uint16_t, N> v) {
+  return Vec128<uint16_t, N>{wasm_i16x8_shl(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> ShiftRight(
+    const Vec128<uint16_t, N> v) {
+  return Vec128<uint16_t, N>{wasm_u16x8_shr(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> ShiftLeft(
+    const Vec128<uint32_t, N> v) {
+  return Vec128<uint32_t, N>{wasm_i32x4_shl(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> ShiftRight(
+    const Vec128<uint32_t, N> v) {
+  return Vec128<uint32_t, N>{wasm_u32x4_shr(v.raw, kBits)};
+}
+
+// Signed
+template <int kBits, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> ShiftLeft(
+    const Vec128<int16_t, N> v) {
+  return Vec128<int16_t, N>{wasm_i16x8_shl(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> ShiftRight(
+    const Vec128<int16_t, N> v) {
+  return Vec128<int16_t, N>{wasm_i16x8_shr(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> ShiftLeft(
+    const Vec128<int32_t, N> v) {
+  return Vec128<int32_t, N>{wasm_i32x4_shl(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> ShiftRight(
+    const Vec128<int32_t, N> v) {
+  return Vec128<int32_t, N>{wasm_i32x4_shr(v.raw, kBits)};
+}
+
+// ------------------------------ Shift lanes by same variable #bits
+
+// Unsigned (no u8)
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> ShiftLeftSame(
+    const Vec128<uint16_t, N> v, const int bits) {
+  return Vec128<uint16_t, N>{wasm_i16x8_shl(v.raw, bits)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> ShiftRightSame(
+    const Vec128<uint16_t, N> v, const int bits) {
+  return Vec128<uint16_t, N>{wasm_u16x8_shr(v.raw, bits)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> ShiftLeftSame(
+    const Vec128<uint32_t, N> v, const int bits) {
+  return Vec128<uint32_t, N>{wasm_i32x4_shl(v.raw, bits)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> ShiftRightSame(
+    const Vec128<uint32_t, N> v, const int bits) {
+  return Vec128<uint32_t, N>{wasm_u32x4_shr(v.raw, bits)};
+}
+
+// Signed (no i8)
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> ShiftLeftSame(
+    const Vec128<int16_t, N> v, const int bits) {
+  return Vec128<int16_t, N>{wasm_i16x8_shl(v.raw, bits)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> ShiftRightSame(
+    const Vec128<int16_t, N> v, const int bits) {
+  return Vec128<int16_t, N>{wasm_i16x8_shr(v.raw, bits)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> ShiftLeftSame(
+    const Vec128<int32_t, N> v, const int bits) {
+  return Vec128<int32_t, N>{wasm_i32x4_shl(v.raw, bits)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> ShiftRightSame(
+    const Vec128<int32_t, N> v, const int bits) {
+  return Vec128<int32_t, N>{wasm_i32x4_shr(v.raw, bits)};
+}
+
+// ------------------------------ Shift lanes by independent variable #bits
+
+// Unsupported.
+
+// ------------------------------ Minimum
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> Min(const Vec128<uint8_t, N> a,
+                                                const Vec128<uint8_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) uint8_t a_lanes[16];
+  alignas(16) uint8_t b_lanes[16];
+  alignas(16) uint8_t output[16];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 16; ++i) {
+    output[i] = std::min(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<uint8_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> Min(const Vec128<uint16_t, N> a,
+                                                 const Vec128<uint16_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) uint16_t a_lanes[8];
+  alignas(16) uint16_t b_lanes[8];
+  alignas(16) uint16_t output[8];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 8; ++i) {
+    output[i] = std::min(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<uint16_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> Min(const Vec128<uint32_t, N> a,
+                                                 const Vec128<uint32_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) uint32_t a_lanes[4];
+  alignas(16) uint32_t b_lanes[4];
+  alignas(16) uint32_t output[4];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::min(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<uint32_t, N>{wasm_v128_load(output)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> Min(const Vec128<int8_t, N> a,
+                                               const Vec128<int8_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) int8_t a_lanes[16];
+  alignas(16) int8_t b_lanes[16];
+  alignas(16) int8_t output[16];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 16; ++i) {
+    output[i] = std::min(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<int8_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> Min(const Vec128<int16_t, N> a,
+                                                const Vec128<int16_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) int16_t a_lanes[8];
+  alignas(16) int16_t b_lanes[8];
+  alignas(16) int16_t output[8];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 8; ++i) {
+    output[i] = std::min(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<int16_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> Min(const Vec128<int32_t, N> a,
+                                                const Vec128<int32_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) int32_t a_lanes[4];
+  alignas(16) int32_t b_lanes[4];
+  alignas(16) int32_t output[4];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::min(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<int32_t, N>{wasm_v128_load(output)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Min(const Vec128<float, N> a,
+                                              const Vec128<float, N> b) {
+  return Vec128<float, N>{__builtin_wasm_min_f32x4(a.raw, b.raw)};
+}
+
+// ------------------------------ Maximum
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> Max(const Vec128<uint8_t, N> a,
+                                                const Vec128<uint8_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) uint8_t a_lanes[16];
+  alignas(16) uint8_t b_lanes[16];
+  alignas(16) uint8_t output[16];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 16; ++i) {
+    output[i] = std::max(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<uint8_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> Max(const Vec128<uint16_t, N> a,
+                                                 const Vec128<uint16_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) uint16_t a_lanes[8];
+  alignas(16) uint16_t b_lanes[8];
+  alignas(16) uint16_t output[8];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 8; ++i) {
+    output[i] = std::max(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<uint16_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> Max(const Vec128<uint32_t, N> a,
+                                                 const Vec128<uint32_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) uint32_t a_lanes[4];
+  alignas(16) uint32_t b_lanes[4];
+  alignas(16) uint32_t output[4];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::max(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<uint32_t, N>{wasm_v128_load(output)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> Max(const Vec128<int8_t, N> a,
+                                               const Vec128<int8_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) int8_t a_lanes[16];
+  alignas(16) int8_t b_lanes[16];
+  alignas(16) int8_t output[16];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 16; ++i) {
+    output[i] = std::max(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<int8_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> Max(const Vec128<int16_t, N> a,
+                                                const Vec128<int16_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) int16_t a_lanes[8];
+  alignas(16) int16_t b_lanes[8];
+  alignas(16) int16_t output[8];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 8; ++i) {
+    output[i] = std::max(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<int16_t, N>{wasm_v128_load(output)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> Max(const Vec128<int32_t, N> a,
+                                                const Vec128<int32_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) int32_t a_lanes[4];
+  alignas(16) int32_t b_lanes[4];
+  alignas(16) int32_t output[4];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::max(a_lanes[i], b_lanes[i]);
+  }
+  return Vec128<int32_t, N>{wasm_v128_load(output)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Max(const Vec128<float, N> a,
+                                              const Vec128<float, N> b) {
+  return Vec128<float, N>{__builtin_wasm_max_f32x4(a.raw, b.raw)};
+}
+
+// Returns the closest value to v within [lo, hi].
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> Clamp(const Vec128<T, N> v,
+                                            const Vec128<T, N> lo,
+                                            const Vec128<T, N> hi) {
+  return Min(Max(lo, v), hi);
+}
+
+// ------------------------------ Integer multiplication
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> operator*(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{wasm_i16x8_mul(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t, N> operator*(
+    const Vec128<uint32_t, N> a, const Vec128<uint32_t, N> b) {
+  return Vec128<uint32_t, N>{wasm_i32x4_mul(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> operator*(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{wasm_i16x8_mul(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> operator*(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Vec128<int32_t, N>{wasm_i32x4_mul(a.raw, b.raw)};
+}
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// Returns the upper 16 bits of a * b in each lane.
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> MulHigh(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) uint16_t a_lanes[8];
+  alignas(16) uint16_t b_lanes[8];
+  alignas(16) uint16_t c_lanes[8];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 8; ++i) {
+    uint32_t ab = static_cast<uint32_t>(a_lanes[i]) * b_lanes[i];
+    c_lanes[i] = static_cast<uint16_t>(ab >> 16);
+  }
+  return Vec128<uint16_t>{wasm_v128_load(c_lanes)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> MulHigh(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) int16_t a_lanes[8];
+  alignas(16) int16_t b_lanes[8];
+  alignas(16) int16_t c_lanes[8];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 8; ++i) {
+    int32_t ab = static_cast<int32_t>(a_lanes[i]) * b_lanes[i];
+    c_lanes[i] = static_cast<int16_t>(ab >> 16);
+  }
+  return Vec128<int16_t>{wasm_v128_load(c_lanes)};
+}
+
+}  // namespace ext
+
+// Multiplies even lanes (0, 2 ..) and places the double-wide result into
+// even and the upper half into its odd neighbor lane.
+HWY_ATTR_WASM HWY_INLINE Vec128<int64_t> MulEven(const Vec128<int32_t> a,
+                                                 const Vec128<int32_t> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) int32_t a_lanes[4];
+  alignas(16) int32_t b_lanes[4];
+  alignas(16) int64_t c_lanes[2];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 2; ++i) {
+    c_lanes[i] = static_cast<int64_t>(a_lanes[2 * i]) * b_lanes[2 * i];
+  }
+  return Vec128<int64_t>{wasm_v128_load(c_lanes)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<uint64_t> MulEven(const Vec128<uint32_t> a,
+                                                  const Vec128<uint32_t> b) {
+  // TODO(eustas): replace, when implemented in WASM.
+  alignas(16) uint32_t a_lanes[4];
+  alignas(16) uint32_t b_lanes[4];
+  alignas(16) uint64_t c_lanes[2];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 2; ++i) {
+    c_lanes[i] = static_cast<uint64_t>(a_lanes[2 * i]) * b_lanes[2 * i];
+  }
+  return Vec128<uint64_t>{wasm_v128_load(c_lanes)};
+}
+
+// ------------------------------ Floating-point negate
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Neg(const Vec128<float, N> v) {
+  const Desc<float, N> df;
+  const Desc<uint32_t, N> du;
+  const auto sign = BitCast(df, Set(du, 0x80000000u));
+  return v ^ sign;
+}
+
+// ------------------------------ Floating-point mul / div
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> operator*(Vec128<float, N> a,
+                                                    Vec128<float, N> b) {
+  return Vec128<float, N>{wasm_f32x4_mul(a.raw, b.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> operator/(const Vec128<float, N> a,
+                                                    const Vec128<float, N> b) {
+  return Vec128<float, N>{wasm_f32x4_div(a.raw, b.raw)};
+}
+
+// Approximate reciprocal
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> ApproximateReciprocal(
+    const Vec128<float, N> v) {
+  // TODO(eustas): replace, when implemented in WASM.
+  const Vec128<float, N> one = Vec128<float, N>{wasm_f32x4_splat(1.0f)};
+  return one / v;
+}
+
+namespace ext {
+// Absolute value of difference.
+HWY_ATTR_WASM HWY_INLINE Vec128<float> AbsDiff(const Vec128<float> a,
+                                               const Vec128<float> b) {
+  const auto mask =
+      BitCast(Full128<float>(), Set(Full128<uint32_t>(), 0x7FFFFFFFu));
+  return Vec128<float>{wasm_v128_and(mask.raw, (a - b).raw)};
+}
+}  // namespace ext
+
+// ------------------------------ Floating-point multiply-add variants
+
+// Returns mul * x + add
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> MulAdd(const Vec128<float, N> mul,
+                                                 const Vec128<float, N> x,
+                                                 const Vec128<float, N> add) {
+  // TODO(eustas): replace, when implemented in WASM.
+  // TODO(eustas): is it wasm_f32x4_qfma?
+  return mul * x + add;
+}
+
+// Returns add - mul * x
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> NegMulAdd(
+    const Vec128<float, N> mul, const Vec128<float, N> x,
+    const Vec128<float, N> add) {
+  // TODO(eustas): replace, when implemented in WASM.
+  return add - mul * x;
+}
+
+// Slightly more expensive on ARM (extra negate)
+namespace ext {
+
+// Returns mul * x - sub
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> MulSub(const Vec128<float, N> mul,
+                                                 const Vec128<float, N> x,
+                                                 const Vec128<float, N> sub) {
+  // TODO(eustas): replace, when implemented in WASM.
+  // TODO(eustas): is it wasm_f32x4_qfms?
+  return mul * x - sub;
+}
+
+// Returns -mul * x - sub
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> NegMulSub(
+    const Vec128<float, N> mul, const Vec128<float, N> x,
+    const Vec128<float, N> sub) {
+  // TODO(eustas): replace, when implemented in WASM.
+  return Neg(mul) * x - sub;
+}
+
+}  // namespace ext
+
+// ------------------------------ Floating-point square root
+
+// Full precision square root
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Sqrt(const Vec128<float, N> v) {
+  alignas(16) float input[4];
+  alignas(16) float output[4];
+  wasm_v128_store(input, v.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::sqrtf(input[i]);
+  }
+  return Vec128<float>{wasm_v128_load(output)};
+  // TODO(eustas): requires target feature "unimplemented-simd128"
+  // return Vec128<float, N>{wasm_f32x4_sqrt(v.raw)};
+}
+
+// Approximate reciprocal square root
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> ApproximateReciprocalSqrt(
+    const Vec128<float, N> v) {
+  // TODO(eustas): find cheaper a way to calculate this.
+  const Vec128<float, N> one = Vec128<float, N>{wasm_f32x4_splat(1.0f)};
+  return one / Sqrt(v);
+}
+
+// ------------------------------ Floating-point rounding
+
+// Toward nearest integer, ties to even
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Round(const Vec128<float, N> v) {
+  // TODO(eustas): workaround does not work - wasm_i32x4_trunc_saturate_f32x4
+  //               limits feasible input to +-2^31.
+  // TODO(eustas): how to do "ties to even"?
+  // TODO(eustas): 8 ops; isn't it cheaper to store/convert/load?
+  // const __f32x4 c00 = wasm_f32x4_splat(0.0f);
+  // const __f32x4 corr = wasm_f32x4_convert_i32x4(wasm_f32x4_le(v.raw, c00));
+  // const __f32x4 c05 = wasm_f32x4_splat(0.5f);
+  // +0.5 for non-negative lane, -0.5 for other.
+  // const __f32x4 delta = wasm_f32x4_add(c05, corr);
+  // Shift input by 0.5 away from 0.
+  // const __f32x4 fixed = wasm_f32x4_add(v.raw, delta);
+  // const __i32x4 result = wasm_i32x4_trunc_saturate_f32x4(fixed);
+  // return Vec128<float, N>{wasm_f32x4_convert_i32x4(result)};
+  alignas(16) float input[4];
+  alignas(16) float output[4];
+  wasm_v128_store(input, v.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::round(input[i]);
+  }
+  return Vec128<float>{wasm_v128_load(output)};
+}
+
+// Toward zero, aka truncate
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Trunc(const Vec128<float, N> v) {
+  // TODO(eustas): impossible
+  // const __i32x4 result = wasm_i32x4_trunc_saturate_f32x4(v.raw);
+  // return Vec128<float, N>{wasm_f32x4_convert_i32x4(result)};
+  alignas(16) float input[4];
+  alignas(16) float output[4];
+  wasm_v128_store(input, v.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::trunc(input[i]);
+  }
+  return Vec128<float>{wasm_v128_load(output)};
+}
+
+// Toward +infinity, aka ceiling
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Ceil(const Vec128<float, N> v) {
+  // TODO(eustas): impossible
+  // const __i32x4 truncated = wasm_i32x4_trunc_saturate_f32x4(v.raw);
+  // const __f32x4 draft = wasm_f32x4_convert_i32x4(truncated);
+  // -1 for positive with fractional part, 0 for negative and integer.
+  // const __i32x4 delta = wasm_f32x4_gt(v.raw, draft);
+  // const __f32x4 corr = wasm_f32x4_convert_i32x4(delta);
+  // return Vec128<float, N>{wasm_f32x4_sub(draft, corr)};
+  alignas(16) float input[4];
+  alignas(16) float output[4];
+  wasm_v128_store(input, v.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::ceil(input[i]);
+  }
+  return Vec128<float>{wasm_v128_load(output)};
+}
+
+// Toward -infinity, aka floor
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> Floor(const Vec128<float, N> v) {
+  // TODO(eustas): impossible
+  // const __i32x4 truncated = wasm_i32x4_trunc_saturate_f32x4(v.raw);
+  // const __f32x4 draft = wasm_f32x4_convert_i32x4(truncated);
+  // -1 for negative with fractional part, 0 for positive and integer.
+  // const __i32x4 delta = wasm_f32x4_lt(v.raw, draft);
+  // const __f32x4 corr = wasm_f32x4_convert_i32x4(delta);
+  // return Vec128<float, N>{wasm_f32x4_add(draft, corr)};
+  alignas(16) float input[4];
+  alignas(16) float output[4];
+  wasm_v128_store(input, v.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[i] = std::floor(input[i]);
+  }
+  return Vec128<float>{wasm_v128_load(output)};
+}
+
+// ================================================== COMPARE
+
+// Comparisons fill a lane with 1-bits if the condition is true, else 0.
+
+// ------------------------------ Equality
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<uint8_t, N> operator==(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Mask128<uint8_t, N>{wasm_i8x16_eq(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<uint16_t, N> operator==(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Mask128<uint16_t, N>{wasm_i16x8_eq(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<uint32_t, N> operator==(
+    const Vec128<uint32_t, N> a, const Vec128<uint32_t, N> b) {
+  return Mask128<uint32_t, N>{wasm_i32x4_eq(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<int8_t, N> operator==(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Mask128<int8_t, N>{wasm_i8x16_eq(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<int16_t, N> operator==(Vec128<int16_t, N> a,
+                                                        Vec128<int16_t, N> b) {
+  return Mask128<int16_t, N>{wasm_i16x8_eq(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<int32_t, N> operator==(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Mask128<int32_t, N>{wasm_i32x4_eq(a.raw, b.raw)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<float, N> operator==(
+    const Vec128<float, N> a, const Vec128<float, N> b) {
+  return Mask128<float, N>{wasm_f32x4_eq(a.raw, b.raw)};
+}
+
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<T, N> TestBit(Vec128<T, N> v,
+                                               Vec128<T, N> bit) {
+  static_assert(!IsFloat<T>(), "Only integer vectors supported");
+  return (v & bit) == bit;
+}
+
+// ------------------------------ Strict inequality
+
+// Signed/float <
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<int8_t, N> operator<(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Mask128<int8_t, N>{wasm_i8x16_lt(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<int16_t, N> operator<(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Mask128<int16_t, N>{wasm_i16x8_lt(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<int32_t, N> operator<(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Mask128<int32_t, N>{wasm_i32x4_lt(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<float, N> operator<(const Vec128<float, N> a,
+                                                     const Vec128<float, N> b) {
+  return Mask128<float, N>{wasm_f32x4_lt(a.raw, b.raw)};
+}
+
+// Signed/float >
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<int8_t, N> operator>(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Mask128<int8_t, N>{wasm_i8x16_gt(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<int16_t, N> operator>(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Mask128<int16_t, N>{wasm_i16x8_gt(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<int32_t, N> operator>(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Mask128<int32_t, N>{wasm_i32x4_gt(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<float, N> operator>(const Vec128<float, N> a,
+                                                     const Vec128<float, N> b) {
+  return Mask128<float, N>{wasm_f32x4_gt(a.raw, b.raw)};
+}
+
+// ------------------------------ Weak inequality
+
+// Float <= >=
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<float, N> operator<=(
+    const Vec128<float, N> a, const Vec128<float, N> b) {
+  return Mask128<float, N>{wasm_f32x4_le(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<float, N> operator>=(
+    const Vec128<float, N> a, const Vec128<float, N> b) {
+  return Mask128<float, N>{wasm_f32x4_ge(a.raw, b.raw)};
+}
+
+// ================================================== LOGICAL
+
+// ------------------------------ Bitwise AND
+
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> operator&(Vec128<T, N> a,
+                                                Vec128<T, N> b) {
+  return Vec128<T, N>{wasm_v128_and(a.raw, b.raw)};
+}
+
+// ------------------------------ Bitwise AND-NOT
+
+// Returns ~not_mask & mask.
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> AndNot(Vec128<T, N> not_mask,
+                                             Vec128<T, N> mask) {
+  return Vec128<T, N>{wasm_v128_and(wasm_v128_not(not_mask.raw), mask.raw)};
+}
+
+// ------------------------------ Bitwise OR
+
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> operator|(Vec128<T, N> a,
+                                                Vec128<T, N> b) {
+  return Vec128<T, N>{wasm_v128_or(a.raw, b.raw)};
+}
+
+// ------------------------------ Bitwise XOR
+
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> operator^(Vec128<T, N> a,
+                                                Vec128<T, N> b) {
+  return Vec128<T, N>{wasm_v128_xor(a.raw, b.raw)};
+}
+
+// ------------------------------ Mask
+
+// Mask and Vec are the same (true = FF..FF).
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Mask128<T, N> MaskFromVec(const Vec128<T, N> v) {
+  return Mask128<T, N>{v.raw};
+}
+
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> VecFromMask(const Mask128<T, N> v) {
+  return Vec128<T, N>{v.raw};
+}
+
+// mask ? yes : no
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> IfThenElse(Mask128<T, N> mask,
+                                                 Vec128<T, N> yes,
+                                                 Vec128<T, N> no) {
+  return Vec128<T, N>{wasm_v128_bitselect(yes.raw, no.raw, mask.raw)};
+}
+
+// mask ? yes : 0
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> IfThenElseZero(Mask128<T, N> mask,
+                                                     Vec128<T, N> yes) {
+  return yes & VecFromMask(mask);
+}
+
+// mask ? 0 : no
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> IfThenZeroElse(Mask128<T, N> mask,
+                                                     Vec128<T, N> no) {
+  return AndNot(VecFromMask(mask), no);
+}
+
+template <typename T, size_t N, HWY_IF_FLOAT(T)>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) {
+  const Desc<T, N> d;
+  return IfThenElse(MaskFromVec(v), Zero(d), v);
+}
+
+// ================================================== MEMORY
+
+// ------------------------------ Load
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> Load(Full128<T> /* tag */,
+                                        const T* HWY_RESTRICT aligned) {
+  return Vec128<T>{wasm_v128_load(aligned)};
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> LoadU(Full128<T> /* tag */,
+                                         const T* HWY_RESTRICT p) {
+  return Vec128<T>{wasm_v128_load(p)};
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, 8 / sizeof(T)> Load(
+    Desc<T, 8 / sizeof(T)> /* tag */, const T* HWY_RESTRICT p) {
+  // TODO(eustas): In WASM memory model do we care about overflow?
+  // TODO(eustas): Use memcpy for safe load?
+  // TODO(eustas): Use memcpy + zero init to set the upper lanes to 0.
+  return Vec128<T, 8 / sizeof(T)>{wasm_v128_load(p)};
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, 4 / sizeof(T)> Load(
+    Desc<T, 4 / sizeof(T)> /* tag */, const T* HWY_RESTRICT p) {
+  return Vec128<T, 4 / sizeof(T)>{wasm_v128_load(p)};
+}
+
+// 128-bit SIMD => nothing to duplicate, same as an unaligned load.
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> LoadDup128(Full128<T> /* tag */,
+                                              const T* HWY_RESTRICT p) {
+  return Vec128<T>{wasm_v128_load(p)};
+}
+
+// ------------------------------ Store
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE void Store(Vec128<T> v, Full128<T> /* tag */,
+                                    T* HWY_RESTRICT aligned) {
+  wasm_v128_store(aligned, v.raw);
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE void StoreU(Vec128<T> v, Full128<T> /* tag */,
+                                     T* HWY_RESTRICT p) {
+  wasm_v128_store(p, v.raw);
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE void Store(Vec128<T, 8 / sizeof(T)> v,
+                                    Desc<T, 8 / sizeof(T)> /* tag */,
+                                    T* HWY_RESTRICT p) {
+  CopyBytes<8>(&v, p);
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE void Store(Vec128<T, 4 / sizeof(T)> v,
+                                    Desc<T, 4 / sizeof(T)> /* tag */,
+                                    T* HWY_RESTRICT p) {
+  CopyBytes<4>(&v, p);
+}
+HWY_ATTR_WASM HWY_INLINE void Store(const Vec128<float, 1> v,
+                                    Desc<float, 1> /* tag */,
+                                    float* HWY_RESTRICT p) {
+  *p = wasm_f32x4_extract_lane(v.raw, 0);
+}
+
+// ------------------------------ Non-temporal stores
+
+// Same as aligned stores on non-x86.
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE void Stream(Vec128<T> v, Full128<T> /* tag */,
+                                     T* HWY_RESTRICT aligned) {
+  wasm_v128_store(aligned, v.raw);
+}
+
+// ------------------------------ Gather
+
+// Unsupported.
+
+// ================================================== SWIZZLE
+
+// ------------------------------ Extract lane
+
+// Gets the single value stored in a vector/part.
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE uint16_t GetLane(const Vec128<uint16_t, N> v) {
+  return wasm_i16x8_extract_lane(v.raw, 0);
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE int16_t GetLane(const Vec128<int16_t, N> v) {
+  return wasm_i16x8_extract_lane(v.raw, 0);
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE uint32_t GetLane(const Vec128<uint32_t, N> v) {
+  return wasm_i32x4_extract_lane(v.raw, 0);
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE int32_t GetLane(const Vec128<int32_t, N> v) {
+  return wasm_i32x4_extract_lane(v.raw, 0);
+}
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE float GetLane(const Vec128<float, N> v) {
+  return wasm_f32x4_extract_lane(v.raw, 0);
+}
+
+// ------------------------------ Extract half
+
+// Returns upper/lower half of a vector.
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N / 2> LowerHalf(Vec128<T, N> v) {
+  return Vec128<T, N / 2>{v.raw};
+}
+
+// These copy hi into lo (smaller instruction encoding than shifts).
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, 8 / sizeof(T)> UpperHalf(Vec128<T> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<T, 8 / sizeof(T)>{wasm_v8x16_shuffle(v.raw, v.raw, 8, 9, 10, 11,
+                                                     12, 13, 14, 15, 8, 9, 10,
+                                                     11, 12, 13, 14, 15)};
+}
+template <>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, 2> UpperHalf(Vec128<float> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<float, 2>{wasm_v8x16_shuffle(v.raw, v.raw, 24, 25, 26, 27, 28,
+                                             29, 30, 31, 8, 9, 10, 11, 12, 13,
+                                             14, 15)};
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, 8 / sizeof(T)> GetHalf(Lower /* tag */,
+                                                          Vec128<T> v) {
+  return LowerHalf(v);
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, 8 / sizeof(T)> GetHalf(Upper /* tag */,
+                                                          Vec128<T> v) {
+  return UpperHalf(v);
+}
+
+// ------------------------------ Shift vector by constant #bytes
+
+// 0x01..0F, kBytes = 1 => 0x02..0F00
+template <int kBytes, typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> ShiftLeftBytes(const Vec128<T> v) {
+  static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
+  const __i8x16 zero = wasm_i8x16_splat(0);
+  switch (kBytes) {
+    case 0:
+      return v;
+
+    case 1:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 0, 1, 2, 3, 4, 5, 6,
+                                          7, 8, 9, 10, 11, 12, 13, 14)};
+
+    case 2:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 0, 1, 2, 3, 4, 5,
+                                          6, 7, 8, 9, 10, 11, 12, 13)};
+
+    case 3:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 0, 1, 2, 3,
+                                          4, 5, 6, 7, 8, 9, 10, 11, 12)};
+
+    case 4:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 0, 1, 2,
+                                          3, 4, 5, 6, 7, 8, 9, 10, 11)};
+
+    case 5:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 0, 1,
+                                          2, 3, 4, 5, 6, 7, 8, 9, 10)};
+
+    case 6:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          0, 1, 2, 3, 4, 5, 6, 7, 8, 9)};
+
+    case 7:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          16, 0, 1, 2, 3, 4, 5, 6, 7, 8)};
+
+    case 8:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          16, 16, 0, 1, 2, 3, 4, 5, 6, 7)};
+
+    case 9:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          16, 16, 16, 0, 1, 2, 3, 4, 5, 6)};
+
+    case 10:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          16, 16, 16, 16, 0, 1, 2, 3, 4, 5)};
+
+    case 11:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          16, 16, 16, 16, 16, 0, 1, 2, 3, 4)};
+
+    case 12:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          16, 16, 16, 16, 16, 16, 0, 1, 2, 3)};
+
+    case 13:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          16, 16, 16, 16, 16, 16, 16, 0, 1, 2)};
+
+    case 14:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          16, 16, 16, 16, 16, 16, 16, 16, 0,
+                                          1)};
+
+    case 15:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 16, 16, 16, 16, 16, 16,
+                                          16, 16, 16, 16, 16, 16, 16, 16, 16,
+                                          0)};
+  }
+  return Vec128<T>{zero};
+}
+
+template <int kLanes, typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> ShiftLeftLanes(const Vec128<T> v) {
+  return ShiftLeftBytes<kLanes * sizeof(T)>(v);
+}
+
+// 0x01..0F, kBytes = 1 => 0x0001..0E
+template <int kBytes, typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> ShiftRightBytes(const Vec128<T> v) {
+  static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
+  const __i8x16 zero = wasm_i8x16_splat(0);
+  switch (kBytes) {
+    case 0:
+      return v;
+
+    case 1:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 1, 2, 3, 4, 5, 6, 7, 8,
+                                          9, 10, 11, 12, 13, 14, 15, 16)};
+
+    case 2:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 2, 3, 4, 5, 6, 7, 8, 9,
+                                          10, 11, 12, 13, 14, 15, 16, 16)};
+
+    case 3:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 3, 4, 5, 6, 7, 8, 9, 10,
+                                          11, 12, 13, 14, 15, 16, 16, 16)};
+
+    case 4:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 4, 5, 6, 7, 8, 9, 10, 11,
+                                          12, 13, 14, 15, 16, 16, 16, 16)};
+
+    case 5:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 5, 6, 7, 8, 9, 10, 11,
+                                          12, 13, 14, 15, 16, 16, 16, 16, 16)};
+
+    case 6:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 6, 7, 8, 9, 10, 11, 12,
+                                          13, 14, 15, 16, 16, 16, 16, 16, 16)};
+
+    case 7:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 7, 8, 9, 10, 11, 12, 13,
+                                          14, 15, 16, 16, 16, 16, 16, 16, 16)};
+
+    case 8:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 8, 9, 10, 11, 12, 13, 14,
+                                          15, 16, 16, 16, 16, 16, 16, 16, 16)};
+
+    case 9:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 9, 10, 11, 12, 13, 14,
+                                          15, 16, 16, 16, 16, 16, 16, 16, 16,
+                                          16)};
+
+    case 10:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 10, 11, 12, 13, 14, 15,
+                                          16, 16, 16, 16, 16, 16, 16, 16, 16,
+                                          16)};
+
+    case 11:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 11, 12, 13, 14, 15, 16,
+                                          16, 16, 16, 16, 16, 16, 16, 16, 16,
+                                          16)};
+
+    case 12:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 12, 13, 14, 15, 16, 16,
+                                          16, 16, 16, 16, 16, 16, 16, 16, 16,
+                                          16)};
+
+    case 13:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 13, 14, 15, 16, 16, 16,
+                                          16, 16, 16, 16, 16, 16, 16, 16, 16,
+                                          16)};
+
+    case 14:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 14, 15, 16, 16, 16, 16,
+                                          16, 16, 16, 16, 16, 16, 16, 16, 16,
+                                          16)};
+
+    case 15:
+      return Vec128<T>{wasm_v8x16_shuffle(v.raw, zero, 15, 16, 16, 16, 16, 16,
+                                          16, 16, 16, 16, 16, 16, 16, 16, 16,
+                                          16)};
+  }
+  return Vec128<T>{zero};
+}
+
+template <int kLanes, typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> ShiftRightLanes(const Vec128<T> v) {
+  return ShiftRightBytes<kLanes * sizeof(T)>(v);
+}
+
+// ------------------------------ Extract from 2x 128-bit at constant offset
+
+// Extracts 128 bits from <hi, lo> by skipping the least-significant kBytes.
+template <int kBytes, typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> CombineShiftRightBytes(const Vec128<T> hi,
+                                                          const Vec128<T> lo) {
+  const auto l = ShiftRightBytes<kBytes>(lo);
+  const auto h = ShiftLeftBytes<16 - kBytes>(hi);
+  return Vec128<T>{wasm_v128_or(l.raw, h.raw)};
+}
+
+// ------------------------------ Broadcast/splat any lane
+
+// Unsigned
+template <int kLane>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t> Broadcast(const Vec128<uint16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  constexpr int l0 = kLane * 2;
+  constexpr int l1 = kLane * 2 + 1;
+  return Vec128<uint16_t>{wasm_v8x16_shuffle(v.raw, v.raw, l0, l1, l0, l1, l0,
+                                             l1, l0, l1, l0, l1, l0, l1, l0, l1,
+                                             l0, l1)};
+}
+template <int kLane>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> Broadcast(const Vec128<uint32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  constexpr int l0 = kLane * 4;
+  constexpr int l1 = kLane * 4 + 1;
+  constexpr int l2 = kLane * 4 + 2;
+  constexpr int l3 = kLane * 4 + 3;
+  return Vec128<uint32_t>{wasm_v8x16_shuffle(v.raw, v.raw, l0, l1, l2, l3, l0,
+                                             l1, l2, l3, l0, l1, l2, l3, l0, l1,
+                                             l2, l3)};
+}
+
+// Signed
+template <int kLane>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t> Broadcast(const Vec128<int16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  constexpr int l0 = kLane * 2;
+  constexpr int l1 = kLane * 2 + 1;
+  return Vec128<int16_t>{wasm_v8x16_shuffle(v.raw, v.raw, l0, l1, l0, l1, l0,
+                                            l1, l0, l1, l0, l1, l0, l1, l0, l1,
+                                            l0, l1)};
+}
+template <int kLane>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> Broadcast(const Vec128<int32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  constexpr int l0 = kLane * 4;
+  constexpr int l1 = kLane * 4 + 1;
+  constexpr int l2 = kLane * 4 + 2;
+  constexpr int l3 = kLane * 4 + 3;
+  return Vec128<int32_t>{wasm_v8x16_shuffle(v.raw, v.raw, l0, l1, l2, l3, l0,
+                                            l1, l2, l3, l0, l1, l2, l3, l0, l1,
+                                            l2, l3)};
+}
+
+// Float
+template <int kLane>
+HWY_ATTR_WASM HWY_INLINE Vec128<float> Broadcast(const Vec128<float> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  constexpr int l0 = kLane * 4;
+  constexpr int l1 = kLane * 4 + 1;
+  constexpr int l2 = kLane * 4 + 2;
+  constexpr int l3 = kLane * 4 + 3;
+  return Vec128<float>{wasm_v8x16_shuffle(v.raw, v.raw, l0, l1, l2, l3, l0, l1,
+                                          l2, l3, l0, l1, l2, l3, l0, l1, l2,
+                                          l3)};
+}
+
+// ------------------------------ Shuffle bytes with variable indices
+
+// Returns vector of bytes[from[i]]. "from" is also interpreted as bytes:
+// either valid indices in [0, 16) or >= 0x80 to zero the i-th output byte.
+template <typename T, typename TI>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> TableLookupBytes(const Vec128<T> bytes,
+                                                    const Vec128<TI> from) {
+  // TODO(eustas): use swizzle? what about 0x80+ indices?
+  alignas(16) uint8_t control[16];
+  alignas(16) uint8_t input[16];
+  alignas(16) uint8_t output[16];
+  wasm_v128_store(control, from.raw);
+  wasm_v128_store(input, bytes.raw);
+  // TODO(eustas): wasm_v8x16_shuffle does not work: params have to be
+  // constants.
+  for (size_t i = 0; i < 16; ++i) {
+    const int idx = control[i];
+    output[i] = (idx >= 0x80) ? 0 : input[idx];
+  }
+  return Vec128<T>{wasm_v128_load(output)};
+}
+
+// ------------------------------ Hard-coded shuffles
+
+// Notation: let Vec128<int32_t> have lanes 3,2,1,0 (0 is least-significant).
+// Shuffle0321 rotates one lane to the right (the previous least-significant
+// lane is now most-significant). These could also be implemented via
+// CombineShiftRightBytes but the shuffle_abcd notation is more convenient.
+
+// Swap 32-bit halves in 64-bit halves.
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> Shuffle2301(
+    const Vec128<uint32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<uint32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 4, 5, 6, 7, 0, 1, 2,
+                                             3, 12, 13, 14, 15, 8, 9, 10, 11)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> Shuffle2301(const Vec128<int32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<int32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 4, 5, 6, 7, 0, 1, 2,
+                                            3, 12, 13, 14, 15, 8, 9, 10, 11)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<float> Shuffle2301(const Vec128<float> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<float>{wasm_v8x16_shuffle(v.raw, v.raw, 4, 5, 6, 7, 0, 1, 2, 3,
+                                          12, 13, 14, 15, 8, 9, 10, 11)};
+}
+
+// Swap 64-bit halves
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> Shuffle1032(
+    const Vec128<uint32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<uint32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 8, 9, 10, 11, 12, 13,
+                                             14, 15, 0, 1, 2, 3, 4, 5, 6, 7)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> Shuffle1032(const Vec128<int32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<int32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 8, 9, 10, 11, 12, 13,
+                                            14, 15, 0, 1, 2, 3, 4, 5, 6, 7)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<float> Shuffle1032(const Vec128<float> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<float>{wasm_v8x16_shuffle(v.raw, v.raw, 8, 9, 10, 11, 12, 13,
+                                          14, 15, 0, 1, 2, 3, 4, 5, 6, 7)};
+}
+
+// Rotate right 32 bits
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> Shuffle0321(
+    const Vec128<uint32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<uint32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 4, 5, 6, 7, 8, 9, 10,
+                                             11, 12, 13, 14, 15, 0, 1, 2, 3)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> Shuffle0321(const Vec128<int32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<int32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 4, 5, 6, 7, 8, 9, 10,
+                                            11, 12, 13, 14, 15, 0, 1, 2, 3)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<float> Shuffle0321(const Vec128<float> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<float>{wasm_v8x16_shuffle(v.raw, v.raw, 4, 5, 6, 7, 8, 9, 10,
+                                          11, 12, 13, 14, 15, 0, 1, 2, 3)};
+}
+// Rotate left 32 bits
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> Shuffle2103(
+    const Vec128<uint32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<uint32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 12, 13, 14, 15, 0, 1,
+                                             2, 3, 4, 5, 6, 7, 8, 9, 10, 11)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> Shuffle2103(const Vec128<int32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<int32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 12, 13, 14, 15, 0, 1,
+                                            2, 3, 4, 5, 6, 7, 8, 9, 10, 11)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<float> Shuffle2103(const Vec128<float> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<float>{wasm_v8x16_shuffle(v.raw, v.raw, 12, 13, 14, 15, 0, 1, 2,
+                                          3, 4, 5, 6, 7, 8, 9, 10, 11)};
+}
+
+// Reverse
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> Shuffle0123(
+    const Vec128<uint32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<uint32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 12, 13, 14, 15, 8, 9,
+                                             10, 11, 4, 5, 6, 7, 0, 1, 2, 3)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> Shuffle0123(const Vec128<int32_t> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<int32_t>{wasm_v8x16_shuffle(v.raw, v.raw, 12, 13, 14, 15, 8, 9,
+                                            10, 11, 4, 5, 6, 7, 0, 1, 2, 3)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<float> Shuffle0123(const Vec128<float> v) {
+  // TODO(eustas): use swizzle?
+  return Vec128<float>{wasm_v8x16_shuffle(v.raw, v.raw, 12, 13, 14, 15, 8, 9,
+                                          10, 11, 4, 5, 6, 7, 0, 1, 2, 3)};
+}
+
+// ------------------------------ Permute (runtime variable)
+
+// Returned by SetTableIndices for use by TableLookupLanes.
+template <typename T>
+struct permute_sse4 {
+  __v128_u raw;
+};
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE permute_sse4<T> SetTableIndices(Full128<T> d,
+                                                         const int32_t* idx) {
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+  for (size_t i = 0; i < d.N; ++i) {
+    // TODO(eustas): also assume idx[i] >= 0
+    if (idx[i] >= static_cast<int32_t>(d.N)) {
+      printf("SetTableIndices [%zu] = %d >= %zu\n", i, idx[i], d.N);
+      HWY_TRAP();
+    }
+  }
+#else
+  (void)d;
+#endif
+
+  const Full128<uint8_t> d8;
+  alignas(16) uint8_t control[d8.N];
+  for (size_t idx_byte = 0; idx_byte < d8.N; ++idx_byte) {
+    const size_t idx_lane = idx_byte / sizeof(T);
+    const size_t mod = idx_byte % sizeof(T);
+    control[idx_byte] = idx[idx_lane] * sizeof(T) + mod;
+  }
+  // TODO(eustas): permute_sse4?
+  return permute_sse4<T>{Load(d8, control).raw};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> TableLookupLanes(
+    const Vec128<uint32_t> v, const permute_sse4<uint32_t> idx) {
+  return TableLookupBytes(v, Vec128<uint8_t>{idx.raw});
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> TableLookupLanes(
+    const Vec128<int32_t> v, const permute_sse4<int32_t> idx) {
+  return TableLookupBytes(v, Vec128<uint8_t>{idx.raw});
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<float> TableLookupLanes(
+    const Vec128<float> v, const permute_sse4<float> idx) {
+  return TableLookupBytes(v, Vec128<uint8_t>{idx.raw});
+}
+
+// ------------------------------ Interleave lanes
+
+// Interleaves lanes from halves of the 128-bit blocks of "a" (which provides
+// the least-significant lane) and "b". To concatenate two half-width integers
+// into one, use ZipLo/hi instead (also works with scalar).
+
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t> InterleaveLo(const Vec128<uint8_t> a,
+                                                      const Vec128<uint8_t> b) {
+  return Vec128<uint8_t>{wasm_v8x16_shuffle(a.raw, b.raw, 0, 16, 1, 17, 2, 18,
+                                            3, 19, 4, 20, 5, 21, 6, 22, 7, 23)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t> InterleaveLo(
+    const Vec128<uint16_t> a, const Vec128<uint16_t> b) {
+  return Vec128<uint16_t>{wasm_v8x16_shuffle(
+      a.raw, b.raw, 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> InterleaveLo(
+    const Vec128<uint32_t> a, const Vec128<uint32_t> b) {
+  return Vec128<uint32_t>{wasm_v8x16_shuffle(
+      a.raw, b.raw, 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23)};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t> InterleaveLo(const Vec128<int8_t> a,
+                                                     const Vec128<int8_t> b) {
+  return Vec128<int8_t>{wasm_v8x16_shuffle(a.raw, b.raw, 0, 16, 1, 17, 2, 18, 3,
+                                           19, 4, 20, 5, 21, 6, 22, 7, 23)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t> InterleaveLo(const Vec128<int16_t> a,
+                                                      const Vec128<int16_t> b) {
+  return Vec128<int16_t>{wasm_v8x16_shuffle(
+      a.raw, b.raw, 0, 1, 16, 17, 2, 3, 18, 19, 4, 5, 20, 21, 6, 7, 22, 23)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> InterleaveLo(const Vec128<int32_t> a,
+                                                      const Vec128<int32_t> b) {
+  return Vec128<int32_t>{wasm_v8x16_shuffle(
+      a.raw, b.raw, 0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20, 21, 22, 23)};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<float> InterleaveLo(const Vec128<float> a,
+                                                    const Vec128<float> b) {
+  return Vec128<float>{wasm_v8x16_shuffle(a.raw, b.raw, 0, 1, 2, 3, 16, 17, 18,
+                                          19, 4, 5, 6, 7, 20, 21, 22, 23)};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t> InterleaveHi(const Vec128<uint8_t> a,
+                                                      const Vec128<uint8_t> b) {
+  return Vec128<uint8_t>{wasm_v8x16_shuffle(a.raw, b.raw, 8, 24, 9, 25, 10, 26,
+                                            11, 27, 12, 28, 13, 29, 14, 30, 15,
+                                            31)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t> InterleaveHi(
+    const Vec128<uint16_t> a, const Vec128<uint16_t> b) {
+  // TODO(eustas): remove emulation, when segfault is fixed.
+  alignas(16) uint16_t a_lanes[8];
+  alignas(16) uint16_t b_lanes[8];
+  alignas(16) uint16_t output[8];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[2 * i] = a_lanes[4 + i];
+    output[2 * i + 1] = b_lanes[4 + i];
+  }
+  return Vec128<uint16_t>{wasm_v128_load(output)};
+  // return Vec128<uint16_t>{wasm_v8x16_shuffle(a.raw, b.raw,
+  //                                            8,  9, 24, 25,
+  //                                           10, 11, 26, 27,
+  //                                           12, 13, 28, 29,
+  //                                           14, 15, 30, 31)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> InterleaveHi(
+    const Vec128<uint32_t> a, const Vec128<uint32_t> b) {
+  // TODO(eustas): remove emulation, when segfault is fixed.
+  alignas(16) uint32_t a_lanes[4];
+  alignas(16) uint32_t b_lanes[4];
+  alignas(16) uint32_t output[4];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 2; ++i) {
+    output[2 * i] = a_lanes[2 + i];
+    output[2 * i + 1] = b_lanes[2 + i];
+  }
+  return Vec128<uint32_t>{wasm_v128_load(output)};
+  // return Vec128<uint32_t>{wasm_v8x16_shuffle(a.raw, b.raw,
+  //                                            8,  9, 10, 11,
+  //                                           24, 25, 26, 27,
+  //                                           12, 13, 14, 15,
+  //                                           28, 29, 30, 31)};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t> InterleaveHi(const Vec128<int8_t> a,
+                                                     const Vec128<int8_t> b) {
+  return Vec128<int8_t>{wasm_v8x16_shuffle(a.raw, b.raw, 8, 24, 9, 25, 10, 26,
+                                           11, 27, 12, 28, 13, 29, 14, 30, 15,
+                                           31)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t> InterleaveHi(const Vec128<int16_t> a,
+                                                      const Vec128<int16_t> b) {
+  // TODO(eustas): remove emulation, when segfault is fixed.
+  alignas(16) int16_t a_lanes[8];
+  alignas(16) int16_t b_lanes[8];
+  alignas(16) int16_t output[8];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 4; ++i) {
+    output[2 * i] = a_lanes[4 + i];
+    output[2 * i + 1] = b_lanes[4 + i];
+  }
+  return Vec128<int16_t>{wasm_v128_load(output)};
+  // return Vec128<int16_t>{wasm_v8x16_shuffle(a.raw, b.raw,
+  //                                           8,  9, 24, 25,
+  //                                          10, 11, 26, 27,
+  //                                          12, 13, 28, 29,
+  //                                          14, 15, 30, 31)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> InterleaveHi(const Vec128<int32_t> a,
+                                                      const Vec128<int32_t> b) {
+  // TODO(eustas): remove emulation, when segfault is fixed.
+  alignas(16) int32_t a_lanes[4];
+  alignas(16) int32_t b_lanes[4];
+  alignas(16) int32_t output[4];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 2; ++i) {
+    output[2 * i] = a_lanes[2 + i];
+    output[2 * i + 1] = b_lanes[2 + i];
+  }
+  return Vec128<int32_t>{wasm_v128_load(output)};
+  // return Vec128<int32_t>{wasm_v8x16_shuffle(a.raw, b.raw,
+  //                                           8,  9, 10, 11,
+  //                                          24, 25, 26, 27,
+  //                                          12, 13, 14, 15,
+  //                                          28, 29, 30, 31)};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<float> InterleaveHi(const Vec128<float> a,
+                                                    const Vec128<float> b) {
+  // TODO(eustas): remove emulation, when segfault is fixed.
+  alignas(16) int32_t a_lanes[4];
+  alignas(16) int32_t b_lanes[4];
+  alignas(16) int32_t output[4];
+  wasm_v128_store(a_lanes, a.raw);
+  wasm_v128_store(b_lanes, b.raw);
+  for (size_t i = 0; i < 2; ++i) {
+    output[2 * i] = a_lanes[2 + i];
+    output[2 * i + 1] = b_lanes[2 + i];
+  }
+  return Vec128<float>{wasm_v128_load(output)};
+  // return Vec128<float>{wasm_v8x16_shuffle(a.raw, b.raw,
+  //                                         8,  9, 10, 11,
+  //                                        24, 25, 26, 27,
+  //                                        12, 13, 14, 15,
+  //                                        28, 29, 30, 31)};
+}
+
+// ------------------------------ Zip lanes
+
+// Same as interleave_*, except that the return lanes are double-width integers;
+// this is necessary because the single-lane scalar cannot return two values.
+
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t> ZipLo(const Vec128<uint8_t> a,
+                                                const Vec128<uint8_t> b) {
+  return Vec128<uint16_t>{InterleaveLo(a, b).raw};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> ZipLo(const Vec128<uint16_t> a,
+                                                const Vec128<uint16_t> b) {
+  return Vec128<uint32_t>{InterleaveLo(a, b).raw};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t> ZipLo(const Vec128<int8_t> a,
+                                               const Vec128<int8_t> b) {
+  return Vec128<int16_t>{InterleaveLo(a, b).raw};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> ZipLo(const Vec128<int16_t> a,
+                                               const Vec128<int16_t> b) {
+  return Vec128<int32_t>{InterleaveLo(a, b).raw};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t> ZipHi(const Vec128<uint8_t> a,
+                                                const Vec128<uint8_t> b) {
+  return Vec128<uint16_t>{InterleaveHi(a, b).raw};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> ZipHi(const Vec128<uint16_t> a,
+                                                const Vec128<uint16_t> b) {
+  return Vec128<uint32_t>{InterleaveHi(a, b).raw};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t> ZipHi(const Vec128<int8_t> a,
+                                               const Vec128<int8_t> b) {
+  return Vec128<int16_t>{InterleaveHi(a, b).raw};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> ZipHi(const Vec128<int16_t> a,
+                                               const Vec128<int16_t> b) {
+  return Vec128<int32_t>{InterleaveHi(a, b).raw};
+}
+
+// ------------------------------ Blocks
+
+// hiH,hiL loH,loL |-> hiL,loL (= lower halves)
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> ConcatLoLo(const Vec128<T> hi,
+                                              const Vec128<T> lo) {
+  return Vec128<T>{wasm_v8x16_shuffle(lo.raw, hi.raw, 0, 1, 2, 3, 4, 5, 6, 7,
+                                      16, 17, 18, 19, 20, 21, 22, 23)};
+}
+
+// hiH,hiL loH,loL |-> hiH,loH (= upper halves)
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> ConcatHiHi(const Vec128<T> hi,
+                                              const Vec128<T> lo) {
+  return Vec128<T>{wasm_v8x16_shuffle(lo.raw, hi.raw, 8, 9, 10, 11, 12, 13, 14,
+                                      15, 24, 25, 26, 27, 28, 29, 30, 31)};
+}
+
+// hiH,hiL loH,loL |-> hiL,loH (= inner halves)
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> ConcatLoHi(const Vec128<T> hi,
+                                              const Vec128<T> lo) {
+  return CombineShiftRightBytes<8>(hi, lo);
+}
+
+// hiH,hiL loH,loL |-> hiH,loL (= outer halves)
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> ConcatHiLo(const Vec128<T> hi,
+                                              const Vec128<T> lo) {
+  return Vec128<T>{wasm_v8x16_shuffle(lo.raw, hi.raw, 0, 1, 2, 3, 4, 5, 6, 7,
+                                      24, 25, 26, 27, 28, 29, 30, 31)};
+}
+template <>
+HWY_ATTR_WASM HWY_INLINE Vec128<float> ConcatHiLo(const Vec128<float> hi,
+                                                  const Vec128<float> lo) {
+  return Vec128<float>{wasm_v8x16_shuffle(hi.raw, lo.raw, 16, 17, 18, 19, 20,
+                                          21, 22, 23, 8, 9, 10, 11, 12, 13, 14,
+                                          15)};
+}
+
+// ------------------------------ Odd/even lanes
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> odd_even_impl(SizeTag<1> /* tag */,
+                                                 const Vec128<T> a,
+                                                 const Vec128<T> b) {
+  const Full128<T> d;
+  const Full128<uint8_t> d8;
+  alignas(16) constexpr uint8_t mask[16] = {0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0,
+                                            0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0};
+  return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a);
+}
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> odd_even_impl(SizeTag<2> /* tag */,
+                                                 const Vec128<T> a,
+                                                 const Vec128<T> b) {
+  return Vec128<T>{wasm_v8x16_shuffle(a.raw, b.raw, 16, 17, 2, 3, 20, 21, 6, 7,
+                                      24, 25, 10, 11, 28, 29, 14, 15)};
+}
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> odd_even_impl(SizeTag<4> /* tag */,
+                                                 const Vec128<T> a,
+                                                 const Vec128<T> b) {
+  return Vec128<T>{wasm_v8x16_shuffle(a.raw, b.raw, 16, 17, 18, 19, 4, 5, 6, 7,
+                                      24, 25, 26, 27, 12, 13, 14, 15)};
+}
+// TODO(eustas): implement
+// template <typename T>
+// HWY_ATTR_WASM HWY_INLINE Vec128<T> odd_even_impl(SizeTag<8> /* tag */,
+//                                                 const Vec128<T> a,
+//                                                 const Vec128<T> b)
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE Vec128<T> OddEven(const Vec128<T> a,
+                                           const Vec128<T> b) {
+  return odd_even_impl(SizeTag<sizeof(T)>(), a, b);
+}
+template <>
+HWY_ATTR_WASM HWY_INLINE Vec128<float> OddEven<float>(const Vec128<float> a,
+                                                      const Vec128<float> b) {
+  return Vec128<float>{wasm_v8x16_shuffle(a.raw, b.raw, 16, 17, 18, 19, 4, 5, 6,
+                                          7, 24, 25, 26, 27, 12, 13, 14, 15)};
+}
+
+// ================================================== CONVERT
+
+// ------------------------------ Promotions (part w/ narrow lanes -> full)
+
+// Unsigned: zero-extend.
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t> ConvertTo(
+    Full128<uint16_t> /* tag */, const Vec128<uint8_t, 8> v) {
+  return Vec128<uint16_t>{wasm_i16x8_widen_low_u8x16(v.raw)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> ConvertTo(
+    Full128<uint32_t> /* tag */, const Vec128<uint8_t, 4> v) {
+  return Vec128<uint32_t>{
+      wasm_i32x4_widen_low_u16x8(wasm_i16x8_widen_low_u8x16(v.raw))};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t> ConvertTo(Full128<int16_t> /* tag */,
+                                                   const Vec128<uint8_t, 8> v) {
+  return Vec128<int16_t>{wasm_i16x8_widen_low_u8x16(v.raw)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                                   const Vec128<uint8_t, 4> v) {
+  return Vec128<int32_t>{
+      wasm_i32x4_widen_low_u16x8(wasm_i16x8_widen_low_u8x16(v.raw))};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> ConvertTo(
+    Full128<uint32_t> /* tag */, const Vec128<uint16_t, 4> v) {
+  return Vec128<uint32_t>{__builtin_wasm_widen_low_u_i32x4_i16x8(v.raw)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> ConvertTo(
+    Full128<int32_t> /* tag */, const Vec128<uint16_t, 4> v) {
+  return Vec128<int32_t>{__builtin_wasm_widen_low_u_i32x4_i16x8(v.raw)};
+}
+
+HWY_ATTR_WASM HWY_INLINE Vec128<uint32_t> U32FromU8(const Vec128<uint8_t> v) {
+  return Vec128<uint32_t>{
+      wasm_i32x4_widen_low_u16x8(wasm_i16x8_widen_low_u8x16(v.raw))};
+}
+
+// Signed: replicate sign bit.
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t> ConvertTo(Full128<int16_t> /* tag */,
+                                                   const Vec128<int8_t, 8> v) {
+  return Vec128<int16_t>{wasm_i16x8_widen_low_i8x16(v.raw)};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                                   const Vec128<int8_t, 4> v) {
+  return Vec128<int32_t>{
+      wasm_i32x4_widen_low_i16x8(wasm_i16x8_widen_low_i8x16(v.raw))};
+}
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                                   const Vec128<int16_t, 4> v) {
+  return Vec128<int32_t>{wasm_i32x4_widen_low_i16x8(v.raw)};
+}
+
+// ------------------------------ Demotions (full -> part w/ narrow lanes)
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t, N> ConvertTo(
+    Desc<uint16_t, N> /* tag */, const Vec128<int32_t, N> v) {
+  return Vec128<uint16_t, N>{wasm_u16x8_narrow_i32x4(v.raw, v.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> ConvertTo(
+    Desc<uint8_t, N> /* tag */, const Vec128<int32_t> v) {
+  const auto intermediate = wasm_i16x8_narrow_i32x4(v.raw, v.raw);
+  return Vec128<uint8_t, N>{
+      wasm_u8x16_narrow_i16x8(intermediate, intermediate)};
+}
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, N> ConvertTo(
+    Desc<uint8_t, N> /* tag */, const Vec128<int16_t> v) {
+  return Vec128<uint8_t, N>{wasm_u8x16_narrow_i16x8(v.raw, v.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int16_t, N> ConvertTo(
+    Desc<int16_t, N> /* tag */, const Vec128<int32_t> v) {
+  return Vec128<int16_t, N>{wasm_i16x8_narrow_i32x4(v.raw, v.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> ConvertTo(Desc<int8_t, N> /* tag */,
+                                                     const Vec128<int32_t> v) {
+  const auto intermediate = wasm_i16x8_narrow_i32x4(v.raw, v.raw);
+  return Vec128<int8_t, N>{wasm_i8x16_narrow_i16x8(intermediate, intermediate)};
+}
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int8_t, N> ConvertTo(Desc<int8_t, N> /* tag */,
+                                                     const Vec128<int16_t> v) {
+  return Vec128<int8_t, N>{wasm_i8x16_narrow_i16x8(v.raw, v.raw)};
+}
+
+// For already range-limited input [0, 255].
+HWY_ATTR_WASM HWY_INLINE Vec128<uint8_t, 4> U8FromU32(
+    const Vec128<uint32_t> v) {
+  const auto intermediate = wasm_i16x8_narrow_i32x4(v.raw, v.raw);
+  return Vec128<uint8_t, 4>{
+      wasm_u8x16_narrow_i16x8(intermediate, intermediate)};
+}
+
+// ------------------------------ Convert i32 <=> f32
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<float, N> ConvertTo(
+    Desc<float, N> /* tag */, const Vec128<int32_t, N> v) {
+  return Vec128<float, N>{wasm_f32x4_convert_i32x4(v.raw)};
+}
+// Truncates (rounds toward zero).
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> ConvertTo(
+    Desc<int32_t, N> /* tag */, const Vec128<float, N> v) {
+  return Vec128<int32_t, N>{wasm_i32x4_trunc_saturate_f32x4(v.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<int32_t, N> NearestInt(
+    const Vec128<float, N> v) {
+  const __f32x4 c00 = wasm_f32x4_splat(0.0f);
+  const __f32x4 corr = wasm_f32x4_convert_i32x4(wasm_f32x4_le(v.raw, c00));
+  const __f32x4 c05 = wasm_f32x4_splat(0.5f);
+  // +0.5 for non-negative lane, -0.5 for other.
+  const __f32x4 delta = wasm_f32x4_add(c05, corr);
+  // Shift input by 0.5 away from 0.
+  const __f32x4 fixed = wasm_f32x4_add(v.raw, delta);
+  return Vec128<int32_t, N>{wasm_i32x4_trunc_saturate_f32x4(fixed)};
+}
+
+// ================================================== MISC
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// ------------------------------ movemask
+
+// Returns a bit array of the most significant bit of each byte in "v", i.e.
+// sum_i=0..15 of (v[i] >> 7) << i; v[0] is the least-significant byte of "v".
+// This is useful for testing/branching based on comparison results.
+HWY_ATTR_WASM HWY_INLINE uint64_t movemask(const Vec128<uint8_t> v) {
+  const __i8x16 bits = wasm_u8x16_shr(v.raw, 7);
+  const __i8x16 var_shift =
+      wasm_i8x16_make(1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8, 1, 2, 4, 8);
+  const __i8x16 shifted_bits = wasm_i8x16_mul(bits, var_shift);
+  const __i16x8 pairs_hi = wasm_u16x8_shr(shifted_bits, 8);
+  const __i16x8 pairs = wasm_v128_or(shifted_bits, pairs_hi);
+  const __i32x4 quads_hi = wasm_u32x4_shr(pairs, 16);
+  const __i32x4 quads = wasm_v128_or(pairs, quads_hi);
+  const __i32x4 mask = wasm_i32x4_splat(0xF);
+  alignas(16) uint32_t lanes[4];
+  wasm_v128_store(lanes, wasm_v128_and(quads, mask));
+  return lanes[0] | (lanes[1] << 4) | (lanes[2] << 8) | (lanes[3] << 12);
+}
+
+// Returns the most significant bit of each float/double lane (see above).
+HWY_ATTR_WASM HWY_INLINE uint64_t movemask(const Vec128<float> v) {
+  const __i32x4 bits = wasm_u32x4_shr(v.raw, 31);
+  const __i32x4 var_shift = wasm_i32x4_make(1, 2, 4, 8);
+  const __i32x4 shifted_bits = wasm_i32x4_mul(bits, var_shift);
+  alignas(16) uint32_t lanes[4];
+  wasm_v128_store(lanes, shifted_bits);
+  return lanes[0] | lanes[1] | lanes[2] | lanes[3];
+}
+
+// ------------------------------ mask
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE bool AllFalse(const Mask128<T> v) {
+  return !wasm_i8x16_any_true(v.raw);
+}
+HWY_ATTR_WASM HWY_INLINE bool AllFalse(const Mask128<float> v) {
+  return !wasm_i32x4_any_true(v.raw);
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE bool AllTrue(const Mask128<T> v) {
+  return wasm_i8x16_all_true(v.raw);
+}
+HWY_ATTR_WASM HWY_INLINE bool AllTrue(const Mask128<float> v) {
+  return wasm_i32x4_all_true(v.raw);
+}
+
+template <typename T>
+HWY_ATTR_WASM HWY_INLINE size_t CountTrue(const Mask128<T> v) {
+  const __i32x4 mask =
+      wasm_i32x4_make(0x01010101, 0x01010101, 0x02020202, 0x02020202);
+  const __i8x16 shifted_bits = wasm_v128_and(v.raw, mask);
+  alignas(16) uint64_t lanes[2];
+  wasm_v128_store(lanes, shifted_bits);
+  return PopCount(lanes[0] | lanes[1]) / sizeof(T);
+}
+HWY_ATTR_WASM HWY_INLINE size_t CountTrue(const Mask128<float> v) {
+  const __i32x4 var_shift = wasm_i32x4_make(1, 2, 4, 8);
+  const __i32x4 shifted_bits = wasm_v128_and(v.raw, var_shift);
+  alignas(16) uint64_t lanes[2];
+  wasm_v128_store(lanes, shifted_bits);
+  return PopCount(lanes[0] | lanes[1]);
+}
+
+// ------------------------------ minpos
+
+// Returns index and min value in lanes 1 and 0.
+HWY_ATTR_WASM HWY_INLINE Vec128<uint16_t> minpos(const Vec128<uint16_t> v) {
+  // TODO(eustas): optimize.
+  alignas(16) uint16_t lanes[8];
+  wasm_v128_store(lanes, v.raw);
+  uint16_t index = 0;
+  uint16_t min_val = lanes[0];
+  for (size_t i = 0; i < 8; ++i) {
+    uint16_t val = lanes[i];
+    if (val < min_val) {
+      index = i;
+      min_val = val;
+    }
+  }
+  return Vec128<uint16_t>{wasm_i16x8_make(min_val, index, 0, 0, 0, 0, 0, 0)};
+}
+
+// ------------------------------ Horizontal sum (reduction)
+
+// TODO(eustas): optimize
+// Returns 64-bit sums of 8-byte groups.
+HWY_ATTR_WASM HWY_INLINE Vec128<uint64_t> SumsOfU8x8(const Vec128<uint8_t> v) {
+  alignas(16) uint8_t lanes[16];
+  wasm_v128_store(lanes, v.raw);
+  uint32_t sums[2] = {0};
+  for (size_t i = 0; i < 16; ++i) {
+    sums[i / 8] += lanes[i];
+  }
+  return Vec128<uint64_t>{wasm_i32x4_make(sums[0], 0, sums[1], 0)};
+}
+
+// For u32/i32/f32.
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> horz_sum_impl(SizeTag<4> /* tag */,
+                                                    const Vec128<T, N> v3210) {
+  const Vec128<T> v1032 = Shuffle1032(v3210);
+  const Vec128<T> v31_20_31_20 = v3210 + v1032;
+  const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
+  return v20_31_20_31 + v31_20_31_20;
+}
+
+// For u64/i64/f64.
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> horz_sum_impl(SizeTag<8> /* tag */,
+                                                    const Vec128<T, N> v10) {
+  const Vec128<T> v01 = Shuffle01(v10);
+  return v10 + v01;
+}
+
+// Supported for u/i/f 32/64. Returns the sum in each lane.
+template <typename T, size_t N>
+HWY_ATTR_WASM HWY_INLINE Vec128<T, N> SumOfLanes(const Vec128<T, N> v) {
+  return horz_sum_impl(SizeTag<sizeof(T)>(), v);
+}
+
+}  // namespace ext
+
+}  // namespace hwy
+
+#endif  // HWY_WASM_H_
diff --git a/third_party/highway/hwy/x86/avx2.h b/third_party/highway/hwy/x86/avx2.h
new file mode 100644
index 000000000..cbe6fef31
--- /dev/null
+++ b/third_party/highway/hwy/x86/avx2.h
@@ -0,0 +1,2085 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_X86_AVX2_H_
+#define HWY_X86_AVX2_H_
+
+// 256-bit AVX2 vectors and operations.
+// WARNING: most operations do not cross 128-bit block boundaries. In
+// particular, "Broadcast", pack and zip behavior may be surprising.
+
+#include <immintrin.h>
+
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+#include <stdio.h>
+#endif
+
+#include "hwy/shared.h"
+#include "hwy/x86/sse4.h"
+
+#define HWY_ATTR_AVX2 HWY_TARGET_ATTR("bmi,bmi2,avx,avx2,fma")
+
+namespace hwy {
+
+template <typename T>
+struct Raw256 {
+  using type = __m256i;
+};
+template <>
+struct Raw256<float> {
+  using type = __m256;
+};
+template <>
+struct Raw256<double> {
+  using type = __m256d;
+};
+
+template <typename T>
+using Full256 = Desc<T, 32 / sizeof(T)>;
+
+template <typename T>
+class Vec256 {
+  using Raw = typename Raw256<T>::type;
+
+ public:
+  // Compound assignment. Only usable if there is a corresponding non-member
+  // binary operator overload. For example, only f32 and f64 support division.
+  HWY_ATTR_AVX2 HWY_INLINE Vec256& operator*=(const Vec256 other) {
+    return *this = (*this * other);
+  }
+  HWY_ATTR_AVX2 HWY_INLINE Vec256& operator/=(const Vec256 other) {
+    return *this = (*this / other);
+  }
+  HWY_ATTR_AVX2 HWY_INLINE Vec256& operator+=(const Vec256 other) {
+    return *this = (*this + other);
+  }
+  HWY_ATTR_AVX2 HWY_INLINE Vec256& operator-=(const Vec256 other) {
+    return *this = (*this - other);
+  }
+  HWY_ATTR_AVX2 HWY_INLINE Vec256& operator&=(const Vec256 other) {
+    return *this = (*this & other);
+  }
+  HWY_ATTR_AVX2 HWY_INLINE Vec256& operator|=(const Vec256 other) {
+    return *this = (*this | other);
+  }
+  HWY_ATTR_AVX2 HWY_INLINE Vec256& operator^=(const Vec256 other) {
+    return *this = (*this ^ other);
+  }
+
+  Raw raw;
+};
+
+// Integer: FF..FF or 0. Float: MSB, all other bits undefined - see README.
+template <typename T>
+class Mask256 {
+  using Raw = typename Raw256<T>::type;
+
+ public:
+  Raw raw;
+};
+
+// ------------------------------ Cast
+
+HWY_ATTR_AVX2 HWY_INLINE __m256i BitCastToInteger(__m256i v) { return v; }
+HWY_ATTR_AVX2 HWY_INLINE __m256i BitCastToInteger(__m256 v) {
+  return _mm256_castps_si256(v);
+}
+HWY_ATTR_AVX2 HWY_INLINE __m256i BitCastToInteger(__m256d v) {
+  return _mm256_castpd_si256(v);
+}
+
+// cast_to_u8
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> cast_to_u8(Vec256<T> v) {
+  return Vec256<uint8_t>{BitCastToInteger(v.raw)};
+}
+
+// Cannot rely on function overloading because return types differ.
+template <typename T>
+struct BitCastFromInteger256 {
+  HWY_ATTR_AVX2 HWY_INLINE __m256i operator()(__m256i v) { return v; }
+};
+template <>
+struct BitCastFromInteger256<float> {
+  HWY_ATTR_AVX2 HWY_INLINE __m256 operator()(__m256i v) {
+    return _mm256_castsi256_ps(v);
+  }
+};
+template <>
+struct BitCastFromInteger256<double> {
+  HWY_ATTR_AVX2 HWY_INLINE __m256d operator()(__m256i v) {
+    return _mm256_castsi256_pd(v);
+  }
+};
+
+// cast_u8_to
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> cast_u8_to(Full256<T> /* tag */,
+                                              Vec256<uint8_t> v) {
+  return Vec256<T>{BitCastFromInteger256<T>()(v.raw)};
+}
+
+// BitCast
+template <typename T, typename FromT>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> BitCast(Full256<T> d, Vec256<FromT> v) {
+  return cast_u8_to(d, cast_to_u8(v));
+}
+
+// ------------------------------ Set
+
+// Returns an all-zero vector.
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> Zero(Full256<T> /* tag */) {
+  return Vec256<T>{_mm256_setzero_si256()};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Zero(Full256<float> /* tag */) {
+  return Vec256<float>{_mm256_setzero_ps()};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Zero(Full256<double> /* tag */) {
+  return Vec256<double>{_mm256_setzero_pd()};
+}
+
+// Returns a vector with all lanes set to "t".
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> Set(Full256<uint8_t> /* tag */,
+                                             const uint8_t t) {
+  return Vec256<uint8_t>{_mm256_set1_epi8(t)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> Set(Full256<uint16_t> /* tag */,
+                                              const uint16_t t) {
+  return Vec256<uint16_t>{_mm256_set1_epi16(t)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> Set(Full256<uint32_t> /* tag */,
+                                              const uint32_t t) {
+  return Vec256<uint32_t>{_mm256_set1_epi32(t)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> Set(Full256<uint64_t> /* tag */,
+                                              const uint64_t t) {
+  return Vec256<uint64_t>{_mm256_set1_epi64x(t)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> Set(Full256<int8_t> /* tag */,
+                                            const int8_t t) {
+  return Vec256<int8_t>{_mm256_set1_epi8(t)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> Set(Full256<int16_t> /* tag */,
+                                             const int16_t t) {
+  return Vec256<int16_t>{_mm256_set1_epi16(t)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Set(Full256<int32_t> /* tag */,
+                                             const int32_t t) {
+  return Vec256<int32_t>{_mm256_set1_epi32(t)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> Set(Full256<int64_t> /* tag */,
+                                             const int64_t t) {
+  return Vec256<int64_t>{_mm256_set1_epi64x(t)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Set(Full256<float> /* tag */,
+                                           const float t) {
+  return Vec256<float>{_mm256_set1_ps(t)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Set(Full256<double> /* tag */,
+                                            const double t) {
+  return Vec256<double>{_mm256_set1_pd(t)};
+}
+
+template <typename T, typename T2>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> Iota(Full256<T> d, T2 first) {
+  alignas(32) T lanes[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = first + i;
+  }
+  return Load(d, lanes);
+}
+
+HWY_DIAGNOSTICS(push)
+HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized")
+
+// Returns a vector with uninitialized elements.
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> Undefined(Full256<T> /* tag */) {
+#ifdef __clang__
+  return Vec256<T>{_mm256_undefined_si256()};
+#else
+  __m256i raw;
+  return Vec256<T>{raw};
+#endif
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Undefined(Full256<float> /* tag */) {
+#ifdef __clang__
+  return Vec256<float>{_mm256_undefined_ps()};
+#else
+  __m256 raw;
+  return Vec256<float>{raw};
+#endif
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Undefined(Full256<double> /* tag */) {
+#ifdef __clang__
+  return Vec256<double>{_mm256_undefined_pd()};
+#else
+  __m256d raw;
+  return Vec256<double>{raw};
+#endif
+}
+
+HWY_DIAGNOSTICS(pop)
+
+// ================================================== LOGICAL
+
+// ------------------------------ Bitwise AND
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> operator&(Vec256<T> a, Vec256<T> b) {
+  return Vec256<T>{_mm256_and_si256(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> operator&(const Vec256<float> a,
+                                                 const Vec256<float> b) {
+  return Vec256<float>{_mm256_and_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> operator&(const Vec256<double> a,
+                                                  const Vec256<double> b) {
+  return Vec256<double>{_mm256_and_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Bitwise AND-NOT
+
+// Returns ~not_mask & mask.
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> AndNot(Vec256<T> not_mask, Vec256<T> mask) {
+  return Vec256<T>{_mm256_andnot_si256(not_mask.raw, mask.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> AndNot(const Vec256<float> not_mask,
+                                              const Vec256<float> mask) {
+  return Vec256<float>{_mm256_andnot_ps(not_mask.raw, mask.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> AndNot(const Vec256<double> not_mask,
+                                               const Vec256<double> mask) {
+  return Vec256<double>{_mm256_andnot_pd(not_mask.raw, mask.raw)};
+}
+
+// ------------------------------ Bitwise OR
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> operator|(Vec256<T> a, Vec256<T> b) {
+  return Vec256<T>{_mm256_or_si256(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> operator|(const Vec256<float> a,
+                                                 const Vec256<float> b) {
+  return Vec256<float>{_mm256_or_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> operator|(const Vec256<double> a,
+                                                  const Vec256<double> b) {
+  return Vec256<double>{_mm256_or_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Bitwise XOR
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> operator^(Vec256<T> a, Vec256<T> b) {
+  return Vec256<T>{_mm256_xor_si256(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> operator^(const Vec256<float> a,
+                                                 const Vec256<float> b) {
+  return Vec256<float>{_mm256_xor_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> operator^(const Vec256<double> a,
+                                                  const Vec256<double> b) {
+  return Vec256<double>{_mm256_xor_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Mask
+
+// Mask and Vec are the same (true = FF..FF).
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Mask256<T> MaskFromVec(const Vec256<T> v) {
+  return Mask256<T>{v.raw};
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> VecFromMask(const Mask256<T> v) {
+  return Vec256<T>{v.raw};
+}
+
+// mask ? yes : no
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> IfThenElse(const Mask256<T> mask,
+                                              const Vec256<T> yes,
+                                              const Vec256<T> no) {
+  return Vec256<T>{_mm256_blendv_epi8(no.raw, yes.raw, mask.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> IfThenElse(const Mask256<float> mask,
+                                                  const Vec256<float> yes,
+                                                  const Vec256<float> no) {
+  return Vec256<float>{_mm256_blendv_ps(no.raw, yes.raw, mask.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> IfThenElse(const Mask256<double> mask,
+                                                   const Vec256<double> yes,
+                                                   const Vec256<double> no) {
+  return Vec256<double>{_mm256_blendv_pd(no.raw, yes.raw, mask.raw)};
+}
+
+// mask ? yes : 0
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> IfThenElseZero(Mask256<T> mask,
+                                                  Vec256<T> yes) {
+  return yes & VecFromMask(mask);
+}
+
+// mask ? 0 : no
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> IfThenZeroElse(Mask256<T> mask,
+                                                  Vec256<T> no) {
+  return AndNot(VecFromMask(mask), no);
+}
+
+template <typename T, HWY_IF_FLOAT(T)>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> ZeroIfNegative(Vec256<T> v) {
+  const auto zero = Zero(Full256<T>());
+  return IfThenElse(MaskFromVec(v), zero, v);
+}
+
+// ================================================== ARITHMETIC
+
+// ------------------------------ Addition
+
+// Unsigned
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> operator+(const Vec256<uint8_t> a,
+                                                   const Vec256<uint8_t> b) {
+  return Vec256<uint8_t>{_mm256_add_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> operator+(const Vec256<uint16_t> a,
+                                                    const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_add_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> operator+(const Vec256<uint32_t> a,
+                                                    const Vec256<uint32_t> b) {
+  return Vec256<uint32_t>{_mm256_add_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> operator+(const Vec256<uint64_t> a,
+                                                    const Vec256<uint64_t> b) {
+  return Vec256<uint64_t>{_mm256_add_epi64(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> operator+(const Vec256<int8_t> a,
+                                                  const Vec256<int8_t> b) {
+  return Vec256<int8_t>{_mm256_add_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> operator+(const Vec256<int16_t> a,
+                                                   const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_add_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> operator+(const Vec256<int32_t> a,
+                                                   const Vec256<int32_t> b) {
+  return Vec256<int32_t>{_mm256_add_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> operator+(const Vec256<int64_t> a,
+                                                   const Vec256<int64_t> b) {
+  return Vec256<int64_t>{_mm256_add_epi64(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> operator+(const Vec256<float> a,
+                                                 const Vec256<float> b) {
+  return Vec256<float>{_mm256_add_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> operator+(const Vec256<double> a,
+                                                  const Vec256<double> b) {
+  return Vec256<double>{_mm256_add_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Subtraction
+
+// Unsigned
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> operator-(const Vec256<uint8_t> a,
+                                                   const Vec256<uint8_t> b) {
+  return Vec256<uint8_t>{_mm256_sub_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> operator-(const Vec256<uint16_t> a,
+                                                    const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_sub_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> operator-(const Vec256<uint32_t> a,
+                                                    const Vec256<uint32_t> b) {
+  return Vec256<uint32_t>{_mm256_sub_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> operator-(const Vec256<uint64_t> a,
+                                                    const Vec256<uint64_t> b) {
+  return Vec256<uint64_t>{_mm256_sub_epi64(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> operator-(const Vec256<int8_t> a,
+                                                  const Vec256<int8_t> b) {
+  return Vec256<int8_t>{_mm256_sub_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> operator-(const Vec256<int16_t> a,
+                                                   const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_sub_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> operator-(const Vec256<int32_t> a,
+                                                   const Vec256<int32_t> b) {
+  return Vec256<int32_t>{_mm256_sub_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> operator-(const Vec256<int64_t> a,
+                                                   const Vec256<int64_t> b) {
+  return Vec256<int64_t>{_mm256_sub_epi64(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> operator-(const Vec256<float> a,
+                                                 const Vec256<float> b) {
+  return Vec256<float>{_mm256_sub_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> operator-(const Vec256<double> a,
+                                                  const Vec256<double> b) {
+  return Vec256<double>{_mm256_sub_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Saturating addition
+
+// Returns a + b clamped to the destination range.
+
+// Unsigned
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> SaturatedAdd(const Vec256<uint8_t> a,
+                                                      const Vec256<uint8_t> b) {
+  return Vec256<uint8_t>{_mm256_adds_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> SaturatedAdd(
+    const Vec256<uint16_t> a, const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_adds_epu16(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> SaturatedAdd(const Vec256<int8_t> a,
+                                                     const Vec256<int8_t> b) {
+  return Vec256<int8_t>{_mm256_adds_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> SaturatedAdd(const Vec256<int16_t> a,
+                                                      const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_adds_epi16(a.raw, b.raw)};
+}
+
+// ------------------------------ Saturating subtraction
+
+// Returns a - b clamped to the destination range.
+
+// Unsigned
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> SaturatedSub(const Vec256<uint8_t> a,
+                                                      const Vec256<uint8_t> b) {
+  return Vec256<uint8_t>{_mm256_subs_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> SaturatedSub(
+    const Vec256<uint16_t> a, const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_subs_epu16(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> SaturatedSub(const Vec256<int8_t> a,
+                                                     const Vec256<int8_t> b) {
+  return Vec256<int8_t>{_mm256_subs_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> SaturatedSub(const Vec256<int16_t> a,
+                                                      const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_subs_epi16(a.raw, b.raw)};
+}
+
+// ------------------------------ Average
+
+// Returns (a + b + 1) / 2
+
+// Unsigned
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> AverageRound(const Vec256<uint8_t> a,
+                                                      const Vec256<uint8_t> b) {
+  return Vec256<uint8_t>{_mm256_avg_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> AverageRound(
+    const Vec256<uint16_t> a, const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_avg_epu16(a.raw, b.raw)};
+}
+
+// ------------------------------ Absolute value
+
+// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1.
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> Abs(const Vec256<int8_t> v) {
+  return Vec256<int8_t>{_mm256_abs_epi8(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> Abs(const Vec256<int16_t> v) {
+  return Vec256<int16_t>{_mm256_abs_epi16(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Abs(const Vec256<int32_t> v) {
+  return Vec256<int32_t>{_mm256_abs_epi32(v.raw)};
+}
+
+// ------------------------------ Shift lanes by constant #bits
+
+// Unsigned
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> ShiftLeft(const Vec256<uint16_t> v) {
+  return Vec256<uint16_t>{_mm256_slli_epi16(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> ShiftRight(const Vec256<uint16_t> v) {
+  return Vec256<uint16_t>{_mm256_srli_epi16(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> ShiftLeft(const Vec256<uint32_t> v) {
+  return Vec256<uint32_t>{_mm256_slli_epi32(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> ShiftRight(const Vec256<uint32_t> v) {
+  return Vec256<uint32_t>{_mm256_srli_epi32(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> ShiftLeft(const Vec256<uint64_t> v) {
+  return Vec256<uint64_t>{_mm256_slli_epi64(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> ShiftRight(const Vec256<uint64_t> v) {
+  return Vec256<uint64_t>{_mm256_srli_epi64(v.raw, kBits)};
+}
+
+// Signed (no i64 ShiftRight)
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> ShiftLeft(const Vec256<int16_t> v) {
+  return Vec256<int16_t>{_mm256_slli_epi16(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> ShiftRight(const Vec256<int16_t> v) {
+  return Vec256<int16_t>{_mm256_srai_epi16(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> ShiftLeft(const Vec256<int32_t> v) {
+  return Vec256<int32_t>{_mm256_slli_epi32(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> ShiftRight(const Vec256<int32_t> v) {
+  return Vec256<int32_t>{_mm256_srai_epi32(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> ShiftLeft(const Vec256<int64_t> v) {
+  return Vec256<int64_t>{_mm256_slli_epi64(v.raw, kBits)};
+}
+
+// ------------------------------ Shift lanes by independent variable #bits
+
+// Unsigned (no u8,u16)
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> operator<<(
+    const Vec256<uint32_t> v, const Vec256<uint32_t> bits) {
+  return Vec256<uint32_t>{_mm256_sllv_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> operator>>(
+    const Vec256<uint32_t> v, const Vec256<uint32_t> bits) {
+  return Vec256<uint32_t>{_mm256_srlv_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> operator<<(
+    const Vec256<uint64_t> v, const Vec256<uint64_t> bits) {
+  return Vec256<uint64_t>{_mm256_sllv_epi64(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> operator>>(
+    const Vec256<uint64_t> v, const Vec256<uint64_t> bits) {
+  return Vec256<uint64_t>{_mm256_srlv_epi64(v.raw, bits.raw)};
+}
+
+// Signed (no i8,i16,i64)
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> operator<<(
+    const Vec256<int32_t> v, const Vec256<int32_t> bits) {
+  return Vec256<int32_t>{_mm256_sllv_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> operator>>(
+    const Vec256<int32_t> v, const Vec256<int32_t> bits) {
+  return Vec256<int32_t>{_mm256_srav_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> operator<<(
+    const Vec256<int64_t> v, const Vec256<int64_t> bits) {
+  return Vec256<int64_t>{_mm256_sllv_epi64(v.raw, bits.raw)};
+}
+
+// Variable shift for SSE registers.
+HWY_ATTR_AVX2 HWY_INLINE Vec128<uint32_t> operator>>(
+    const Vec128<uint32_t> v, const Vec128<uint32_t> bits) {
+  return Vec128<uint32_t>{_mm_srlv_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec128<uint32_t> operator<<(
+    const Vec128<uint32_t> v, const Vec128<uint32_t> bits) {
+  return Vec128<uint32_t>{_mm_sllv_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec128<uint64_t> operator<<(
+    const Vec128<uint64_t> v, const Vec128<uint64_t> bits) {
+  return Vec128<uint64_t>{_mm_sllv_epi64(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec128<uint64_t> operator>>(
+    const Vec128<uint64_t> v, const Vec128<uint64_t> bits) {
+  return Vec128<uint64_t>{_mm_srlv_epi64(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec128<int32_t> operator<<(
+    const Vec128<int32_t> v, const Vec128<int32_t> bits) {
+  return Vec128<int32_t>{_mm_sllv_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec128<int32_t> operator>>(
+    const Vec128<int32_t> v, const Vec128<int32_t> bits) {
+  return Vec128<int32_t>{_mm_srav_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec128<int64_t> operator<<(
+    const Vec128<int64_t> v, const Vec128<int64_t> bits) {
+  return Vec128<int64_t>{_mm_sllv_epi64(v.raw, bits.raw)};
+}
+
+// ------------------------------ Minimum
+
+// Unsigned (no u64)
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> Min(const Vec256<uint8_t> a,
+                                             const Vec256<uint8_t> b) {
+  return Vec256<uint8_t>{_mm256_min_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> Min(const Vec256<uint16_t> a,
+                                              const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_min_epu16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> Min(const Vec256<uint32_t> a,
+                                              const Vec256<uint32_t> b) {
+  return Vec256<uint32_t>{_mm256_min_epu32(a.raw, b.raw)};
+}
+
+// Signed (no i64)
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> Min(const Vec256<int8_t> a,
+                                            const Vec256<int8_t> b) {
+  return Vec256<int8_t>{_mm256_min_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> Min(const Vec256<int16_t> a,
+                                             const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_min_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Min(const Vec256<int32_t> a,
+                                             const Vec256<int32_t> b) {
+  return Vec256<int32_t>{_mm256_min_epi32(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Min(const Vec256<float> a,
+                                           const Vec256<float> b) {
+  return Vec256<float>{_mm256_min_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Min(const Vec256<double> a,
+                                            const Vec256<double> b) {
+  return Vec256<double>{_mm256_min_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Maximum
+
+// Unsigned (no u64)
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> Max(const Vec256<uint8_t> a,
+                                             const Vec256<uint8_t> b) {
+  return Vec256<uint8_t>{_mm256_max_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> Max(const Vec256<uint16_t> a,
+                                              const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_max_epu16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> Max(const Vec256<uint32_t> a,
+                                              const Vec256<uint32_t> b) {
+  return Vec256<uint32_t>{_mm256_max_epu32(a.raw, b.raw)};
+}
+
+// Signed (no i64)
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> Max(const Vec256<int8_t> a,
+                                            const Vec256<int8_t> b) {
+  return Vec256<int8_t>{_mm256_max_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> Max(const Vec256<int16_t> a,
+                                             const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_max_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Max(const Vec256<int32_t> a,
+                                             const Vec256<int32_t> b) {
+  return Vec256<int32_t>{_mm256_max_epi32(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Max(const Vec256<float> a,
+                                           const Vec256<float> b) {
+  return Vec256<float>{_mm256_max_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Max(const Vec256<double> a,
+                                            const Vec256<double> b) {
+  return Vec256<double>{_mm256_max_pd(a.raw, b.raw)};
+}
+
+// Returns the closest value to v within [lo, hi].
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> Clamp(const Vec256<T> v, const Vec256<T> lo,
+                                         const Vec256<T> hi) {
+  return Min(Max(lo, v), hi);
+}
+
+// ------------------------------ Integer multiplication
+
+// Unsigned
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> operator*(const Vec256<uint16_t> a,
+                                                    const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_mullo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> operator*(const Vec256<uint32_t> a,
+                                                    const Vec256<uint32_t> b) {
+  return Vec256<uint32_t>{_mm256_mullo_epi32(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> operator*(const Vec256<int16_t> a,
+                                                   const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_mullo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> operator*(const Vec256<int32_t> a,
+                                                   const Vec256<int32_t> b) {
+  return Vec256<int32_t>{_mm256_mullo_epi32(a.raw, b.raw)};
+}
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// Returns the upper 16 bits of a * b in each lane.
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> MulHigh(const Vec256<uint16_t> a,
+                                                  const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_mulhi_epu16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> MulHigh(const Vec256<int16_t> a,
+                                                 const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_mulhi_epi16(a.raw, b.raw)};
+}
+
+}  // namespace ext
+
+// Multiplies even lanes (0, 2 ..) and places the double-wide result into
+// even and the upper half into its odd neighbor lane.
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> MulEven(const Vec256<int32_t> a,
+                                                 const Vec256<int32_t> b) {
+  return Vec256<int64_t>{_mm256_mul_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> MulEven(const Vec256<uint32_t> a,
+                                                  const Vec256<uint32_t> b) {
+  return Vec256<uint64_t>{_mm256_mul_epu32(a.raw, b.raw)};
+}
+
+// ------------------------------ Floating-point negate
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Neg(const Vec256<float> v) {
+  const Full256<float> df;
+  const Full256<uint32_t> du;
+  const auto sign = BitCast(df, Set(du, 0x80000000u));
+  return v ^ sign;
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Neg(const Vec256<double> v) {
+  const Full256<double> df;
+  const Full256<uint64_t> du;
+  const auto sign = BitCast(df, Set(du, 0x8000000000000000ull));
+  return v ^ sign;
+}
+
+// ------------------------------ Floating-point mul / div
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> operator*(const Vec256<float> a,
+                                                 const Vec256<float> b) {
+  return Vec256<float>{_mm256_mul_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> operator*(const Vec256<double> a,
+                                                  const Vec256<double> b) {
+  return Vec256<double>{_mm256_mul_pd(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> operator/(const Vec256<float> a,
+                                                 const Vec256<float> b) {
+  return Vec256<float>{_mm256_div_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> operator/(const Vec256<double> a,
+                                                  const Vec256<double> b) {
+  return Vec256<double>{_mm256_div_pd(a.raw, b.raw)};
+}
+
+// Approximate reciprocal
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> ApproximateReciprocal(
+    const Vec256<float> v) {
+  return Vec256<float>{_mm256_rcp_ps(v.raw)};
+}
+
+namespace ext {
+// Absolute value of difference.
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> AbsDiff(const Vec256<float> a,
+                                               const Vec256<float> b) {
+  const auto mask =
+      BitCast(Full256<float>(), Set(Full256<uint32_t>(), 0x7FFFFFFFu));
+  return mask & (a - b);
+}
+}  // namespace ext
+
+// ------------------------------ Floating-point multiply-add variants
+
+// Returns mul * x + add
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> MulAdd(const Vec256<float> mul,
+                                              const Vec256<float> x,
+                                              const Vec256<float> add) {
+  return Vec256<float>{_mm256_fmadd_ps(mul.raw, x.raw, add.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> MulAdd(const Vec256<double> mul,
+                                               const Vec256<double> x,
+                                               const Vec256<double> add) {
+  return Vec256<double>{_mm256_fmadd_pd(mul.raw, x.raw, add.raw)};
+}
+
+// Returns add - mul * x
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> NegMulAdd(const Vec256<float> mul,
+                                                 const Vec256<float> x,
+                                                 const Vec256<float> add) {
+  return Vec256<float>{_mm256_fnmadd_ps(mul.raw, x.raw, add.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> NegMulAdd(const Vec256<double> mul,
+                                                  const Vec256<double> x,
+                                                  const Vec256<double> add) {
+  return Vec256<double>{_mm256_fnmadd_pd(mul.raw, x.raw, add.raw)};
+}
+
+// Slightly more expensive on ARM (extra negate)
+namespace ext {
+
+// Returns mul * x - sub
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> MulSub(const Vec256<float> mul,
+                                              const Vec256<float> x,
+                                              const Vec256<float> sub) {
+  return Vec256<float>{_mm256_fmsub_ps(mul.raw, x.raw, sub.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> MulSub(const Vec256<double> mul,
+                                               const Vec256<double> x,
+                                               const Vec256<double> sub) {
+  return Vec256<double>{_mm256_fmsub_pd(mul.raw, x.raw, sub.raw)};
+}
+
+// Returns -mul * x - sub
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> NegMulSub(const Vec256<float> mul,
+                                                 const Vec256<float> x,
+                                                 const Vec256<float> sub) {
+  return Vec256<float>{_mm256_fnmsub_ps(mul.raw, x.raw, sub.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> NegMulSub(const Vec256<double> mul,
+                                                  const Vec256<double> x,
+                                                  const Vec256<double> sub) {
+  return Vec256<double>{_mm256_fnmsub_pd(mul.raw, x.raw, sub.raw)};
+}
+
+}  // namespace ext
+
+// ------------------------------ Floating-point square root
+
+// Full precision square root
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Sqrt(const Vec256<float> v) {
+  return Vec256<float>{_mm256_sqrt_ps(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Sqrt(const Vec256<double> v) {
+  return Vec256<double>{_mm256_sqrt_pd(v.raw)};
+}
+
+// Approximate reciprocal square root
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> ApproximateReciprocalSqrt(
+    const Vec256<float> v) {
+  return Vec256<float>{_mm256_rsqrt_ps(v.raw)};
+}
+
+// ------------------------------ Floating-point rounding
+
+// Toward nearest integer, tie to even
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Round(const Vec256<float> v) {
+  return Vec256<float>{
+      _mm256_round_ps(v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Round(const Vec256<double> v) {
+  return Vec256<double>{
+      _mm256_round_pd(v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)};
+}
+
+// Toward zero, aka truncate
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Trunc(const Vec256<float> v) {
+  return Vec256<float>{
+      _mm256_round_ps(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Trunc(const Vec256<double> v) {
+  return Vec256<double>{
+      _mm256_round_pd(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)};
+}
+
+// Toward +infinity, aka ceiling
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Ceil(const Vec256<float> v) {
+  return Vec256<float>{
+      _mm256_round_ps(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Ceil(const Vec256<double> v) {
+  return Vec256<double>{
+      _mm256_round_pd(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)};
+}
+
+// Toward -infinity, aka floor
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Floor(const Vec256<float> v) {
+  return Vec256<float>{
+      _mm256_round_ps(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Floor(const Vec256<double> v) {
+  return Vec256<double>{
+      _mm256_round_pd(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)};
+}
+
+// ================================================== COMPARE
+
+// Comparisons fill a lane with 1-bits if the condition is true, else 0.
+
+// ------------------------------ Equality
+
+// Unsigned
+HWY_ATTR_AVX2 HWY_INLINE Mask256<uint8_t> operator==(const Vec256<uint8_t> a,
+                                                     const Vec256<uint8_t> b) {
+  return Mask256<uint8_t>{_mm256_cmpeq_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<uint16_t> operator==(
+    const Vec256<uint16_t> a, const Vec256<uint16_t> b) {
+  return Mask256<uint16_t>{_mm256_cmpeq_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<uint32_t> operator==(
+    const Vec256<uint32_t> a, const Vec256<uint32_t> b) {
+  return Mask256<uint32_t>{_mm256_cmpeq_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<uint64_t> operator==(
+    const Vec256<uint64_t> a, const Vec256<uint64_t> b) {
+  return Mask256<uint64_t>{_mm256_cmpeq_epi64(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int8_t> operator==(const Vec256<int8_t> a,
+                                                    const Vec256<int8_t> b) {
+  return Mask256<int8_t>{_mm256_cmpeq_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int16_t> operator==(const Vec256<int16_t> a,
+                                                     const Vec256<int16_t> b) {
+  return Mask256<int16_t>{_mm256_cmpeq_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int32_t> operator==(const Vec256<int32_t> a,
+                                                     const Vec256<int32_t> b) {
+  return Mask256<int32_t>{_mm256_cmpeq_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int64_t> operator==(const Vec256<int64_t> a,
+                                                     const Vec256<int64_t> b) {
+  return Mask256<int64_t>{_mm256_cmpeq_epi64(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX2 HWY_INLINE Mask256<float> operator==(const Vec256<float> a,
+                                                   const Vec256<float> b) {
+  return Mask256<float>{_mm256_cmp_ps(a.raw, b.raw, _CMP_EQ_OQ)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<double> operator==(const Vec256<double> a,
+                                                    const Vec256<double> b) {
+  return Mask256<double>{_mm256_cmp_pd(a.raw, b.raw, _CMP_EQ_OQ)};
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Mask256<T> TestBit(const Vec256<T> v,
+                                            const Vec256<T> bit) {
+  static_assert(!IsFloat<T>(), "Only integer vectors supported");
+  return (v & bit) == bit;
+}
+
+// ------------------------------ Strict inequality
+
+// Pre-9.3 GCC immintrin.h uses char, which may be unsigned, causing cmpgt_epi8
+// to perform an unsigned comparison instead of the intended signed. Workaround
+// is to cast to an explicitly signed type. See https://godbolt.org/z/PL7Ujy
+#if HWY_COMPILER_GCC != 0 && HWY_COMPILER_GCC < 930
+#define HWY_AVX2_GCC_CMPGT8_WORKAROUND 1
+#else
+#define HWY_AVX2_GCC_CMPGT8_WORKAROUND 0
+#endif
+
+// Signed/float <
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int8_t> operator<(const Vec256<int8_t> a,
+                                                   const Vec256<int8_t> b) {
+#if HWY_AVX2_GCC_CMPGT8_WORKAROUND
+  using i8x32 = signed char __attribute__((__vector_size__(32)));
+  return Mask256<int8_t>{static_cast<__m256i>(reinterpret_cast<i8x32>(a.raw) <
+                                              reinterpret_cast<i8x32>(b.raw))};
+#else
+  return Mask256<int8_t>{_mm256_cmpgt_epi8(b.raw, a.raw)};
+#endif
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int16_t> operator<(const Vec256<int16_t> a,
+                                                    const Vec256<int16_t> b) {
+  return Mask256<int16_t>{_mm256_cmpgt_epi16(b.raw, a.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int32_t> operator<(const Vec256<int32_t> a,
+                                                    const Vec256<int32_t> b) {
+  return Mask256<int32_t>{_mm256_cmpgt_epi32(b.raw, a.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int64_t> operator<(const Vec256<int64_t> a,
+                                                    const Vec256<int64_t> b) {
+  return Mask256<int64_t>{_mm256_cmpgt_epi64(b.raw, a.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<float> operator<(const Vec256<float> a,
+                                                  const Vec256<float> b) {
+  return Mask256<float>{_mm256_cmp_ps(a.raw, b.raw, _CMP_LT_OQ)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<double> operator<(const Vec256<double> a,
+                                                   const Vec256<double> b) {
+  return Mask256<double>{_mm256_cmp_pd(a.raw, b.raw, _CMP_LT_OQ)};
+}
+
+// Signed/float >
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int8_t> operator>(const Vec256<int8_t> a,
+                                                   const Vec256<int8_t> b) {
+#if HWY_AVX2_GCC_CMPGT8_WORKAROUND
+  using i8x32 = signed char __attribute__((__vector_size__(32)));
+  return Mask256<int8_t>{static_cast<__m256i>(reinterpret_cast<i8x32>(a.raw) >
+                                              reinterpret_cast<i8x32>(b.raw))};
+#else
+  return Mask256<int8_t>{_mm256_cmpgt_epi8(a.raw, b.raw)};
+#endif
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int16_t> operator>(const Vec256<int16_t> a,
+                                                    const Vec256<int16_t> b) {
+  return Mask256<int16_t>{_mm256_cmpgt_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int32_t> operator>(const Vec256<int32_t> a,
+                                                    const Vec256<int32_t> b) {
+  return Mask256<int32_t>{_mm256_cmpgt_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<int64_t> operator>(const Vec256<int64_t> a,
+                                                    const Vec256<int64_t> b) {
+  return Mask256<int64_t>{_mm256_cmpgt_epi64(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<float> operator>(const Vec256<float> a,
+                                                  const Vec256<float> b) {
+  return Mask256<float>{_mm256_cmp_ps(a.raw, b.raw, _CMP_GT_OQ)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<double> operator>(const Vec256<double> a,
+                                                   const Vec256<double> b) {
+  return Mask256<double>{_mm256_cmp_pd(a.raw, b.raw, _CMP_GT_OQ)};
+}
+
+// ------------------------------ Weak inequality
+
+// Float <= >=
+HWY_ATTR_AVX2 HWY_INLINE Mask256<float> operator<=(const Vec256<float> a,
+                                                   const Vec256<float> b) {
+  return Mask256<float>{_mm256_cmp_ps(a.raw, b.raw, _CMP_LE_OQ)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<double> operator<=(const Vec256<double> a,
+                                                    const Vec256<double> b) {
+  return Mask256<double>{_mm256_cmp_pd(a.raw, b.raw, _CMP_LE_OQ)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<float> operator>=(const Vec256<float> a,
+                                                   const Vec256<float> b) {
+  return Mask256<float>{_mm256_cmp_ps(a.raw, b.raw, _CMP_GE_OQ)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Mask256<double> operator>=(const Vec256<double> a,
+                                                    const Vec256<double> b) {
+  return Mask256<double>{_mm256_cmp_pd(a.raw, b.raw, _CMP_GE_OQ)};
+}
+
+// ================================================== MEMORY
+
+// ------------------------------ Load
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> Load(Full256<T> /* tag */,
+                                        const T* HWY_RESTRICT aligned) {
+  return Vec256<T>{
+      _mm256_load_si256(reinterpret_cast<const __m256i*>(aligned))};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Load(Full256<float> /* tag */,
+                                            const float* HWY_RESTRICT aligned) {
+  return Vec256<float>{_mm256_load_ps(aligned)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Load(
+    Full256<double> /* tag */, const double* HWY_RESTRICT aligned) {
+  return Vec256<double>{_mm256_load_pd(aligned)};
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> LoadU(Full256<T> /* tag */,
+                                         const T* HWY_RESTRICT p) {
+  return Vec256<T>{_mm256_loadu_si256(reinterpret_cast<const __m256i*>(p))};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> LoadU(Full256<float> /* tag */,
+                                             const float* HWY_RESTRICT p) {
+  return Vec256<float>{_mm256_loadu_ps(p)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> LoadU(Full256<double> /* tag */,
+                                              const double* HWY_RESTRICT p) {
+  return Vec256<double>{_mm256_loadu_pd(p)};
+}
+
+// Loads 128 bit and duplicates into both 128-bit halves. This avoids the
+// 3-cycle cost of moving data between 128-bit halves and avoids port 5.
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> LoadDup128(Full256<T> /* tag */,
+                                              const T* HWY_RESTRICT p) {
+#if HWY_LOADDUP_ASM
+  __m256i out;
+  asm("vbroadcasti128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0]));
+  return Vec256<T>{out};
+#else
+  return Vec256<T>{_mm256_broadcastsi128_si256(LoadU(Full128<T>(), p).raw)};
+#endif
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> LoadDup128(
+    Full256<float> /* tag */, const float* const HWY_RESTRICT p) {
+#if HWY_LOADDUP_ASM
+  __m256 out;
+  asm("vbroadcastf128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0]));
+  return Vec256<float>{out};
+#else
+  return Vec256<float>{_mm256_broadcast_ps(reinterpret_cast<const __m128*>(p))};
+#endif
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> LoadDup128(
+    Full256<double> /* tag */, const double* const HWY_RESTRICT p) {
+#if HWY_LOADDUP_ASM
+  __m256d out;
+  asm("vbroadcastf128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0]));
+  return Vec256<double>{out};
+#else
+  return Vec256<double>{
+      _mm256_broadcast_pd(reinterpret_cast<const __m128d*>(p))};
+#endif
+}
+
+// ------------------------------ Store
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE void Store(Vec256<T> v, Full256<T> /* tag */,
+                                    T* HWY_RESTRICT aligned) {
+  _mm256_store_si256(reinterpret_cast<__m256i*>(aligned), v.raw);
+}
+HWY_ATTR_AVX2 HWY_INLINE void Store(const Vec256<float> v,
+                                    Full256<float> /* tag */,
+                                    float* HWY_RESTRICT aligned) {
+  _mm256_store_ps(aligned, v.raw);
+}
+HWY_ATTR_AVX2 HWY_INLINE void Store(const Vec256<double> v,
+                                    Full256<double> /* tag */,
+                                    double* HWY_RESTRICT aligned) {
+  _mm256_store_pd(aligned, v.raw);
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE void StoreU(Vec256<T> v, Full256<T> /* tag */,
+                                     T* HWY_RESTRICT p) {
+  _mm256_storeu_si256(reinterpret_cast<__m256i*>(p), v.raw);
+}
+HWY_ATTR_AVX2 HWY_INLINE void StoreU(const Vec256<float> v,
+                                     Full256<float> /* tag */,
+                                     float* HWY_RESTRICT p) {
+  _mm256_storeu_ps(p, v.raw);
+}
+HWY_ATTR_AVX2 HWY_INLINE void StoreU(const Vec256<double> v,
+                                     Full256<double> /* tag */,
+                                     double* HWY_RESTRICT p) {
+  _mm256_storeu_pd(p, v.raw);
+}
+
+// ------------------------------ Non-temporal stores
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE void Stream(Vec256<T> v, Full256<T> /* tag */,
+                                     T* HWY_RESTRICT aligned) {
+  _mm256_stream_si256(reinterpret_cast<__m256i*>(aligned), v.raw);
+}
+HWY_ATTR_AVX2 HWY_INLINE void Stream(const Vec256<float> v,
+                                     Full256<float> /* tag */,
+                                     float* HWY_RESTRICT aligned) {
+  _mm256_stream_ps(aligned, v.raw);
+}
+HWY_ATTR_AVX2 HWY_INLINE void Stream(const Vec256<double> v,
+                                     Full256<double> /* tag */,
+                                     double* HWY_RESTRICT aligned) {
+  _mm256_stream_pd(aligned, v.raw);
+}
+
+// ------------------------------ Gather
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> gather_offset_impl(
+    SizeTag<4> /* tag */, Full256<T> /* tag */, const T* HWY_RESTRICT base,
+    const Vec256<int32_t> offset) {
+  return Vec256<T>{_mm256_i32gather_epi32(
+      reinterpret_cast<const int32_t*>(base), offset.raw, 1)};
+}
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> gather_index_impl(
+    SizeTag<4> /* tag */, Full256<T> /* tag */, const T* HWY_RESTRICT base,
+    const Vec256<int32_t> index) {
+  return Vec256<T>{_mm256_i32gather_epi32(
+      reinterpret_cast<const int32_t*>(base), index.raw, 4)};
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> gather_offset_impl(
+    SizeTag<8> /* tag */, Full256<T> /* tag */, const T* HWY_RESTRICT base,
+    const Vec256<int64_t> offset) {
+  return Vec256<T>{_mm256_i64gather_epi64(
+      reinterpret_cast<const GatherIndex64*>(base), offset.raw, 1)};
+}
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> gather_index_impl(
+    SizeTag<8> /* tag */, Full256<T> /* tag */, const T* HWY_RESTRICT base,
+    const Vec256<int64_t> index) {
+  return Vec256<T>{_mm256_i64gather_epi64(
+      reinterpret_cast<const GatherIndex64*>(base), index.raw, 8)};
+}
+
+template <typename T, typename Offset>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> GatherOffset(Full256<T> d,
+                                                const T* HWY_RESTRICT base,
+                                                const Vec256<Offset> offset) {
+  static_assert(sizeof(T) == sizeof(Offset), "SVE requires same size base/ofs");
+  return gather_offset_impl(SizeTag<sizeof(T)>(), d, base, offset);
+}
+template <typename T, typename Index>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> GatherIndex(Full256<T> d,
+                                               const T* HWY_RESTRICT base,
+                                               const Vec256<Index> index) {
+  static_assert(sizeof(T) == sizeof(Index), "SVE requires same size base/idx");
+  return gather_index_impl(SizeTag<sizeof(T)>(), d, base, index);
+}
+
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> GatherOffset<float>(
+    Full256<float> /* tag */, const float* HWY_RESTRICT base,
+    const Vec256<int32_t> offset) {
+  return Vec256<float>{_mm256_i32gather_ps(base, offset.raw, 1)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> GatherIndex<float>(
+    Full256<float> /* tag */, const float* HWY_RESTRICT base,
+    const Vec256<int32_t> index) {
+  return Vec256<float>{_mm256_i32gather_ps(base, index.raw, 4)};
+}
+
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> GatherOffset<double>(
+    Full256<double> /* tag */, const double* HWY_RESTRICT base,
+    const Vec256<int64_t> offset) {
+  return Vec256<double>{_mm256_i64gather_pd(base, offset.raw, 1)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> GatherIndex<double>(
+    Full256<double> /* tag */, const double* HWY_RESTRICT base,
+    const Vec256<int64_t> index) {
+  return Vec256<double>{_mm256_i64gather_pd(base, index.raw, 8)};
+}
+
+}  // namespace ext
+
+// ================================================== SWIZZLE
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE T GetLane(const Vec256<T> v) {
+  return GetLane(LowerHalf(v));
+}
+
+// ------------------------------ Extract half
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec128<T> LowerHalf(Vec256<T> v) {
+  return Vec128<T>{_mm256_castsi256_si128(v.raw)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec128<float> LowerHalf(Vec256<float> v) {
+  return Vec128<float>{_mm256_castps256_ps128(v.raw)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec128<double> LowerHalf(Vec256<double> v) {
+  return Vec128<double>{_mm256_castpd256_pd128(v.raw)};
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec128<T> UpperHalf(Vec256<T> v) {
+  return Vec128<T>{_mm256_extracti128_si256(v.raw, 1)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec128<float> UpperHalf(Vec256<float> v) {
+  return Vec128<float>{_mm256_extractf128_ps(v.raw, 1)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec128<double> UpperHalf(Vec256<double> v) {
+  return Vec128<double>{_mm256_extractf128_pd(v.raw, 1)};
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec128<T> GetHalf(Lower /* tag */, Vec256<T> v) {
+  return LowerHalf(v);
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec128<T> GetHalf(Upper /* tag */, Vec256<T> v) {
+  return UpperHalf(v);
+}
+
+// ------------------------------ Shift vector by constant #bytes
+
+// 0x01..0F, kBytes = 1 => 0x02..0F00
+template <int kBytes, typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> ShiftLeftBytes(const Vec256<T> v) {
+  static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
+  // This is the same operation as _mm256_bslli_epi128.
+  return Vec256<T>{_mm256_slli_si256(v.raw, kBytes)};
+}
+
+template <int kLanes, typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> ShiftLeftLanes(const Vec256<T> v) {
+  return ShiftLeftBytes<kLanes * sizeof(T)>(v);
+}
+
+// 0x01..0F, kBytes = 1 => 0x0001..0E
+template <int kBytes, typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> ShiftRightBytes(const Vec256<T> v) {
+  static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
+  // This is the same operation as _mm256_bsrli_epi128.
+  return Vec256<T>{_mm256_srli_si256(v.raw, kBytes)};
+}
+
+template <int kLanes, typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> ShiftRightLanes(const Vec256<T> v) {
+  return ShiftRightBytes<kLanes * sizeof(T)>(v);
+}
+
+// ------------------------------ Extract from 2x 128-bit at constant offset
+
+// Extracts 128 bits from <hi, lo> by skipping the least-significant kBytes.
+template <int kBytes, typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> CombineShiftRightBytes(const Vec256<T> hi,
+                                                          const Vec256<T> lo) {
+  const Full256<uint8_t> d8;
+  const Vec256<uint8_t> extracted_bytes{
+      _mm256_alignr_epi8(BitCast(d8, hi).raw, BitCast(d8, lo).raw, kBytes)};
+  return BitCast(Full256<T>(), extracted_bytes);
+}
+
+// ------------------------------ Broadcast/splat any lane
+
+// Unsigned
+template <int kLane>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> Broadcast(const Vec256<uint16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  if (kLane < 4) {
+    const __m256i lo = _mm256_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
+    return Vec256<uint16_t>{_mm256_unpacklo_epi64(lo, lo)};
+  } else {
+    const __m256i hi =
+        _mm256_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
+    return Vec256<uint16_t>{_mm256_unpackhi_epi64(hi, hi)};
+  }
+}
+template <int kLane>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> Broadcast(const Vec256<uint32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x55 * kLane)};
+}
+template <int kLane>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> Broadcast(const Vec256<uint64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec256<uint64_t>{_mm256_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
+}
+
+// Signed
+template <int kLane>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> Broadcast(const Vec256<int16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  if (kLane < 4) {
+    const __m256i lo = _mm256_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
+    return Vec256<int16_t>{_mm256_unpacklo_epi64(lo, lo)};
+  } else {
+    const __m256i hi =
+        _mm256_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
+    return Vec256<int16_t>{_mm256_unpackhi_epi64(hi, hi)};
+  }
+}
+template <int kLane>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Broadcast(const Vec256<int32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x55 * kLane)};
+}
+template <int kLane>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> Broadcast(const Vec256<int64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec256<int64_t>{_mm256_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
+}
+
+// Float
+template <int kLane>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Broadcast(Vec256<float> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x55 * kLane)};
+}
+template <int kLane>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Broadcast(const Vec256<double> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec256<double>{_mm256_shuffle_pd(v.raw, v.raw, 15 * kLane)};
+}
+
+// ------------------------------ Hard-coded shuffles
+
+// Notation: let Vec256<int32_t> have lanes 7,6,5,4,3,2,1,0 (0 is
+// least-significant). Shuffle0321 rotates four-lane blocks one lane to the
+// right (the previous least-significant lane is now most-significant =>
+// 47650321). These could also be implemented via CombineShiftRightBytes but
+// the shuffle_abcd notation is more convenient.
+
+// Swap 32-bit halves in 64-bit halves.
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> Shuffle2301(
+    const Vec256<uint32_t> v) {
+  return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0xB1)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Shuffle2301(const Vec256<int32_t> v) {
+  return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0xB1)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Shuffle2301(const Vec256<float> v) {
+  return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0xB1)};
+}
+
+// Swap 64-bit halves
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> Shuffle1032(
+    const Vec256<uint32_t> v) {
+  return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x4E)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Shuffle1032(const Vec256<int32_t> v) {
+  return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x4E)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Shuffle1032(const Vec256<float> v) {
+  // Shorter encoding than _mm256_permute_ps.
+  return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x4E)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> Shuffle01(const Vec256<uint64_t> v) {
+  return Vec256<uint64_t>{_mm256_shuffle_epi32(v.raw, 0x4E)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> Shuffle01(const Vec256<int64_t> v) {
+  return Vec256<int64_t>{_mm256_shuffle_epi32(v.raw, 0x4E)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> Shuffle01(const Vec256<double> v) {
+  // Shorter encoding than _mm256_permute_pd.
+  return Vec256<double>{_mm256_shuffle_pd(v.raw, v.raw, 5)};
+}
+
+// Rotate right 32 bits
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> Shuffle0321(
+    const Vec256<uint32_t> v) {
+  return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x39)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Shuffle0321(const Vec256<int32_t> v) {
+  return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x39)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Shuffle0321(const Vec256<float> v) {
+  return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x39)};
+}
+// Rotate left 32 bits
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> Shuffle2103(
+    const Vec256<uint32_t> v) {
+  return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x93)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Shuffle2103(const Vec256<int32_t> v) {
+  return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x93)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> Shuffle2103(const Vec256<float> v) {
+  return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x93)};
+}
+
+// Reverse
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> Shuffle0123(
+    const Vec256<uint32_t> v) {
+  return Vec256<uint32_t>{_mm256_shuffle_epi32(v.raw, 0x1B)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> Shuffle0123(const Vec256<int32_t> v) {
+  return Vec256<int32_t>{_mm256_shuffle_epi32(v.raw, 0x1B)};
+}
+HWY_ATTR_AVX2
+HWY_INLINE Vec256<float> Shuffle0123(const Vec256<float> v) {
+  return Vec256<float>{_mm256_shuffle_ps(v.raw, v.raw, 0x1B)};
+}
+
+// ------------------------------ Permute (runtime variable)
+
+// Returned by SetTableIndices for use by TableLookupLanes.
+template <typename T>
+struct Permute256 {
+  __m256i raw;
+};
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Permute256<T> SetTableIndices(const Full256<T> d,
+                                                       const int32_t* idx) {
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+  for (size_t i = 0; i < d.N; ++i) {
+    if (idx[i] >= static_cast<int32_t>(d.N)) {
+      printf("SetTableIndices [%zu] = %d >= %zu\n", i, idx[i], d.N);
+      HWY_TRAP();
+    }
+  }
+#else
+  (void)d;
+#endif
+  return Permute256<T>{LoadU(Full256<int32_t>(), idx).raw};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> TableLookupLanes(
+    const Vec256<uint32_t> v, const Permute256<uint32_t> idx) {
+  return Vec256<uint32_t>{_mm256_permutevar8x32_epi32(v.raw, idx.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> TableLookupLanes(
+    const Vec256<int32_t> v, const Permute256<int32_t> idx) {
+  return Vec256<int32_t>{_mm256_permutevar8x32_epi32(v.raw, idx.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> TableLookupLanes(
+    const Vec256<float> v, const Permute256<float> idx) {
+  return Vec256<float>{_mm256_permutevar8x32_ps(v.raw, idx.raw)};
+}
+
+// ------------------------------ Interleave lanes
+
+// Interleaves lanes from halves of the 128-bit blocks of "a" (which provides
+// the least-significant lane) and "b". To concatenate two half-width integers
+// into one, use ZipLo/hi instead (also works with scalar).
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> InterleaveLo(const Vec256<uint8_t> a,
+                                                      const Vec256<uint8_t> b) {
+  return Vec256<uint8_t>{_mm256_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> InterleaveLo(
+    const Vec256<uint16_t> a, const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> InterleaveLo(
+    const Vec256<uint32_t> a, const Vec256<uint32_t> b) {
+  return Vec256<uint32_t>{_mm256_unpacklo_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> InterleaveLo(
+    const Vec256<uint64_t> a, const Vec256<uint64_t> b) {
+  return Vec256<uint64_t>{_mm256_unpacklo_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> InterleaveLo(const Vec256<int8_t> a,
+                                                     const Vec256<int8_t> b) {
+  return Vec256<int8_t>{_mm256_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> InterleaveLo(const Vec256<int16_t> a,
+                                                      const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> InterleaveLo(const Vec256<int32_t> a,
+                                                      const Vec256<int32_t> b) {
+  return Vec256<int32_t>{_mm256_unpacklo_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> InterleaveLo(const Vec256<int64_t> a,
+                                                      const Vec256<int64_t> b) {
+  return Vec256<int64_t>{_mm256_unpacklo_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> InterleaveLo(const Vec256<float> a,
+                                                    const Vec256<float> b) {
+  return Vec256<float>{_mm256_unpacklo_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> InterleaveLo(const Vec256<double> a,
+                                                     const Vec256<double> b) {
+  return Vec256<double>{_mm256_unpacklo_pd(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint8_t> InterleaveHi(const Vec256<uint8_t> a,
+                                                      const Vec256<uint8_t> b) {
+  return Vec256<uint8_t>{_mm256_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> InterleaveHi(
+    const Vec256<uint16_t> a, const Vec256<uint16_t> b) {
+  return Vec256<uint16_t>{_mm256_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> InterleaveHi(
+    const Vec256<uint32_t> a, const Vec256<uint32_t> b) {
+  return Vec256<uint32_t>{_mm256_unpackhi_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> InterleaveHi(
+    const Vec256<uint64_t> a, const Vec256<uint64_t> b) {
+  return Vec256<uint64_t>{_mm256_unpackhi_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int8_t> InterleaveHi(const Vec256<int8_t> a,
+                                                     const Vec256<int8_t> b) {
+  return Vec256<int8_t>{_mm256_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> InterleaveHi(const Vec256<int16_t> a,
+                                                      const Vec256<int16_t> b) {
+  return Vec256<int16_t>{_mm256_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> InterleaveHi(const Vec256<int32_t> a,
+                                                      const Vec256<int32_t> b) {
+  return Vec256<int32_t>{_mm256_unpackhi_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> InterleaveHi(const Vec256<int64_t> a,
+                                                      const Vec256<int64_t> b) {
+  return Vec256<int64_t>{_mm256_unpackhi_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> InterleaveHi(const Vec256<float> a,
+                                                    const Vec256<float> b) {
+  return Vec256<float>{_mm256_unpackhi_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> InterleaveHi(const Vec256<double> a,
+                                                     const Vec256<double> b) {
+  return Vec256<double>{_mm256_unpackhi_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Zip lanes
+
+// Same as interleave_*, except that the return lanes are double-width integers;
+// this is necessary because the single-lane scalar cannot return two values.
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> ZipLo(const Vec256<uint8_t> a,
+                                                const Vec256<uint8_t> b) {
+  return Vec256<uint16_t>{_mm256_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> ZipLo(const Vec256<uint16_t> a,
+                                                const Vec256<uint16_t> b) {
+  return Vec256<uint32_t>{_mm256_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> ZipLo(const Vec256<uint32_t> a,
+                                                const Vec256<uint32_t> b) {
+  return Vec256<uint64_t>{_mm256_unpacklo_epi32(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> ZipLo(const Vec256<int8_t> a,
+                                               const Vec256<int8_t> b) {
+  return Vec256<int16_t>{_mm256_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> ZipLo(const Vec256<int16_t> a,
+                                               const Vec256<int16_t> b) {
+  return Vec256<int32_t>{_mm256_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> ZipLo(const Vec256<int32_t> a,
+                                               const Vec256<int32_t> b) {
+  return Vec256<int64_t>{_mm256_unpacklo_epi32(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> ZipHi(const Vec256<uint8_t> a,
+                                                const Vec256<uint8_t> b) {
+  return Vec256<uint16_t>{_mm256_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> ZipHi(const Vec256<uint16_t> a,
+                                                const Vec256<uint16_t> b) {
+  return Vec256<uint32_t>{_mm256_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> ZipHi(const Vec256<uint32_t> a,
+                                                const Vec256<uint32_t> b) {
+  return Vec256<uint64_t>{_mm256_unpackhi_epi32(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> ZipHi(const Vec256<int8_t> a,
+                                               const Vec256<int8_t> b) {
+  return Vec256<int16_t>{_mm256_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> ZipHi(const Vec256<int16_t> a,
+                                               const Vec256<int16_t> b) {
+  return Vec256<int32_t>{_mm256_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> ZipHi(const Vec256<int32_t> a,
+                                               const Vec256<int32_t> b) {
+  return Vec256<int64_t>{_mm256_unpackhi_epi32(a.raw, b.raw)};
+}
+
+// ------------------------------ Blocks
+
+// hiH,hiL loH,loL |-> hiL,loL (= lower halves)
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> ConcatLoLo(const Vec256<T> hi,
+                                              const Vec256<T> lo) {
+  return Vec256<T>{_mm256_permute2x128_si256(lo.raw, hi.raw, 0x20)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> ConcatLoLo(const Vec256<float> hi,
+                                                  const Vec256<float> lo) {
+  return Vec256<float>{_mm256_permute2f128_ps(lo.raw, hi.raw, 0x20)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> ConcatLoLo(const Vec256<double> hi,
+                                                   const Vec256<double> lo) {
+  return Vec256<double>{_mm256_permute2f128_pd(lo.raw, hi.raw, 0x20)};
+}
+
+// hiH,hiL loH,loL |-> hiH,loH (= upper halves)
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> ConcatHiHi(const Vec256<T> hi,
+                                              const Vec256<T> lo) {
+  return Vec256<T>{_mm256_permute2x128_si256(lo.raw, hi.raw, 0x31)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> ConcatHiHi(const Vec256<float> hi,
+                                                  const Vec256<float> lo) {
+  return Vec256<float>{_mm256_permute2f128_ps(lo.raw, hi.raw, 0x31)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> ConcatHiHi(const Vec256<double> hi,
+                                                   const Vec256<double> lo) {
+  return Vec256<double>{_mm256_permute2f128_pd(lo.raw, hi.raw, 0x31)};
+}
+
+// hiH,hiL loH,loL |-> hiL,loH (= inner halves / swap blocks)
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> ConcatLoHi(const Vec256<T> hi,
+                                              const Vec256<T> lo) {
+  return Vec256<T>{_mm256_permute2x128_si256(lo.raw, hi.raw, 0x21)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> ConcatLoHi(const Vec256<float> hi,
+                                                  const Vec256<float> lo) {
+  return Vec256<float>{_mm256_permute2f128_ps(lo.raw, hi.raw, 0x21)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> ConcatLoHi(const Vec256<double> hi,
+                                                   const Vec256<double> lo) {
+  return Vec256<double>{_mm256_permute2f128_pd(lo.raw, hi.raw, 0x21)};
+}
+
+// hiH,hiL loH,loL |-> hiH,loL (= outer halves)
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> ConcatHiLo(const Vec256<T> hi,
+                                              const Vec256<T> lo) {
+  return Vec256<T>{_mm256_blend_epi32(hi.raw, lo.raw, 0x0F)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> ConcatHiLo(const Vec256<float> hi,
+                                                  const Vec256<float> lo) {
+  return Vec256<float>{_mm256_blend_ps(hi.raw, lo.raw, 0x0F)};
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> ConcatHiLo(const Vec256<double> hi,
+                                                   const Vec256<double> lo) {
+  return Vec256<double>{_mm256_blend_pd(hi.raw, lo.raw, 3)};
+}
+
+// ------------------------------ Odd/even lanes
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> odd_even_impl(SizeTag<1> /* tag */,
+                                                 const Vec256<T> a,
+                                                 const Vec256<T> b) {
+  const Full256<T> d;
+  const Full256<uint8_t> d8;
+  alignas(32) constexpr uint8_t mask[16] = {0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0,
+                                            0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0};
+  return IfThenElse(MaskFromVec(BitCast(d, LoadDup128(d8, mask))), b, a);
+}
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> odd_even_impl(SizeTag<2> /* tag */,
+                                                 const Vec256<T> a,
+                                                 const Vec256<T> b) {
+  return Vec256<T>{_mm256_blend_epi16(a.raw, b.raw, 0x55)};
+}
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> odd_even_impl(SizeTag<4> /* tag */,
+                                                 const Vec256<T> a,
+                                                 const Vec256<T> b) {
+  return Vec256<T>{_mm256_blend_epi32(a.raw, b.raw, 0x55)};
+}
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> odd_even_impl(SizeTag<8> /* tag */,
+                                                 const Vec256<T> a,
+                                                 const Vec256<T> b) {
+  return Vec256<T>{_mm256_blend_epi32(a.raw, b.raw, 0x33)};
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> OddEven(const Vec256<T> a,
+                                           const Vec256<T> b) {
+  return odd_even_impl(SizeTag<sizeof(T)>(), a, b);
+}
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> OddEven<float>(const Vec256<float> a,
+                                                      const Vec256<float> b) {
+  return Vec256<float>{_mm256_blend_ps(a.raw, b.raw, 0x55)};
+}
+
+template <>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> OddEven<double>(
+    const Vec256<double> a, const Vec256<double> b) {
+  return Vec256<double>{_mm256_blend_pd(a.raw, b.raw, 5)};
+}
+
+// ================================================== CONVERT
+
+// ------------------------------ Shuffle bytes with variable indices
+
+// Returns vector of bytes[from[i]]. "from" is also interpreted as bytes:
+// either valid indices in [0, 16) or >= 0x80 to zero the i-th output byte.
+template <typename T, typename TI>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> TableLookupBytes(const Vec256<T> bytes,
+                                                    const Vec256<TI> from) {
+  return Vec256<T>{_mm256_shuffle_epi8(bytes.raw, from.raw)};
+}
+
+// ------------------------------ Promotions (part w/ narrow lanes -> full)
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<double> ConvertTo(Full256<double> /* tag */,
+                                                  const Vec128<float, 4> v) {
+  return Vec256<double>{_mm256_cvtps_pd(v.raw)};
+}
+
+// Unsigned: zero-extend.
+// Note: these have 3 cycle latency; if inputs are already split across the
+// 128 bit blocks (in their upper/lower halves), then ZipHi/lo would be faster.
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint16_t> ConvertTo(Full256<uint16_t> /* tag */,
+                                                    Vec128<uint8_t> v) {
+  return Vec256<uint16_t>{_mm256_cvtepu8_epi16(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> ConvertTo(Full256<uint32_t> /* tag */,
+                                                    Vec128<uint8_t, 8> v) {
+  return Vec256<uint32_t>{_mm256_cvtepu8_epi32(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> ConvertTo(Full256<int16_t> /* tag */,
+                                                   Vec128<uint8_t> v) {
+  return Vec256<int16_t>{_mm256_cvtepu8_epi16(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> ConvertTo(Full256<int32_t> /* tag */,
+                                                   Vec128<uint8_t, 8> v) {
+  return Vec256<int32_t>{_mm256_cvtepu8_epi32(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> ConvertTo(Full256<uint32_t> /* tag */,
+                                                    Vec128<uint16_t> v) {
+  return Vec256<uint32_t>{_mm256_cvtepu16_epi32(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> ConvertTo(Full256<int32_t> /* tag */,
+                                                   Vec128<uint16_t> v) {
+  return Vec256<int32_t>{_mm256_cvtepu16_epi32(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> ConvertTo(Full256<uint64_t> /* tag */,
+                                                    Vec128<uint32_t> v) {
+  return Vec256<uint64_t>{_mm256_cvtepu32_epi64(v.raw)};
+}
+
+// Special case for "v" with all blocks equal (e.g. from LoadDup128):
+// single-cycle latency instead of 3.
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint32_t> U32FromU8(const Vec256<uint8_t> v) {
+  const Full256<uint32_t> d32;
+  alignas(32) static constexpr uint32_t k32From8[8] = {
+      0xFFFFFF00UL, 0xFFFFFF01UL, 0xFFFFFF02UL, 0xFFFFFF03UL,
+      0xFFFFFF04UL, 0xFFFFFF05UL, 0xFFFFFF06UL, 0xFFFFFF07UL};
+  return TableLookupBytes(BitCast(d32, v), Load(d32, k32From8));
+}
+
+// Signed: replicate sign bit.
+// Note: these have 3 cycle latency; if inputs are already split across the
+// 128 bit blocks (in their upper/lower halves), then ZipHi/lo followed by
+// signed shift would be faster.
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int16_t> ConvertTo(Full256<int16_t> /* tag */,
+                                                   Vec128<int8_t> v) {
+  return Vec256<int16_t>{_mm256_cvtepi8_epi16(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> ConvertTo(Full256<int32_t> /* tag */,
+                                                   Vec128<int8_t, 8> v) {
+  return Vec256<int32_t>{_mm256_cvtepi8_epi32(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> ConvertTo(Full256<int32_t> /* tag */,
+                                                   Vec128<int16_t> v) {
+  return Vec256<int32_t>{_mm256_cvtepi16_epi32(v.raw)};
+}
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int64_t> ConvertTo(Full256<int64_t> /* tag */,
+                                                   Vec128<int32_t> v) {
+  return Vec256<int64_t>{_mm256_cvtepi32_epi64(v.raw)};
+}
+
+// ------------------------------ Demotions (full -> part w/ narrow lanes)
+
+HWY_ATTR_AVX2 HWY_INLINE Vec128<uint16_t> ConvertTo(Full128<uint16_t> /* tag */,
+                                                    const Vec256<int32_t> v) {
+  const __m256i u16 = _mm256_packus_epi32(v.raw, v.raw);
+  // Concatenating lower halves of both 128-bit blocks afterward is more
+  // efficient than an extra input with low block = high block of v.
+  return Vec128<uint16_t>{
+      _mm256_castsi256_si128(_mm256_permute4x64_epi64(u16, 0x88))};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec128<uint8_t, 8> ConvertTo(
+    Desc<uint8_t, 8> /* tag */, const Vec256<int32_t> v) {
+  const __m256i u16_blocks = _mm256_packus_epi32(v.raw, v.raw);
+  // Concatenate lower 64 bits of each 128-bit block
+  const __m256i u16_concat = _mm256_permute4x64_epi64(u16_blocks, 0x88);
+  const __m128i u16 = _mm256_castsi256_si128(u16_concat);
+  return Vec128<uint8_t, 8>{_mm_packus_epi16(u16, u16)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec128<int16_t> ConvertTo(Full128<int16_t> /* tag */,
+                                                   const Vec256<int32_t> v) {
+  const __m256i i16 = _mm256_packs_epi32(v.raw, v.raw);
+  return Vec128<int16_t>{
+      _mm256_castsi256_si128(_mm256_permute4x64_epi64(i16, 0x88))};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec128<int8_t, 8> ConvertTo(Desc<int8_t, 8> /* tag */,
+                                                     const Vec256<int32_t> v) {
+  const __m256i i16_blocks = _mm256_packs_epi32(v.raw, v.raw);
+  // Concatenate lower 64 bits of each 128-bit block
+  const __m256i i16_concat = _mm256_permute4x64_epi64(i16_blocks, 0x88);
+  const __m128i i16 = _mm256_castsi256_si128(i16_concat);
+  return Vec128<int8_t, 8>{_mm_packs_epi16(i16, i16)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec128<uint8_t> ConvertTo(Full128<uint8_t> /* tag */,
+                                                   const Vec256<int16_t> v) {
+  const __m256i u8 = _mm256_packus_epi16(v.raw, v.raw);
+  return Vec128<uint8_t>{
+      _mm256_castsi256_si128(_mm256_permute4x64_epi64(u8, 0x88))};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec128<int8_t> ConvertTo(Full128<int8_t> /* tag */,
+                                                  const Vec256<int16_t> v) {
+  const __m256i i8 = _mm256_packs_epi16(v.raw, v.raw);
+  return Vec128<int8_t>{
+      _mm256_castsi256_si128(_mm256_permute4x64_epi64(i8, 0x88))};
+}
+
+// For already range-limited input [0, 255].
+HWY_ATTR_AVX2 HWY_INLINE Vec128<uint8_t, 8> U8FromU32(
+    const Vec256<uint32_t> v) {
+  const Full256<uint32_t> d32;
+  alignas(32) static constexpr uint32_t k8From32[8] = {
+      0x0C080400u, ~0u, ~0u, ~0u, ~0u, 0x0C080400u, ~0u, ~0u};
+  // Place first four bytes in lo[0], remaining 4 in hi[1].
+  const auto quad = TableLookupBytes(v, Load(d32, k8From32));
+  // Interleave both quadruplets - OR instead of unpack reduces port5 pressure.
+  const auto lo = LowerHalf(quad);
+  const auto hi = UpperHalf(quad);
+  const auto pair = LowerHalf(lo | hi);
+  return BitCast(Desc<uint8_t, 8>(), pair);
+}
+
+// ------------------------------ Convert i32 <=> f32
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<float> ConvertTo(Full256<float> /* tag */,
+                                                 const Vec256<int32_t> v) {
+  return Vec256<float>{_mm256_cvtepi32_ps(v.raw)};
+}
+// Truncates (rounds toward zero).
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> ConvertTo(Full256<int32_t> /* tag */,
+                                                   const Vec256<float> v) {
+  return Vec256<int32_t>{_mm256_cvttps_epi32(v.raw)};
+}
+
+HWY_ATTR_AVX2 HWY_INLINE Vec256<int32_t> NearestInt(const Vec256<float> v) {
+  return Vec256<int32_t>{_mm256_cvtps_epi32(v.raw)};
+}
+
+// ================================================== MISC
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// ------------------------------ movemask
+
+// Returns a bit array of the most significant bit of each byte in "v", i.e.
+// sum_i=0..31 of (v[i] >> 7) << i; v[0] is the least-significant byte of "v".
+// This is useful for testing/branching based on comparison results.
+HWY_ATTR_AVX2 HWY_INLINE uint64_t movemask(const Vec256<uint8_t> v) {
+  // Prevent sign-extension of 32-bit masks because the intrinsic returns int.
+  return static_cast<uint32_t>(_mm256_movemask_epi8(v.raw));
+}
+
+// Returns the most significant bit of each float/double lane (see above).
+HWY_ATTR_AVX2 HWY_INLINE uint64_t movemask(const Vec256<float> v) {
+  return static_cast<unsigned>(_mm256_movemask_ps(v.raw));
+}
+HWY_ATTR_AVX2 HWY_INLINE uint64_t movemask(const Vec256<double> v) {
+  return static_cast<unsigned>(_mm256_movemask_pd(v.raw));
+}
+
+// ------------------------------ mask
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE bool AllFalse(const Mask256<T> v) {
+  // Cheaper than PTEST, which is 2 uop / 3L.
+  const auto bytes = BitCast(Full256<uint8_t>(), VecFromMask(v));
+  return movemask(bytes) == 0;
+}
+HWY_ATTR_AVX2 HWY_INLINE bool AllFalse(const Mask256<float> v) {
+  return movemask(VecFromMask(v)) == 0;
+}
+HWY_ATTR_AVX2 HWY_INLINE bool AllFalse(const Mask256<double> v) {
+  return movemask(VecFromMask(v)) == 0;
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE bool AllTrue(const Mask256<T> v) {
+  const auto bytes = BitCast(Full256<uint8_t>(), VecFromMask(v));
+  return movemask(bytes) == 0xFFFFFFFFu;
+}
+HWY_ATTR_AVX2 HWY_INLINE bool AllTrue(const Mask256<float> v) {
+  return movemask(VecFromMask(v)) == 0xFF;
+}
+HWY_ATTR_AVX2 HWY_INLINE bool AllTrue(const Mask256<double> v) {
+  return movemask(VecFromMask(v)) == 0xF;
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE size_t CountTrue(const Mask256<T> v) {
+  // Integer vectors: only have movemask for u8, so divide by number of bytes.
+  const auto bytes = BitCast(Full256<uint8_t>(), VecFromMask(v));
+  return PopCount(movemask(bytes)) / sizeof(T);
+}
+HWY_ATTR_AVX2 HWY_INLINE size_t CountTrue(const Mask256<float> v) {
+  return PopCount(movemask(VecFromMask(v)));
+}
+HWY_ATTR_AVX2 HWY_INLINE size_t CountTrue(const Mask256<double> v) {
+  return PopCount(movemask(VecFromMask(v)));
+}
+
+// ------------------------------ Horizontal sum (reduction)
+
+// Returns 64-bit sums of 8-byte groups.
+HWY_ATTR_AVX2 HWY_INLINE Vec256<uint64_t> SumsOfU8x8(const Vec256<uint8_t> v) {
+  return Vec256<uint64_t>{_mm256_sad_epu8(v.raw, _mm256_setzero_si256())};
+}
+
+// Returns sum{lane[i]} in each lane. "v3210" is a replicated 128-bit block.
+// Same logic as x86_sse4.h, but with Vec256 arguments.
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> horz_sum_impl(SizeTag<4> /* tag */,
+                                                 const Vec256<T> v3210) {
+  const auto v1032 = Shuffle1032(v3210);
+  const auto v31_20_31_20 = v3210 + v1032;
+  const auto v20_31_20_31 = Shuffle0321(v31_20_31_20);
+  return v20_31_20_31 + v31_20_31_20;
+}
+
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> horz_sum_impl(SizeTag<8> /* tag */,
+                                                 const Vec256<T> v10) {
+  const auto v01 = Shuffle01(v10);
+  return v10 + v01;
+}
+
+// Supported for {uif}32x8, {uif}64x4. Returns the sum in each lane.
+template <typename T>
+HWY_ATTR_AVX2 HWY_INLINE Vec256<T> SumOfLanes(const Vec256<T> vHL) {
+  const Vec256<T> vLH = ConcatLoHi(vHL, vHL);
+  return horz_sum_impl(SizeTag<sizeof(T)>(), vLH + vHL);
+}
+
+}  // namespace ext
+
+// TODO(janwas): wrappers for all intrinsics (in x86 namespace).
+}  // namespace hwy
+
+#endif  // HWY_X86_AVX2_H_
diff --git a/third_party/highway/hwy/x86/avx512.h b/third_party/highway/hwy/x86/avx512.h
new file mode 100644
index 000000000..62c2b2720
--- /dev/null
+++ b/third_party/highway/hwy/x86/avx512.h
@@ -0,0 +1,2346 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_X86_AVX512_H_
+#define HWY_X86_AVX512_H_
+
+// 512-bit AVX512 vectors and operations.
+// WARNING: most operations do not cross 128-bit block boundaries. In
+// particular, "Broadcast", pack and zip behavior may be surprising.
+
+#include <immintrin.h>
+
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+#include <stdio.h>
+#endif
+
+#include "hwy/shared.h"
+#include "hwy/x86/avx2.h"
+
+// Must include contents of HWY_ATTR_AVX2 because an AVX-512 test may call
+// AVX2 functions (e.g. when converting to half-vectors).
+#define HWY_ATTR_AVX512 \
+  HWY_TARGET_ATTR("bmi,bmi2,avx,avx2,fma,avx512f,avx512vl,avx512dq,avx512bw")
+
+namespace hwy {
+
+template <typename T>
+struct Raw512 {
+  using type = __m512i;
+};
+template <>
+struct Raw512<float> {
+  using type = __m512;
+};
+template <>
+struct Raw512<double> {
+  using type = __m512d;
+};
+
+template <typename T>
+using Full512 = Desc<T, 64 / sizeof(T)>;
+
+template <typename T>
+class Vec512 {
+  using Raw = typename Raw512<T>::type;
+
+ public:
+  // Compound assignment. Only usable if there is a corresponding non-member
+  // binary operator overload. For example, only f32 and f64 support division.
+  HWY_ATTR_AVX512 HWY_INLINE Vec512& operator*=(const Vec512 other) {
+    return *this = (*this * other);
+  }
+  HWY_ATTR_AVX512 HWY_INLINE Vec512& operator/=(const Vec512 other) {
+    return *this = (*this / other);
+  }
+  HWY_ATTR_AVX512 HWY_INLINE Vec512& operator+=(const Vec512 other) {
+    return *this = (*this + other);
+  }
+  HWY_ATTR_AVX512 HWY_INLINE Vec512& operator-=(const Vec512 other) {
+    return *this = (*this - other);
+  }
+  HWY_ATTR_AVX512 HWY_INLINE Vec512& operator&=(const Vec512 other) {
+    return *this = (*this & other);
+  }
+  HWY_ATTR_AVX512 HWY_INLINE Vec512& operator|=(const Vec512 other) {
+    return *this = (*this | other);
+  }
+  HWY_ATTR_AVX512 HWY_INLINE Vec512& operator^=(const Vec512 other) {
+    return *this = (*this ^ other);
+  }
+
+  Raw raw;
+};
+
+// Template arg: sizeof(lane type)
+template <size_t size>
+struct RawMask512 {};
+template <>
+struct RawMask512<1> {
+  using type = __mmask64;
+};
+template <>
+struct RawMask512<2> {
+  using type = __mmask32;
+};
+template <>
+struct RawMask512<4> {
+  using type = __mmask16;
+};
+template <>
+struct RawMask512<8> {
+  using type = __mmask8;
+};
+
+// Mask register: one bit per lane.
+template <typename T>
+class Mask512 {
+  using Raw = typename RawMask512<sizeof(T)>::type;
+
+ public:
+  Raw raw;
+};
+
+// ------------------------------ Cast
+
+HWY_ATTR_AVX512 HWY_INLINE __m512i BitCastToInteger(__m512i v) { return v; }
+HWY_ATTR_AVX512 HWY_INLINE __m512i BitCastToInteger(__m512 v) {
+  return _mm512_castps_si512(v);
+}
+HWY_ATTR_AVX512 HWY_INLINE __m512i BitCastToInteger(__m512d v) {
+  return _mm512_castpd_si512(v);
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> cast_to_u8(Vec512<T> v) {
+  return Vec512<uint8_t>{BitCastToInteger(v.raw)};
+}
+
+// Cannot rely on function overloading because return types differ.
+template <typename T>
+struct BitCastFromInteger512 {
+  HWY_ATTR_AVX512 HWY_INLINE __m512i operator()(__m512i v) { return v; }
+};
+template <>
+struct BitCastFromInteger512<float> {
+  HWY_ATTR_AVX512 HWY_INLINE __m512 operator()(__m512i v) {
+    return _mm512_castsi512_ps(v);
+  }
+};
+template <>
+struct BitCastFromInteger512<double> {
+  HWY_ATTR_AVX512 HWY_INLINE __m512d operator()(__m512i v) {
+    return _mm512_castsi512_pd(v);
+  }
+};
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> cast_u8_to(Full512<T> /* tag */,
+                                                Vec512<uint8_t> v) {
+  return Vec512<T>{BitCastFromInteger512<T>()(v.raw)};
+}
+
+template <typename T, typename FromT>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> BitCast(Full512<T> d, Vec512<FromT> v) {
+  return cast_u8_to(d, cast_to_u8(v));
+}
+
+// ------------------------------ Set
+
+// Returns an all-zero vector.
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> Zero(Full512<T> /* tag */) {
+  return Vec512<T>{_mm512_setzero_si512()};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Zero(Full512<float> /* tag */) {
+  return Vec512<float>{_mm512_setzero_ps()};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Zero(Full512<double> /* tag */) {
+  return Vec512<double>{_mm512_setzero_pd()};
+}
+
+// Returns a vector with all lanes set to "t".
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> Set(Full512<uint8_t> /* tag */,
+                                               const uint8_t t) {
+  return Vec512<uint8_t>{_mm512_set1_epi8(t)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> Set(Full512<uint16_t> /* tag */,
+                                                const uint16_t t) {
+  return Vec512<uint16_t>{_mm512_set1_epi16(t)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> Set(Full512<uint32_t> /* tag */,
+                                                const uint32_t t) {
+  return Vec512<uint32_t>{_mm512_set1_epi32(t)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> Set(Full512<uint64_t> /* tag */,
+                                                const uint64_t t) {
+  return Vec512<uint64_t>{_mm512_set1_epi64(t)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> Set(Full512<int8_t> /* tag */,
+                                              const int8_t t) {
+  return Vec512<int8_t>{_mm512_set1_epi8(t)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> Set(Full512<int16_t> /* tag */,
+                                               const int16_t t) {
+  return Vec512<int16_t>{_mm512_set1_epi16(t)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Set(Full512<int32_t> /* tag */,
+                                               const int32_t t) {
+  return Vec512<int32_t>{_mm512_set1_epi32(t)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> Set(Full512<int64_t> /* tag */,
+                                               const int64_t t) {
+  return Vec512<int64_t>{_mm512_set1_epi64(t)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Set(Full512<float> /* tag */,
+                                             const float t) {
+  return Vec512<float>{_mm512_set1_ps(t)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Set(Full512<double> /* tag */,
+                                              const double t) {
+  return Vec512<double>{_mm512_set1_pd(t)};
+}
+
+template <typename T, typename T2>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> Iota(Full512<T> d, const T2 first) {
+  alignas(64) T lanes[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = first + i;
+  }
+  return Load(d, lanes);
+}
+
+HWY_DIAGNOSTICS(push)
+HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized")
+
+// Returns a vector with uninitialized elements.
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> Undefined(Full512<T> /* tag */) {
+#ifdef __clang__
+  return Vec512<T>{_mm512_undefined_epi32()};
+#else
+  __m512i raw;
+  return Vec512<T>{raw};
+#endif
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Undefined(Full512<float> /* tag */) {
+#ifdef __clang__
+  return Vec512<float>{_mm512_undefined_ps()};
+#else
+  __m512 raw;
+  return Vec512<float>{raw};
+#endif
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Undefined(Full512<double> /* tag */) {
+#ifdef __clang__
+  return Vec512<double>{_mm512_undefined_pd()};
+#else
+  __m512d raw;
+  return Vec512<double>{raw};
+#endif
+}
+
+HWY_DIAGNOSTICS(pop)
+
+// ================================================== LOGICAL
+
+// ------------------------------ Bitwise AND
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> operator&(const Vec512<T> a,
+                                               const Vec512<T> b) {
+  return Vec512<T>{_mm512_and_si512(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> operator&(const Vec512<float> a,
+                                                   const Vec512<float> b) {
+  return Vec512<float>{_mm512_and_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> operator&(const Vec512<double> a,
+                                                    const Vec512<double> b) {
+  return Vec512<double>{_mm512_and_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Bitwise AND-NOT
+
+// Returns ~not_mask & mask.
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> AndNot(const Vec512<T> not_mask,
+                                            const Vec512<T> mask) {
+  return Vec512<T>{_mm512_andnot_si512(not_mask.raw, mask.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> AndNot(const Vec512<float> not_mask,
+                                                const Vec512<float> mask) {
+  return Vec512<float>{_mm512_andnot_ps(not_mask.raw, mask.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> AndNot(const Vec512<double> not_mask,
+                                                 const Vec512<double> mask) {
+  return Vec512<double>{_mm512_andnot_pd(not_mask.raw, mask.raw)};
+}
+
+// ------------------------------ Bitwise OR
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> operator|(const Vec512<T> a,
+                                               const Vec512<T> b) {
+  return Vec512<T>{_mm512_or_si512(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> operator|(const Vec512<float> a,
+                                                   const Vec512<float> b) {
+  return Vec512<float>{_mm512_or_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> operator|(const Vec512<double> a,
+                                                    const Vec512<double> b) {
+  return Vec512<double>{_mm512_or_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Bitwise XOR
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> operator^(const Vec512<T> a,
+                                               const Vec512<T> b) {
+  return Vec512<T>{_mm512_xor_si512(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> operator^(const Vec512<float> a,
+                                                   const Vec512<float> b) {
+  return Vec512<float>{_mm512_xor_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> operator^(const Vec512<double> a,
+                                                    const Vec512<double> b) {
+  return Vec512<double>{_mm512_xor_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Select/blend
+
+// Returns mask ? b : a.
+
+// Templates for signed/unsigned integer of a particular size.
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElseImpl(SizeTag<1> /* tag */,
+                                                    const Mask512<T> mask,
+                                                    const Vec512<T> yes,
+                                                    const Vec512<T> no) {
+  return Vec512<T>{_mm512_mask_mov_epi8(no.raw, mask.raw, yes.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElseImpl(SizeTag<2> /* tag */,
+                                                    const Mask512<T> mask,
+                                                    const Vec512<T> yes,
+                                                    const Vec512<T> no) {
+  return Vec512<T>{_mm512_mask_mov_epi16(no.raw, mask.raw, yes.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElseImpl(SizeTag<4> /* tag */,
+                                                    const Mask512<T> mask,
+                                                    const Vec512<T> yes,
+                                                    const Vec512<T> no) {
+  return Vec512<T>{_mm512_mask_mov_epi32(no.raw, mask.raw, yes.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElseImpl(SizeTag<8> /* tag */,
+                                                    const Mask512<T> mask,
+                                                    const Vec512<T> yes,
+                                                    const Vec512<T> no) {
+  return Vec512<T>{_mm512_mask_mov_epi64(no.raw, mask.raw, yes.raw)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElse(const Mask512<T> mask,
+                                                const Vec512<T> yes,
+                                                const Vec512<T> no) {
+  return IfThenElseImpl(SizeTag<sizeof(T)>(), mask, yes, no);
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> IfThenElse(const Mask512<float> mask,
+                                                    const Vec512<float> yes,
+                                                    const Vec512<float> no) {
+  return Vec512<float>{_mm512_mask_mov_ps(no.raw, mask.raw, yes.raw)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> IfThenElse(const Mask512<double> mask,
+                                                     const Vec512<double> yes,
+                                                     const Vec512<double> no) {
+  return Vec512<double>{_mm512_mask_mov_pd(no.raw, mask.raw, yes.raw)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElseZeroImpl(SizeTag<1> /* tag */,
+                                                        const Mask512<T> mask,
+                                                        const Vec512<T> yes) {
+  return Vec512<T>{_mm512_maskz_mov_epi8(mask.raw, yes.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElseZeroImpl(SizeTag<2> /* tag */,
+                                                        const Mask512<T> mask,
+                                                        const Vec512<T> yes) {
+  return Vec512<T>{_mm512_maskz_mov_epi16(mask.raw, yes.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElseZeroImpl(SizeTag<4> /* tag */,
+                                                        const Mask512<T> mask,
+                                                        const Vec512<T> yes) {
+  return Vec512<T>{_mm512_maskz_mov_epi32(mask.raw, yes.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElseZeroImpl(SizeTag<8> /* tag */,
+                                                        const Mask512<T> mask,
+                                                        const Vec512<T> yes) {
+  return Vec512<T>{_mm512_maskz_mov_epi64(mask.raw, yes.raw)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenElseZero(const Mask512<T> mask,
+                                                    const Vec512<T> yes) {
+  return IfThenElseZeroImpl(SizeTag<sizeof(T)>(), mask, yes);
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> IfThenElseZero(
+    const Mask512<float> mask, const Vec512<float> yes) {
+  return Vec512<float>{_mm512_maskz_mov_ps(mask.raw, yes.raw)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> IfThenElseZero(
+    const Mask512<double> mask, const Vec512<double> yes) {
+  return Vec512<double>{_mm512_maskz_mov_pd(mask.raw, yes.raw)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenZeroElseImpl(SizeTag<1> /* tag */,
+                                                        const Mask512<T> mask,
+                                                        const Vec512<T> no) {
+  // xor_epi8/16 are missing, but we have sub, which is just as fast for u8/16.
+  return Vec512<T>{_mm512_mask_sub_epi8(no.raw, mask.raw, no.raw, no.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenZeroElseImpl(SizeTag<2> /* tag */,
+                                                        const Mask512<T> mask,
+                                                        const Vec512<T> no) {
+  return Vec512<T>{_mm512_mask_sub_epi16(no.raw, mask.raw, no.raw, no.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenZeroElseImpl(SizeTag<4> /* tag */,
+                                                        const Mask512<T> mask,
+                                                        const Vec512<T> no) {
+  return Vec512<T>{_mm512_mask_xor_epi32(no.raw, mask.raw, no.raw, no.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenZeroElseImpl(SizeTag<8> /* tag */,
+                                                        const Mask512<T> mask,
+                                                        const Vec512<T> no) {
+  return Vec512<T>{_mm512_mask_xor_epi64(no.raw, mask.raw, no.raw, no.raw)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> IfThenZeroElse(const Mask512<T> mask,
+                                                    const Vec512<T> no) {
+  return IfThenZeroElseImpl(SizeTag<sizeof(T)>(), mask, no);
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> IfThenZeroElse(
+    const Mask512<float> mask, const Vec512<float> no) {
+  return Vec512<float>{_mm512_mask_xor_ps(no.raw, mask.raw, no.raw, no.raw)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> IfThenZeroElse(
+    const Mask512<double> mask, const Vec512<double> no) {
+  return Vec512<double>{_mm512_mask_xor_pd(no.raw, mask.raw, no.raw, no.raw)};
+}
+
+template <typename T, HWY_IF_FLOAT(T)>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> ZeroIfNegative(const Vec512<T> v) {
+  const auto zero = Zero(Full512<T>());
+  return IfThenElse(MaskFromVec(v), zero, v);
+}
+
+// ================================================== ARITHMETIC
+
+// ------------------------------ Addition
+
+// Unsigned
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> operator+(const Vec512<uint8_t> a,
+                                                     const Vec512<uint8_t> b) {
+  return Vec512<uint8_t>{_mm512_add_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> operator+(
+    const Vec512<uint16_t> a, const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_add_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> operator+(
+    const Vec512<uint32_t> a, const Vec512<uint32_t> b) {
+  return Vec512<uint32_t>{_mm512_add_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> operator+(
+    const Vec512<uint64_t> a, const Vec512<uint64_t> b) {
+  return Vec512<uint64_t>{_mm512_add_epi64(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> operator+(const Vec512<int8_t> a,
+                                                    const Vec512<int8_t> b) {
+  return Vec512<int8_t>{_mm512_add_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> operator+(const Vec512<int16_t> a,
+                                                     const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_add_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> operator+(const Vec512<int32_t> a,
+                                                     const Vec512<int32_t> b) {
+  return Vec512<int32_t>{_mm512_add_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> operator+(const Vec512<int64_t> a,
+                                                     const Vec512<int64_t> b) {
+  return Vec512<int64_t>{_mm512_add_epi64(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> operator+(const Vec512<float> a,
+                                                   const Vec512<float> b) {
+  return Vec512<float>{_mm512_add_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> operator+(const Vec512<double> a,
+                                                    const Vec512<double> b) {
+  return Vec512<double>{_mm512_add_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Subtraction
+
+// Unsigned
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> operator-(const Vec512<uint8_t> a,
+                                                     const Vec512<uint8_t> b) {
+  return Vec512<uint8_t>{_mm512_sub_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> operator-(
+    const Vec512<uint16_t> a, const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_sub_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> operator-(
+    const Vec512<uint32_t> a, const Vec512<uint32_t> b) {
+  return Vec512<uint32_t>{_mm512_sub_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> operator-(
+    const Vec512<uint64_t> a, const Vec512<uint64_t> b) {
+  return Vec512<uint64_t>{_mm512_sub_epi64(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> operator-(const Vec512<int8_t> a,
+                                                    const Vec512<int8_t> b) {
+  return Vec512<int8_t>{_mm512_sub_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> operator-(const Vec512<int16_t> a,
+                                                     const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_sub_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> operator-(const Vec512<int32_t> a,
+                                                     const Vec512<int32_t> b) {
+  return Vec512<int32_t>{_mm512_sub_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> operator-(const Vec512<int64_t> a,
+                                                     const Vec512<int64_t> b) {
+  return Vec512<int64_t>{_mm512_sub_epi64(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> operator-(const Vec512<float> a,
+                                                   const Vec512<float> b) {
+  return Vec512<float>{_mm512_sub_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> operator-(const Vec512<double> a,
+                                                    const Vec512<double> b) {
+  return Vec512<double>{_mm512_sub_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Saturating addition
+
+// Returns a + b clamped to the destination range.
+
+// Unsigned
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> SaturatedAdd(
+    const Vec512<uint8_t> a, const Vec512<uint8_t> b) {
+  return Vec512<uint8_t>{_mm512_adds_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> SaturatedAdd(
+    const Vec512<uint16_t> a, const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_adds_epu16(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> SaturatedAdd(const Vec512<int8_t> a,
+                                                       const Vec512<int8_t> b) {
+  return Vec512<int8_t>{_mm512_adds_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> SaturatedAdd(
+    const Vec512<int16_t> a, const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_adds_epi16(a.raw, b.raw)};
+}
+
+// ------------------------------ Saturating subtraction
+
+// Returns a - b clamped to the destination range.
+
+// Unsigned
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> SaturatedSub(
+    const Vec512<uint8_t> a, const Vec512<uint8_t> b) {
+  return Vec512<uint8_t>{_mm512_subs_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> SaturatedSub(
+    const Vec512<uint16_t> a, const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_subs_epu16(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> SaturatedSub(const Vec512<int8_t> a,
+                                                       const Vec512<int8_t> b) {
+  return Vec512<int8_t>{_mm512_subs_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> SaturatedSub(
+    const Vec512<int16_t> a, const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_subs_epi16(a.raw, b.raw)};
+}
+
+// ------------------------------ Average
+
+// Returns (a + b + 1) / 2
+
+// Unsigned
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> AverageRound(
+    const Vec512<uint8_t> a, const Vec512<uint8_t> b) {
+  return Vec512<uint8_t>{_mm512_avg_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> AverageRound(
+    const Vec512<uint16_t> a, const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_avg_epu16(a.raw, b.raw)};
+}
+
+// ------------------------------ Absolute value
+
+// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1.
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> Abs(const Vec512<int8_t> v) {
+  return Vec512<int8_t>{_mm512_abs_epi8(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> Abs(const Vec512<int16_t> v) {
+  return Vec512<int16_t>{_mm512_abs_epi16(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Abs(const Vec512<int32_t> v) {
+  return Vec512<int32_t>{_mm512_abs_epi32(v.raw)};
+}
+
+// ------------------------------ Shift lanes by constant #bits
+
+// Unsigned
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> ShiftLeft(
+    const Vec512<uint16_t> v) {
+  return Vec512<uint16_t>{_mm512_slli_epi16(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> ShiftRight(
+    const Vec512<uint16_t> v) {
+  return Vec512<uint16_t>{_mm512_srli_epi16(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> ShiftLeft(
+    const Vec512<uint32_t> v) {
+  return Vec512<uint32_t>{_mm512_slli_epi32(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> ShiftRight(
+    const Vec512<uint32_t> v) {
+  return Vec512<uint32_t>{_mm512_srli_epi32(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> ShiftLeft(
+    const Vec512<uint64_t> v) {
+  return Vec512<uint64_t>{_mm512_slli_epi64(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> ShiftRight(
+    const Vec512<uint64_t> v) {
+  return Vec512<uint64_t>{_mm512_srli_epi64(v.raw, kBits)};
+}
+
+// Signed (no i64 ShiftRight)
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> ShiftLeft(const Vec512<int16_t> v) {
+  return Vec512<int16_t>{_mm512_slli_epi16(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> ShiftRight(const Vec512<int16_t> v) {
+  return Vec512<int16_t>{_mm512_srai_epi16(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> ShiftLeft(const Vec512<int32_t> v) {
+  return Vec512<int32_t>{_mm512_slli_epi32(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> ShiftRight(const Vec512<int32_t> v) {
+  return Vec512<int32_t>{_mm512_srai_epi32(v.raw, kBits)};
+}
+template <int kBits>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> ShiftLeft(const Vec512<int64_t> v) {
+  return Vec512<int64_t>{_mm512_slli_epi64(v.raw, kBits)};
+}
+
+// ------------------------------ Shift lanes by independent variable #bits
+
+// Unsigned (no u8,u16)
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> operator<<(
+    const Vec512<uint32_t> v, const Vec512<uint32_t> bits) {
+  return Vec512<uint32_t>{_mm512_sllv_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> operator>>(
+    const Vec512<uint32_t> v, const Vec512<uint32_t> bits) {
+  return Vec512<uint32_t>{_mm512_srlv_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> operator<<(
+    const Vec512<uint64_t> v, const Vec512<uint64_t> bits) {
+  return Vec512<uint64_t>{_mm512_sllv_epi64(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> operator>>(
+    const Vec512<uint64_t> v, const Vec512<uint64_t> bits) {
+  return Vec512<uint64_t>{_mm512_srlv_epi64(v.raw, bits.raw)};
+}
+
+// Signed (no i8,i16,i64)
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> operator<<(
+    const Vec512<int32_t> v, const Vec512<int32_t> bits) {
+  return Vec512<int32_t>{_mm512_sllv_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> operator>>(
+    const Vec512<int32_t> v, const Vec512<int32_t> bits) {
+  return Vec512<int32_t>{_mm512_srav_epi32(v.raw, bits.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> operator<<(
+    const Vec512<int64_t> v, const Vec512<int64_t> bits) {
+  return Vec512<int64_t>{_mm512_sllv_epi64(v.raw, bits.raw)};
+}
+
+// ------------------------------ Minimum
+
+// Unsigned (no u64)
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> Min(const Vec512<uint8_t> a,
+                                               const Vec512<uint8_t> b) {
+  return Vec512<uint8_t>{_mm512_min_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> Min(const Vec512<uint16_t> a,
+                                                const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_min_epu16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> Min(const Vec512<uint32_t> a,
+                                                const Vec512<uint32_t> b) {
+  return Vec512<uint32_t>{_mm512_min_epu32(a.raw, b.raw)};
+}
+
+// Signed (no i64)
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> Min(const Vec512<int8_t> a,
+                                              const Vec512<int8_t> b) {
+  return Vec512<int8_t>{_mm512_min_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> Min(const Vec512<int16_t> a,
+                                               const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_min_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Min(const Vec512<int32_t> a,
+                                               const Vec512<int32_t> b) {
+  return Vec512<int32_t>{_mm512_min_epi32(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Min(const Vec512<float> a,
+                                             const Vec512<float> b) {
+  return Vec512<float>{_mm512_min_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Min(const Vec512<double> a,
+                                              const Vec512<double> b) {
+  return Vec512<double>{_mm512_min_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Maximum
+
+// Unsigned (no u64)
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> Max(const Vec512<uint8_t> a,
+                                               const Vec512<uint8_t> b) {
+  return Vec512<uint8_t>{_mm512_max_epu8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> Max(const Vec512<uint16_t> a,
+                                                const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_max_epu16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> Max(const Vec512<uint32_t> a,
+                                                const Vec512<uint32_t> b) {
+  return Vec512<uint32_t>{_mm512_max_epu32(a.raw, b.raw)};
+}
+
+// Signed (no i64)
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> Max(const Vec512<int8_t> a,
+                                              const Vec512<int8_t> b) {
+  return Vec512<int8_t>{_mm512_max_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> Max(const Vec512<int16_t> a,
+                                               const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_max_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Max(const Vec512<int32_t> a,
+                                               const Vec512<int32_t> b) {
+  return Vec512<int32_t>{_mm512_max_epi32(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Max(const Vec512<float> a,
+                                             const Vec512<float> b) {
+  return Vec512<float>{_mm512_max_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Max(const Vec512<double> a,
+                                              const Vec512<double> b) {
+  return Vec512<double>{_mm512_max_pd(a.raw, b.raw)};
+}
+
+// Returns the closest value to v within [lo, hi].
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> Clamp(const Vec512<T> v,
+                                           const Vec512<T> lo,
+                                           const Vec512<T> hi) {
+  return Min(Max(lo, v), hi);
+}
+
+// ------------------------------ Integer multiplication
+
+// Unsigned
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> operator*(
+    const Vec512<uint16_t> a, const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_mullo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> operator*(
+    const Vec512<uint32_t> a, const Vec512<uint32_t> b) {
+  return Vec512<uint32_t>{_mm512_mullo_epi32(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> operator*(const Vec512<int16_t> a,
+                                                     const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_mullo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> operator*(const Vec512<int32_t> a,
+                                                     const Vec512<int32_t> b) {
+  return Vec512<int32_t>{_mm512_mullo_epi32(a.raw, b.raw)};
+}
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// Returns the upper 16 bits of a * b in each lane.
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> MulHigh(const Vec512<uint16_t> a,
+                                                    const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_mulhi_epu16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> MulHigh(const Vec512<int16_t> a,
+                                                   const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_mulhi_epi16(a.raw, b.raw)};
+}
+
+}  // namespace ext
+
+// Multiplies even lanes (0, 2 ..) and places the double-wide result into
+// even and the upper half into its odd neighbor lane.
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> MulEven(const Vec512<int32_t> a,
+                                                   const Vec512<int32_t> b) {
+  return Vec512<int64_t>{_mm512_mul_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> MulEven(const Vec512<uint32_t> a,
+                                                    const Vec512<uint32_t> b) {
+  return Vec512<uint64_t>{_mm512_mul_epu32(a.raw, b.raw)};
+}
+
+// ------------------------------ Floating-point negate
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Neg(const Vec512<float> v) {
+  const Full512<float> df;
+  const Full512<uint32_t> du;
+  const auto sign = BitCast(df, Set(du, 0x80000000u));
+  return v ^ sign;
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Neg(const Vec512<double> v) {
+  const Full512<double> df;
+  const Full512<uint64_t> du;
+  const auto sign = BitCast(df, Set(du, 0x8000000000000000ull));
+  return v ^ sign;
+}
+
+// ------------------------------ Floating-point mul / div
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> operator*(const Vec512<float> a,
+                                                   const Vec512<float> b) {
+  return Vec512<float>{_mm512_mul_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> operator*(const Vec512<double> a,
+                                                    const Vec512<double> b) {
+  return Vec512<double>{_mm512_mul_pd(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> operator/(const Vec512<float> a,
+                                                   const Vec512<float> b) {
+  return Vec512<float>{_mm512_div_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> operator/(const Vec512<double> a,
+                                                    const Vec512<double> b) {
+  return Vec512<double>{_mm512_div_pd(a.raw, b.raw)};
+}
+
+// Approximate reciprocal
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> ApproximateReciprocal(
+    const Vec512<float> v) {
+  return Vec512<float>{_mm512_rcp14_ps(v.raw)};
+}
+
+namespace ext {
+// Absolute value of difference.
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> AbsDiff(const Vec512<float> a,
+                                                 const Vec512<float> b) {
+  return Vec512<float>{_mm512_abs_ps((a - b).raw)};
+}
+}  // namespace ext
+
+// ------------------------------ Floating-point multiply-add variants
+
+// Returns mul * x + add
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> MulAdd(const Vec512<float> mul,
+                                                const Vec512<float> x,
+                                                const Vec512<float> add) {
+  return Vec512<float>{_mm512_fmadd_ps(mul.raw, x.raw, add.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> MulAdd(const Vec512<double> mul,
+                                                 const Vec512<double> x,
+                                                 const Vec512<double> add) {
+  return Vec512<double>{_mm512_fmadd_pd(mul.raw, x.raw, add.raw)};
+}
+
+// Returns add - mul * x
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> NegMulAdd(const Vec512<float> mul,
+                                                   const Vec512<float> x,
+                                                   const Vec512<float> add) {
+  return Vec512<float>{_mm512_fnmadd_ps(mul.raw, x.raw, add.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> NegMulAdd(const Vec512<double> mul,
+                                                    const Vec512<double> x,
+                                                    const Vec512<double> add) {
+  return Vec512<double>{_mm512_fnmadd_pd(mul.raw, x.raw, add.raw)};
+}
+
+// Slightly more expensive on ARM (extra negate)
+namespace ext {
+
+// Returns mul * x - sub
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> MulSub(const Vec512<float> mul,
+                                                const Vec512<float> x,
+                                                const Vec512<float> sub) {
+  return Vec512<float>{_mm512_fmsub_ps(mul.raw, x.raw, sub.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> MulSub(const Vec512<double> mul,
+                                                 const Vec512<double> x,
+                                                 const Vec512<double> sub) {
+  return Vec512<double>{_mm512_fmsub_pd(mul.raw, x.raw, sub.raw)};
+}
+
+// Returns -mul * x - sub
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> NegMulSub(const Vec512<float> mul,
+                                                   const Vec512<float> x,
+                                                   const Vec512<float> sub) {
+  return Vec512<float>{_mm512_fnmsub_ps(mul.raw, x.raw, sub.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> NegMulSub(const Vec512<double> mul,
+                                                    const Vec512<double> x,
+                                                    const Vec512<double> sub) {
+  return Vec512<double>{_mm512_fnmsub_pd(mul.raw, x.raw, sub.raw)};
+}
+
+}  // namespace ext
+
+// ------------------------------ Floating-point square root
+
+// Full precision square root
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Sqrt(const Vec512<float> v) {
+  return Vec512<float>{_mm512_sqrt_ps(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Sqrt(const Vec512<double> v) {
+  return Vec512<double>{_mm512_sqrt_pd(v.raw)};
+}
+
+// Approximate reciprocal square root
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> ApproximateReciprocalSqrt(
+    const Vec512<float> v) {
+  return Vec512<float>{_mm512_rsqrt14_ps(v.raw)};
+}
+
+// ------------------------------ Floating-point rounding
+
+// Toward nearest integer, tie to even
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Round(const Vec512<float> v) {
+  return Vec512<float>{_mm512_roundscale_ps(
+      v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Round(const Vec512<double> v) {
+  return Vec512<double>{_mm512_roundscale_pd(
+      v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)};
+}
+
+// Toward zero, aka truncate
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Trunc(const Vec512<float> v) {
+  return Vec512<float>{
+      _mm512_roundscale_ps(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Trunc(const Vec512<double> v) {
+  return Vec512<double>{
+      _mm512_roundscale_pd(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)};
+}
+
+// Toward +infinity, aka ceiling
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Ceil(const Vec512<float> v) {
+  return Vec512<float>{
+      _mm512_roundscale_ps(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Ceil(const Vec512<double> v) {
+  return Vec512<double>{
+      _mm512_roundscale_pd(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)};
+}
+
+// Toward -infinity, aka floor
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Floor(const Vec512<float> v) {
+  return Vec512<float>{
+      _mm512_roundscale_ps(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Floor(const Vec512<double> v) {
+  return Vec512<double>{
+      _mm512_roundscale_pd(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)};
+}
+
+// ================================================== COMPARE
+
+// Comparisons fill a lane with 1-bits if the condition is true, else 0.
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> TestBitImpl(SizeTag<1> /*tag*/,
+                                                  const Vec512<T> v,
+                                                  const Vec512<T> bit) {
+  return Mask512<T>{_mm512_test_epi8_mask(v.raw, bit.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> TestBitImpl(SizeTag<2> /*tag*/,
+                                                  const Vec512<T> v,
+                                                  const Vec512<T> bit) {
+  return Mask512<T>{_mm512_test_epi16_mask(v.raw, bit.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> TestBitImpl(SizeTag<4> /*tag*/,
+                                                  const Vec512<T> v,
+                                                  const Vec512<T> bit) {
+  return Mask512<T>{_mm512_test_epi32_mask(v.raw, bit.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> TestBitImpl(SizeTag<8> /*tag*/,
+                                                  const Vec512<T> v,
+                                                  const Vec512<T> bit) {
+  return Mask512<T>{_mm512_test_epi64_mask(v.raw, bit.raw)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> TestBit(const Vec512<T> v,
+                                              const Vec512<T> bit) {
+  static_assert(!IsFloat<T>(), "Only integer vectors supported");
+  return TestBitImpl(SizeTag<sizeof(T)>(), v, bit);
+}
+
+// ------------------------------ Equality
+
+// Unsigned
+HWY_ATTR_AVX512 HWY_INLINE Mask512<uint8_t> operator==(
+    const Vec512<uint8_t> a, const Vec512<uint8_t> b) {
+  return Mask512<uint8_t>{_mm512_cmpeq_epi8_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<uint16_t> operator==(
+    const Vec512<uint16_t> a, const Vec512<uint16_t> b) {
+  return Mask512<uint16_t>{_mm512_cmpeq_epi16_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<uint32_t> operator==(
+    const Vec512<uint32_t> a, const Vec512<uint32_t> b) {
+  return Mask512<uint32_t>{_mm512_cmpeq_epi32_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<uint64_t> operator==(
+    const Vec512<uint64_t> a, const Vec512<uint64_t> b) {
+  return Mask512<uint64_t>{_mm512_cmpeq_epi64_mask(a.raw, b.raw)};
+}
+
+// Signed
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int8_t> operator==(const Vec512<int8_t> a,
+                                                      const Vec512<int8_t> b) {
+  return Mask512<int8_t>{_mm512_cmpeq_epi8_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int16_t> operator==(
+    const Vec512<int16_t> a, const Vec512<int16_t> b) {
+  return Mask512<int16_t>{_mm512_cmpeq_epi16_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int32_t> operator==(
+    const Vec512<int32_t> a, const Vec512<int32_t> b) {
+  return Mask512<int32_t>{_mm512_cmpeq_epi32_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int64_t> operator==(
+    const Vec512<int64_t> a, const Vec512<int64_t> b) {
+  return Mask512<int64_t>{_mm512_cmpeq_epi64_mask(a.raw, b.raw)};
+}
+
+// Float
+HWY_ATTR_AVX512 HWY_INLINE Mask512<float> operator==(const Vec512<float> a,
+                                                     const Vec512<float> b) {
+  return Mask512<float>{_mm512_cmp_ps_mask(a.raw, b.raw, _CMP_EQ_OQ)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<double> operator==(const Vec512<double> a,
+                                                      const Vec512<double> b) {
+  return Mask512<double>{_mm512_cmp_pd_mask(a.raw, b.raw, _CMP_EQ_OQ)};
+}
+
+// ------------------------------ Strict inequality
+
+// Signed/float <
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int8_t> operator<(const Vec512<int8_t> a,
+                                                     const Vec512<int8_t> b) {
+  return Mask512<int8_t>{_mm512_cmpgt_epi8_mask(b.raw, a.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int16_t> operator<(const Vec512<int16_t> a,
+                                                      const Vec512<int16_t> b) {
+  return Mask512<int16_t>{_mm512_cmpgt_epi16_mask(b.raw, a.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int32_t> operator<(const Vec512<int32_t> a,
+                                                      const Vec512<int32_t> b) {
+  return Mask512<int32_t>{_mm512_cmpgt_epi32_mask(b.raw, a.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int64_t> operator<(const Vec512<int64_t> a,
+                                                      const Vec512<int64_t> b) {
+  return Mask512<int64_t>{_mm512_cmpgt_epi64_mask(b.raw, a.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<float> operator<(const Vec512<float> a,
+                                                    const Vec512<float> b) {
+  return Mask512<float>{_mm512_cmp_ps_mask(a.raw, b.raw, _CMP_LT_OQ)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<double> operator<(const Vec512<double> a,
+                                                     const Vec512<double> b) {
+  return Mask512<double>{_mm512_cmp_pd_mask(a.raw, b.raw, _CMP_LT_OQ)};
+}
+
+// Signed/float >
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int8_t> operator>(const Vec512<int8_t> a,
+                                                     const Vec512<int8_t> b) {
+  return Mask512<int8_t>{_mm512_cmpgt_epi8_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int16_t> operator>(const Vec512<int16_t> a,
+                                                      const Vec512<int16_t> b) {
+  return Mask512<int16_t>{_mm512_cmpgt_epi16_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int32_t> operator>(const Vec512<int32_t> a,
+                                                      const Vec512<int32_t> b) {
+  return Mask512<int32_t>{_mm512_cmpgt_epi32_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<int64_t> operator>(const Vec512<int64_t> a,
+                                                      const Vec512<int64_t> b) {
+  return Mask512<int64_t>{_mm512_cmpgt_epi64_mask(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<float> operator>(const Vec512<float> a,
+                                                    const Vec512<float> b) {
+  return Mask512<float>{_mm512_cmp_ps_mask(a.raw, b.raw, _CMP_GT_OQ)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<double> operator>(const Vec512<double> a,
+                                                     const Vec512<double> b) {
+  return Mask512<double>{_mm512_cmp_pd_mask(a.raw, b.raw, _CMP_GT_OQ)};
+}
+
+// ------------------------------ Weak inequality
+
+// Float <= >=
+HWY_ATTR_AVX512 HWY_INLINE Mask512<float> operator<=(const Vec512<float> a,
+                                                     const Vec512<float> b) {
+  return Mask512<float>{_mm512_cmp_ps_mask(a.raw, b.raw, _CMP_LE_OQ)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<double> operator<=(const Vec512<double> a,
+                                                      const Vec512<double> b) {
+  return Mask512<double>{_mm512_cmp_pd_mask(a.raw, b.raw, _CMP_LE_OQ)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<float> operator>=(const Vec512<float> a,
+                                                     const Vec512<float> b) {
+  return Mask512<float>{_mm512_cmp_ps_mask(a.raw, b.raw, _CMP_GE_OQ)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<double> operator>=(const Vec512<double> a,
+                                                      const Vec512<double> b) {
+  return Mask512<double>{_mm512_cmp_pd_mask(a.raw, b.raw, _CMP_GE_OQ)};
+}
+
+// ------------------------------ Mask
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> MaskFromVecImpl(SizeTag<1> /*tag*/,
+                                                      const Vec512<T> v) {
+  return Mask512<T>{_mm512_movepi8_mask(v.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> MaskFromVecImpl(SizeTag<2> /*tag*/,
+                                                      const Vec512<T> v) {
+  return Mask512<T>{_mm512_movepi16_mask(v.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> MaskFromVecImpl(SizeTag<4> /*tag*/,
+                                                      const Vec512<T> v) {
+  return Mask512<T>{_mm512_movepi32_mask(v.raw)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> MaskFromVecImpl(SizeTag<8> /*tag*/,
+                                                      const Vec512<T> v) {
+  return Mask512<T>{_mm512_movepi64_mask(v.raw)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Mask512<T> MaskFromVec(const Vec512<T> v) {
+  return MaskFromVecImpl(SizeTag<sizeof(T)>(), v);
+}
+// There do not seem to be native floating-point versions of these instructions.
+HWY_ATTR_AVX512 HWY_INLINE Mask512<float> MaskFromVec(const Vec512<float> v) {
+  return Mask512<float>{MaskFromVec(BitCast(Full512<int32_t>(), v)).raw};
+}
+HWY_ATTR_AVX512 HWY_INLINE Mask512<double> MaskFromVec(const Vec512<double> v) {
+  return Mask512<double>{MaskFromVec(BitCast(Full512<int64_t>(), v)).raw};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> VecFromMask(
+    const Mask512<uint8_t> v) {
+  return Vec512<uint8_t>{_mm512_movm_epi8(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> VecFromMask(const Mask512<int8_t> v) {
+  return Vec512<int8_t>{_mm512_movm_epi8(v.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> VecFromMask(
+    const Mask512<uint16_t> v) {
+  return Vec512<uint16_t>{_mm512_movm_epi16(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> VecFromMask(
+    const Mask512<int16_t> v) {
+  return Vec512<int16_t>{_mm512_movm_epi16(v.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> VecFromMask(
+    const Mask512<uint32_t> v) {
+  return Vec512<uint32_t>{_mm512_movm_epi32(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> VecFromMask(
+    const Mask512<int32_t> v) {
+  return Vec512<int32_t>{_mm512_movm_epi32(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> VecFromMask(const Mask512<float> v) {
+  return Vec512<float>{_mm512_castsi512_ps(_mm512_movm_epi32(v.raw))};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> VecFromMask(
+    const Mask512<uint64_t> v) {
+  return Vec512<uint64_t>{_mm512_movm_epi64(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> VecFromMask(
+    const Mask512<int64_t> v) {
+  return Vec512<int64_t>{_mm512_movm_epi64(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> VecFromMask(const Mask512<double> v) {
+  return Vec512<double>{_mm512_castsi512_pd(_mm512_movm_epi64(v.raw))};
+}
+
+// ================================================== MEMORY
+
+// ------------------------------ Load
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> Load(Full512<T> /* tag */,
+                                          const T* HWY_RESTRICT aligned) {
+  return Vec512<T>{
+      _mm512_load_si512(reinterpret_cast<const __m512i*>(aligned))};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Load(
+    Full512<float> /* tag */, const float* HWY_RESTRICT aligned) {
+  return Vec512<float>{_mm512_load_ps(aligned)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Load(
+    Full512<double> /* tag */, const double* HWY_RESTRICT aligned) {
+  return Vec512<double>{_mm512_load_pd(aligned)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> LoadU(Full512<T> /* tag */,
+                                           const T* HWY_RESTRICT p) {
+  return Vec512<T>{_mm512_loadu_si512(reinterpret_cast<const __m512i*>(p))};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> LoadU(Full512<float> /* tag */,
+                                               const float* HWY_RESTRICT p) {
+  return Vec512<float>{_mm512_loadu_ps(p)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> LoadU(Full512<double> /* tag */,
+                                                const double* HWY_RESTRICT p) {
+  return Vec512<double>{_mm512_loadu_pd(p)};
+}
+
+// Loads 128 bit and duplicates into both 128-bit halves. This avoids the
+// 3-cycle cost of moving data between 128-bit halves and avoids port 5.
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> LoadDup128(Full512<T> /* tag */,
+                                                const T* const HWY_RESTRICT p) {
+  // Clang 3.9 generates VINSERTF128 which is slower, but inline assembly leads
+  // to "invalid output size for constraint" without -mavx512:
+  // https://gcc.godbolt.org/z/-Jt_-F
+#if HWY_LOADDUP_ASM
+  __m512i out;
+  asm("vbroadcasti128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0]));
+  return Vec512<T>{out};
+#else
+  const auto x4 = LoadU(Full128<T>(), p);
+  return Vec512<T>{_mm512_broadcast_i32x4(x4.raw)};
+#endif
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> LoadDup128(
+    Full512<float> /* tag */, const float* const HWY_RESTRICT p) {
+#if HWY_LOADDUP_ASM
+  __m512 out;
+  asm("vbroadcastf128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0]));
+  return Vec512<float>{out};
+#else
+  const __m128 x4 = _mm_loadu_ps(p);
+  return Vec512<float>{_mm512_broadcast_f32x4(x4)};
+#endif
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> LoadDup128(
+    Full512<double> /* tag */, const double* const HWY_RESTRICT p) {
+#if HWY_LOADDUP_ASM
+  __m512d out;
+  asm("vbroadcastf128 %1, %[reg]" : [ reg ] "=x"(out) : "m"(p[0]));
+  return Vec512<double>{out};
+#else
+  const __m128d x2 = _mm_loadu_pd(p);
+  return Vec512<double>{_mm512_broadcast_f64x2(x2)};
+#endif
+}
+
+// ------------------------------ Store
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE void Store(const Vec512<T> v, Full512<T> /* tag */,
+                                      T* HWY_RESTRICT aligned) {
+  _mm512_store_si512(reinterpret_cast<__m512i*>(aligned), v.raw);
+}
+HWY_ATTR_AVX512 HWY_INLINE void Store(const Vec512<float> v,
+                                      Full512<float> /* tag */,
+                                      float* HWY_RESTRICT aligned) {
+  _mm512_store_ps(aligned, v.raw);
+}
+HWY_ATTR_AVX512 HWY_INLINE void Store(const Vec512<double> v,
+                                      Full512<double> /* tag */,
+                                      double* HWY_RESTRICT aligned) {
+  _mm512_store_pd(aligned, v.raw);
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE void StoreU(const Vec512<T> v, Full512<T> /* tag */,
+                                       T* HWY_RESTRICT p) {
+  _mm512_storeu_si512(reinterpret_cast<__m512i*>(p), v.raw);
+}
+HWY_ATTR_AVX512 HWY_INLINE void StoreU(const Vec512<float> v,
+                                       Full512<float> /* tag */,
+                                       float* HWY_RESTRICT p) {
+  _mm512_storeu_ps(p, v.raw);
+}
+HWY_ATTR_AVX512 HWY_INLINE void StoreU(const Vec512<double> v, Full512<double>,
+                                       double* HWY_RESTRICT p) {
+  _mm512_storeu_pd(p, v.raw);
+}
+
+// ------------------------------ Non-temporal stores
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE void Stream(const Vec512<T> v, Full512<T> /* tag */,
+                                       T* HWY_RESTRICT aligned) {
+  _mm512_stream_si512(reinterpret_cast<__m512i*>(aligned), v.raw);
+}
+HWY_ATTR_AVX512 HWY_INLINE void Stream(const Vec512<float> v,
+                                       Full512<float> /* tag */,
+                                       float* HWY_RESTRICT aligned) {
+  _mm512_stream_ps(aligned, v.raw);
+}
+HWY_ATTR_AVX512 HWY_INLINE void Stream(const Vec512<double> v, Full512<double>,
+                                       double* HWY_RESTRICT aligned) {
+  _mm512_stream_pd(aligned, v.raw);
+}
+
+// ------------------------------ Gather
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> gather_offset_impl(
+    SizeTag<4> /* tag */, Full512<T> /* tag */, const T* HWY_RESTRICT base,
+    const Vec512<int32_t> offset) {
+  return Vec512<T>{_mm512_i32gather_epi32(offset.raw, base, 1)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> gather_index_impl(
+    SizeTag<4> /* tag */, Full512<T> /* tag */, const T* HWY_RESTRICT base,
+    const Vec512<int32_t> index) {
+  return Vec512<T>{_mm512_i32gather_epi32(index.raw, base, 4)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> gather_offset_impl(
+    SizeTag<8> /* tag */, Full512<T> /* tag */, const T* HWY_RESTRICT base,
+    const Vec512<int64_t> offset) {
+  return Vec512<T>{_mm512_i64gather_epi64(offset.raw, base, 1)};
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> gather_index_impl(
+    SizeTag<8> /* tag */, Full512<T> /* tag */, const T* HWY_RESTRICT base,
+    const Vec512<int64_t> index) {
+  return Vec512<T>{_mm512_i64gather_epi64(index.raw, base, 8)};
+}
+
+template <typename T, typename Offset>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> GatherOffset(Full512<T> d,
+                                                  const T* HWY_RESTRICT base,
+                                                  const Vec512<Offset> offset) {
+  static_assert(sizeof(T) == sizeof(Offset), "SVE requires same size base/ofs");
+  return gather_offset_impl(SizeTag<sizeof(T)>(), d, base, offset);
+}
+template <typename T, typename Index>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> GatherIndex(Full512<T> d,
+                                                 const T* HWY_RESTRICT base,
+                                                 const Vec512<Index> index) {
+  static_assert(sizeof(T) == sizeof(Index), "SVE requires same size base/idx");
+  return gather_index_impl(SizeTag<sizeof(T)>(), d, base, index);
+}
+
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> GatherOffset<float>(
+    Full512<float> /* tag */, const float* HWY_RESTRICT base,
+    const Vec512<int32_t> offset) {
+  return Vec512<float>{_mm512_i32gather_ps(offset.raw, base, 1)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> GatherIndex<float>(
+    Full512<float> /* tag */, const float* HWY_RESTRICT base,
+    const Vec512<int32_t> index) {
+  return Vec512<float>{_mm512_i32gather_ps(index.raw, base, 4)};
+}
+
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> GatherOffset<double>(
+    Full512<double> /* tag */, const double* HWY_RESTRICT base,
+    const Vec512<int64_t> offset) {
+  return Vec512<double>{_mm512_i64gather_pd(offset.raw, base, 1)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> GatherIndex<double>(
+    Full512<double> /* tag */, const double* HWY_RESTRICT base,
+    const Vec512<int64_t> index) {
+  return Vec512<double>{_mm512_i64gather_pd(index.raw, base, 8)};
+}
+
+}  // namespace ext
+
+// ================================================== SWIZZLE
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE T GetLane(const Vec512<T> v) {
+  return GetLane(LowerHalf(v));
+}
+
+// ------------------------------ Extract half
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec256<T> LowerHalf(Vec512<T> v) {
+  return Vec256<T>{_mm512_castsi512_si256(v.raw)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec256<float> LowerHalf(Vec512<float> v) {
+  return Vec256<float>{_mm512_castps512_ps256(v.raw)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec256<double> LowerHalf(Vec512<double> v) {
+  return Vec256<double>{_mm512_castpd512_pd256(v.raw)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec256<T> UpperHalf(Vec512<T> v) {
+  return Vec256<T>{_mm512_extracti32x8_epi32(v.raw, 1)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec256<float> UpperHalf(Vec512<float> v) {
+  return Vec256<float>{_mm512_extractf32x8_ps(v.raw, 1)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec256<double> UpperHalf(Vec512<double> v) {
+  return Vec256<double>{_mm512_extractf64x4_pd(v.raw, 1)};
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec256<T> GetHalf(Lower /* tag */, Vec512<T> v) {
+  return LowerHalf(v);
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec256<T> GetHalf(Upper /* tag */, Vec512<T> v) {
+  return UpperHalf(v);
+}
+
+// ------------------------------ Shift vector by constant #bytes
+
+// 0x01..0F, kBytes = 1 => 0x02..0F00
+template <int kBytes, typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> ShiftLeftBytes(const Vec512<T> v) {
+  static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
+  return Vec512<T>{_mm512_bslli_epi128(v.raw, kBytes)};
+}
+
+template <int kLanes, typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> ShiftLeftLanes(const Vec512<T> v) {
+  return ShiftLeftBytes<kLanes * sizeof(T)>(v);
+}
+
+// 0x01..0F, kBytes = 1 => 0x0001..0E
+template <int kBytes, typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> ShiftRightBytes(const Vec512<T> v) {
+  static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
+  return Vec512<T>{_mm512_bsrli_epi128(v.raw, kBytes)};
+}
+
+template <int kLanes, typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> ShiftRightLanes(const Vec512<T> v) {
+  return ShiftRightBytes<kLanes * sizeof(T)>(v);
+}
+
+// ------------------------------ Extract from 2x 128-bit at constant offset
+
+// Extracts 128 bits from <hi, lo> by skipping the least-significant kBytes.
+template <int kBytes, typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> CombineShiftRightBytes(
+    const Vec512<T> hi, const Vec512<T> lo) {
+  const Full512<uint8_t> d8;
+  const Vec512<uint8_t> extracted_bytes{
+      _mm512_alignr_epi8(BitCast(d8, hi).raw, BitCast(d8, lo).raw, kBytes)};
+  return BitCast(Full512<T>(), extracted_bytes);
+}
+
+// ------------------------------ Broadcast/splat any lane
+
+// Unsigned
+template <int kLane>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> Broadcast(
+    const Vec512<uint16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  if (kLane < 4) {
+    const __m512i lo = _mm512_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
+    return Vec512<uint16_t>{_mm512_unpacklo_epi64(lo, lo)};
+  } else {
+    const __m512i hi =
+        _mm512_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
+    return Vec512<uint16_t>{_mm512_unpackhi_epi64(hi, hi)};
+  }
+}
+template <int kLane>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> Broadcast(
+    const Vec512<uint32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  constexpr _MM_PERM_ENUM perm = static_cast<_MM_PERM_ENUM>(0x55 * kLane);
+  return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, perm)};
+}
+template <int kLane>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> Broadcast(
+    const Vec512<uint64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  constexpr _MM_PERM_ENUM perm = kLane ? _MM_PERM_DCDC : _MM_PERM_BABA;
+  return Vec512<uint64_t>{_mm512_shuffle_epi32(v.raw, perm)};
+}
+
+// Signed
+template <int kLane>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> Broadcast(const Vec512<int16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  if (kLane < 4) {
+    const __m512i lo = _mm512_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
+    return Vec512<int16_t>{_mm512_unpacklo_epi64(lo, lo)};
+  } else {
+    const __m512i hi =
+        _mm512_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
+    return Vec512<int16_t>{_mm512_unpackhi_epi64(hi, hi)};
+  }
+}
+template <int kLane>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Broadcast(const Vec512<int32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  constexpr _MM_PERM_ENUM perm = static_cast<_MM_PERM_ENUM>(0x55 * kLane);
+  return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, perm)};
+}
+template <int kLane>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> Broadcast(const Vec512<int64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  constexpr _MM_PERM_ENUM perm = kLane ? _MM_PERM_DCDC : _MM_PERM_BABA;
+  return Vec512<int64_t>{_mm512_shuffle_epi32(v.raw, perm)};
+}
+
+// Float
+template <int kLane>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Broadcast(const Vec512<float> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  constexpr _MM_PERM_ENUM perm = static_cast<_MM_PERM_ENUM>(0x55 * kLane);
+  return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, perm)};
+}
+template <int kLane>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Broadcast(const Vec512<double> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  constexpr _MM_PERM_ENUM perm = static_cast<_MM_PERM_ENUM>(0xFF * kLane);
+  return Vec512<double>{_mm512_shuffle_pd(v.raw, v.raw, perm)};
+}
+
+// ------------------------------ Hard-coded shuffles
+
+// Notation: let Vec512<int32_t> have lanes 7,6,5,4,3,2,1,0 (0 is
+// least-significant). Shuffle0321 rotates four-lane blocks one lane to the
+// right (the previous least-significant lane is now most-significant =>
+// 47650321). These could also be implemented via CombineShiftRightBytes but
+// the shuffle_abcd notation is more convenient.
+
+// Swap 32-bit halves in 64-bit halves.
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> Shuffle2301(
+    const Vec512<uint32_t> v) {
+  return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_CDAB)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Shuffle2301(
+    const Vec512<int32_t> v) {
+  return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_CDAB)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Shuffle2301(const Vec512<float> v) {
+  return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_CDAB)};
+}
+
+// Swap 64-bit halves
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> Shuffle1032(
+    const Vec512<uint32_t> v) {
+  return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_BADC)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Shuffle1032(
+    const Vec512<int32_t> v) {
+  return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_BADC)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Shuffle1032(const Vec512<float> v) {
+  // Shorter encoding than _mm512_permute_ps.
+  return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_BADC)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> Shuffle01(
+    const Vec512<uint64_t> v) {
+  return Vec512<uint64_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_BADC)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> Shuffle01(const Vec512<int64_t> v) {
+  return Vec512<int64_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_BADC)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> Shuffle01(const Vec512<double> v) {
+  // Shorter encoding than _mm512_permute_pd.
+  return Vec512<double>{_mm512_shuffle_pd(v.raw, v.raw, _MM_PERM_BBBB)};
+}
+
+// Rotate right 32 bits
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> Shuffle0321(
+    const Vec512<uint32_t> v) {
+  return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_ADCB)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Shuffle0321(
+    const Vec512<int32_t> v) {
+  return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_ADCB)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Shuffle0321(const Vec512<float> v) {
+  return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_ADCB)};
+}
+// Rotate left 32 bits
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> Shuffle2103(
+    const Vec512<uint32_t> v) {
+  return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_CBAD)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Shuffle2103(
+    const Vec512<int32_t> v) {
+  return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_CBAD)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Shuffle2103(const Vec512<float> v) {
+  return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_CBAD)};
+}
+
+// Reverse
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> Shuffle0123(
+    const Vec512<uint32_t> v) {
+  return Vec512<uint32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_ABCD)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> Shuffle0123(
+    const Vec512<int32_t> v) {
+  return Vec512<int32_t>{_mm512_shuffle_epi32(v.raw, _MM_PERM_ABCD)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> Shuffle0123(const Vec512<float> v) {
+  return Vec512<float>{_mm512_shuffle_ps(v.raw, v.raw, _MM_PERM_ABCD)};
+}
+
+// ------------------------------ Permute (runtime variable)
+
+// Returned by SetTableIndices for use by TableLookupLanes.
+template <typename T>
+struct Permute512 {
+  __m512i raw;
+};
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Permute512<T> SetTableIndices(const Full512<T> d,
+                                                         const int32_t* idx) {
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+  for (size_t i = 0; i < d.N; ++i) {
+    if (idx[i] >= static_cast<int32_t>(d.N)) {
+      printf("SetTableIndices [%zu] = %d >= %zu\n", i, idx[i], d.N);
+      HWY_TRAP();
+    }
+  }
+#else
+  (void)d;
+#endif
+  return Permute512<T>{LoadU(Full512<int32_t>(), idx).raw};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> TableLookupLanes(
+    const Vec512<uint32_t> v, const Permute512<uint32_t> idx) {
+  return Vec512<uint32_t>{_mm512_permutexvar_epi32(idx.raw, v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> TableLookupLanes(
+    const Vec512<int32_t> v, const Permute512<int32_t> idx) {
+  return Vec512<int32_t>{_mm512_permutexvar_epi32(idx.raw, v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> TableLookupLanes(
+    const Vec512<float> v, const Permute512<float> idx) {
+  return Vec512<float>{_mm512_permutexvar_ps(idx.raw, v.raw)};
+}
+
+// ------------------------------ Interleave lanes
+
+// Interleaves lanes from halves of the 128-bit blocks of "a" (which provides
+// the least-significant lane) and "b". To concatenate two half-width integers
+// into one, use ZipLo/hi instead (also works with scalar).
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> InterleaveLo(
+    const Vec512<uint8_t> a, const Vec512<uint8_t> b) {
+  return Vec512<uint8_t>{_mm512_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> InterleaveLo(
+    const Vec512<uint16_t> a, const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> InterleaveLo(
+    const Vec512<uint32_t> a, const Vec512<uint32_t> b) {
+  return Vec512<uint32_t>{_mm512_unpacklo_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> InterleaveLo(
+    const Vec512<uint64_t> a, const Vec512<uint64_t> b) {
+  return Vec512<uint64_t>{_mm512_unpacklo_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> InterleaveLo(const Vec512<int8_t> a,
+                                                       const Vec512<int8_t> b) {
+  return Vec512<int8_t>{_mm512_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> InterleaveLo(
+    const Vec512<int16_t> a, const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> InterleaveLo(
+    const Vec512<int32_t> a, const Vec512<int32_t> b) {
+  return Vec512<int32_t>{_mm512_unpacklo_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> InterleaveLo(
+    const Vec512<int64_t> a, const Vec512<int64_t> b) {
+  return Vec512<int64_t>{_mm512_unpacklo_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> InterleaveLo(const Vec512<float> a,
+                                                      const Vec512<float> b) {
+  return Vec512<float>{_mm512_unpacklo_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> InterleaveLo(const Vec512<double> a,
+                                                       const Vec512<double> b) {
+  return Vec512<double>{_mm512_unpacklo_pd(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint8_t> InterleaveHi(
+    const Vec512<uint8_t> a, const Vec512<uint8_t> b) {
+  return Vec512<uint8_t>{_mm512_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> InterleaveHi(
+    const Vec512<uint16_t> a, const Vec512<uint16_t> b) {
+  return Vec512<uint16_t>{_mm512_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> InterleaveHi(
+    const Vec512<uint32_t> a, const Vec512<uint32_t> b) {
+  return Vec512<uint32_t>{_mm512_unpackhi_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> InterleaveHi(
+    const Vec512<uint64_t> a, const Vec512<uint64_t> b) {
+  return Vec512<uint64_t>{_mm512_unpackhi_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int8_t> InterleaveHi(const Vec512<int8_t> a,
+                                                       const Vec512<int8_t> b) {
+  return Vec512<int8_t>{_mm512_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> InterleaveHi(
+    const Vec512<int16_t> a, const Vec512<int16_t> b) {
+  return Vec512<int16_t>{_mm512_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> InterleaveHi(
+    const Vec512<int32_t> a, const Vec512<int32_t> b) {
+  return Vec512<int32_t>{_mm512_unpackhi_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> InterleaveHi(
+    const Vec512<int64_t> a, const Vec512<int64_t> b) {
+  return Vec512<int64_t>{_mm512_unpackhi_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> InterleaveHi(const Vec512<float> a,
+                                                      const Vec512<float> b) {
+  return Vec512<float>{_mm512_unpackhi_ps(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> InterleaveHi(const Vec512<double> a,
+                                                       const Vec512<double> b) {
+  return Vec512<double>{_mm512_unpackhi_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Zip lanes
+
+// Same as interleave_*, except that the return lanes are double-width integers;
+// this is necessary because the single-lane scalar cannot return two values.
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> ZipLo(const Vec512<uint8_t> a,
+                                                  const Vec512<uint8_t> b) {
+  return Vec512<uint16_t>{_mm512_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> ZipLo(const Vec512<uint16_t> a,
+                                                  const Vec512<uint16_t> b) {
+  return Vec512<uint32_t>{_mm512_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> ZipLo(const Vec512<uint32_t> a,
+                                                  const Vec512<uint32_t> b) {
+  return Vec512<uint64_t>{_mm512_unpacklo_epi32(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> ZipLo(const Vec512<int8_t> a,
+                                                 const Vec512<int8_t> b) {
+  return Vec512<int16_t>{_mm512_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> ZipLo(const Vec512<int16_t> a,
+                                                 const Vec512<int16_t> b) {
+  return Vec512<int32_t>{_mm512_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> ZipLo(const Vec512<int32_t> a,
+                                                 const Vec512<int32_t> b) {
+  return Vec512<int64_t>{_mm512_unpacklo_epi32(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> ZipHi(const Vec512<uint8_t> a,
+                                                  const Vec512<uint8_t> b) {
+  return Vec512<uint16_t>{_mm512_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> ZipHi(const Vec512<uint16_t> a,
+                                                  const Vec512<uint16_t> b) {
+  return Vec512<uint32_t>{_mm512_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> ZipHi(const Vec512<uint32_t> a,
+                                                  const Vec512<uint32_t> b) {
+  return Vec512<uint64_t>{_mm512_unpackhi_epi32(a.raw, b.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> ZipHi(const Vec512<int8_t> a,
+                                                 const Vec512<int8_t> b) {
+  return Vec512<int16_t>{_mm512_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> ZipHi(const Vec512<int16_t> a,
+                                                 const Vec512<int16_t> b) {
+  return Vec512<int32_t>{_mm512_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> ZipHi(const Vec512<int32_t> a,
+                                                 const Vec512<int32_t> b) {
+  return Vec512<int64_t>{_mm512_unpackhi_epi32(a.raw, b.raw)};
+}
+
+// ------------------------------ Blocks
+
+// hiH,hiL loH,loL |-> hiL,loL (= lower halves)
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> ConcatLoLo(const Vec512<T> hi,
+                                                const Vec512<T> lo) {
+  return Vec512<T>{_mm512_shuffle_i32x4(lo.raw, hi.raw, _MM_PERM_BABA)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> ConcatLoLo(const Vec512<float> hi,
+                                                    const Vec512<float> lo) {
+  return Vec512<float>{_mm512_shuffle_f32x4(lo.raw, hi.raw, _MM_PERM_BABA)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> ConcatLoLo(const Vec512<double> hi,
+                                                     const Vec512<double> lo) {
+  return Vec512<double>{_mm512_shuffle_f64x2(lo.raw, hi.raw, _MM_PERM_BABA)};
+}
+
+// hiH,hiL loH,loL |-> hiH,loH (= upper halves)
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> ConcatHiHi(const Vec512<T> hi,
+                                                const Vec512<T> lo) {
+  return Vec512<T>{_mm512_shuffle_i32x4(lo.raw, hi.raw, _MM_PERM_DCDC)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> ConcatHiHi(const Vec512<float> hi,
+                                                    const Vec512<float> lo) {
+  return Vec512<float>{_mm512_shuffle_f32x4(lo.raw, hi.raw, _MM_PERM_DCDC)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> ConcatHiHi(const Vec512<double> hi,
+                                                     const Vec512<double> lo) {
+  return Vec512<double>{_mm512_shuffle_f64x2(lo.raw, hi.raw, _MM_PERM_DCDC)};
+}
+
+// hiH,hiL loH,loL |-> hiL,loH (= inner halves / swap blocks)
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> ConcatLoHi(const Vec512<T> hi,
+                                                const Vec512<T> lo) {
+  return Vec512<T>{_mm512_shuffle_i32x4(lo.raw, hi.raw, 0x4E)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> ConcatLoHi(const Vec512<float> hi,
+                                                    const Vec512<float> lo) {
+  return Vec512<float>{_mm512_shuffle_f32x4(lo.raw, hi.raw, 0x4E)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> ConcatLoHi(const Vec512<double> hi,
+                                                     const Vec512<double> lo) {
+  return Vec512<double>{_mm512_shuffle_f64x2(lo.raw, hi.raw, 0x4E)};
+}
+
+// hiH,hiL loH,loL |-> hiH,loL (= outer halves)
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> ConcatHiLo(const Vec512<T> hi,
+                                                const Vec512<T> lo) {
+  // There are no imm8 blend in AVX512. Use blend16 because 32-bit masks
+  // are efficiently loaded from 32-bit regs.
+  const __mmask32 mask = /*_cvtu32_mask32 */ (0x0000FFFF);
+  return Vec512<T>{_mm512_mask_blend_epi16(mask, hi.raw, lo.raw)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> ConcatHiLo(const Vec512<float> hi,
+                                                    const Vec512<float> lo) {
+  const __mmask16 mask = /*_cvtu32_mask16 */ (0x00FF);
+  return Vec512<float>{_mm512_mask_blend_ps(mask, hi.raw, lo.raw)};
+}
+template <>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> ConcatHiLo(const Vec512<double> hi,
+                                                     const Vec512<double> lo) {
+  const __mmask8 mask = /*_cvtu32_mask8 */ (0x0F);
+  return Vec512<double>{_mm512_mask_blend_pd(mask, hi.raw, lo.raw)};
+}
+
+// ------------------------------ Odd/even lanes
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> OddEven(const Vec512<T> a,
+                                             const Vec512<T> b) {
+  constexpr size_t s = sizeof(T);
+  constexpr int shift = s == 1 ? 0 : s == 2 ? 32 : s == 4 ? 48 : 56;
+  return IfThenElse(Mask512<T>{0x5555555555555555ull >> shift}, b, a);
+}
+
+// ================================================== CONVERT
+
+// ------------------------------ Shuffle bytes with variable indices
+
+// Returns vector of bytes[from[i]]. "from" is also interpreted as bytes:
+// either valid indices in [0, 16) or >= 0x80 to zero the i-th output byte.
+template <typename T, typename TI>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> TableLookupBytes(const Vec512<T> bytes,
+                                                      const Vec512<TI> from) {
+  return Vec512<T>{_mm512_shuffle_epi8(bytes.raw, from.raw)};
+}
+
+// ------------------------------ Promotions (part w/ narrow lanes -> full)
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<double> ConvertTo(Full512<double> /* tag */,
+                                                    Vec256<float> v) {
+  return Vec512<double>{_mm512_cvtps_pd(v.raw)};
+}
+
+// Unsigned: zero-extend.
+// Note: these have 3 cycle latency; if inputs are already split across the
+// 128 bit blocks (in their upper/lower halves), then ZipHi/lo would be faster.
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint16_t> ConvertTo(
+    Full512<uint16_t> /* tag */, Vec256<uint8_t> v) {
+  return Vec512<uint16_t>{_mm512_cvtepu8_epi16(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> ConvertTo(
+    Full512<uint32_t> /* tag */, Vec128<uint8_t> v) {
+  return Vec512<uint32_t>{_mm512_cvtepu8_epi32(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> ConvertTo(Full512<int16_t> /* tag */,
+                                                     Vec256<uint8_t> v) {
+  return Vec512<int16_t>{_mm512_cvtepu8_epi16(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> ConvertTo(Full512<int32_t> /* tag */,
+                                                     Vec128<uint8_t> v) {
+  return Vec512<int32_t>{_mm512_cvtepu8_epi32(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> ConvertTo(
+    Full512<uint32_t> /* tag */, Vec256<uint16_t> v) {
+  return Vec512<uint32_t>{_mm512_cvtepu16_epi32(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> ConvertTo(Full512<int32_t> /* tag */,
+                                                     Vec256<uint16_t> v) {
+  return Vec512<int32_t>{_mm512_cvtepu16_epi32(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> ConvertTo(
+    Full512<uint64_t> /* tag */, Vec256<uint32_t> v) {
+  return Vec512<uint64_t>{_mm512_cvtepu32_epi64(v.raw)};
+}
+
+// Special case for "v" with all blocks equal (e.g. from LoadDup128):
+// single-cycle latency instead of 3.
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint32_t> U32FromU8(const Vec512<uint8_t> v) {
+  const Full512<uint32_t> d32;
+  alignas(64) static constexpr uint32_t k32From8[16] = {
+      0xFFFFFF00UL, 0xFFFFFF01UL, 0xFFFFFF02UL, 0xFFFFFF03UL,
+      0xFFFFFF04UL, 0xFFFFFF05UL, 0xFFFFFF06UL, 0xFFFFFF07UL,
+      0xFFFFFF08UL, 0xFFFFFF09UL, 0xFFFFFF0AUL, 0xFFFFFF0BUL,
+      0xFFFFFF0CUL, 0xFFFFFF0DUL, 0xFFFFFF0EUL, 0xFFFFFF0FUL};
+  return TableLookupBytes(BitCast(d32, v), Load(d32, k32From8));
+}
+
+// Signed: replicate sign bit.
+// Note: these have 3 cycle latency; if inputs are already split across the
+// 128 bit blocks (in their upper/lower halves), then ZipHi/lo followed by
+// signed shift would be faster.
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int16_t> ConvertTo(Full512<int16_t> /* tag */,
+                                                     Vec256<int8_t> v) {
+  return Vec512<int16_t>{_mm512_cvtepi8_epi16(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> ConvertTo(Full512<int32_t> /* tag */,
+                                                     Vec128<int8_t> v) {
+  return Vec512<int32_t>{_mm512_cvtepi8_epi32(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> ConvertTo(Full512<int32_t> /* tag */,
+                                                     Vec256<int16_t> v) {
+  return Vec512<int32_t>{_mm512_cvtepi16_epi32(v.raw)};
+}
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int64_t> ConvertTo(Full512<int64_t> /* tag */,
+                                                     Vec256<int32_t> v) {
+  return Vec512<int64_t>{_mm512_cvtepi32_epi64(v.raw)};
+}
+
+// ------------------------------ Demotions (full -> part w/ narrow lanes)
+
+HWY_ATTR_AVX512 HWY_INLINE Vec256<uint16_t> ConvertTo(
+    Full256<uint16_t> /* tag */, const Vec512<int32_t> v) {
+  const Vec512<uint16_t> u16{_mm512_packus_epi32(v.raw, v.raw)};
+
+  // Compress even u64 lanes into 256 bit.
+  alignas(64) static constexpr uint64_t kLanes[8] = {0, 2, 4, 6, 0, 2, 4, 6};
+  const auto idx64 = Load(Full512<uint64_t>(), kLanes);
+  const Vec512<uint16_t> even{_mm512_permutexvar_epi64(idx64.raw, u16.raw)};
+  return LowerHalf(even);
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec128<uint8_t, 16> ConvertTo(
+    Full128<uint8_t> /* tag */, const Vec512<int32_t> v) {
+  const Vec512<uint16_t> u16{_mm512_packus_epi32(v.raw, v.raw)};
+  const Vec512<uint8_t> u8{_mm512_packus_epi16(u16.raw, u16.raw)};
+
+  alignas(64) static constexpr uint32_t kLanes[16] = {0, 4, 8, 12, 0, 4, 8, 12,
+                                                      0, 4, 8, 12, 0, 4, 8, 12};
+  const auto idx32 = Load(Full512<uint32_t>(), kLanes);
+  const Vec512<uint8_t> fixed{_mm512_permutexvar_epi32(idx32.raw, u8.raw)};
+  return LowerHalf(LowerHalf(fixed));
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec256<int16_t> ConvertTo(Full256<int16_t> /* tag */,
+                                                     const Vec512<int32_t> v) {
+  const Vec512<int16_t> i16{_mm512_packs_epi32(v.raw, v.raw)};
+
+  // Compress even u64 lanes into 256 bit.
+  alignas(64) static constexpr uint64_t kLanes[8] = {0, 2, 4, 6, 0, 2, 4, 6};
+  const auto idx64 = Load(Full512<uint64_t>(), kLanes);
+  const Vec512<int16_t> even{_mm512_permutexvar_epi64(idx64.raw, i16.raw)};
+  return LowerHalf(even);
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec128<int8_t, 16> ConvertTo(
+    Full128<int8_t> /* tag */, const Vec512<int32_t> v) {
+  const Vec512<int16_t> i16{_mm512_packs_epi32(v.raw, v.raw)};
+  const Vec512<int8_t> i8{_mm512_packs_epi16(i16.raw, i16.raw)};
+
+  alignas(64) static constexpr uint32_t kLanes[16] = {0, 4, 8, 12, 0, 4, 8, 12,
+                                                      0, 4, 8, 12, 0, 4, 8, 12};
+  const auto idx32 = Load(Full512<uint32_t>(), kLanes);
+  const Vec512<int8_t> fixed{_mm512_permutexvar_epi32(idx32.raw, i8.raw)};
+  return LowerHalf(LowerHalf(fixed));
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec256<uint8_t> ConvertTo(Full256<uint8_t> /* tag */,
+                                                     const Vec512<int16_t> v) {
+  const Vec512<uint8_t> u8{_mm512_packus_epi16(v.raw, v.raw)};
+
+  // Compress even u64 lanes into 256 bit.
+  alignas(64) static constexpr uint64_t kLanes[8] = {0, 2, 4, 6, 0, 2, 4, 6};
+  const auto idx64 = Load(Full512<uint64_t>(), kLanes);
+  const Vec512<uint8_t> even{_mm512_permutexvar_epi64(idx64.raw, u8.raw)};
+  return LowerHalf(even);
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec256<int8_t> ConvertTo(Full256<int8_t> /* tag */,
+                                                    const Vec512<int16_t> v) {
+  const Vec512<int8_t> u8{_mm512_packs_epi16(v.raw, v.raw)};
+
+  // Compress even u64 lanes into 256 bit.
+  alignas(64) static constexpr uint64_t kLanes[8] = {0, 2, 4, 6, 0, 2, 4, 6};
+  const auto idx64 = Load(Full512<uint64_t>(), kLanes);
+  const Vec512<int8_t> even{_mm512_permutexvar_epi64(idx64.raw, u8.raw)};
+  return LowerHalf(even);
+}
+
+// For already range-limited input [0, 255].
+HWY_ATTR_AVX512 HWY_INLINE Vec128<uint8_t, 16> U8FromU32(
+    const Vec512<uint32_t> v) {
+  const Full512<uint32_t> d32;
+  // In each 128 bit block, gather the lower byte of 4 uint32_t lanes into the
+  // lowest 4 bytes.
+  alignas(64) static constexpr uint32_t k8From32[4] = {0x0C080400u, ~0u, ~0u,
+                                                       ~0u};
+  const auto quads = TableLookupBytes(v, LoadDup128(d32, k8From32));
+  // Gather the lowest 4 bytes of 4 128-bit blocks.
+  alignas(64) static constexpr uint32_t kIndex32[16] = {
+      0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12, 0, 4, 8, 12};
+  const Vec512<uint8_t> bytes{
+      _mm512_permutexvar_epi32(Load(d32, kIndex32).raw, quads.raw)};
+  return LowerHalf(LowerHalf(bytes));
+}
+
+// ------------------------------ Convert i32 <=> f32
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<float> ConvertTo(Full512<float> /* tag */,
+                                                   const Vec512<int32_t> v) {
+  return Vec512<float>{_mm512_cvtepi32_ps(v.raw)};
+}
+// Truncates (rounds toward zero).
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> ConvertTo(Full512<int32_t> /* tag */,
+                                                     const Vec512<float> v) {
+  return Vec512<int32_t>{_mm512_cvttps_epi32(v.raw)};
+}
+
+HWY_ATTR_AVX512 HWY_INLINE Vec512<int32_t> NearestInt(const Vec512<float> v) {
+  return Vec512<int32_t>{_mm512_cvtps_epi32(v.raw)};
+}
+
+// ================================================== MISC
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// ------------------------------ movemask
+
+// Returns a bit array of the most significant bit of each byte in "v", i.e.
+// sum_i=0..31 of (v[i] >> 7) << i; v[0] is the least-significant byte of "v".
+// This is useful for testing/branching based on comparison results.
+HWY_ATTR_AVX512 HWY_INLINE uint64_t movemask(const Vec512<uint8_t> v) {
+  return _mm512_cmplt_epi8_mask(v.raw, _mm512_setzero_si512());
+}
+
+// Returns the most significant bit of each float/double lane (see above).
+HWY_ATTR_AVX512 HWY_INLINE uint64_t movemask(const Vec512<float> v) {
+  // cmp < 0 doesn't set if -0.0 and <= -0.0 sets despite 0.0 input.
+  return _mm512_test_epi32_mask(_mm512_castps_si512(v.raw),
+                                _mm512_set1_epi32(-0x7fffffff - 1));
+}
+HWY_ATTR_AVX512 HWY_INLINE uint64_t movemask(const Vec512<double> v) {
+  // cmp < 0 doesn't set if -0.0 and <= -0.0 sets despite 0.0 input.
+  return _mm512_test_epi64_mask(_mm512_castpd_si512(v.raw),
+                                _mm512_set1_epi64(-0x7fffffffffffffff - 1));
+}
+
+// ------------------------------ mask
+
+// For Clang and GCC, KORTEST wasn't added until recently.
+#if HWY_COMPILER_GCC >= 700 || HWY_COMPILER_CLANG >= 800 || \
+    (HWY_COMPILER_GCC == 0 && HWY_COMPILER_CLANG == 0)
+#define HWY_COMPILER_HAS_KORTEST 1
+#else
+#define HWY_COMPILER_HAS_KORTEST 0
+#endif
+
+// Beware: the suffix indicates the number of mask bits, not lane size!
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllFalseImpl(SizeTag<1> /*tag*/,
+                                             const Mask512<T> v) {
+#if HWY_COMPILER_HAS_KORTEST
+  return _kortestz_mask64_u8(v.raw, v.raw);
+#else
+  return v.raw == 0;
+#endif
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllFalseImpl(SizeTag<2> /*tag*/,
+                                             const Mask512<T> v) {
+#if HWY_COMPILER_HAS_KORTEST
+  return _kortestz_mask32_u8(v.raw, v.raw);
+#else
+  return v.raw == 0;
+#endif
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllFalseImpl(SizeTag<4> /*tag*/,
+                                             const Mask512<T> v) {
+#if HWY_COMPILER_HAS_KORTEST
+  return _kortestz_mask16_u8(v.raw, v.raw);
+#else
+  return v.raw == 0;
+#endif
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllFalseImpl(SizeTag<8> /*tag*/,
+                                             const Mask512<T> v) {
+#if HWY_COMPILER_HAS_KORTEST
+  return _kortestz_mask8_u8(v.raw, v.raw);
+#else
+  return v.raw == 0;
+#endif
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllFalse(const Mask512<T> v) {
+  return AllFalseImpl(SizeTag<sizeof(T)>(), v);
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllTrueImpl(SizeTag<1> /*tag*/,
+                                            const Mask512<T> v) {
+#if HWY_COMPILER_HAS_KORTEST
+  return _kortestc_mask64_u8(v.raw, v.raw);
+#else
+  return v.raw == 0xFFFFFFFFFFFFFFFFull;
+#endif
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllTrueImpl(SizeTag<2> /*tag*/,
+                                            const Mask512<T> v) {
+#if HWY_COMPILER_HAS_KORTEST
+  return _kortestc_mask32_u8(v.raw, v.raw);
+#else
+  return v.raw == 0xFFFFFFFFull;
+#endif
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllTrueImpl(SizeTag<4> /*tag*/,
+                                            const Mask512<T> v) {
+#if HWY_COMPILER_HAS_KORTEST
+  return _kortestc_mask16_u8(v.raw, v.raw);
+#else
+  return v.raw == 0xFFFFull;
+#endif
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllTrueImpl(SizeTag<8> /*tag*/,
+                                            const Mask512<T> v) {
+#if HWY_COMPILER_HAS_KORTEST
+  return _kortestc_mask8_u8(v.raw, v.raw);
+#else
+  return v.raw == 0xFFull;
+#endif
+}
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE bool AllTrue(const Mask512<T> v) {
+  return AllTrueImpl(SizeTag<sizeof(T)>(), v);
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE size_t CountTrue(const Mask512<T> v) {
+  return PopCount(v.raw);
+}
+
+// ------------------------------ Horizontal sum (reduction)
+
+// Returns 64-bit sums of 8-byte groups.
+HWY_ATTR_AVX512 HWY_INLINE Vec512<uint64_t> SumsOfU8x8(
+    const Vec512<uint8_t> v) {
+  return Vec512<uint64_t>{_mm512_sad_epu8(v.raw, _mm512_setzero_si512())};
+}
+
+// Returns sum{lane[i]} in each lane. "v3210" is a replicated 128-bit block.
+// Same logic as x86_sse4.h, but with Vec512 arguments.
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> horz_sum_impl(SizeTag<4> /* tag */,
+                                                   const Vec512<T> v3210) {
+  const auto v1032 = Shuffle1032(v3210);
+  const auto v31_20_31_20 = v3210 + v1032;
+  const auto v20_31_20_31 = Shuffle0321(v31_20_31_20);
+  return v20_31_20_31 + v31_20_31_20;
+}
+
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> horz_sum_impl(SizeTag<8> /* tag */,
+                                                   const Vec512<T> v10) {
+  const auto v01 = Shuffle01(v10);
+  return v10 + v01;
+}
+
+// Swaps adjacent 128-bit blocks.
+HWY_ATTR_AVX512 HWY_INLINE __m512i Blocks2301(__m512i v) {
+  return _mm512_shuffle_i32x4(v, v, _MM_SHUFFLE(2, 3, 0, 1));
+}
+HWY_ATTR_AVX512 HWY_INLINE __m512 Blocks2301(__m512 v) {
+  return _mm512_shuffle_f32x4(v, v, _MM_SHUFFLE(2, 3, 0, 1));
+}
+HWY_ATTR_AVX512 HWY_INLINE __m512d Blocks2301(__m512d v) {
+  return _mm512_shuffle_f64x2(v, v, _MM_SHUFFLE(2, 3, 0, 1));
+}
+
+// Swaps upper/lower pairs of 128-bit blocks.
+HWY_ATTR_AVX512 HWY_INLINE __m512i Blocks1032(__m512i v) {
+  return _mm512_shuffle_i32x4(v, v, _MM_SHUFFLE(1, 0, 3, 2));
+}
+HWY_ATTR_AVX512 HWY_INLINE __m512 Blocks1032(__m512 v) {
+  return _mm512_shuffle_f32x4(v, v, _MM_SHUFFLE(1, 0, 3, 2));
+}
+HWY_ATTR_AVX512 HWY_INLINE __m512d Blocks1032(__m512d v) {
+  return _mm512_shuffle_f64x2(v, v, _MM_SHUFFLE(1, 0, 3, 2));
+}
+
+// Supported for {uif}32x8, {uif}64x4. Returns the sum in each lane.
+template <typename T>
+HWY_ATTR_AVX512 HWY_INLINE Vec512<T> SumOfLanes(const Vec512<T> v3210) {
+  // Sum of all 128-bit blocks in each 128-bit block.
+  const Vec512<T> v2301{Blocks2301(v3210.raw)};
+  const Vec512<T> v32_32_10_10 = v2301 + v3210;
+  const Vec512<T> v10_10_32_32{Blocks1032(v32_32_10_10.raw)};
+  return horz_sum_impl(SizeTag<sizeof(T)>(), v32_32_10_10 + v10_10_32_32);
+}
+
+}  // namespace ext
+
+// TODO(janwas): wrappers for all intrinsics (in x86 namespace).
+}  // namespace hwy
+
+#endif  // HWY_X86_AVX512_H_
diff --git a/third_party/highway/hwy/x86/sse4.h b/third_party/highway/hwy/x86/sse4.h
new file mode 100644
index 000000000..c67313717
--- /dev/null
+++ b/third_party/highway/hwy/x86/sse4.h
@@ -0,0 +1,2182 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef HWY_X86_SSE4_H_
+#define HWY_X86_SSE4_H_
+
+// 128-bit SSE4 vectors and operations.
+
+#include <smmintrin.h>
+
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+#include <stdio.h>
+#endif
+
+#include "hwy/shared.h"
+
+#define HWY_ATTR_SSE4 HWY_TARGET_ATTR("sse4.1")
+
+namespace hwy {
+
+template <typename T>
+struct Raw128 {
+  using type = __m128i;
+};
+template <>
+struct Raw128<float> {
+  using type = __m128;
+};
+template <>
+struct Raw128<double> {
+  using type = __m128d;
+};
+
+template <typename T>
+using Full128 = Desc<T, 16 / sizeof(T)>;
+
+template <typename T, size_t N = 16 / sizeof(T)>
+class Vec128 {
+  using Raw = typename Raw128<T>::type;
+
+ public:
+  // Compound assignment. Only usable if there is a corresponding non-member
+  // binary operator overload. For example, only f32 and f64 support division.
+  HWY_ATTR_SSE4 HWY_INLINE Vec128& operator*=(const Vec128 other) {
+    return *this = (*this * other);
+  }
+  HWY_ATTR_SSE4 HWY_INLINE Vec128& operator/=(const Vec128 other) {
+    return *this = (*this / other);
+  }
+  HWY_ATTR_SSE4 HWY_INLINE Vec128& operator+=(const Vec128 other) {
+    return *this = (*this + other);
+  }
+  HWY_ATTR_SSE4 HWY_INLINE Vec128& operator-=(const Vec128 other) {
+    return *this = (*this - other);
+  }
+  HWY_ATTR_SSE4 HWY_INLINE Vec128& operator&=(const Vec128 other) {
+    return *this = (*this & other);
+  }
+  HWY_ATTR_SSE4 HWY_INLINE Vec128& operator|=(const Vec128 other) {
+    return *this = (*this | other);
+  }
+  HWY_ATTR_SSE4 HWY_INLINE Vec128& operator^=(const Vec128 other) {
+    return *this = (*this ^ other);
+  }
+
+  Raw raw;
+};
+
+// Integer: FF..FF or 0. Float: MSB, all other bits undefined - see README.
+template <typename T, size_t N = 16 / sizeof(T)>
+class Mask128 {
+  using Raw = typename Raw128<T>::type;
+
+ public:
+  Raw raw;
+};
+
+// ------------------------------ Cast
+
+HWY_ATTR_SSE4 HWY_INLINE __m128i BitCastToInteger(__m128i v) { return v; }
+HWY_ATTR_SSE4 HWY_INLINE __m128i BitCastToInteger(__m128 v) {
+  return _mm_castps_si128(v);
+}
+HWY_ATTR_SSE4 HWY_INLINE __m128i BitCastToInteger(__m128d v) {
+  return _mm_castpd_si128(v);
+}
+
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N * sizeof(T)> cast_to_u8(
+    Vec128<T, N> v) {
+  return Vec128<uint8_t, N * sizeof(T)>{BitCastToInteger(v.raw)};
+}
+
+// Cannot rely on function overloading because return types differ.
+template <typename T>
+struct BitCastFromInteger128 {
+  HWY_ATTR_SSE4 HWY_INLINE __m128i operator()(__m128i v) { return v; }
+};
+template <>
+struct BitCastFromInteger128<float> {
+  HWY_ATTR_SSE4 HWY_INLINE __m128 operator()(__m128i v) {
+    return _mm_castsi128_ps(v);
+  }
+};
+template <>
+struct BitCastFromInteger128<double> {
+  HWY_ATTR_SSE4 HWY_INLINE __m128d operator()(__m128i v) {
+    return _mm_castsi128_pd(v);
+  }
+};
+
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> cast_u8_to(
+    Desc<T, N> /* tag */, Vec128<uint8_t, N * sizeof(T)> v) {
+  return Vec128<T, N>{BitCastFromInteger128<T>()(v.raw)};
+}
+
+template <typename T, size_t N, typename FromT>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> BitCast(
+    Desc<T, N> d, Vec128<FromT, N * sizeof(T) / sizeof(FromT)> v) {
+  return cast_u8_to(d, cast_to_u8(v));
+}
+
+// ------------------------------ Set
+
+// Returns an all-zero vector/part.
+template <typename T, size_t N, HWY_IF128(T, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> Zero(Desc<T, N> /* tag */) {
+  return Vec128<T, N>{_mm_setzero_si128()};
+}
+template <size_t N, HWY_IF128(float, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Zero(Desc<float, N> /* tag */) {
+  return Vec128<float, N>{_mm_setzero_ps()};
+}
+template <size_t N, HWY_IF128(double, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Zero(Desc<double, N> /* tag */) {
+  return Vec128<double, N>{_mm_setzero_pd()};
+}
+
+// Returns a vector/part with all lanes set to "t".
+template <size_t N, HWY_IF128(uint8_t, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> Set(Desc<uint8_t, N> /* tag */,
+                                                const uint8_t t) {
+  return Vec128<uint8_t, N>{_mm_set1_epi8(t)};
+}
+template <size_t N, HWY_IF128(uint16_t, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> Set(Desc<uint16_t, N> /* tag */,
+                                                 const uint16_t t) {
+  return Vec128<uint16_t, N>{_mm_set1_epi16(t)};
+}
+template <size_t N, HWY_IF128(uint32_t, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> Set(Desc<uint32_t, N> /* tag */,
+                                                 const uint32_t t) {
+  return Vec128<uint32_t, N>{_mm_set1_epi32(t)};
+}
+template <size_t N, HWY_IF128(uint64_t, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t, N> Set(Desc<uint64_t, N> /* tag */,
+                                                 const uint64_t t) {
+  return Vec128<uint64_t, N>{_mm_set1_epi64x(t)};
+}
+template <size_t N, HWY_IF128(int8_t, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> Set(Desc<int8_t, N> /* tag */,
+                                               const int8_t t) {
+  return Vec128<int8_t, N>{_mm_set1_epi8(t)};
+}
+template <size_t N, HWY_IF128(int16_t, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> Set(Desc<int16_t, N> /* tag */,
+                                                const int16_t t) {
+  return Vec128<int16_t, N>{_mm_set1_epi16(t)};
+}
+template <size_t N, HWY_IF128(int32_t, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> Set(Desc<int32_t, N> /* tag */,
+                                                const int32_t t) {
+  return Vec128<int32_t, N>{_mm_set1_epi32(t)};
+}
+template <size_t N, HWY_IF128(int64_t, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t, N> Set(Desc<int64_t, N> /* tag */,
+                                                const int64_t t) {
+  return Vec128<int64_t, N>{_mm_set1_epi64x(t)};
+}
+template <size_t N, HWY_IF128(float, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Set(Desc<float, N> /* tag */,
+                                              const float t) {
+  return Vec128<float, N>{_mm_set1_ps(t)};
+}
+template <size_t N, HWY_IF128(double, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Set(Desc<double, N> /* tag */,
+                                               const double t) {
+  return Vec128<double, N>{_mm_set1_pd(t)};
+}
+
+// Returns a vector with lane i=[0, N) set to "first" + i. Unique per-lane
+// values are required to detect lane-crossing bugs.
+template <typename T, size_t N, typename T2, HWY_IF128(T, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> Iota(Desc<T, N> d, const T2 first) {
+  alignas(16) T lanes[d.N];
+  for (size_t i = 0; i < d.N; ++i) {
+    lanes[i] = first + i;
+  }
+  return Load(d, lanes);
+}
+
+HWY_DIAGNOSTICS(push)
+HWY_DIAGNOSTICS_OFF(disable : 4700, ignored "-Wuninitialized")
+
+// Returns a vector with uninitialized elements.
+template <typename T, size_t N, HWY_IF128(T, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> Undefined(Desc<T, N> /* tag */) {
+#ifdef __clang__
+  return Vec128<T, N>{_mm_undefined_si128()};
+#else
+  __m128i raw;
+  return Vec128<T, N>{raw};
+#endif
+}
+template <size_t N, HWY_IF128(float, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Undefined(Desc<float, N> /* tag */) {
+#ifdef __clang__
+  return Vec128<float, N>{_mm_undefined_ps()};
+#else
+  __m128 raw;
+  return Vec128<float, N>{raw};
+#endif
+}
+template <size_t N, HWY_IF128(double, N)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Undefined(
+    Desc<double, N> /* tag */) {
+#ifdef __clang__
+  return Vec128<double, N>{_mm_undefined_pd()};
+#else
+  __m128d raw;
+  return Vec128<double, N>{raw};
+#endif
+}
+
+HWY_DIAGNOSTICS(pop)
+
+// ================================================== ARITHMETIC
+
+// ------------------------------ Addition
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> operator+(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{_mm_add_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> operator+(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{_mm_add_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> operator+(
+    const Vec128<uint32_t, N> a, const Vec128<uint32_t, N> b) {
+  return Vec128<uint32_t, N>{_mm_add_epi32(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t, N> operator+(
+    const Vec128<uint64_t, N> a, const Vec128<uint64_t, N> b) {
+  return Vec128<uint64_t, N>{_mm_add_epi64(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> operator+(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{_mm_add_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> operator+(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{_mm_add_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> operator+(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Vec128<int32_t, N>{_mm_add_epi32(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t, N> operator+(
+    const Vec128<int64_t, N> a, const Vec128<int64_t, N> b) {
+  return Vec128<int64_t, N>{_mm_add_epi64(a.raw, b.raw)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> operator+(const Vec128<float, N> a,
+                                                    const Vec128<float, N> b) {
+  return Vec128<float, N>{_mm_add_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> operator+(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Vec128<double, N>{_mm_add_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Subtraction
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> operator-(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{_mm_sub_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> operator-(Vec128<uint16_t, N> a,
+                                                       Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{_mm_sub_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> operator-(
+    const Vec128<uint32_t, N> a, const Vec128<uint32_t, N> b) {
+  return Vec128<uint32_t, N>{_mm_sub_epi32(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t, N> operator-(
+    const Vec128<uint64_t, N> a, const Vec128<uint64_t, N> b) {
+  return Vec128<uint64_t, N>{_mm_sub_epi64(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> operator-(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{_mm_sub_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> operator-(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{_mm_sub_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> operator-(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Vec128<int32_t, N>{_mm_sub_epi32(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t, N> operator-(
+    const Vec128<int64_t, N> a, const Vec128<int64_t, N> b) {
+  return Vec128<int64_t, N>{_mm_sub_epi64(a.raw, b.raw)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> operator-(const Vec128<float, N> a,
+                                                    const Vec128<float, N> b) {
+  return Vec128<float, N>{_mm_sub_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> operator-(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Vec128<double, N>{_mm_sub_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Saturating addition
+
+// Returns a + b clamped to the destination range.
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> SaturatedAdd(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{_mm_adds_epu8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> SaturatedAdd(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{_mm_adds_epu16(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> SaturatedAdd(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{_mm_adds_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> SaturatedAdd(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{_mm_adds_epi16(a.raw, b.raw)};
+}
+
+// ------------------------------ Saturating subtraction
+
+// Returns a - b clamped to the destination range.
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> SaturatedSub(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{_mm_subs_epu8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> SaturatedSub(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{_mm_subs_epu16(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> SaturatedSub(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{_mm_subs_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> SaturatedSub(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{_mm_subs_epi16(a.raw, b.raw)};
+}
+
+// ------------------------------ Average
+
+// Returns (a + b + 1) / 2
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> AverageRound(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{_mm_avg_epu8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> AverageRound(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{_mm_avg_epu16(a.raw, b.raw)};
+}
+
+// ------------------------------ Absolute value
+
+// Returns absolute value, except that LimitsMin() maps to LimitsMax() + 1.
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> Abs(const Vec128<int8_t, N> v) {
+  return Vec128<int8_t, N>{_mm_abs_epi8(v.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> Abs(const Vec128<int16_t, N> v) {
+  return Vec128<int16_t, N>{_mm_abs_epi16(v.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> Abs(const Vec128<int32_t, N> v) {
+  return Vec128<int32_t, N>{_mm_abs_epi32(v.raw)};
+}
+
+// ------------------------------ Shift lanes by constant #bits
+
+// Unsigned
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> ShiftLeft(
+    const Vec128<uint16_t, N> v) {
+  return Vec128<uint16_t, N>{_mm_slli_epi16(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> ShiftRight(
+    const Vec128<uint16_t, N> v) {
+  return Vec128<uint16_t, N>{_mm_srli_epi16(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> ShiftLeft(
+    const Vec128<uint32_t, N> v) {
+  return Vec128<uint32_t, N>{_mm_slli_epi32(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> ShiftRight(
+    const Vec128<uint32_t, N> v) {
+  return Vec128<uint32_t, N>{_mm_srli_epi32(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t, N> ShiftLeft(
+    const Vec128<uint64_t, N> v) {
+  return Vec128<uint64_t, N>{_mm_slli_epi64(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t, N> ShiftRight(
+    const Vec128<uint64_t, N> v) {
+  return Vec128<uint64_t, N>{_mm_srli_epi64(v.raw, kBits)};
+}
+
+// Signed (no i64 ShiftRight)
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> ShiftLeft(
+    const Vec128<int16_t, N> v) {
+  return Vec128<int16_t, N>{_mm_slli_epi16(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> ShiftRight(
+    const Vec128<int16_t, N> v) {
+  return Vec128<int16_t, N>{_mm_srai_epi16(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> ShiftLeft(
+    const Vec128<int32_t, N> v) {
+  return Vec128<int32_t, N>{_mm_slli_epi32(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> ShiftRight(
+    const Vec128<int32_t, N> v) {
+  return Vec128<int32_t, N>{_mm_srai_epi32(v.raw, kBits)};
+}
+template <int kBits, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t, N> ShiftLeft(
+    const Vec128<int64_t, N> v) {
+  return Vec128<int64_t, N>{_mm_slli_epi64(v.raw, kBits)};
+}
+
+// ------------------------------ Shift lanes by same variable #bits
+
+// Unsigned (no u8)
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> ShiftLeftSame(
+    const Vec128<uint16_t, N> v, const int bits) {
+  return Vec128<uint16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> ShiftRightSame(
+    const Vec128<uint16_t, N> v, const int bits) {
+  return Vec128<uint16_t, N>{_mm_srl_epi16(v.raw, _mm_cvtsi32_si128(bits))};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> ShiftLeftSame(
+    const Vec128<uint32_t, N> v, const int bits) {
+  return Vec128<uint32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> ShiftRightSame(
+    const Vec128<uint32_t, N> v, const int bits) {
+  return Vec128<uint32_t, N>{_mm_srl_epi32(v.raw, _mm_cvtsi32_si128(bits))};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t, N> ShiftLeftSame(
+    const Vec128<uint64_t, N> v, const int bits) {
+  return Vec128<uint64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t, N> ShiftRightSame(
+    const Vec128<uint64_t, N> v, const int bits) {
+  return Vec128<uint64_t, N>{_mm_srl_epi64(v.raw, _mm_cvtsi32_si128(bits))};
+}
+
+// Signed (no i8,i64)
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> ShiftLeftSame(
+    const Vec128<int16_t, N> v, const int bits) {
+  return Vec128<int16_t, N>{_mm_sll_epi16(v.raw, _mm_cvtsi32_si128(bits))};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> ShiftRightSame(
+    const Vec128<int16_t, N> v, const int bits) {
+  return Vec128<int16_t, N>{_mm_sra_epi16(v.raw, _mm_cvtsi32_si128(bits))};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> ShiftLeftSame(
+    const Vec128<int32_t, N> v, const int bits) {
+  return Vec128<int32_t, N>{_mm_sll_epi32(v.raw, _mm_cvtsi32_si128(bits))};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> ShiftRightSame(
+    const Vec128<int32_t, N> v, const int bits) {
+  return Vec128<int32_t, N>{_mm_sra_epi32(v.raw, _mm_cvtsi32_si128(bits))};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t, N> ShiftLeftSame(
+    const Vec128<int64_t, N> v, const int bits) {
+  return Vec128<int64_t, N>{_mm_sll_epi64(v.raw, _mm_cvtsi32_si128(bits))};
+}
+
+// ------------------------------ Shift lanes by independent variable #bits
+
+// Unsupported.
+
+// ------------------------------ Minimum
+
+// Unsigned (no u64)
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> Min(const Vec128<uint8_t, N> a,
+                                                const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{_mm_min_epu8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> Min(const Vec128<uint16_t, N> a,
+                                                 const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{_mm_min_epu16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> Min(const Vec128<uint32_t, N> a,
+                                                 const Vec128<uint32_t, N> b) {
+  return Vec128<uint32_t, N>{_mm_min_epu32(a.raw, b.raw)};
+}
+
+// Signed (no i64)
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> Min(const Vec128<int8_t, N> a,
+                                               const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{_mm_min_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> Min(const Vec128<int16_t, N> a,
+                                                const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{_mm_min_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> Min(const Vec128<int32_t, N> a,
+                                                const Vec128<int32_t, N> b) {
+  return Vec128<int32_t, N>{_mm_min_epi32(a.raw, b.raw)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Min(const Vec128<float, N> a,
+                                              const Vec128<float, N> b) {
+  return Vec128<float, N>{_mm_min_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Min(const Vec128<double, N> a,
+                                               const Vec128<double, N> b) {
+  return Vec128<double, N>{_mm_min_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Maximum
+
+// Unsigned (no u64)
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> Max(const Vec128<uint8_t, N> a,
+                                                const Vec128<uint8_t, N> b) {
+  return Vec128<uint8_t, N>{_mm_max_epu8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> Max(const Vec128<uint16_t, N> a,
+                                                 const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{_mm_max_epu16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> Max(const Vec128<uint32_t, N> a,
+                                                 const Vec128<uint32_t, N> b) {
+  return Vec128<uint32_t, N>{_mm_max_epu32(a.raw, b.raw)};
+}
+
+// Signed (no i64)
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> Max(const Vec128<int8_t, N> a,
+                                               const Vec128<int8_t, N> b) {
+  return Vec128<int8_t, N>{_mm_max_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> Max(const Vec128<int16_t, N> a,
+                                                const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{_mm_max_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> Max(const Vec128<int32_t, N> a,
+                                                const Vec128<int32_t, N> b) {
+  return Vec128<int32_t, N>{_mm_max_epi32(a.raw, b.raw)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Max(const Vec128<float, N> a,
+                                              const Vec128<float, N> b) {
+  return Vec128<float, N>{_mm_max_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Max(const Vec128<double, N> a,
+                                               const Vec128<double, N> b) {
+  return Vec128<double, N>{_mm_max_pd(a.raw, b.raw)};
+}
+
+// Returns the closest value to v within [lo, hi].
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> Clamp(const Vec128<T, N> v,
+                                            const Vec128<T, N> lo,
+                                            const Vec128<T, N> hi) {
+  return Min(Max(lo, v), hi);
+}
+
+// ------------------------------ Integer multiplication
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> operator*(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{_mm_mullo_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t, N> operator*(
+    const Vec128<uint32_t, N> a, const Vec128<uint32_t, N> b) {
+  return Vec128<uint32_t, N>{_mm_mullo_epi32(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> operator*(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{_mm_mullo_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> operator*(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Vec128<int32_t, N>{_mm_mullo_epi32(a.raw, b.raw)};
+}
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// Returns the upper 16 bits of a * b in each lane.
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> MulHigh(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Vec128<uint16_t, N>{_mm_mulhi_epu16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> MulHigh(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Vec128<int16_t, N>{_mm_mulhi_epi16(a.raw, b.raw)};
+}
+
+}  // namespace ext
+
+// Multiplies even lanes (0, 2 ..) and places the double-wide result into
+// even and the upper half into its odd neighbor lane.
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t> MulEven(const Vec128<int32_t> a,
+                                                 const Vec128<int32_t> b) {
+  return Vec128<int64_t>{_mm_mul_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t> MulEven(const Vec128<uint32_t> a,
+                                                  const Vec128<uint32_t> b) {
+  return Vec128<uint64_t>{_mm_mul_epu32(a.raw, b.raw)};
+}
+
+// ------------------------------ Floating-point negate
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Neg(const Vec128<float, N> v) {
+  const Desc<float, N> df;
+  const Desc<uint32_t, N> du;
+  const auto sign = BitCast(df, Set(du, 0x80000000u));
+  return v ^ sign;
+}
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Neg(const Vec128<double, N> v) {
+  const Desc<double, N> df;
+  const Desc<uint64_t, N> du;
+  const auto sign = BitCast(df, Set(du, 0x8000000000000000ull));
+  return v ^ sign;
+}
+
+// ------------------------------ Floating-point mul / div
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> operator*(Vec128<float, N> a,
+                                                    Vec128<float, N> b) {
+  return Vec128<float, N>{_mm_mul_ps(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, 1> operator*(const Vec128<float, 1> a,
+                                                    const Vec128<float, 1> b) {
+  return Vec128<float, 1>{_mm_mul_ss(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> operator*(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Vec128<double, N>{_mm_mul_pd(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, 1> operator*(
+    const Vec128<double, 1> a, const Vec128<double, 1> b) {
+  return Vec128<double, 1>{_mm_mul_sd(a.raw, b.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> operator/(const Vec128<float, N> a,
+                                                    const Vec128<float, N> b) {
+  return Vec128<float, N>{_mm_div_ps(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, 1> operator/(const Vec128<float, 1> a,
+                                                    const Vec128<float, 1> b) {
+  return Vec128<float, 1>{_mm_div_ss(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> operator/(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Vec128<double, N>{_mm_div_pd(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, 1> operator/(
+    const Vec128<double, 1> a, const Vec128<double, 1> b) {
+  return Vec128<double, 1>{_mm_div_sd(a.raw, b.raw)};
+}
+
+// Approximate reciprocal
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> ApproximateReciprocal(
+    const Vec128<float, N> v) {
+  return Vec128<float, N>{_mm_rcp_ps(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, 1> ApproximateReciprocal(
+    const Vec128<float, 1> v) {
+  return Vec128<float, 1>{_mm_rcp_ss(v.raw)};
+}
+
+namespace ext {
+// Absolute value of difference.
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> AbsDiff(const Vec128<float> a,
+                                               const Vec128<float> b) {
+  const auto mask =
+      BitCast(Full128<float>(), Set(Full128<uint32_t>(), 0x7FFFFFFFu));
+  return Vec128<float>{_mm_and_ps(mask.raw, (a - b).raw)};
+}
+}  // namespace ext
+
+// ------------------------------ Floating-point multiply-add variants
+
+// Returns mul * x + add
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> MulAdd(const Vec128<float, N> mul,
+                                                 const Vec128<float, N> x,
+                                                 const Vec128<float, N> add) {
+  return mul * x + add;
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> MulAdd(const Vec128<double, N> mul,
+                                                  const Vec128<double, N> x,
+                                                  const Vec128<double, N> add) {
+  return mul * x + add;
+}
+
+// Returns add - mul * x
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> NegMulAdd(
+    const Vec128<float, N> mul, const Vec128<float, N> x,
+    const Vec128<float, N> add) {
+  return add - mul * x;
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> NegMulAdd(
+    const Vec128<double, N> mul, const Vec128<double, N> x,
+    const Vec128<double, N> add) {
+  return add - mul * x;
+}
+
+// Slightly more expensive on ARM (extra negate)
+namespace ext {
+
+// Returns mul * x - sub
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> MulSub(const Vec128<float, N> mul,
+                                                 const Vec128<float, N> x,
+                                                 const Vec128<float, N> sub) {
+  return mul * x - sub;
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> MulSub(const Vec128<double, N> mul,
+                                                  const Vec128<double, N> x,
+                                                  const Vec128<double, N> sub) {
+  return mul * x - sub;
+}
+
+// Returns -mul * x - sub
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> NegMulSub(
+    const Vec128<float, N> mul, const Vec128<float, N> x,
+    const Vec128<float, N> sub) {
+  return Neg(mul) * x - sub;
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> NegMulSub(
+    const Vec128<double, N> mul, const Vec128<double, N> x,
+    const Vec128<double, N> sub) {
+  return Neg(mul) * x - sub;
+}
+
+}  // namespace ext
+
+// ------------------------------ Floating-point square root
+
+// Full precision square root
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Sqrt(const Vec128<float, N> v) {
+  return Vec128<float, N>{_mm_sqrt_ps(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, 1> Sqrt(const Vec128<float, 1> v) {
+  return Vec128<float, 1>{_mm_sqrt_ss(v.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Sqrt(const Vec128<double, N> v) {
+  return Vec128<double, N>{_mm_sqrt_pd(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, 1> Sqrt(const Vec128<double, 1> v) {
+  return Vec128<double, 1>{_mm_sqrt_sd(_mm_setzero_pd(), v.raw)};
+}
+
+// Approximate reciprocal square root
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> ApproximateReciprocalSqrt(
+    const Vec128<float, N> v) {
+  return Vec128<float, N>{_mm_rsqrt_ps(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, 1> ApproximateReciprocalSqrt(
+    const Vec128<float, 1> v) {
+  return Vec128<float, 1>{_mm_rsqrt_ss(v.raw)};
+}
+
+// ------------------------------ Floating-point rounding
+
+// Toward nearest integer, ties to even
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Round(const Vec128<float, N> v) {
+  return Vec128<float, N>{
+      _mm_round_ps(v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Round(const Vec128<double, N> v) {
+  return Vec128<double, N>{
+      _mm_round_pd(v.raw, _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC)};
+}
+
+// Toward zero, aka truncate
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Trunc(const Vec128<float, N> v) {
+  return Vec128<float, N>{
+      _mm_round_ps(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Trunc(const Vec128<double, N> v) {
+  return Vec128<double, N>{
+      _mm_round_pd(v.raw, _MM_FROUND_TO_ZERO | _MM_FROUND_NO_EXC)};
+}
+
+// Toward +infinity, aka ceiling
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Ceil(const Vec128<float, N> v) {
+  return Vec128<float, N>{
+      _mm_round_ps(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Ceil(const Vec128<double, N> v) {
+  return Vec128<double, N>{
+      _mm_round_pd(v.raw, _MM_FROUND_TO_POS_INF | _MM_FROUND_NO_EXC)};
+}
+
+// Toward -infinity, aka floor
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> Floor(const Vec128<float, N> v) {
+  return Vec128<float, N>{
+      _mm_round_ps(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> Floor(const Vec128<double, N> v) {
+  return Vec128<double, N>{
+      _mm_round_pd(v.raw, _MM_FROUND_TO_NEG_INF | _MM_FROUND_NO_EXC)};
+}
+
+// ================================================== COMPARE
+
+// Comparisons fill a lane with 1-bits if the condition is true, else 0.
+
+// ------------------------------ Equality
+
+// Unsigned
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<uint8_t, N> operator==(
+    const Vec128<uint8_t, N> a, const Vec128<uint8_t, N> b) {
+  return Mask128<uint8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<uint16_t, N> operator==(
+    const Vec128<uint16_t, N> a, const Vec128<uint16_t, N> b) {
+  return Mask128<uint16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<uint32_t, N> operator==(
+    const Vec128<uint32_t, N> a, const Vec128<uint32_t, N> b) {
+  return Mask128<uint32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<uint64_t, N> operator==(
+    const Vec128<uint64_t, N> a, const Vec128<uint64_t, N> b) {
+  return Mask128<uint64_t, N>{_mm_cmpeq_epi64(a.raw, b.raw)};
+}
+
+// Signed
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int8_t, N> operator==(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Mask128<int8_t, N>{_mm_cmpeq_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int16_t, N> operator==(Vec128<int16_t, N> a,
+                                                        Vec128<int16_t, N> b) {
+  return Mask128<int16_t, N>{_mm_cmpeq_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int32_t, N> operator==(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Mask128<int32_t, N>{_mm_cmpeq_epi32(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int64_t, N> operator==(
+    const Vec128<int64_t, N> a, const Vec128<int64_t, N> b) {
+  return Mask128<int64_t, N>{_mm_cmpeq_epi64(a.raw, b.raw)};
+}
+
+// Float
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<float, N> operator==(
+    const Vec128<float, N> a, const Vec128<float, N> b) {
+  return Mask128<float, N>{_mm_cmpeq_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<double, N> operator==(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Mask128<double, N>{_mm_cmpeq_pd(a.raw, b.raw)};
+}
+
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<T, N> TestBit(Vec128<T, N> v,
+                                               Vec128<T, N> bit) {
+  static_assert(!IsFloat<T>(), "Only integer vectors supported");
+  return (v & bit) == bit;
+}
+
+// ------------------------------ Strict inequality
+
+// Signed/float <
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int8_t, N> operator<(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Mask128<int8_t, N>{_mm_cmpgt_epi8(b.raw, a.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int16_t, N> operator<(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Mask128<int16_t, N>{_mm_cmpgt_epi16(b.raw, a.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int32_t, N> operator<(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Mask128<int32_t, N>{_mm_cmpgt_epi32(b.raw, a.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<float, N> operator<(const Vec128<float, N> a,
+                                                     const Vec128<float, N> b) {
+  return Mask128<float, N>{_mm_cmplt_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<double, N> operator<(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Mask128<double, N>{_mm_cmplt_pd(a.raw, b.raw)};
+}
+
+// Signed/float >
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int8_t, N> operator>(
+    const Vec128<int8_t, N> a, const Vec128<int8_t, N> b) {
+  return Mask128<int8_t, N>{_mm_cmpgt_epi8(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int16_t, N> operator>(
+    const Vec128<int16_t, N> a, const Vec128<int16_t, N> b) {
+  return Mask128<int16_t, N>{_mm_cmpgt_epi16(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<int32_t, N> operator>(
+    const Vec128<int32_t, N> a, const Vec128<int32_t, N> b) {
+  return Mask128<int32_t, N>{_mm_cmpgt_epi32(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<float, N> operator>(const Vec128<float, N> a,
+                                                     const Vec128<float, N> b) {
+  return Mask128<float, N>{_mm_cmpgt_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<double, N> operator>(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Mask128<double, N>{_mm_cmpgt_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Weak inequality
+
+// Float <= >=
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<float, N> operator<=(
+    const Vec128<float, N> a, const Vec128<float, N> b) {
+  return Mask128<float, N>{_mm_cmple_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<double, N> operator<=(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Mask128<double, N>{_mm_cmple_pd(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<float, N> operator>=(
+    const Vec128<float, N> a, const Vec128<float, N> b) {
+  return Mask128<float, N>{_mm_cmpge_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<double, N> operator>=(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Mask128<double, N>{_mm_cmpge_pd(a.raw, b.raw)};
+}
+
+// ================================================== LOGICAL
+
+// ------------------------------ Bitwise AND
+
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> operator&(Vec128<T, N> a,
+                                                Vec128<T, N> b) {
+  return Vec128<T, N>{_mm_and_si128(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> operator&(const Vec128<float, N> a,
+                                                    const Vec128<float, N> b) {
+  return Vec128<float, N>{_mm_and_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> operator&(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Vec128<double, N>{_mm_and_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Bitwise AND-NOT
+
+// Returns ~not_mask & mask.
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> AndNot(Vec128<T, N> not_mask,
+                                             Vec128<T, N> mask) {
+  return Vec128<T, N>{_mm_andnot_si128(not_mask.raw, mask.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> AndNot(
+    const Vec128<float, N> not_mask, const Vec128<float, N> mask) {
+  return Vec128<float, N>{_mm_andnot_ps(not_mask.raw, mask.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> AndNot(
+    const Vec128<double, N> not_mask, const Vec128<double, N> mask) {
+  return Vec128<double, N>{_mm_andnot_pd(not_mask.raw, mask.raw)};
+}
+
+// ------------------------------ Bitwise OR
+
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> operator|(Vec128<T, N> a,
+                                                Vec128<T, N> b) {
+  return Vec128<T, N>{_mm_or_si128(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> operator|(const Vec128<float, N> a,
+                                                    const Vec128<float, N> b) {
+  return Vec128<float, N>{_mm_or_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> operator|(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Vec128<double, N>{_mm_or_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Bitwise XOR
+
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> operator^(Vec128<T, N> a,
+                                                Vec128<T, N> b) {
+  return Vec128<T, N>{_mm_xor_si128(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> operator^(const Vec128<float, N> a,
+                                                    const Vec128<float, N> b) {
+  return Vec128<float, N>{_mm_xor_ps(a.raw, b.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> operator^(
+    const Vec128<double, N> a, const Vec128<double, N> b) {
+  return Vec128<double, N>{_mm_xor_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Mask
+
+// Mask and Vec are the same (true = FF..FF).
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Mask128<T, N> MaskFromVec(const Vec128<T, N> v) {
+  return Mask128<T, N>{v.raw};
+}
+
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> VecFromMask(const Mask128<T, N> v) {
+  return Vec128<T, N>{v.raw};
+}
+
+// mask ? yes : no
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> IfThenElse(Mask128<T, N> mask,
+                                                 Vec128<T, N> yes,
+                                                 Vec128<T, N> no) {
+  return Vec128<T, N>{_mm_blendv_epi8(no.raw, yes.raw, mask.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> IfThenElse(
+    const Mask128<float, N> mask, const Vec128<float, N> yes,
+    const Vec128<float, N> no) {
+  return Vec128<float, N>{_mm_blendv_ps(no.raw, yes.raw, mask.raw)};
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, N> IfThenElse(
+    const Mask128<double, N> mask, const Vec128<double, N> yes,
+    const Vec128<double, N> no) {
+  return Vec128<double, N>{_mm_blendv_pd(no.raw, yes.raw, mask.raw)};
+}
+
+// mask ? yes : 0
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> IfThenElseZero(Mask128<T, N> mask,
+                                                     Vec128<T, N> yes) {
+  return yes & VecFromMask(mask);
+}
+
+// mask ? 0 : no
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> IfThenZeroElse(Mask128<T, N> mask,
+                                                     Vec128<T, N> no) {
+  return AndNot(VecFromMask(mask), no);
+}
+
+template <typename T, size_t N, HWY_IF_FLOAT(T)>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> ZeroIfNegative(Vec128<T, N> v) {
+  const Desc<T, N> d;
+  return IfThenElse(MaskFromVec(v), Zero(d), v);
+}
+
+// ================================================== MEMORY
+
+// ------------------------------ Load
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> Load(Full128<T> /* tag */,
+                                        const T* HWY_RESTRICT aligned) {
+  return Vec128<T>{_mm_load_si128(reinterpret_cast<const __m128i*>(aligned))};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> Load(Full128<float> /* tag */,
+                                            const float* HWY_RESTRICT aligned) {
+  return Vec128<float>{_mm_load_ps(aligned)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double> Load(
+    Full128<double> /* tag */, const double* HWY_RESTRICT aligned) {
+  return Vec128<double>{_mm_load_pd(aligned)};
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> LoadU(Full128<T> /* tag */,
+                                         const T* HWY_RESTRICT p) {
+  return Vec128<T>{_mm_loadu_si128(reinterpret_cast<const __m128i*>(p))};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> LoadU(Full128<float> /* tag */,
+                                             const float* HWY_RESTRICT p) {
+  return Vec128<float>{_mm_loadu_ps(p)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double> LoadU(Full128<double> /* tag */,
+                                              const double* HWY_RESTRICT p) {
+  return Vec128<double>{_mm_loadu_pd(p)};
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, 8 / sizeof(T)> Load(
+    Desc<T, 8 / sizeof(T)> /* tag */, const T* HWY_RESTRICT p) {
+  return Vec128<T, 8 / sizeof(T)>{
+      _mm_loadl_epi64(reinterpret_cast<const __m128i*>(p))};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, 2> Load(Desc<float, 2> /* tag */,
+                                               const float* HWY_RESTRICT p) {
+  const __m128 hi = _mm_setzero_ps();
+  return Vec128<float, 2>{_mm_loadl_pi(hi, reinterpret_cast<const __m64*>(p))};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, 1> Load(Desc<double, 1> /* tag */,
+                                                const double* HWY_RESTRICT p) {
+  const __m128d hi = _mm_setzero_pd();
+  return Vec128<double, 1>{_mm_loadl_pd(hi, p)};
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, 4 / sizeof(T)> Load(
+    Desc<T, 4 / sizeof(T)> /* tag */, const T* HWY_RESTRICT p) {
+  // TODO(janwas): load_ss?
+  int32_t bits;
+  CopyBytes<4>(p, &bits);
+  return Vec128<T, 4 / sizeof(T)>{_mm_cvtsi32_si128(bits)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, 1> Load(Desc<float, 1> /* tag */,
+                                               const float* HWY_RESTRICT p) {
+  return Vec128<float, 1>{_mm_load_ss(p)};
+}
+
+// 128-bit SIMD => nothing to duplicate, same as an unaligned load.
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> LoadDup128(Full128<T> d,
+                                              const T* HWY_RESTRICT p) {
+  return LoadU(d, p);
+}
+
+// ------------------------------ Store
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE void Store(Vec128<T> v, Full128<T> /* tag */,
+                                    T* HWY_RESTRICT aligned) {
+  _mm_store_si128(reinterpret_cast<__m128i*>(aligned), v.raw);
+}
+HWY_ATTR_SSE4 HWY_INLINE void Store(const Vec128<float> v,
+                                    Full128<float> /* tag */,
+                                    float* HWY_RESTRICT aligned) {
+  _mm_store_ps(aligned, v.raw);
+}
+HWY_ATTR_SSE4 HWY_INLINE void Store(const Vec128<double> v,
+                                    Full128<double> /* tag */,
+                                    double* HWY_RESTRICT aligned) {
+  _mm_store_pd(aligned, v.raw);
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE void StoreU(Vec128<T> v, Full128<T> /* tag */,
+                                     T* HWY_RESTRICT p) {
+  _mm_storeu_si128(reinterpret_cast<__m128i*>(p), v.raw);
+}
+HWY_ATTR_SSE4 HWY_INLINE void StoreU(const Vec128<float> v,
+                                     Full128<float> /* tag */,
+                                     float* HWY_RESTRICT p) {
+  _mm_storeu_ps(p, v.raw);
+}
+HWY_ATTR_SSE4 HWY_INLINE void StoreU(const Vec128<double> v,
+                                     Full128<double> /* tag */,
+                                     double* HWY_RESTRICT p) {
+  _mm_storeu_pd(p, v.raw);
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE void Store(Vec128<T, 8 / sizeof(T)> v,
+                                    Desc<T, 8 / sizeof(T)> /* tag */,
+                                    T* HWY_RESTRICT p) {
+  // Avoid _mm_storel_epi64 - triggers clang-analyzer uninitialized-memory error
+  // (apparently a false alarm because MSAN does not fail).
+  CopyBytes<8>(&v, p);
+}
+HWY_ATTR_SSE4 HWY_INLINE void Store(const Vec128<float, 2> v,
+                                    Desc<float, 2> /* tag */,
+                                    float* HWY_RESTRICT p) {
+  _mm_storel_pi(reinterpret_cast<__m64*>(p), v.raw);
+}
+HWY_ATTR_SSE4 HWY_INLINE void Store(const Vec128<double, 1> v,
+                                    Desc<double, 1> /* tag */,
+                                    double* HWY_RESTRICT p) {
+  _mm_storel_pd(p, v.raw);
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE void Store(Vec128<T, 4 / sizeof(T)> v,
+                                    Desc<T, 4 / sizeof(T)> /* tag */,
+                                    T* HWY_RESTRICT p) {
+  // _mm_storeu_si32 is documented but unavailable in Clang; CopyBytes generates
+  // bad code; type-punning is unsafe; this actually generates MOVD.
+  _mm_store_ss(reinterpret_cast<float * HWY_RESTRICT>(p),
+               _mm_castsi128_ps(v.raw));
+}
+HWY_ATTR_SSE4 HWY_INLINE void Store(const Vec128<float, 1> v,
+                                    Desc<float, 1> /* tag */,
+                                    float* HWY_RESTRICT p) {
+  _mm_store_ss(p, v.raw);
+}
+
+// ------------------------------ Non-temporal stores
+
+// Same as aligned stores on non-x86.
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE void Stream(Vec128<T> v, Full128<T> /* tag */,
+                                     T* HWY_RESTRICT aligned) {
+  _mm_stream_si128(reinterpret_cast<__m128i*>(aligned), v.raw);
+}
+HWY_ATTR_SSE4 HWY_INLINE void Stream(const Vec128<float> v,
+                                     Full128<float> /* tag */,
+                                     float* HWY_RESTRICT aligned) {
+  _mm_stream_ps(aligned, v.raw);
+}
+HWY_ATTR_SSE4 HWY_INLINE void Stream(const Vec128<double> v,
+                                     Full128<double> /* tag */,
+                                     double* HWY_RESTRICT aligned) {
+  _mm_stream_pd(aligned, v.raw);
+}
+
+// ------------------------------ Gather
+
+// Unsupported.
+
+// ================================================== SWIZZLE
+
+// ------------------------------ Extract lane
+
+// Gets the single value stored in a vector/part.
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE uint16_t GetLane(const Vec128<uint16_t, N> v) {
+  return _mm_cvtsi128_si32(v.raw) & 0xFFFF;
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE int16_t GetLane(const Vec128<int16_t, N> v) {
+  return _mm_cvtsi128_si32(v.raw) & 0xFFFF;
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE uint32_t GetLane(const Vec128<uint32_t, N> v) {
+  return _mm_cvtsi128_si32(v.raw);
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE int32_t GetLane(const Vec128<int32_t, N> v) {
+  return _mm_cvtsi128_si32(v.raw);
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE float GetLane(const Vec128<float, N> v) {
+  return _mm_cvtss_f32(v.raw);
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE uint64_t GetLane(const Vec128<uint64_t, N> v) {
+  return _mm_cvtsi128_si64(v.raw);
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE int64_t GetLane(const Vec128<int64_t, N> v) {
+  return _mm_cvtsi128_si64(v.raw);
+}
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE double GetLane(const Vec128<double, N> v) {
+  return _mm_cvtsd_f64(v.raw);
+}
+
+// ------------------------------ Extract half
+
+// Returns upper/lower half of a vector.
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N / 2> LowerHalf(Vec128<T, N> v) {
+  return Vec128<T, N / 2>{v.raw};
+}
+
+// These copy hi into lo (smaller instruction encoding than shifts).
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, 8 / sizeof(T)> UpperHalf(Vec128<T> v) {
+  return Vec128<T, 8 / sizeof(T)>{_mm_unpackhi_epi64(v.raw, v.raw)};
+}
+template <>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, 2> UpperHalf(Vec128<float> v) {
+  return Vec128<float, 2>{_mm_movehl_ps(v.raw, v.raw)};
+}
+template <>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double, 1> UpperHalf(Vec128<double> v) {
+  return Vec128<double, 1>{_mm_unpackhi_pd(v.raw, v.raw)};
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, 8 / sizeof(T)> GetHalf(Lower /* tag */,
+                                                          Vec128<T> v) {
+  return LowerHalf(v);
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, 8 / sizeof(T)> GetHalf(Upper /* tag */,
+                                                          Vec128<T> v) {
+  return UpperHalf(v);
+}
+
+// ------------------------------ Shift vector by constant #bytes
+
+// 0x01..0F, kBytes = 1 => 0x02..0F00
+template <int kBytes, typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> ShiftLeftBytes(const Vec128<T> v) {
+  static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
+  return Vec128<T>{_mm_slli_si128(v.raw, kBytes)};
+}
+
+template <int kLanes, typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> ShiftLeftLanes(const Vec128<T> v) {
+  return ShiftLeftBytes<kLanes * sizeof(T)>(v);
+}
+
+// 0x01..0F, kBytes = 1 => 0x0001..0E
+template <int kBytes, typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> ShiftRightBytes(const Vec128<T> v) {
+  static_assert(0 <= kBytes && kBytes <= 16, "Invalid kBytes");
+  return Vec128<T>{_mm_srli_si128(v.raw, kBytes)};
+}
+
+template <int kLanes, typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> ShiftRightLanes(const Vec128<T> v) {
+  return ShiftRightBytes<kLanes * sizeof(T)>(v);
+}
+
+// ------------------------------ Extract from 2x 128-bit at constant offset
+
+// Extracts 128 bits from <hi, lo> by skipping the least-significant kBytes.
+template <int kBytes, typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> CombineShiftRightBytes(const Vec128<T> hi,
+                                                          const Vec128<T> lo) {
+  const Full128<uint8_t> d8;
+  const Vec128<uint8_t> extracted_bytes{
+      _mm_alignr_epi8(BitCast(d8, hi).raw, BitCast(d8, lo).raw, kBytes)};
+  return BitCast(Full128<T>(), extracted_bytes);
+}
+
+// ------------------------------ Broadcast/splat any lane
+
+// Unsigned
+template <int kLane>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t> Broadcast(const Vec128<uint16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  if (kLane < 4) {
+    const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
+    return Vec128<uint16_t>{_mm_unpacklo_epi64(lo, lo)};
+  } else {
+    const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
+    return Vec128<uint16_t>{_mm_unpackhi_epi64(hi, hi)};
+  }
+}
+template <int kLane>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> Broadcast(const Vec128<uint32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)};
+}
+template <int kLane>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t> Broadcast(const Vec128<uint64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec128<uint64_t>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
+}
+
+// Signed
+template <int kLane>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t> Broadcast(const Vec128<int16_t> v) {
+  static_assert(0 <= kLane && kLane < 8, "Invalid lane");
+  if (kLane < 4) {
+    const __m128i lo = _mm_shufflelo_epi16(v.raw, (0x55 * kLane) & 0xFF);
+    return Vec128<int16_t>{_mm_unpacklo_epi64(lo, lo)};
+  } else {
+    const __m128i hi = _mm_shufflehi_epi16(v.raw, (0x55 * (kLane - 4)) & 0xFF);
+    return Vec128<int16_t>{_mm_unpackhi_epi64(hi, hi)};
+  }
+}
+template <int kLane>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> Broadcast(const Vec128<int32_t> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x55 * kLane)};
+}
+template <int kLane>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t> Broadcast(const Vec128<int64_t> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec128<int64_t>{_mm_shuffle_epi32(v.raw, kLane ? 0xEE : 0x44)};
+}
+
+// Float
+template <int kLane>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> Broadcast(const Vec128<float> v) {
+  static_assert(0 <= kLane && kLane < 4, "Invalid lane");
+  return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x55 * kLane)};
+}
+template <int kLane>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double> Broadcast(const Vec128<double> v) {
+  static_assert(0 <= kLane && kLane < 2, "Invalid lane");
+  return Vec128<double>{_mm_shuffle_pd(v.raw, v.raw, 3 * kLane)};
+}
+
+// ------------------------------ Shuffle bytes with variable indices
+
+// Returns vector of bytes[from[i]]. "from" is also interpreted as bytes:
+// either valid indices in [0, 16) or >= 0x80 to zero the i-th output byte.
+template <typename T, typename TI>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> TableLookupBytes(const Vec128<T> bytes,
+                                                    const Vec128<TI> from) {
+  return Vec128<T>{_mm_shuffle_epi8(bytes.raw, from.raw)};
+}
+
+// ------------------------------ Hard-coded shuffles
+
+// Notation: let Vec128<int32_t> have lanes 3,2,1,0 (0 is least-significant).
+// Shuffle0321 rotates one lane to the right (the previous least-significant
+// lane is now most-significant). These could also be implemented via
+// CombineShiftRightBytes but the shuffle_abcd notation is more convenient.
+
+// Swap 32-bit halves in 64-bit halves.
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> Shuffle2301(
+    const Vec128<uint32_t> v) {
+  return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0xB1)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> Shuffle2301(const Vec128<int32_t> v) {
+  return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0xB1)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> Shuffle2301(const Vec128<float> v) {
+  return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0xB1)};
+}
+
+// Swap 64-bit halves
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> Shuffle1032(
+    const Vec128<uint32_t> v) {
+  return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> Shuffle1032(const Vec128<int32_t> v) {
+  return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> Shuffle1032(const Vec128<float> v) {
+  return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x4E)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t> Shuffle01(const Vec128<uint64_t> v) {
+  return Vec128<uint64_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t> Shuffle01(const Vec128<int64_t> v) {
+  return Vec128<int64_t>{_mm_shuffle_epi32(v.raw, 0x4E)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double> Shuffle01(const Vec128<double> v) {
+  return Vec128<double>{_mm_shuffle_pd(v.raw, v.raw, 1)};
+}
+
+// Rotate right 32 bits
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> Shuffle0321(
+    const Vec128<uint32_t> v) {
+  return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x39)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> Shuffle0321(const Vec128<int32_t> v) {
+  return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x39)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> Shuffle0321(const Vec128<float> v) {
+  return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x39)};
+}
+// Rotate left 32 bits
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> Shuffle2103(
+    const Vec128<uint32_t> v) {
+  return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x93)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> Shuffle2103(const Vec128<int32_t> v) {
+  return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x93)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> Shuffle2103(const Vec128<float> v) {
+  return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x93)};
+}
+
+// Reverse
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> Shuffle0123(
+    const Vec128<uint32_t> v) {
+  return Vec128<uint32_t>{_mm_shuffle_epi32(v.raw, 0x1B)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> Shuffle0123(const Vec128<int32_t> v) {
+  return Vec128<int32_t>{_mm_shuffle_epi32(v.raw, 0x1B)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> Shuffle0123(const Vec128<float> v) {
+  return Vec128<float>{_mm_shuffle_ps(v.raw, v.raw, 0x1B)};
+}
+
+// ------------------------------ Permute (runtime variable)
+
+// Returned by SetTableIndices for use by TableLookupLanes.
+template <typename T>
+struct permute_sse4 {
+  __m128i raw;
+};
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE permute_sse4<T> SetTableIndices(Full128<T> d,
+                                                         const int32_t* idx) {
+#if !defined(NDEBUG) || defined(ADDRESS_SANITIZER)
+  for (size_t i = 0; i < d.N; ++i) {
+    if (idx[i] >= static_cast<int32_t>(d.N)) {
+      printf("SetTableIndices [%zu] = %d >= %zu\n", i, idx[i], d.N);
+      HWY_TRAP();
+    }
+  }
+#else
+  (void)d;
+#endif
+
+  const Full128<uint8_t> d8;
+  alignas(16) uint8_t control[d8.N];
+  for (size_t idx_byte = 0; idx_byte < d8.N; ++idx_byte) {
+    const size_t idx_lane = idx_byte / sizeof(T);
+    const size_t mod = idx_byte % sizeof(T);
+    control[idx_byte] = idx[idx_lane] * sizeof(T) + mod;
+  }
+  return permute_sse4<T>{Load(d8, control).raw};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> TableLookupLanes(
+    const Vec128<uint32_t> v, const permute_sse4<uint32_t> idx) {
+  return TableLookupBytes(v, Vec128<uint8_t>{idx.raw});
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> TableLookupLanes(
+    const Vec128<int32_t> v, const permute_sse4<int32_t> idx) {
+  return TableLookupBytes(v, Vec128<uint8_t>{idx.raw});
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> TableLookupLanes(
+    const Vec128<float> v, const permute_sse4<float> idx) {
+  const Full128<int32_t> di;
+  const Full128<float> df;
+  return BitCast(df,
+                 TableLookupBytes(BitCast(di, v), Vec128<uint8_t>{idx.raw}));
+}
+
+// ------------------------------ Interleave lanes
+
+// Interleaves lanes from halves of the 128-bit blocks of "a" (which provides
+// the least-significant lane) and "b". To concatenate two half-width integers
+// into one, use ZipLo/hi instead (also works with scalar).
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t> InterleaveLo(const Vec128<uint8_t> a,
+                                                      const Vec128<uint8_t> b) {
+  return Vec128<uint8_t>{_mm_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t> InterleaveLo(
+    const Vec128<uint16_t> a, const Vec128<uint16_t> b) {
+  return Vec128<uint16_t>{_mm_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> InterleaveLo(
+    const Vec128<uint32_t> a, const Vec128<uint32_t> b) {
+  return Vec128<uint32_t>{_mm_unpacklo_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t> InterleaveLo(
+    const Vec128<uint64_t> a, const Vec128<uint64_t> b) {
+  return Vec128<uint64_t>{_mm_unpacklo_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t> InterleaveLo(const Vec128<int8_t> a,
+                                                     const Vec128<int8_t> b) {
+  return Vec128<int8_t>{_mm_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t> InterleaveLo(const Vec128<int16_t> a,
+                                                      const Vec128<int16_t> b) {
+  return Vec128<int16_t>{_mm_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> InterleaveLo(const Vec128<int32_t> a,
+                                                      const Vec128<int32_t> b) {
+  return Vec128<int32_t>{_mm_unpacklo_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t> InterleaveLo(const Vec128<int64_t> a,
+                                                      const Vec128<int64_t> b) {
+  return Vec128<int64_t>{_mm_unpacklo_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> InterleaveLo(const Vec128<float> a,
+                                                    const Vec128<float> b) {
+  return Vec128<float>{_mm_unpacklo_ps(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double> InterleaveLo(const Vec128<double> a,
+                                                     const Vec128<double> b) {
+  return Vec128<double>{_mm_unpacklo_pd(a.raw, b.raw)};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t> InterleaveHi(const Vec128<uint8_t> a,
+                                                      const Vec128<uint8_t> b) {
+  return Vec128<uint8_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t> InterleaveHi(
+    const Vec128<uint16_t> a, const Vec128<uint16_t> b) {
+  return Vec128<uint16_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> InterleaveHi(
+    const Vec128<uint32_t> a, const Vec128<uint32_t> b) {
+  return Vec128<uint32_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t> InterleaveHi(
+    const Vec128<uint64_t> a, const Vec128<uint64_t> b) {
+  return Vec128<uint64_t>{_mm_unpackhi_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t> InterleaveHi(const Vec128<int8_t> a,
+                                                     const Vec128<int8_t> b) {
+  return Vec128<int8_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t> InterleaveHi(const Vec128<int16_t> a,
+                                                      const Vec128<int16_t> b) {
+  return Vec128<int16_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> InterleaveHi(const Vec128<int32_t> a,
+                                                      const Vec128<int32_t> b) {
+  return Vec128<int32_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t> InterleaveHi(const Vec128<int64_t> a,
+                                                      const Vec128<int64_t> b) {
+  return Vec128<int64_t>{_mm_unpackhi_epi64(a.raw, b.raw)};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> InterleaveHi(const Vec128<float> a,
+                                                    const Vec128<float> b) {
+  return Vec128<float>{_mm_unpackhi_ps(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double> InterleaveHi(const Vec128<double> a,
+                                                     const Vec128<double> b) {
+  return Vec128<double>{_mm_unpackhi_pd(a.raw, b.raw)};
+}
+
+// ------------------------------ Zip lanes
+
+// Same as interleave_*, except that the return lanes are double-width integers;
+// this is necessary because the single-lane scalar cannot return two values.
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t> ZipLo(const Vec128<uint8_t> a,
+                                                const Vec128<uint8_t> b) {
+  return Vec128<uint16_t>{_mm_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> ZipLo(const Vec128<uint16_t> a,
+                                                const Vec128<uint16_t> b) {
+  return Vec128<uint32_t>{_mm_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t> ZipLo(const Vec128<uint32_t> a,
+                                                const Vec128<uint32_t> b) {
+  return Vec128<uint64_t>{_mm_unpacklo_epi32(a.raw, b.raw)};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t> ZipLo(const Vec128<int8_t> a,
+                                               const Vec128<int8_t> b) {
+  return Vec128<int16_t>{_mm_unpacklo_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> ZipLo(const Vec128<int16_t> a,
+                                               const Vec128<int16_t> b) {
+  return Vec128<int32_t>{_mm_unpacklo_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t> ZipLo(const Vec128<int32_t> a,
+                                               const Vec128<int32_t> b) {
+  return Vec128<int64_t>{_mm_unpacklo_epi32(a.raw, b.raw)};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t> ZipHi(const Vec128<uint8_t> a,
+                                                const Vec128<uint8_t> b) {
+  return Vec128<uint16_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> ZipHi(const Vec128<uint16_t> a,
+                                                const Vec128<uint16_t> b) {
+  return Vec128<uint32_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t> ZipHi(const Vec128<uint32_t> a,
+                                                const Vec128<uint32_t> b) {
+  return Vec128<uint64_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t> ZipHi(const Vec128<int8_t> a,
+                                               const Vec128<int8_t> b) {
+  return Vec128<int16_t>{_mm_unpackhi_epi8(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> ZipHi(const Vec128<int16_t> a,
+                                               const Vec128<int16_t> b) {
+  return Vec128<int32_t>{_mm_unpackhi_epi16(a.raw, b.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t> ZipHi(const Vec128<int32_t> a,
+                                               const Vec128<int32_t> b) {
+  return Vec128<int64_t>{_mm_unpackhi_epi32(a.raw, b.raw)};
+}
+
+// ------------------------------ Blocks
+
+// hiH,hiL loH,loL |-> hiL,loL (= lower halves)
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> ConcatLoLo(const Vec128<T> hi,
+                                              const Vec128<T> lo) {
+  const Full128<uint64_t> d64;
+  return BitCast(Full128<T>(),
+                 InterleaveLo(BitCast(d64, lo), BitCast(d64, hi)));
+}
+
+// hiH,hiL loH,loL |-> hiH,loH (= upper halves)
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> ConcatHiHi(const Vec128<T> hi,
+                                              const Vec128<T> lo) {
+  const Full128<uint64_t> d64;
+  return BitCast(Full128<T>(),
+                 InterleaveHi(BitCast(d64, lo), BitCast(d64, hi)));
+}
+
+// hiH,hiL loH,loL |-> hiL,loH (= inner halves)
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> ConcatLoHi(const Vec128<T> hi,
+                                              const Vec128<T> lo) {
+  return CombineShiftRightBytes<8>(hi, lo);
+}
+
+// hiH,hiL loH,loL |-> hiH,loL (= outer halves)
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> ConcatHiLo(const Vec128<T> hi,
+                                              const Vec128<T> lo) {
+  return Vec128<T>{_mm_blend_epi16(hi.raw, lo.raw, 0x0F)};
+}
+template <>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> ConcatHiLo(const Vec128<float> hi,
+                                                  const Vec128<float> lo) {
+  return Vec128<float>{_mm_blend_ps(hi.raw, lo.raw, 3)};
+}
+template <>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double> ConcatHiLo(const Vec128<double> hi,
+                                                   const Vec128<double> lo) {
+  return Vec128<double>{_mm_blend_pd(hi.raw, lo.raw, 1)};
+}
+
+// ------------------------------ Odd/even lanes
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> odd_even_impl(SizeTag<1> /* tag */,
+                                                 const Vec128<T> a,
+                                                 const Vec128<T> b) {
+  const Full128<T> d;
+  const Full128<uint8_t> d8;
+  alignas(16) constexpr uint8_t mask[16] = {0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0,
+                                            0xFF, 0, 0xFF, 0, 0xFF, 0, 0xFF, 0};
+  return IfThenElse(MaskFromVec(BitCast(d, Load(d8, mask))), b, a);
+}
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> odd_even_impl(SizeTag<2> /* tag */,
+                                                 const Vec128<T> a,
+                                                 const Vec128<T> b) {
+  return Vec128<T>{_mm_blend_epi16(a.raw, b.raw, 0x55)};
+}
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> odd_even_impl(SizeTag<4> /* tag */,
+                                                 const Vec128<T> a,
+                                                 const Vec128<T> b) {
+  return Vec128<T>{_mm_blend_epi16(a.raw, b.raw, 0x33)};
+}
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> odd_even_impl(SizeTag<8> /* tag */,
+                                                 const Vec128<T> a,
+                                                 const Vec128<T> b) {
+  return Vec128<T>{_mm_blend_epi16(a.raw, b.raw, 0x0F)};
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T> OddEven(const Vec128<T> a,
+                                           const Vec128<T> b) {
+  return odd_even_impl(SizeTag<sizeof(T)>(), a, b);
+}
+template <>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float> OddEven<float>(const Vec128<float> a,
+                                                      const Vec128<float> b) {
+  return Vec128<float>{_mm_blend_ps(a.raw, b.raw, 5)};
+}
+
+template <>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double> OddEven<double>(
+    const Vec128<double> a, const Vec128<double> b) {
+  return Vec128<double>{_mm_blend_pd(a.raw, b.raw, 1)};
+}
+
+// ================================================== CONVERT
+
+// ------------------------------ Promotions (part w/ narrow lanes -> full)
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<double> ConvertTo(Full128<double> /* tag */,
+                                                  const Vec128<float, 2> v) {
+  return Vec128<double>{_mm_cvtps_pd(v.raw)};
+}
+
+// Unsigned: zero-extend.
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t> ConvertTo(
+    Full128<uint16_t> /* tag */, const Vec128<uint8_t, 8> v) {
+  return Vec128<uint16_t>{_mm_cvtepu8_epi16(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> ConvertTo(
+    Full128<uint32_t> /* tag */, const Vec128<uint8_t, 4> v) {
+  return Vec128<uint32_t>{_mm_cvtepu8_epi32(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t> ConvertTo(Full128<int16_t> /* tag */,
+                                                   const Vec128<uint8_t, 8> v) {
+  return Vec128<int16_t>{_mm_cvtepu8_epi16(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                                   const Vec128<uint8_t, 4> v) {
+  return Vec128<int32_t>{_mm_cvtepu8_epi32(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> ConvertTo(
+    Full128<uint32_t> /* tag */, const Vec128<uint16_t, 4> v) {
+  return Vec128<uint32_t>{_mm_cvtepu16_epi32(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> ConvertTo(
+    Full128<int32_t> /* tag */, const Vec128<uint16_t, 4> v) {
+  return Vec128<int32_t>{_mm_cvtepu16_epi32(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t> ConvertTo(
+    Full128<uint64_t> /* tag */, const Vec128<uint32_t, 2> v) {
+  return Vec128<uint64_t>{_mm_cvtepu32_epi64(v.raw)};
+}
+
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint32_t> U32FromU8(const Vec128<uint8_t> v) {
+  return Vec128<uint32_t>{_mm_cvtepu8_epi32(v.raw)};
+}
+
+// Signed: replicate sign bit.
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t> ConvertTo(Full128<int16_t> /* tag */,
+                                                   const Vec128<int8_t, 8> v) {
+  return Vec128<int16_t>{_mm_cvtepi8_epi16(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                                   const Vec128<int8_t, 4> v) {
+  return Vec128<int32_t>{_mm_cvtepi8_epi32(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t> ConvertTo(Full128<int32_t> /* tag */,
+                                                   const Vec128<int16_t, 4> v) {
+  return Vec128<int32_t>{_mm_cvtepi16_epi32(v.raw)};
+}
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int64_t> ConvertTo(Full128<int64_t> /* tag */,
+                                                   const Vec128<int32_t, 2> v) {
+  return Vec128<int64_t>{_mm_cvtepi32_epi64(v.raw)};
+}
+
+// ------------------------------ Demotions (full -> part w/ narrow lanes)
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t, N> ConvertTo(
+    Desc<uint16_t, N> /* tag */, const Vec128<int32_t, N> v) {
+  return Vec128<uint16_t, N>{_mm_packus_epi32(v.raw, v.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> ConvertTo(
+    Desc<uint8_t, N> /* tag */, const Vec128<int32_t> v) {
+  const __m128i u16 = _mm_packus_epi32(v.raw, v.raw);
+  return Vec128<uint8_t, N>{_mm_packus_epi16(u16, u16)};
+}
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, N> ConvertTo(
+    Desc<uint8_t, N> /* tag */, const Vec128<int16_t> v) {
+  return Vec128<uint8_t, N>{_mm_packus_epi16(v.raw, v.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int16_t, N> ConvertTo(
+    Desc<int16_t, N> /* tag */, const Vec128<int32_t> v) {
+  return Vec128<int16_t, N>{_mm_packs_epi32(v.raw, v.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> ConvertTo(Desc<int8_t, N> /* tag */,
+                                                     const Vec128<int32_t> v) {
+  const __m128i i16 = _mm_packs_epi32(v.raw, v.raw);
+  return Vec128<int8_t, N>{_mm_packs_epi16(i16, i16)};
+}
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int8_t, N> ConvertTo(Desc<int8_t, N> /* tag */,
+                                                     const Vec128<int16_t> v) {
+  return Vec128<int8_t, N>{_mm_packs_epi16(v.raw, v.raw)};
+}
+
+// For already range-limited input [0, 255].
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint8_t, 4> U8FromU32(
+    const Vec128<uint32_t> v) {
+  const Full128<uint32_t> d32;
+  const Full128<uint8_t> d8;
+  alignas(16) static constexpr uint32_t k8From32[4] = {
+      0x0C080400u, 0x0C080400u, 0x0C080400u, 0x0C080400u};
+  // Also replicate bytes into all 32 bit lanes for safety.
+  const auto quad = TableLookupBytes(v, Load(d32, k8From32));
+  return LowerHalf(LowerHalf(BitCast(d8, quad)));
+}
+
+// ------------------------------ Convert i32 <=> f32
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<float, N> ConvertTo(
+    Desc<float, N> /* tag */, const Vec128<int32_t, N> v) {
+  return Vec128<float, N>{_mm_cvtepi32_ps(v.raw)};
+}
+// Truncates (rounds toward zero).
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> ConvertTo(
+    Desc<int32_t, N> /* tag */, const Vec128<float, N> v) {
+  return Vec128<int32_t, N>{_mm_cvttps_epi32(v.raw)};
+}
+
+template <size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<int32_t, N> NearestInt(
+    const Vec128<float, N> v) {
+  return Vec128<int32_t, N>{_mm_cvtps_epi32(v.raw)};
+}
+
+// ================================================== MISC
+
+// "Extensions": useful but not quite performance-portable operations. We add
+// functions to this namespace in multiple places.
+namespace ext {
+
+// ------------------------------ movemask
+
+// Returns a bit array of the most significant bit of each byte in "v", i.e.
+// sum_i=0..15 of (v[i] >> 7) << i; v[0] is the least-significant byte of "v".
+// This is useful for testing/branching based on comparison results.
+HWY_ATTR_SSE4 HWY_INLINE uint64_t movemask(const Vec128<uint8_t> v) {
+  return static_cast<unsigned>(_mm_movemask_epi8(v.raw));
+}
+
+// Returns the most significant bit of each float/double lane (see above).
+HWY_ATTR_SSE4 HWY_INLINE uint64_t movemask(const Vec128<float> v) {
+  return static_cast<unsigned>(_mm_movemask_ps(v.raw));
+}
+HWY_ATTR_SSE4 HWY_INLINE uint64_t movemask(const Vec128<double> v) {
+  return static_cast<unsigned>(_mm_movemask_pd(v.raw));
+}
+
+// ------------------------------ mask
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE bool AllFalse(const Mask128<T> v) {
+  // Cheaper than PTEST, which is 2 uop / 3L.
+  const auto bytes = BitCast(Full128<uint8_t>(), VecFromMask(v));
+  return movemask(bytes) == 0;
+}
+HWY_ATTR_SSE4 HWY_INLINE bool AllFalse(const Mask128<float> v) {
+  return movemask(VecFromMask(v)) == 0;
+}
+HWY_ATTR_SSE4 HWY_INLINE bool AllFalse(const Mask128<double> v) {
+  return movemask(VecFromMask(v)) == 0;
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE bool AllTrue(const Mask128<T> v) {
+  const auto bytes = BitCast(Full128<uint8_t>(), VecFromMask(v));
+  return movemask(bytes) == 0xFFFF;
+}
+HWY_ATTR_SSE4 HWY_INLINE bool AllTrue(const Mask128<float> v) {
+  return movemask(VecFromMask(v)) == 0xF;
+}
+HWY_ATTR_SSE4 HWY_INLINE bool AllTrue(const Mask128<double> v) {
+  return movemask(VecFromMask(v)) == 3;
+}
+
+template <typename T>
+HWY_ATTR_SSE4 HWY_INLINE size_t CountTrue(const Mask128<T> v) {
+  // Integer vectors: only have movemask for u8, so divide by number of bytes.
+  const auto bytes = BitCast(Full128<uint8_t>(), VecFromMask(v));
+  return PopCount(movemask(bytes)) / sizeof(T);
+}
+HWY_ATTR_SSE4 HWY_INLINE size_t CountTrue(const Mask128<float> v) {
+  return PopCount(movemask(VecFromMask(v)));
+}
+HWY_ATTR_SSE4 HWY_INLINE size_t CountTrue(const Mask128<double> v) {
+  return PopCount(movemask(VecFromMask(v)));
+}
+
+// ------------------------------ minpos
+
+// Returns index and min value in lanes 1 and 0.
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint16_t> minpos(const Vec128<uint16_t> v) {
+  return Vec128<uint16_t>{_mm_minpos_epu16(v.raw)};
+}
+
+// ------------------------------ Horizontal sum (reduction)
+
+// Returns 64-bit sums of 8-byte groups.
+HWY_ATTR_SSE4 HWY_INLINE Vec128<uint64_t> SumsOfU8x8(const Vec128<uint8_t> v) {
+  return Vec128<uint64_t>{_mm_sad_epu8(v.raw, _mm_setzero_si128())};
+}
+
+// For u32/i32/f32.
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> horz_sum_impl(SizeTag<4> /* tag */,
+                                                    const Vec128<T, N> v3210) {
+  const Vec128<T> v1032 = Shuffle1032(v3210);
+  const Vec128<T> v31_20_31_20 = v3210 + v1032;
+  const Vec128<T> v20_31_20_31 = Shuffle0321(v31_20_31_20);
+  return v20_31_20_31 + v31_20_31_20;
+}
+
+// For u64/i64/f64.
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> horz_sum_impl(SizeTag<8> /* tag */,
+                                                    const Vec128<T, N> v10) {
+  const Vec128<T> v01 = Shuffle01(v10);
+  return v10 + v01;
+}
+
+// Supported for u/i/f 32/64. Returns the sum in each lane.
+template <typename T, size_t N>
+HWY_ATTR_SSE4 HWY_INLINE Vec128<T, N> SumOfLanes(const Vec128<T, N> v) {
+  return horz_sum_impl(SizeTag<sizeof(T)>(), v);
+}
+
+}  // namespace ext
+
+// TODO(janwas): wrappers for all intrinsics (in x86 namespace).
+}  // namespace hwy
+
+#endif  // HWY_X86_SSE4_H_
diff --git a/third_party/highway/instruction_matrix.pdf b/third_party/highway/instruction_matrix.pdf
new file mode 100644
index 000000000..23608f361
Binary files /dev/null and b/third_party/highway/instruction_matrix.pdf differ
diff --git a/third_party/highway/quick_reference.md b/third_party/highway/quick_reference.md
new file mode 100644
index 000000000..7d5625a15
--- /dev/null
+++ b/third_party/highway/quick_reference.md
@@ -0,0 +1,533 @@
+# API synopsis / quick reference
+
+[TOC]
+
+## Headers
+
+*   When targeting a single CPU (chosen at compile-time): include
+    `hwy/static_targets.h`.
+*   When generating implementations for multiple targets: include
+    `hwy/foreach_target.h` per the example in README.md.
+*   For dispatching to the appropriate implementation for the current CPU:
+    include `hwy/runtime_dispatch.h`.
+
+## Preprocessor macros
+
+*   `HWY_ATTR` must be prefixed to any function declaration/definition into
+    which Highway functions are (transitively) inlined.
+
+*   `HWY_BITS`: how many bits in a full vector, or 0 if `NONE`.
+    Example: use `#if HWY_BITS == 0` to provide an alternative to functions
+    such as `TableLookupBytes` which are not supported by scalar.h.
+
+*   `HWY_ALIGN`: Ensures an array is aligned and suitable for Load()/Store()
+    functions. Example: `HWY_ALIGN T lanes[d.N];`
+
+*   `HWY_FUNC` expands to a target-specific function name; the same source code
+    is compiled multiple times to generate implementations for each target.
+
+*   `HWY_NAMESPACE` expands to a target-specific namespace name in which
+    target-specific functions called by HWY_FUNC must reside.
+
+## Vector and descriptor types
+
+SIMD vectors consist of one or more 'lanes' of the same built-in type `T =
+uint##_t, int##_t, float or double` for `## = 8, 16, 32, 64`. Highway provides
+data types for vectors of unspecified sizes `N`, where N is a power of two
+in the range [1, kMaxVectorSize / sizeof(T)].
+
+Highway relies on "descriptors" (zero-sized class templates) to select the
+function overload matching a given `T` and `N`. Users typically define a
+descriptor lvalue `d` using aliases:
+
+*   `const HWY_FULL(T) d;` for the largest available N;
+*   `const HWY_CAPPED(T, N) d;` for up to `N` lanes.
+
+The API does not document the names of the vector types. Instead, user code
+relies on type deduction (`auto`). Examples:
+
+*   Zero-initialize: `auto v0 = Zero(d);`
+*   Uninitialized/output variables: `auto v = Undefined(d); Func(&v);`
+*   Function arguments or return types: `VT<D> Func2(VT<D> arg);`
+
+## Operations
+
+Let `V` denote a `D::N`-lane vector (where `D::N` is a power of two).
+Operations limited to certain types have prefixes `V`: `u8/16` or `uif` for
+unsigned/signed/floating-point types.
+
+### Initialization
+
+*   <code>V **Zero**(D)</code>: returns D::N-lane vector or scalar with all bits
+    set to 0.
+*   <code>V **Set**(D, T)</code>: returns D::N-lane vector or scalar with all
+    lanes equal to the given value of type `T`.
+*   <code>V **Iota**(D, T)</code>: returns D::N-lane vector or scalar where the
+    lane with index `i` has the given value of type `T` plus `i`. The least
+    significant lane has index 0.
+*   <code>V **Undefined**(D)</code>: returns uninitialized D::N-lane vector or
+    scalar.
+
+### Arithmetic
+
+*   <code>V **operator+**(V a, V b)</code>: returns `a[i] + b[i]` (mod 2^bits).
+*   <code>V **operator-**(V a, V b)</code>: returns `a[i] - b[i]` (mod 2^bits).
+
+*   `V`: `ui8/16` \
+    <code>V **SaturatedAdd**(V a, V b)</code> returns `a[i] + b[i]` saturated
+    to the minimum/maximum representable value.
+*   `V`: `ui8/16` \
+    <code>V **SaturatedSub**(V a, V b)</code> returns `a[i] - b[i]` saturated
+    to the minimum/maximum representable value.
+
+*   `V`: `u8/16` \
+    <code>V **AverageRound**(V a, V b)</code> returns `(a[i] + b[i] + 1) / 2`.
+
+*   `V`: `i8/16/32` \
+    <code>V **Abs**(V a)</code> returns the absolute value of `a[i]`;
+    `LimitsMin()` maps to `LimitsMax() + 1`.
+
+*   `V`: `ui8/16/32`, `f` \
+    <code>V **Min**(V a, V b)</code>: returns `min(a[i], b[i])`.
+
+*   `V`: `ui8/16/32`, `f` \
+    <code>V **Max**(V a, V b)</code>: returns `max(a[i], b[i])`.
+
+*   `V`: `ui8/16/32`, `f` \
+    <code>V **Clamp**(V a, V lo, V hi)</code>: returns `a[i]` clamped to
+    `[lo[i], hi[i]]`.
+
+*   `V`: `f` \
+    <code>V **operator/**(V a, V b)</code>: returns `a[i] / b[i]` in each lane.
+
+*   `V`: `f` \
+    <code>V **Sqrt**(V a)</code>: returns `sqrt(a[i])`.
+
+*   `V`: `f32` \
+    <code>V **ApproximateReciprocalSqrt**(V a)</code>: returns an approximation
+    of `1.0 / sqrt(a[i])`. `sqrt(a) ~= ApproximateReciprocalSqrt(a) * a`. x86
+    and PPC provide 12-bit approximations but the error on ARM is closer to 1%.
+
+*   `V`: `f32` \
+    <code>V **ApproximateReciprocal**(V a)</code>: returns an approximation of
+    `1.0 / a[i]`.
+
+*   `V`: `f32` \
+    <code>V **ext::AbsDiff**(V a, V b)</code>: returns `|a[i] - b[i]|` in each
+    lane.
+
+#### Multiply
+
+*   `V`: `ui16/32` \
+    <code>V <b>operator*</b>(V a, V b)</code>: returns the lower half of
+    `a[i] * b[i]` in each lane.
+
+*   `V`: `f` \
+    <code>V <b>operator*</b>(V a, V b)</code>: returns `a[i] * b[i]` in each
+    lane.
+
+*   `V`: `i16` \
+    <code>V **ext::MulHigh**(V a, V b)</code>: returns the upper half of
+    `a[i] * b[i]` in each lane.
+
+*   `V`: `ui32` \
+    <code>V **MulEven**(V a, V b)</code>: returns double-wide result of
+    `a[i] * b[i]` for every even `i`, in lanes `i` (lower) and `i + 1` (upper).
+
+#### Fused multiply-add
+
+When supported, these functions are more precise and faster than separate
+multiplication followed by addition.
+
+*   `V`: `f` \
+    <code>V **MulAdd**(V a, V b, V c)</code>: returns `a[i] * b[i] + c[i]`.
+
+*   `V`: `f` \
+    <code>V **NegMulAdd**(V a, V b, V c)</code>: returns `-a[i] * b[i] + c[i]`.
+
+*   `V`: `f` \
+    <code>V **ext::MulSub**(V a, V b, V c)</code>: returns `a[i] * b[i] - c[i]`.
+
+*   `V`: `f` \
+    <code>V **ext::NegMulSub**(V a, V b, V c)</code>: returns
+    `-a[i] * b[i] - c[i]`.
+
+#### Shifts
+
+**Note**: it is generally fastest to shift by a compile-time constant number of
+bits. ARM requires the count be less than the lane size.
+
+*   `V`: `ui16/32/64` \
+    <code>V **ShiftLeft**&lt;int&gt;(V a)</code> returns `a[i] <<` a
+    compile-time constant count.
+
+*   `V`: `u16/32/64`, `i16/32` \
+    <code>V **ShiftRight**&lt;int&gt;(V a)</code> returns `a[i] >>` a compile-time
+    constant count. Inserts zero or sign bit(s) depending on `V`.
+
+**Note**: independent shifts are only available if `HWY_HAS_VARIABLE_SHIFT`:
+
+*   `V`: `ui32/64` \
+    <code>V **operator<<**(V a, V b)</code> returns `a[i] << b[i]`, which is
+    zero when `b[i] >= sizeof(T)*8`.
+
+*   `V`: `u32/64`, `i32` \
+    <code>V **operator>>**(V a, V b)</code> returns `a[i] >> b[i]`, which is
+    zero when `b[i] >= sizeof(T)*8`. Inserts zero or sign bit(s).
+
+**Note**: the following are only provided if `!HWY_HAS_VARIABLE_SHIFT`:
+
+*   `V`: `ui16/32/64` \
+    <code>V **ShiftLeftSame**(V a, int bits)</code> returns `a[i] << bits`.
+
+*   `V`: `u16/32/64`, `i16/32` \
+    <code>V **ShiftRightSame**(V a, int bits)</code> returns `a[i] >> bits`.
+    Inserts 0 or sign bit(s).
+
+#### Floating-point rounding
+
+*   `V`: `f` \
+    <code>V **Round**(V a)</code>: returns `a[i]` rounded towards the nearest
+    integer, with ties to even.
+
+*   `V`: `f` \
+    <code>V **Trunc**(V a)</code>: returns `a[i]` rounded towards zero
+    (truncate).
+
+*   `V`: `f` \
+    <code>V **Ceil**(V a)</code>: returns `a[i]` rounded towards positive
+    infinity (ceiling).
+
+*   `V`: `f` \
+    <code>V **Floor**(V a)</code>: returns `a[i]` rounded towards negative
+    infinity.
+
+### Logical
+
+These operate on individual bits within each lane, even for floating-point
+vectors.
+
+*   <code>V **operator&**(V a, V b)</code>: returns `a[i] & b[i]`.
+
+*   <code>V **operator|**(V a, V b)</code>: returns `a[i] | b[i]`.
+
+*   <code>V **operator^**(V a, V b)</code>: returns `a[i] ^ b[i]`.
+
+*   <code>V **AndNot**(V a, V b)</code>: returns `~a[i] & b[i]`.
+
+### Masks
+
+Let `M` denote a mask capable of storing true/false for each lane.
+
+*   <code>M **MaskFromVec**(V v)</code>: returns false in lane `i` if
+    `v[i] == 0`, or true if `v[i]` has all bits set.
+
+*   <code>V **VecFromMask**(M m)</code>: returns 0 in lane `i` if
+    `m[i] == false`, otherwise all bits set.
+
+*   <code>V **IfThenElse**(M mask, V yes, V no)</code>:
+    returns `mask[i] ? yes[i] : no[i]`.
+*   <code>V **IfThenElseZero**(M mask, V yes)</code>:
+    returns `mask[i] ? yes[i] : 0`.
+*   <code>V **IfThenZeroElse**(M mask, V no)</code>:
+    returns `mask[i] ? 0 : no[i]`.
+
+*   <code>V **ZeroIfNegative**(V v)</code>: returns `v[i] < 0 ? 0 : v[i]`.
+
+*   <code>bool **ext::AllTrue**(M m)</code>: returns whether all `m[i]` are
+    true.
+*   <code>bool **ext::AllFalse**(M m)</code>: returns whether all `m[i]` are
+    false.
+
+*   <code>size_t **ext::CountTrue**(M m)</code>: returns how many of `m[i]` are
+    true [0, N].
+
+### Comparisons
+
+These return a mask (see above) indicating whether the condition is true.
+
+*   <code>M **operator==**(V a, V b)</code>: returns `a[i] == b[i]`.
+
+*   `V`: `if` \
+    <code>M **operator&lt;**(V a, V b)</code>: returns `a[i] < b[i]`.
+*   `V`: `if` \
+    <code>M **operator&gt;**(V a, V b)</code>: returns `a[i] > b[i]`.
+
+*   `V`: `f` \
+    <code>M **operator&lt;=**(V a, V b)</code>: returns `a[i] <= b[i]`.
+*   `V`: `f` \
+    <code>M **operator&gt;=**(V a, V b)</code>: returns `a[i] >= b[i]`.
+
+*   `V`: `ui` \
+    <code>M **TestBit**(V v, V bit)</code>: returns `(v[i] & bit[i]) == bit[i]`.
+    `bit[i]` must have exactly one bit set.
+
+### Memory
+
+Memory operands are little-endian, otherwise their order would depend on the
+lane configuration. Pointers are the addresses of `N` consecutive `T` values,
+either naturally-aligned (`aligned`) or possibly unaligned (`p`).
+
+#### Load
+
+*   <code>VT&lt;D&gt; **Load**(D, const D::T* aligned)</code>: returns
+    `aligned[i]`.
+*   <code>VT&lt;D&gt; **LoadU**(D, const D::T* p)</code>: returns `p[i]`.
+
+*   <code>VT&lt;D&gt; **LoadDup128**(D, const D::T* p)</code>: returns one
+    128-bit block loaded from `p` and broadcasted into all 128-bit block\[s\].
+    This enables a `ConvertTo` overload that avoids a 3-cycle overhead on
+    AVX2/AVX-512. This is faster than broadcasting single values and useful for
+    specifying constants without having to know the (maximum) vector length.
+
+#### Gather
+
+**Note**: only available if `HWY_HAS_GATHER`:
+
+*   `V`,`VI`: (`uif32,i32`), (`uif64,i64`) \
+    <code>VT&lt;D&gt; **GatherOffset**(D, const D::T* base, VI offsets)</code>.
+    Returns elements of base selected by signed/possibly repeated *byte*
+    `offsets[i]`.
+
+*   `V`,`VI`: (`uif32,i32`), (`uif64,i64`) \
+    <code>VT&lt;D&gt; **GatherIndex**(D, const D::T* base, VI indices)</code>.
+    Returns vector of `base[indices[i]]`. Indices are signed and need not be
+    unique.
+
+#### Store
+
+*   <code>void **Store**(VT&lt;D&gt; a, D, D::T* aligned)</code>: copies `a[i]` into
+    `aligned[i]`.
+*   <code>void **StoreU**(VT&lt;D&gt; a, D, D::T* p)</code>: copies `a[i]` into
+    `p[i]`.
+
+### Cache control
+
+*   <code>void **Stream**(VT&lt;D&gt; a, D, const D::T* aligned)</code>: copies `a[i]`
+    into `aligned[i]` with non-temporal hint on x86 (for good performance, call
+    for all consecutive vectors within the same cache line).
+
+*   `T`: `u32/64` \
+    <code>void **Stream**(T, T* aligned)</code>: copies `T` into `*aligned` with
+    non-temporal hint on x86.
+
+*   <code>void **LoadFence**()</code>: delays subsequent loads until prior loads
+    are visible. Also a full fence on Intel CPUs. No effect on non-x86.
+
+*   <code>void **StoreFence**()</code>: ensures previous non-temporal stores are
+    visible. No effect on non-x86.
+
+*   <code>void **FlushCacheline**(const void* p)</code>: invalidates and flushes
+    the cache line containing "p". No effect on non-x86.
+
+*   <code>void **Prefetch**(const T* p)</code>: begins loading the cache line
+    containing "p".
+
+### Type conversion
+
+*   <code>VT&lt;D&gt; **BitCast**(D, V)</code>: returns the bits of `V` reinterpreted
+    as type `VT<D>`.
+
+*   `V`,`D`: (`u8,i16`), (`u8,i32`), (`u16,i32`), (`i8,i16`), (`i8,i32`),
+    (`i16,i32`), (`f32,f64`) \
+    <code>VT&lt;D&gt; **ConvertTo**(D, V part)</code>: returns `part[i]` zero- or
+    sign-extended to the wider `D::T` type.
+
+*   `V`,`D`: (`u8,u32`) \
+    <code>VT&lt;D&gt; **U32FromU8**(V)</code>: special-case `u8` to `u32` conversion
+    when all blocks of `V` are identical, e.g. from `LoadDup128`.
+
+*   `V`,`D`: (`u32,u8`) \
+    <code>VT&lt;D&gt; **U8FromU32**(V)</code>: special-case `u32` to `u8` conversion
+    when all lanes of `V` are already clamped to `[0, 256)`.
+
+*   `V`,`D`: (`i16,i8`), (`i32,i8`), (`i32,i16`), (`i16,u8`), (`i32,u8`),
+    (`i32,u16`) \
+    <code>VT&lt;D&gt; **ConvertTo**(D, V a)</code>: returns `a[i]` after packing with
+    signed/unsigned saturation, i.e. a vector with narrower lane type
+    `D::T`.
+
+*   `V`,`D`: (`i32`,`f32`) \
+    <code>VT&lt;D&gt; **ConvertTo**(D, V)</code>: converts an int32_t value to float.
+
+*   `V`,`D`: (`f32`,`i32`) \
+    <code>VT&lt;D&gt; **ConvertTo**(D, V)</code>: rounds float towards zero and
+    converts the value to int32_t.
+
+*   `V`: `f32`; `Ret`: `i32` \
+    <code>Ret **NearestInt**(V a)</code>: returns the integer nearest to `a[i]`.
+
+### Swizzle
+
+*   <code>T **GetLane**(V)</code>: returns lane 0 within `V`. This is useful
+    for extracting `ext::SumOfLanes` results.
+
+*   <code>V2 **GetHalf**(Upper/Lower, V)</code>: returns upper or lower half of
+    the vector `V`. When a specific half is needed, `V2 UpperHalf(V)` and
+    `V2 LowerHalf(V)` are more convenient alternatives.
+
+*   <code>V **OddEven**(V a, V b)</code>: returns a vector whose odd lanes are
+    taken from `a` and the even lanes from `b`.
+
+**Note**: if vectors are larger than 128 bits, the following operations split
+their operands into independently processed 128-bit *blocks*.
+
+*   `V`: `ui16/32/64`, `f` \
+    <code>V **Broadcast**&lt;int i&gt;(V)</code>: returns individual *blocks*,
+    each with lanes set to `input_block[i]`, `i = [0, 16/sizeof(T))`.
+
+*   `Ret`: double-width `u/i`; `V`: `u8/16/32`, `i8/16/32` \
+    <code>Ret **ZipLo**(V a, V b)</code>: returns the same bits as InterleaveLo,
+    except that `Ret` is a vector with double-width lanes (required in order to
+    use this operation with `scalar`).
+
+**Note**: the following are only available for full vectors (`N > 1), and split
+their operands into independently processed 128-bit *blocks*:
+
+*   `Ret`: double-width u/i; `V`: `u8/16/32`, `i8/16/32` \
+    <code>Ret **ZipHi**(V a, V b)</code>: returns the same bits as InterleaveHi,
+    except that `Ret` is a vector with double-width lanes (required in order to
+    use this operation with `scalar`).
+
+*   `V`: `ui` \
+    <code>V **ShiftLeftBytes**&lt;int&gt;(V)</code>: returns the result of
+    shifting independent *blocks* left by `int` bytes \[1, 15\].
+
+*   `V`: `ui` \
+    <code>V **ShiftLeftLanes**&lt;int&gt;(V)</code>: returns the result of
+    shifting independent *blocks* left by `int` lanes \[1, 15\].
+
+*   `V`: `ui` \
+    <code>V **ShiftRightBytes**&lt;int&gt;(V)</code>: returns the result of
+    shifting independent *blocks* right by `int` bytes \[1, 15\].
+
+*   `V`: `ui` \
+    <code>V **ShiftRightLanes**&lt;int&gt;(V)</code>: returns the result of
+    shifting independent *blocks* right by `int` lanes \[1, 15\].
+
+*   `V`: `ui` \
+    <code>V **CombineShiftRightBytes**&lt;int&gt;(V hi, V lo)</code>: returns
+    the result of shifting two concatenated *blocks* `hi[i] || lo[i]` right by
+    `int` bytes \[1, 15\].
+
+*   `V`: `ui`; `VI`: `ui` \
+    <code>V **TableLookupBytes**(V bytes, VI from)</code>: returns *blocks* with
+    `bytes[from[i]]`, or zero if bit 7 of byte `from[i]` is set.
+
+*   `V`: `uif32` \
+    <code>V **Shuffle2301**(V)</code>: returns *blocks* with 32-bit halves
+    swapped inside 64-bit halves.
+
+*   `V`: `uif32` \
+    <code>V **Shuffle1032**(V)</code>: returns *blocks* with 64-bit halves
+    swapped.
+
+*   `V`: `uif64` \
+    <code>V **Shuffle01**(V)</code>: returns *blocks* with 64-bit halves
+    swapped.
+
+*   `V`: `uif32` \
+    <code>V **Shuffle0321**(V)</code>: returns *blocks* rotated right (toward
+    the lower end) by 32 bits.
+
+*   `V`: `uif32` \
+    <code>V **Shuffle2103**(V)</code>: returns *blocks* rotated left (toward the
+    upper end) by 32 bits.
+
+*   `V`: `uif32` \
+    <code>V **Shuffle0123**(V)</code>: returns *blocks* with lanes in reverse
+    order.
+
+*   <code>V **InterleaveLo**(V a, V b)</code>: returns *blocks* with alternating
+    lanes from the lower halves of `a` and `b` (`a[0]` in the least-significant
+    lane).
+
+*   <code>V **InterleaveHi**(V a, V b)</code>: returns *blocks* with alternating
+    lanes from the upper halves of `a` and `b` (`a[N/2]` in the
+    least-significant lane).
+
+**Note**: the following operations cross block boundaries, which is typically
+more expensive on AVX2/AVX-512 than within-block operations.
+
+*   <code>V **ConcatLoLo**(V hi, V lo)</code>: returns the concatenation of the
+    lower halves of `hi` and `lo` without splitting into blocks.
+
+*   <code>V **ConcatHiHi**(V hi, V lo)</code>: returns the concatenation of the
+    upper halves of `hi` and `lo` without splitting into blocks.
+
+*   <code>V **ConcatLoHi**(V hi, V lo)</code>: returns the inner half of the
+    concatenation of `hi` and `lo` without splitting into blocks. Useful for
+    swapping the two blocks in 256-bit vectors.
+
+*   <code>V **ConcatHiLo**(V hi, V lo)</code>: returns the outer quarters of the
+    concatenation of `hi` and `lo` without splitting into blocks. Unlike the
+    other variants, this does not incur a block-crossing penalty on AVX2.
+
+*   `V`: `uif32` \
+    <code>V **TableLookupLanes**(V a, VI)</code> returns a vector of
+    `a[indices[i]]`, where `VI` is from `SetTableIndices(D, &indices[0])`.
+
+*   <code>VI **SetTableIndices**(D, int* idx)</code> prepares for
+    `TableLookupLanes` with lane indices `idx = [0, d.N)` (need not be unique).
+
+### Misc
+
+**Note**: the following are only available for full vectors (`N > 1`):
+
+*   `V`: `u8`, `f` \
+    <code>uint64_t **ext::movemask**(V a)</code>: returns sum of
+    `upper_bit(a[i]) << i`.
+
+*   `V`: `u8`; `Ret`: `u64` \
+    <code>Ret **ext::SumsOfU8x8**(V)</code>: returns the sums of 8 consecutive
+    bytes in each 64-bit lane.
+
+*   `V`: `uif32/64` \
+    <code>V **ext::SumOfLanes**(V v)</code>: returns the sum of all lanes in
+    each lane; to obtain the result, use `GetLane(horz_sum_result)`.
+
+## Advanced macros
+
+Let `Target` denote an instruction set: `NONE/SSE4/AVX2/AVX512/PPC8/ARM8/WASM`.
+
+*   `HWY_Target=##` are powers of two uniquely identifying `Target`.
+*   `HWY_STATIC_TARGETS=##`, defined within `static_targets.h`, indicates which
+    HWY_Target(s) may be used without runtime dispatch.
+*   `HWY_RUNTIME_TARGETS=##`, defined within `runtime_targets.h`, indicates
+    which specializations are generated (and only called if supported).
+
+*   `HWY_LANES_OR_0(T)`: how many lanes of type `T` in a full vector, or 0 if
+    `NONE`. Intended for use by HWY_FULL/CAPPED. Note that this cannot be used
+    in #if conditions because it uses sizeof.
+
+*   `HWY_HAS_GATHER`: whether the current target supports gather().
+*   `HWY_HAS_VARIABLE_SHIFT`: whether the current target supports variable
+    shifts, i.e. per-lane shift amounts (v1 << v2).
+*   `HWY_HAS_INT64`: whether the current target supports 64-bit integers.
+*   `HWY_HAS_CMP64`: whether the current target supports 64-bit signed
+    comparisons.
+*   `HWY_HAS_DOUBLE`: whether the current target supports double-precision
+    vectors.
+
+*   `HWY_IDE` is 0 except for purposes of IDE parsers; adding it to conditions
+    such as `#if HWY_BITS == 0 || HWY_IDE` avoids code appearing greyed out.
+
+## Compiler support
+
+Clang and GCC require e.g. -mavx2 flags in order to use SIMD intrinsics.
+However, this enables AVX2 instructions in the entire translation unit, which
+may violate the one-definition rule and cause crashes. Instead, we use
+target-specific attribute annotations: any function using SIMD must be prefixed
+with `HWY_ATTR`. These are supported by GCC 4.9 or Clang 3.9 and unnecessary in
+MSVC.
+
+Immediates (compile-time constants) are specified as template arguments to avoid
+constant-propagation issues with Clang on ARM.
+
+## Type traits
+
+*   `IsFloat<T>()` returns true if the T is a floating-point type.
+*   `IsSigned<T>()` returns true if the T is a signed or floating-point type.
+*   `LimitsMin/Max<T>()` return the smallest/largest value representable in T.
+*   `SizeTag<N>` is an empty struct, used to select overloaded functions
+    appropriate for N bytes.
diff --git a/third_party/lcms b/third_party/lcms
new file mode 160000
index 000000000..65c63bf54
--- /dev/null
+++ b/third_party/lcms
@@ -0,0 +1 @@
+Subproject commit 65c63bf549d78253c14b30b3d62cb668bbbe612c
diff --git a/third_party/lcms2.cmake b/third_party/lcms2.cmake
new file mode 100644
index 000000000..cd85fb26b
--- /dev/null
+++ b/third_party/lcms2.cmake
@@ -0,0 +1,54 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_library(lcms2 STATIC
+  lcms/src/cmsalpha.c
+  lcms/src/cmscam02.c
+  lcms/src/cmscgats.c
+  lcms/src/cmscnvrt.c
+  lcms/src/cmserr.c
+  lcms/src/cmsgamma.c
+  lcms/src/cmsgmt.c
+  lcms/src/cmshalf.c
+  lcms/src/cmsintrp.c
+  lcms/src/cmsio0.c
+  lcms/src/cmsio1.c
+  lcms/src/cmslut.c
+  lcms/src/cmsmd5.c
+  lcms/src/cmsmtrx.c
+  lcms/src/cmsnamed.c
+  lcms/src/cmsopt.c
+  lcms/src/cmspack.c
+  lcms/src/cmspcs.c
+  lcms/src/cmsplugin.c
+  lcms/src/cmsps2.c
+  lcms/src/cmssamp.c
+  lcms/src/cmssm.c
+  lcms/src/cmstypes.c
+  lcms/src/cmsvirt.c
+  lcms/src/cmswtpnt.c
+  lcms/src/cmsxform.c
+  lcms/src/lcms2_internal.h
+)
+target_include_directories(lcms2
+    PUBLIC "${CMAKE_CURRENT_LIST_DIR}/lcms/include")
+# By default LCMS uses sizeof(void*) for memory alignment, but in arm 32-bits we
+# can't access doubles not aligned to 8 bytes. This forces the alignment to 8
+# bytes.
+target_compile_definitions(lcms2
+  PRIVATE "-DCMS_PTR_ALIGNMENT=8")
+
+if (NOT "${JPEGXL_EMSCRIPTEN}")
+set_property(TARGET lcms2 PROPERTY POSITION_INDEPENDENT_CODE ON)
+endif()
diff --git a/third_party/lodepng b/third_party/lodepng
new file mode 160000
index 000000000..9652b3617
--- /dev/null
+++ b/third_party/lodepng
@@ -0,0 +1 @@
+Subproject commit 9652b36175737fbec20c3cfbfcaaa4b4807ea26f
diff --git a/third_party/lodepng.cmake b/third_party/lodepng.cmake
new file mode 100644
index 000000000..e19089dd3
--- /dev/null
+++ b/third_party/lodepng.cmake
@@ -0,0 +1,18 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_library(lodepng STATIC
+  lodepng/lodepng.cpp
+  lodepng/lodepng.h
+)
diff --git a/third_party/sjpeg b/third_party/sjpeg
new file mode 160000
index 000000000..868ab558f
--- /dev/null
+++ b/third_party/sjpeg
@@ -0,0 +1 @@
+Subproject commit 868ab558fad70fcbe8863ba4e85179eeb81cc840
diff --git a/third_party/sjpeg.cmake b/third_party/sjpeg.cmake
new file mode 100644
index 000000000..152a86ebe
--- /dev/null
+++ b/third_party/sjpeg.cmake
@@ -0,0 +1,23 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# We need to CACHE the SJPEG_BUILD_EXAMPLES to not be removed by the option()
+# inside SJPEG.
+set(SJPEG_BUILD_EXAMPLES NO CACHE BOOL "Examples")
+# SJPEG uses OpenGL which throws a warning if multiple options are installed.
+# This setting makes it prefer the new version.
+set(OpenGL_GL_PREFERENCE GLVND)
+
+add_subdirectory(sjpeg EXCLUDE_FROM_ALL)
+target_include_directories(sjpeg PUBLIC "${CMAKE_CURRENT_LIST_DIR}/sjpeg/src/")
diff --git a/third_party/skcms b/third_party/skcms
new file mode 160000
index 000000000..64374756e
--- /dev/null
+++ b/third_party/skcms
@@ -0,0 +1 @@
+Subproject commit 64374756e03700d649f897dbd98c95e78c30c7da
diff --git a/third_party/skcms.cmake b/third_party/skcms.cmake
new file mode 100644
index 000000000..50b8a566e
--- /dev/null
+++ b/third_party/skcms.cmake
@@ -0,0 +1,20 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+add_library(skcms STATIC skcms/skcms.cc)
+target_include_directories(skcms PUBLIC "${CMAKE_CURRENT_LIST_DIR}/skcms/")
+
+if (NOT "${JPEGXL_EMSCRIPTEN}")
+set_property(TARGET skcms PROPERTY POSITION_INDEPENDENT_CODE ON)
+endif()
diff --git a/third_party/testdata/dots/ellipses.png b/third_party/testdata/dots/ellipses.png
new file mode 100644
index 000000000..a536988d2
Binary files /dev/null and b/third_party/testdata/dots/ellipses.png differ
diff --git a/third_party/testdata/imagecompression.info/LICENSE.txt b/third_party/testdata/imagecompression.info/LICENSE.txt
new file mode 100644
index 000000000..9e4b54810
--- /dev/null
+++ b/third_party/testdata/imagecompression.info/LICENSE.txt
@@ -0,0 +1,8 @@
+These Images are available without any prohibitive copyright restrictions.
+
+These images are (c) there respective owners. You are granted full redistribution and publication rights on these images provided:
+
+1. The origin of the pictures must not be misrepresented; you must not claim that you took the original pictures. If you use, publish or redistribute them, an acknowledgment would be appreciated but is not required.
+2. Altered versions must be plainly marked as such, and must not be misinterpreted as being the originals.
+3. No payment is required for distribution of this material, it must be available freely under the conditions stated here. That is, it is prohibited to sell the material.
+4. This notice may not be removed or altered from any distribution.
diff --git a/third_party/testdata/imagecompression.info/flower_foveon.png b/third_party/testdata/imagecompression.info/flower_foveon.png
new file mode 100644
index 000000000..b7a21da5e
Binary files /dev/null and b/third_party/testdata/imagecompression.info/flower_foveon.png differ
diff --git a/third_party/testdata/jxl/color_management/sRGB-D2700.icc b/third_party/testdata/jxl/color_management/sRGB-D2700.icc
new file mode 100644
index 000000000..348c92926
Binary files /dev/null and b/third_party/testdata/jxl/color_management/sRGB-D2700.icc differ
diff --git a/third_party/testdata/pngsuite/PngSuite.LICENSE b/third_party/testdata/pngsuite/PngSuite.LICENSE
new file mode 100644
index 000000000..8d4d1d077
--- /dev/null
+++ b/third_party/testdata/pngsuite/PngSuite.LICENSE
@@ -0,0 +1,9 @@
+PngSuite
+--------
+
+Permission to use, copy, modify and distribute these images for any
+purpose and without fee is hereby granted.
+
+
+(c) Willem van Schaik, 1996, 2011
+
diff --git a/third_party/testdata/pngsuite/PngSuite.README b/third_party/testdata/pngsuite/PngSuite.README
new file mode 100644
index 000000000..3ef8f24db
--- /dev/null
+++ b/third_party/testdata/pngsuite/PngSuite.README
@@ -0,0 +1,25 @@
+        PNGSUITE
+----------------
+
+        testset for PNG-(de)coders
+        created by Willem van Schaik
+------------------------------------
+
+This is a collection of graphics images created to test the png applications
+like viewers, converters and editors. All (as far as that is possible)
+formats supported by the PNG standard are represented.
+
+The suite consists of the following files:
+
+-	PngSuite.README		- this file
+-	PngSuite.LICENSE	- the PngSuite is freeware
+-	PngSuite.png		- image with PngSuite logo
+-	PngSuite.tgz		- archive of all PNG testfiles
+-	PngSuite.zip		- same in .zip format for PCs
+
+
+--------
+    (c) Willem van Schaik
+	willem@schaik.com
+        Calgary, April 2011
+
diff --git a/third_party/testdata/pngsuite/README.md b/third_party/testdata/pngsuite/README.md
new file mode 100644
index 000000000..0b8cae7cd
--- /dev/null
+++ b/third_party/testdata/pngsuite/README.md
@@ -0,0 +1,15 @@
+Subset of images taken from: http://www.schaik.com/pngsuite/
+
+Please refer to the original [README](PngSuite.README) file and
+[LICENSE](PngSuite.LICENSE) file in this directory.
+
+Descriptions:
+
+*   ccwn2c08 - chroma chunk w:0.3127,0.3290 r:0.64,0.33 g:0.30,0.60 b:0.15,0.06
+*   ccwn3p08 - chroma chunk w:0.3127,0.3290 r:0.64,0.33 g:0.30,0.60 b:0.15,0.06
+*   ct1n0g04 - with textual data
+*   ctjn0g04 - international UTF-8, japanese
+*   ctzn0g04 - with compressed textual data
+*   exif2c08 - chunk with jpeg exif data
+*   g04n2c08 - color, file-gamma = 0.45
+*   g10n3p04 - paletted, file-gamma = 1.00
diff --git a/third_party/testdata/pngsuite/ccwn2c08.png b/third_party/testdata/pngsuite/ccwn2c08.png
new file mode 100644
index 000000000..47c24817b
Binary files /dev/null and b/third_party/testdata/pngsuite/ccwn2c08.png differ
diff --git a/third_party/testdata/pngsuite/ccwn3p08.png b/third_party/testdata/pngsuite/ccwn3p08.png
new file mode 100644
index 000000000..8bb2c1098
Binary files /dev/null and b/third_party/testdata/pngsuite/ccwn3p08.png differ
diff --git a/third_party/testdata/pngsuite/ct1n0g04.png b/third_party/testdata/pngsuite/ct1n0g04.png
new file mode 100644
index 000000000..3ba110aa7
Binary files /dev/null and b/third_party/testdata/pngsuite/ct1n0g04.png differ
diff --git a/third_party/testdata/pngsuite/ctjn0g04.png b/third_party/testdata/pngsuite/ctjn0g04.png
new file mode 100644
index 000000000..a77b8d2fe
Binary files /dev/null and b/third_party/testdata/pngsuite/ctjn0g04.png differ
diff --git a/third_party/testdata/pngsuite/ctzn0g04.png b/third_party/testdata/pngsuite/ctzn0g04.png
new file mode 100644
index 000000000..b4401c9cf
Binary files /dev/null and b/third_party/testdata/pngsuite/ctzn0g04.png differ
diff --git a/third_party/testdata/pngsuite/exif2c08.png b/third_party/testdata/pngsuite/exif2c08.png
new file mode 100644
index 000000000..56eb73499
Binary files /dev/null and b/third_party/testdata/pngsuite/exif2c08.png differ
diff --git a/third_party/testdata/pngsuite/g04n2c08.png b/third_party/testdata/pngsuite/g04n2c08.png
new file mode 100644
index 000000000..a652b0ce8
Binary files /dev/null and b/third_party/testdata/pngsuite/g04n2c08.png differ
diff --git a/third_party/testdata/pngsuite/g10n3p04.png b/third_party/testdata/pngsuite/g10n3p04.png
new file mode 100644
index 000000000..1b6a6be2c
Binary files /dev/null and b/third_party/testdata/pngsuite/g10n3p04.png differ
diff --git a/third_party/testdata/position_encoding/bike_dots.txt b/third_party/testdata/position_encoding/bike_dots.txt
new file mode 100644
index 000000000..3e6195d97
--- /dev/null
+++ b/third_party/testdata/position_encoding/bike_dots.txt
@@ -0,0 +1,156 @@
+155 2048 2560
+0 6 593
+0 10 600
+0 21 2541
+0 32 2546
+0 34 1176
+0 44 2550
+0 54 2554
+0 65 2558
+0 128 2126
+0 133 2229
+0 155 2070
+0 158 2052
+0 166 2134
+0 217 2173
+0 225 2085
+0 288 2400
+0 307 2103
+0 347 2203
+0 368 1416
+0 368 2101
+0 370 1429
+0 392 1740
+0 454 2134
+0 537 2141
+0 648 2170
+0 648 2188
+0 727 1772
+0 759 2029
+0 793 2001
+0 795 1992
+0 837 1435
+0 870 2057
+0 889 2005
+0 938 2062
+0 961 2032
+0 966 2004
+0 978 2017
+0 986 2011
+0 992 2027
+0 1001 1384
+0 1188 1351
+0 1260 1457
+0 1282 1992
+0 1304 1465
+0 1309 2317
+0 1361 1196
+0 1384 2320
+0 1389 2049
+0 1402 2053
+0 1526 636
+0 1561 2013
+0 1585 1478
+0 1761 634
+0 1771 1438
+0 1788 1218
+0 1791 2306
+0 1818 1261
+0 1843 1227
+0 1996 1240
+0 1998 1273
+1 17 614
+1 64 2327
+1 87 2148
+1 226 2009
+1 264 2041
+1 269 2025
+1 354 2204
+1 373 1710
+1 413 1409
+1 477 2115
+1 486 2121
+1 611 1688
+1 647 1967
+1 960 2012
+1 1203 1439
+1 1229 1462
+1 1742 2341
+2 634 1892
+2 976 2279
+2 1039 2237
+2 1056 2312
+2 1070 2301
+2 1071 2340
+2 1079 2329
+2 1081 2278
+2 1155 2287
+2 1416 1348
+2 1535 1279
+2 1550 2296
+2 1571 1298
+2 1664 1535
+2 1682 1276
+2 1752 2087
+2 1804 1330
+2 1949 1254
+2 1987 1259
+2 2015 2190
+2 2021 1291
+2 2043 1285
+3 810 1275
+3 820 2293
+3 821 1289
+3 827 1255
+3 1405 1417
+3 1431 2105
+3 1635 1330
+3 1668 1644
+3 1670 1653
+4 805 1264
+4 807 1292
+4 1046 2405
+4 1254 2394
+4 1573 1419
+4 1576 1372
+4 1667 1344
+4 1784 1454
+4 1872 1293
+4 1974 1460
+4 1977 2289
+5 806 2280
+5 1050 2234
+5 1412 1291
+5 1540 2294
+5 1755 2290
+5 1993 2136
+5 2045 1314
+6 747 2285
+6 814 1303
+6 1115 2337
+6 1215 2339
+6 1562 2292
+6 1979 1407
+7 1657 1507
+8 1785 1608
+9 274 2000
+9 445 2082
+9 908 2232
+9 915 2347
+9 990 2285
+9 1114 2314
+9 1349 2357
+9 1484 2210
+9 1542 2436
+9 1555 2408
+9 1787 2060
+9 1798 1967
+9 1858 1996
+9 1894 1999
+10 1864 1984
+11 794 2210
+11 1120 2281
+11 1193 2311
+12 1061 2304
+13 843 2307
+14 1715 2441
diff --git a/third_party/testdata/position_encoding/keong_dots.txt b/third_party/testdata/position_encoding/keong_dots.txt
new file mode 100644
index 000000000..c0811bd63
--- /dev/null
+++ b/third_party/testdata/position_encoding/keong_dots.txt
@@ -0,0 +1,23 @@
+22 2500 2500
+0 505 32
+0 966 1480
+0 1624 160
+1 761 78
+2 531 351
+2 771 1787
+2 903 1748
+2 1041 2210
+2 1179 2133
+2 1535 2080
+2 1879 2462
+3 661 1657
+3 702 1627
+3 779 1423
+3 1492 2056
+4 2231 1947
+4 2276 2317
+4 2291 2123
+4 2331 2122
+4 2396 1981
+5 1952 2063
+6 1103 2248
diff --git a/third_party/testdata/position_encoding/non_square.txt b/third_party/testdata/position_encoding/non_square.txt
new file mode 100644
index 000000000..603d6898b
--- /dev/null
+++ b/third_party/testdata/position_encoding/non_square.txt
@@ -0,0 +1,6 @@
+5 2500 4
+0 5 1
+0 505 3
+0 510 0
+1 761 1
+2 2000 3
diff --git a/third_party/testdata/position_encoding/repeated_dots.txt b/third_party/testdata/position_encoding/repeated_dots.txt
new file mode 100644
index 000000000..f956e1985
--- /dev/null
+++ b/third_party/testdata/position_encoding/repeated_dots.txt
@@ -0,0 +1,6 @@
+5 2500 2500
+0 505 32
+0 505 33
+0 504 32
+1 761 78
+2 761 78
diff --git a/third_party/testdata/position_encoding/woman_dots.txt b/third_party/testdata/position_encoding/woman_dots.txt
new file mode 100644
index 000000000..57137131b
--- /dev/null
+++ b/third_party/testdata/position_encoding/woman_dots.txt
@@ -0,0 +1,693 @@
+692 2048 2560
+0 221 912
+0 358 2494
+0 372 1251
+0 409 1238
+0 413 1155
+0 480 1501
+0 485 1476
+0 802 1974
+0 829 1500
+0 829 1523
+0 838 1458
+0 838 1529
+0 839 1435
+0 841 1474
+0 842 1426
+0 846 1442
+0 850 1433
+0 854 1449
+0 857 1440
+0 859 1464
+0 861 1432
+0 862 1456
+0 866 1447
+0 869 1439
+0 877 1447
+0 878 1819
+0 882 1431
+0 885 1460
+0 899 1443
+0 903 2200
+0 931 687
+0 946 1435
+0 963 59
+0 1014 1432
+0 1038 1433
+0 1069 1432
+0 1076 1440
+0 1096 1444
+0 1124 70
+0 1129 1435
+0 1149 1425
+0 1214 1438
+0 1215 1412
+0 1243 1428
+0 1255 1409
+0 1265 674
+0 1266 1398
+0 1307 1394
+0 1353 1371
+0 1356 1363
+0 1386 112
+0 1439 1296
+0 1656 1907
+0 1667 1926
+0 1693 1879
+0 1694 2258
+0 1702 1841
+0 1702 1873
+0 1705 1881
+0 1707 1827
+0 1714 1874
+0 1717 1883
+0 1719 1843
+0 1727 1837
+0 1743 1848
+0 1750 1759
+0 1750 1768
+0 1751 1751
+0 1760 1756
+0 1766 2243
+0 1767 2250
+0 1772 1745
+0 1775 1737
+0 1780 2248
+0 1785 1737
+0 1798 1780
+0 1800 1731
+0 1807 1739
+0 1811 1732
+0 1812 2286
+0 1814 1725
+0 1843 1716
+0 1852 1718
+1 428 1702
+1 435 1641
+1 460 1926
+1 460 1957
+1 461 1697
+1 472 2344
+1 482 1793
+1 483 1667
+1 486 2138
+1 488 1754
+1 499 2525
+1 503 1653
+1 515 2476
+1 527 2478
+1 528 2494
+1 531 2485
+1 532 2386
+1 534 1411
+1 536 1387
+1 545 1828
+1 548 1294
+1 559 2463
+1 561 717
+1 561 2551
+1 566 1201
+1 566 2529
+1 567 682
+1 580 1163
+1 581 2312
+1 585 1172
+1 586 1258
+1 587 1198
+1 597 909
+1 598 2332
+1 604 2537
+1 611 669
+1 612 2544
+1 629 601
+1 631 2413
+1 638 524
+1 640 2338
+1 641 633
+1 656 583
+1 666 531
+1 668 2116
+1 670 2250
+1 710 1910
+1 734 304
+1 776 1548
+1 806 262
+1 838 1201
+1 845 1601
+1 857 1129
+1 863 1564
+1 867 1180
+1 868 1112
+1 872 1548
+1 882 1517
+1 916 1461
+1 972 1545
+1 1026 1671
+1 1059 1635
+1 1083 88
+1 1140 1665
+1 1165 1515
+1 1228 1766
+1 1277 1615
+1 1287 1746
+1 1350 1632
+1 1423 1642
+1 1425 1615
+1 1428 1634
+1 1435 1647
+1 1446 221
+1 1453 1684
+1 1457 1543
+1 1471 1413
+1 1476 1774
+1 1478 1255
+1 1494 1679
+1 1507 1596
+1 1511 1631
+1 1566 1780
+1 1573 394
+1 1579 1721
+1 1584 1385
+1 1590 1432
+1 1618 559
+1 1648 1469
+1 1656 1097
+1 1678 1540
+1 1687 1494
+1 1709 1493
+1 1710 884
+1 1722 1534
+1 1732 1601
+1 1744 1544
+1 1757 1574
+1 1775 1384
+1 1776 1572
+1 1788 1409
+1 1803 1629
+2 458 1672
+2 493 2448
+2 502 2503
+2 544 1305
+2 552 1323
+2 552 2391
+2 553 2446
+2 570 2508
+2 611 1136
+2 648 1728
+2 651 814
+2 665 2320
+2 671 991
+2 672 563
+2 674 392
+2 692 1077
+2 742 1481
+2 746 291
+2 751 351
+2 761 930
+2 768 292
+2 788 373
+2 807 1758
+2 823 191
+2 853 1298
+2 865 1604
+2 1006 1689
+2 1039 99
+2 1057 1655
+2 1085 1688
+2 1095 1690
+2 1126 1674
+2 1139 1687
+2 1152 1724
+2 1168 1717
+2 1191 178
+2 1201 1657
+2 1204 1744
+2 1215 1721
+2 1260 1744
+2 1280 1777
+2 1290 1684
+2 1362 207
+2 1395 189
+2 1402 184
+2 1491 324
+2 1504 419
+2 1509 361
+2 1511 898
+2 1514 484
+2 1514 1313
+2 1529 418
+2 1533 897
+2 1595 690
+2 1620 1404
+2 1684 849
+2 1689 840
+2 1690 876
+2 1692 904
+2 1701 1011
+2 1704 846
+2 1744 1107
+2 1798 1516
+3 447 1968
+3 450 1687
+3 474 1773
+3 500 2516
+3 520 1295
+3 561 1439
+3 562 694
+3 572 1583
+3 575 1397
+3 599 1309
+3 600 1344
+3 601 1198
+3 611 986
+3 612 1370
+3 617 1235
+3 618 955
+3 619 728
+3 622 1098
+3 629 979
+3 653 1449
+3 663 728
+3 668 961
+3 675 427
+3 684 1461
+3 690 1986
+3 692 1688
+3 701 622
+3 704 323
+3 704 698
+3 707 679
+3 712 300
+3 712 2011
+3 714 325
+3 720 622
+3 724 1016
+3 735 2355
+3 736 2302
+3 738 1239
+3 743 663
+3 744 1415
+3 745 489
+3 748 1313
+3 750 644
+3 753 1051
+3 754 876
+3 755 1331
+3 757 904
+3 772 1825
+3 794 252
+3 794 1738
+3 798 272
+3 803 312
+3 821 530
+3 825 1707
+3 829 233
+3 841 1369
+3 843 1212
+3 849 1150
+3 852 1235
+3 860 206
+3 877 1648
+3 880 1199
+3 891 1611
+3 900 1664
+3 901 205
+3 912 295
+3 931 1468
+3 958 178
+3 990 1496
+3 1005 157
+3 1016 183
+3 1017 1670
+3 1027 191
+3 1030 174
+3 1037 128
+3 1043 1694
+3 1052 1695
+3 1056 135
+3 1075 1677
+3 1095 137
+3 1099 1701
+3 1112 1660
+3 1117 158
+3 1124 158
+3 1125 1718
+3 1143 104
+3 1143 1699
+3 1156 1657
+3 1162 1691
+3 1174 1683
+3 1178 1720
+3 1183 123
+3 1191 1654
+3 1193 1699
+3 1194 1738
+3 1203 1680
+3 1205 1717
+3 1220 1739
+3 1227 1700
+3 1232 111
+3 1247 1686
+3 1249 1766
+3 1267 1691
+3 1275 131
+3 1318 1772
+3 1322 143
+3 1324 1787
+3 1339 1730
+3 1341 258
+3 1359 1769
+3 1362 1735
+3 1376 303
+3 1381 226
+3 1393 142
+3 1394 346
+3 1410 264
+3 1434 237
+3 1438 339
+3 1490 851
+3 1493 379
+3 1503 606
+3 1504 1328
+3 1515 461
+3 1524 593
+3 1526 552
+3 1541 470
+3 1544 380
+3 1568 750
+3 1578 1003
+3 1586 1365
+3 1595 866
+3 1600 767
+3 1610 658
+3 1624 776
+3 1625 1313
+3 1641 1421
+3 1669 1154
+3 1672 2338
+3 1682 1177
+3 1685 1194
+3 1695 1309
+3 1709 1281
+3 1710 833
+3 1734 1087
+3 1738 1184
+3 1741 1343
+3 1749 1122
+3 1777 1372
+3 1780 1412
+3 1801 1485
+4 424 1656
+4 428 1743
+4 465 2063
+4 491 2108
+4 513 2490
+4 532 1274
+4 537 2530
+4 545 2481
+4 546 2375
+4 559 2559
+4 561 1816
+4 562 1321
+4 570 2555
+4 581 1373
+4 602 751
+4 609 680
+4 646 2473
+4 671 2091
+4 687 1696
+4 699 1659
+4 702 1945
+4 795 237
+4 841 1382
+4 845 1162
+4 872 1535
+4 1021 171
+4 1024 112
+4 1115 1639
+4 1135 1675
+4 1162 1647
+4 1247 1655
+4 1284 1766
+4 1426 199
+4 1443 278
+4 1494 1369
+4 1528 338
+4 1601 376
+4 1670 810
+4 1675 1177
+4 1692 865
+4 1697 1504
+4 1709 1042
+4 1714 1298
+4 1737 1313
+4 1753 1400
+5 422 1710
+5 460 1980
+5 461 1885
+5 478 2080
+5 481 2022
+5 481 2095
+5 481 2120
+5 490 2421
+5 500 1714
+5 507 2459
+5 508 2483
+5 519 1758
+5 527 2434
+5 528 1314
+5 530 2445
+5 536 2472
+5 553 1822
+5 583 1387
+5 603 861
+5 619 685
+5 623 740
+5 628 763
+5 650 654
+5 652 2396
+5 653 2377
+5 656 1707
+5 656 2335
+5 662 2294
+5 664 2128
+5 667 413
+5 667 2272
+5 685 400
+5 691 345
+5 713 745
+5 719 1887
+5 730 759
+5 748 303
+5 755 302
+5 762 1270
+5 773 1855
+5 792 1441
+5 797 1751
+5 801 1741
+5 811 1269
+5 812 1747
+5 840 214
+5 844 1358
+5 845 1341
+5 856 1322
+5 857 1200
+5 871 1696
+5 952 1473
+5 972 1692
+5 986 1667
+5 986 1686
+5 996 1687
+5 997 1668
+5 999 188
+5 1015 1690
+5 1019 1650
+5 1033 1693
+5 1055 1624
+5 1065 1625
+5 1068 1656
+5 1074 1627
+5 1085 1627
+5 1096 153
+5 1100 1680
+5 1115 151
+5 1119 1705
+5 1129 1685
+5 1131 1631
+5 1131 1664
+5 1134 1642
+5 1137 1653
+5 1140 1632
+5 1143 1644
+5 1148 152
+5 1149 1690
+5 1152 1646
+5 1158 1635
+5 1160 1670
+5 1163 1704
+5 1165 1659
+5 1168 1636
+5 1170 1673
+5 1171 1650
+5 1178 132
+5 1178 1638
+5 1187 1641
+5 1190 1722
+5 1206 1646
+5 1216 1696
+5 1216 1749
+5 1223 1748
+5 1237 1758
+5 1243 1673
+5 1244 1732
+5 1254 1686
+5 1254 1730
+5 1256 1765
+5 1263 1707
+5 1272 1750
+5 1273 1760
+5 1284 181
+5 1298 1709
+5 1301 1723
+5 1307 173
+5 1309 143
+5 1314 133
+5 1344 1774
+5 1359 299
+5 1387 428
+5 1389 182
+5 1465 980
+5 1498 1303
+5 1509 536
+5 1518 1343
+5 1520 514
+5 1527 454
+5 1531 405
+5 1532 1326
+5 1632 1008
+5 1654 718
+5 1660 1008
+5 1682 1403
+5 1689 969
+5 1693 982
+5 1705 946
+5 1713 1024
+5 1713 1437
+5 1726 1447
+5 1728 1125
+5 1730 1229
+5 1731 1175
+5 1736 1389
+5 1768 1357
+5 1773 1417
+5 1779 1432
+6 418 1772
+6 423 1762
+6 436 1782
+6 437 1798
+6 458 1911
+6 462 1817
+6 480 2417
+6 482 1642
+6 493 2430
+6 505 1645
+6 531 2513
+6 537 1342
+6 537 1441
+6 537 2462
+6 542 1177
+6 542 1360
+6 547 1275
+6 568 1081
+6 570 2169
+6 589 957
+6 591 1157
+6 613 706
+6 630 2392
+6 642 650
+6 642 666
+6 646 2453
+6 647 2536
+6 661 2064
+6 663 552
+6 678 2027
+6 688 1949
+6 725 1870
+6 741 1830
+6 775 1604
+6 792 1711
+6 903 1495
+6 990 1610
+6 990 2249
+6 1007 1601
+6 1013 1611
+6 1023 1613
+6 1102 1620
+6 1167 1626
+6 1214 1640
+6 1299 1675
+6 1310 1681
+6 1444 1703
+6 1482 1698
+6 1514 327
+6 1542 1883
+6 1544 350
+6 1629 1465
+6 1671 1544
+6 1674 1498
+6 1683 1509
+6 1701 1784
+6 1710 1538
+7 170 1027
+7 279 958
+7 401 1524
+7 451 1068
+7 457 1054
+7 487 1501
+7 502 2559
+7 533 2019
+7 575 2048
+7 803 1550
+7 891 1437
+7 928 1427
+7 957 1438
+7 1005 1422
+7 1016 1423
+7 1034 1424
+7 1061 1433
+7 1112 1449
+7 1142 1422
+7 1418 1310
+7 1691 1871
+7 1708 1858
+7 1708 2255
+7 1724 2259
+7 1743 2187
+7 1752 2246
+7 1755 2175
+7 1770 1753
+7 1797 1738
+7 1907 2170
+8 501 2466
+8 505 2474
+8 1346 1142
+9 1533 1316
+10 504 2493
+10 515 2499
+10 524 2468
+10 547 2518
+10 550 2458
+10 557 2524
+10 574 2535
+10 676 2005
+10 1030 1680
+10 1124 1641
+10 1148 1711
+10 1179 1673
+10 1702 1341
+11 787 2300
+11 863 1587
+11 902 2308
+11 1013 2367
+12 858 1596
+13 784 2448
diff --git a/third_party/testdata/raw.pixls/DJI-FC6310-16bit_709_v4_krita.png b/third_party/testdata/raw.pixls/DJI-FC6310-16bit_709_v4_krita.png
new file mode 100644
index 000000000..4d00d4501
Binary files /dev/null and b/third_party/testdata/raw.pixls/DJI-FC6310-16bit_709_v4_krita.png differ
diff --git a/third_party/testdata/raw.pixls/DJI-FC6310-16bit_srgb8_v4_krita.png b/third_party/testdata/raw.pixls/DJI-FC6310-16bit_srgb8_v4_krita.png
new file mode 100644
index 000000000..b011444ed
Binary files /dev/null and b/third_party/testdata/raw.pixls/DJI-FC6310-16bit_srgb8_v4_krita.png differ
diff --git a/third_party/testdata/raw.pixls/Google-Pixel2XL-16bit_acescg_g1_v4_krita.png b/third_party/testdata/raw.pixls/Google-Pixel2XL-16bit_acescg_g1_v4_krita.png
new file mode 100644
index 000000000..ab19700a8
Binary files /dev/null and b/third_party/testdata/raw.pixls/Google-Pixel2XL-16bit_acescg_g1_v4_krita.png differ
diff --git a/third_party/testdata/raw.pixls/Google-Pixel2XL-16bit_srgb8_v4_krita.png b/third_party/testdata/raw.pixls/Google-Pixel2XL-16bit_srgb8_v4_krita.png
new file mode 100644
index 000000000..a1624c2bd
Binary files /dev/null and b/third_party/testdata/raw.pixls/Google-Pixel2XL-16bit_srgb8_v4_krita.png differ
diff --git a/third_party/testdata/raw.pixls/HUAWEI-EVA-L09-16bit_709_g1_dt.png b/third_party/testdata/raw.pixls/HUAWEI-EVA-L09-16bit_709_g1_dt.png
new file mode 100644
index 000000000..3d11b6764
Binary files /dev/null and b/third_party/testdata/raw.pixls/HUAWEI-EVA-L09-16bit_709_g1_dt.png differ
diff --git a/third_party/testdata/raw.pixls/HUAWEI-EVA-L09-16bit_srgb8_dt.png b/third_party/testdata/raw.pixls/HUAWEI-EVA-L09-16bit_srgb8_dt.png
new file mode 100644
index 000000000..acbd3a48d
Binary files /dev/null and b/third_party/testdata/raw.pixls/HUAWEI-EVA-L09-16bit_srgb8_dt.png differ
diff --git a/third_party/testdata/raw.pixls/LICENSE.md b/third_party/testdata/raw.pixls/LICENSE.md
new file mode 100644
index 000000000..25fee1261
--- /dev/null
+++ b/third_party/testdata/raw.pixls/LICENSE.md
@@ -0,0 +1,114 @@
+### CC0 1.0 Universal
+
+#### Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator and
+subsequent owner(s) (each and all, an "owner") of an original work of authorship
+and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for the
+purpose of contributing to a commons of creative, cultural and scientific works
+("Commons") that the public can reliably and without fear of later claims of
+infringement build upon, modify, incorporate in other works, reuse and
+redistribute as freely as possible in any form whatsoever and for any purposes,
+including without limitation commercial purposes. These owners may contribute to
+the Commons to promote the ideal of a free culture and the further production of
+creative, cultural and scientific works, or to gain reputation or greater
+distribution for their Work in part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any expectation of
+additional consideration or compensation, the person associating CC0 with a Work
+(the "Affirmer"), to the extent that he or she is an owner of Copyright and
+Related Rights in the Work, voluntarily elects to apply CC0 to the Work and
+publicly distribute the Work under its terms, with knowledge of his or her
+Copyright and Related Rights in the Work and the meaning and intended legal
+effect of CC0 on those rights.
+
+**1. Copyright and Related Rights.** A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and Related
+Rights"). Copyright and Related Rights include, but are not limited to, the
+following:
+
+i. the right to reproduce, adapt, distribute, perform, display, communicate, and
+translate a Work;
+
+ii. moral rights retained by the original author(s) and/or performer(s);
+
+iii. publicity and privacy rights pertaining to a person's image or likeness
+depicted in a Work;
+
+iv. rights protecting against unfair competition in regards to a Work, subject
+to the limitations in paragraph 4(a), below;
+
+v. rights protecting the extraction, dissemination, use and reuse of data in a
+Work;
+
+vi. database rights (such as those arising under Directive 96/9/EC of the
+European Parliament and of the Council of 11 March 1996 on the legal protection
+of databases, and under any national implementation thereof, including any
+amended or successor version of such directive); and
+
+vii. other similar, equivalent or corresponding rights throughout the world
+based on applicable law or treaty, and any national implementations thereof.
+
+**2. Waiver.** To the greatest extent permitted by, but not in contravention of,
+applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and
+unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and
+Related Rights and associated claims and causes of action, whether now known or
+unknown (including existing as well as future claims and causes of action), in
+the Work (i) in all territories worldwide, (ii) for the maximum duration
+provided by applicable law or treaty (including future time extensions), (iii)
+in any current or future medium and for any number of copies, and (iv) for any
+purpose whatsoever, including without limitation commercial, advertising or
+promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit
+of each member of the public at large and to the detriment of Affirmer's heirs
+and successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public as
+contemplated by Affirmer's express Statement of Purpose.
+
+**3. Public License Fallback.** Should any part of the Waiver for any reason be
+judged legally invalid or ineffective under applicable law, then the Waiver
+shall be preserved to the maximum extent permitted taking into account
+Affirmer's express Statement of Purpose. In addition, to the extent the Waiver
+is so judged Affirmer hereby grants to each affected person a royalty-free, non
+transferable, non sublicensable, non exclusive, irrevocable and unconditional
+license to exercise Affirmer's Copyright and Related Rights in the Work (i) in
+all territories worldwide, (ii) for the maximum duration provided by applicable
+law or treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional purposes
+(the "License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any reason
+be judged legally invalid or ineffective under applicable law, such partial
+invalidity or ineffectiveness shall not invalidate the remainder of the License,
+and in such case Affirmer hereby affirms that he or she will not (i) exercise
+any of his or her remaining Copyright and Related Rights in the Work or (ii)
+assert any associated claims and causes of action with respect to the Work, in
+either case contrary to Affirmer's express Statement of Purpose.
+
+**4. Limitations and Disclaimers.**
+
+a. No trademark or patent rights held by Affirmer are waived, abandoned,
+surrendered, licensed or otherwise affected by this document.
+
+b. Affirmer offers the Work as-is and makes no representations or warranties of
+any kind concerning the Work, express, implied, statutory or otherwise,
+including without limitation warranties of title, merchantability, fitness for a
+particular purpose, non infringement, or the absence of latent or other defects,
+accuracy, or the present or absence of errors, whether or not discoverable, all
+to the greatest extent permissible under applicable law.
+
+c. Affirmer disclaims responsibility for clearing rights of other persons that
+may apply to the Work or any use thereof, including without limitation any
+person's Copyright and Related Rights in the Work. Further, Affirmer disclaims
+responsibility for obtaining any necessary consents, permissions or other rights
+required for any use of the Work.
+
+d. Affirmer understands and acknowledges that Creative Commons is not a party to
+this document and has no duty or obligation with respect to this CC0 or use of
+the Work.
+
+For more information, please see
+<http://creativecommons.org/publicdomain/zero/1.0/>
diff --git a/third_party/testdata/raw.pixls/Nikon-D300-12bit_2020_g1_dt.png b/third_party/testdata/raw.pixls/Nikon-D300-12bit_2020_g1_dt.png
new file mode 100644
index 000000000..9b9330274
Binary files /dev/null and b/third_party/testdata/raw.pixls/Nikon-D300-12bit_2020_g1_dt.png differ
diff --git a/third_party/testdata/raw.pixls/Nikon-D300-12bit_srgb8_dt.png b/third_party/testdata/raw.pixls/Nikon-D300-12bit_srgb8_dt.png
new file mode 100644
index 000000000..ec5ca3fbc
Binary files /dev/null and b/third_party/testdata/raw.pixls/Nikon-D300-12bit_srgb8_dt.png differ
diff --git a/third_party/testdata/raw.pixls/README.md b/third_party/testdata/raw.pixls/README.md
new file mode 100644
index 000000000..6069e89e4
--- /dev/null
+++ b/third_party/testdata/raw.pixls/README.md
@@ -0,0 +1,4 @@
+Images taken from https://raw.pixls.us/
+
+The original pictures were released under a Creative Commons Zero license. See
+[LICENSE.md](LICENSE.md) in this directory for details.
diff --git a/third_party/testdata/raw.pixls/Sony-DSC-RX1RM2-14bit_709_v4_krita.png b/third_party/testdata/raw.pixls/Sony-DSC-RX1RM2-14bit_709_v4_krita.png
new file mode 100644
index 000000000..23fbd5fb7
Binary files /dev/null and b/third_party/testdata/raw.pixls/Sony-DSC-RX1RM2-14bit_709_v4_krita.png differ
diff --git a/third_party/testdata/raw.pixls/Sony-DSC-RX1RM2-14bit_srgb8_v4_krita.png b/third_party/testdata/raw.pixls/Sony-DSC-RX1RM2-14bit_srgb8_v4_krita.png
new file mode 100644
index 000000000..9ac7cdc3d
Binary files /dev/null and b/third_party/testdata/raw.pixls/Sony-DSC-RX1RM2-14bit_srgb8_v4_krita.png differ
diff --git a/third_party/testdata/wesaturate/500px/cvo9xd_keong_macan_grayscale.png b/third_party/testdata/wesaturate/500px/cvo9xd_keong_macan_grayscale.png
new file mode 100644
index 000000000..063fc7dc2
Binary files /dev/null and b/third_party/testdata/wesaturate/500px/cvo9xd_keong_macan_grayscale.png differ
diff --git a/third_party/testdata/wesaturate/500px/cvo9xd_keong_macan_srgb8.png b/third_party/testdata/wesaturate/500px/cvo9xd_keong_macan_srgb8.png
new file mode 100644
index 000000000..871bdfa30
Binary files /dev/null and b/third_party/testdata/wesaturate/500px/cvo9xd_keong_macan_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/500px/tmshre_riaphotographs_alpha.png b/third_party/testdata/wesaturate/500px/tmshre_riaphotographs_alpha.png
new file mode 100644
index 000000000..798e02462
Binary files /dev/null and b/third_party/testdata/wesaturate/500px/tmshre_riaphotographs_alpha.png differ
diff --git a/third_party/testdata/wesaturate/500px/tmshre_riaphotographs_srgb8.png b/third_party/testdata/wesaturate/500px/tmshre_riaphotographs_srgb8.png
new file mode 100644
index 000000000..a9397614d
Binary files /dev/null and b/third_party/testdata/wesaturate/500px/tmshre_riaphotographs_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/500px/u76c0g_bliznaca_srgb8.png b/third_party/testdata/wesaturate/500px/u76c0g_bliznaca_srgb8.png
new file mode 100644
index 000000000..6521ae57e
Binary files /dev/null and b/third_party/testdata/wesaturate/500px/u76c0g_bliznaca_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/Nikon-D3-14bit_2020_g1.png b/third_party/testdata/wesaturate/64px/Nikon-D3-14bit_2020_g1.png
new file mode 100644
index 000000000..90f5428cb
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/Nikon-D3-14bit_2020_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/Nikon-D3-14bit_srgb8.png b/third_party/testdata/wesaturate/64px/Nikon-D3-14bit_srgb8.png
new file mode 100644
index 000000000..302359e12
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/Nikon-D3-14bit_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/a2d1un_nkitzmiller_2020_g1.png b/third_party/testdata/wesaturate/64px/a2d1un_nkitzmiller_2020_g1.png
new file mode 100644
index 000000000..580b69be2
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/a2d1un_nkitzmiller_2020_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/a2d1un_nkitzmiller_srgb8.png b/third_party/testdata/wesaturate/64px/a2d1un_nkitzmiller_srgb8.png
new file mode 100644
index 000000000..637cecfdc
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/a2d1un_nkitzmiller_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/cvo9xd_keong_macan_2020_g1.png b/third_party/testdata/wesaturate/64px/cvo9xd_keong_macan_2020_g1.png
new file mode 100644
index 000000000..3fddd74f3
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/cvo9xd_keong_macan_2020_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/cvo9xd_keong_macan_srgb8.png b/third_party/testdata/wesaturate/64px/cvo9xd_keong_macan_srgb8.png
new file mode 100644
index 000000000..b2605db6e
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/cvo9xd_keong_macan_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/l1qnb5_nkitzmiller_2020_g1.png b/third_party/testdata/wesaturate/64px/l1qnb5_nkitzmiller_2020_g1.png
new file mode 100644
index 000000000..5e09b81a3
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/l1qnb5_nkitzmiller_2020_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/l1qnb5_nkitzmiller_srgb8.png b/third_party/testdata/wesaturate/64px/l1qnb5_nkitzmiller_srgb8.png
new file mode 100644
index 000000000..546a6983c
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/l1qnb5_nkitzmiller_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/phu1or_alfann24_2020_g1.png b/third_party/testdata/wesaturate/64px/phu1or_alfann24_2020_g1.png
new file mode 100644
index 000000000..3cca25e55
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/phu1or_alfann24_2020_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/phu1or_alfann24_srgb8.png b/third_party/testdata/wesaturate/64px/phu1or_alfann24_srgb8.png
new file mode 100644
index 000000000..cbb25f039
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/phu1or_alfann24_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/q3a0b3_d17ws_709_g1.png b/third_party/testdata/wesaturate/64px/q3a0b3_d17ws_709_g1.png
new file mode 100644
index 000000000..ea207b842
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/q3a0b3_d17ws_709_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/q3a0b3_d17ws_srgb8.png b/third_party/testdata/wesaturate/64px/q3a0b3_d17ws_srgb8.png
new file mode 100644
index 000000000..eb835f2bd
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/q3a0b3_d17ws_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/ra0ed45_alfann24_709_g1.png b/third_party/testdata/wesaturate/64px/ra0ed45_alfann24_709_g1.png
new file mode 100644
index 000000000..cfbf52c78
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/ra0ed45_alfann24_709_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/ra0ed45_alfann24_srgb8.png b/third_party/testdata/wesaturate/64px/ra0ed45_alfann24_srgb8.png
new file mode 100644
index 000000000..d0063704e
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/ra0ed45_alfann24_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/t0gho7_orlaustral_709_g1.png b/third_party/testdata/wesaturate/64px/t0gho7_orlaustral_709_g1.png
new file mode 100644
index 000000000..bea947dfa
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/t0gho7_orlaustral_709_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/t0gho7_orlaustral_srgb8.png b/third_party/testdata/wesaturate/64px/t0gho7_orlaustral_srgb8.png
new file mode 100644
index 000000000..5af24bb6c
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/t0gho7_orlaustral_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/tmshre_riaphotographs_709_g1.png b/third_party/testdata/wesaturate/64px/tmshre_riaphotographs_709_g1.png
new file mode 100644
index 000000000..97854161b
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/tmshre_riaphotographs_709_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/tmshre_riaphotographs_srgb8.png b/third_party/testdata/wesaturate/64px/tmshre_riaphotographs_srgb8.png
new file mode 100644
index 000000000..63536eada
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/tmshre_riaphotographs_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/u76c0g_bliznaca_709_g1.png b/third_party/testdata/wesaturate/64px/u76c0g_bliznaca_709_g1.png
new file mode 100644
index 000000000..8c73f9be0
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/u76c0g_bliznaca_709_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/u76c0g_bliznaca_srgb8.png b/third_party/testdata/wesaturate/64px/u76c0g_bliznaca_srgb8.png
new file mode 100644
index 000000000..000c68c69
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/u76c0g_bliznaca_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/64px/vgqcws_vin_709_g1.png b/third_party/testdata/wesaturate/64px/vgqcws_vin_709_g1.png
new file mode 100644
index 000000000..dfd48c903
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/vgqcws_vin_709_g1.png differ
diff --git a/third_party/testdata/wesaturate/64px/vgqcws_vin_srgb8.png b/third_party/testdata/wesaturate/64px/vgqcws_vin_srgb8.png
new file mode 100644
index 000000000..2f911b009
Binary files /dev/null and b/third_party/testdata/wesaturate/64px/vgqcws_vin_srgb8.png differ
diff --git a/third_party/testdata/wesaturate/LICENSE.md b/third_party/testdata/wesaturate/LICENSE.md
new file mode 100644
index 000000000..25fee1261
--- /dev/null
+++ b/third_party/testdata/wesaturate/LICENSE.md
@@ -0,0 +1,114 @@
+### CC0 1.0 Universal
+
+#### Statement of Purpose
+
+The laws of most jurisdictions throughout the world automatically confer
+exclusive Copyright and Related Rights (defined below) upon the creator and
+subsequent owner(s) (each and all, an "owner") of an original work of authorship
+and/or a database (each, a "Work").
+
+Certain owners wish to permanently relinquish those rights to a Work for the
+purpose of contributing to a commons of creative, cultural and scientific works
+("Commons") that the public can reliably and without fear of later claims of
+infringement build upon, modify, incorporate in other works, reuse and
+redistribute as freely as possible in any form whatsoever and for any purposes,
+including without limitation commercial purposes. These owners may contribute to
+the Commons to promote the ideal of a free culture and the further production of
+creative, cultural and scientific works, or to gain reputation or greater
+distribution for their Work in part through the use and efforts of others.
+
+For these and/or other purposes and motivations, and without any expectation of
+additional consideration or compensation, the person associating CC0 with a Work
+(the "Affirmer"), to the extent that he or she is an owner of Copyright and
+Related Rights in the Work, voluntarily elects to apply CC0 to the Work and
+publicly distribute the Work under its terms, with knowledge of his or her
+Copyright and Related Rights in the Work and the meaning and intended legal
+effect of CC0 on those rights.
+
+**1. Copyright and Related Rights.** A Work made available under CC0 may be
+protected by copyright and related or neighboring rights ("Copyright and Related
+Rights"). Copyright and Related Rights include, but are not limited to, the
+following:
+
+i. the right to reproduce, adapt, distribute, perform, display, communicate, and
+translate a Work;
+
+ii. moral rights retained by the original author(s) and/or performer(s);
+
+iii. publicity and privacy rights pertaining to a person's image or likeness
+depicted in a Work;
+
+iv. rights protecting against unfair competition in regards to a Work, subject
+to the limitations in paragraph 4(a), below;
+
+v. rights protecting the extraction, dissemination, use and reuse of data in a
+Work;
+
+vi. database rights (such as those arising under Directive 96/9/EC of the
+European Parliament and of the Council of 11 March 1996 on the legal protection
+of databases, and under any national implementation thereof, including any
+amended or successor version of such directive); and
+
+vii. other similar, equivalent or corresponding rights throughout the world
+based on applicable law or treaty, and any national implementations thereof.
+
+**2. Waiver.** To the greatest extent permitted by, but not in contravention of,
+applicable law, Affirmer hereby overtly, fully, permanently, irrevocably and
+unconditionally waives, abandons, and surrenders all of Affirmer's Copyright and
+Related Rights and associated claims and causes of action, whether now known or
+unknown (including existing as well as future claims and causes of action), in
+the Work (i) in all territories worldwide, (ii) for the maximum duration
+provided by applicable law or treaty (including future time extensions), (iii)
+in any current or future medium and for any number of copies, and (iv) for any
+purpose whatsoever, including without limitation commercial, advertising or
+promotional purposes (the "Waiver"). Affirmer makes the Waiver for the benefit
+of each member of the public at large and to the detriment of Affirmer's heirs
+and successors, fully intending that such Waiver shall not be subject to
+revocation, rescission, cancellation, termination, or any other legal or
+equitable action to disrupt the quiet enjoyment of the Work by the public as
+contemplated by Affirmer's express Statement of Purpose.
+
+**3. Public License Fallback.** Should any part of the Waiver for any reason be
+judged legally invalid or ineffective under applicable law, then the Waiver
+shall be preserved to the maximum extent permitted taking into account
+Affirmer's express Statement of Purpose. In addition, to the extent the Waiver
+is so judged Affirmer hereby grants to each affected person a royalty-free, non
+transferable, non sublicensable, non exclusive, irrevocable and unconditional
+license to exercise Affirmer's Copyright and Related Rights in the Work (i) in
+all territories worldwide, (ii) for the maximum duration provided by applicable
+law or treaty (including future time extensions), (iii) in any current or future
+medium and for any number of copies, and (iv) for any purpose whatsoever,
+including without limitation commercial, advertising or promotional purposes
+(the "License"). The License shall be deemed effective as of the date CC0 was
+applied by Affirmer to the Work. Should any part of the License for any reason
+be judged legally invalid or ineffective under applicable law, such partial
+invalidity or ineffectiveness shall not invalidate the remainder of the License,
+and in such case Affirmer hereby affirms that he or she will not (i) exercise
+any of his or her remaining Copyright and Related Rights in the Work or (ii)
+assert any associated claims and causes of action with respect to the Work, in
+either case contrary to Affirmer's express Statement of Purpose.
+
+**4. Limitations and Disclaimers.**
+
+a. No trademark or patent rights held by Affirmer are waived, abandoned,
+surrendered, licensed or otherwise affected by this document.
+
+b. Affirmer offers the Work as-is and makes no representations or warranties of
+any kind concerning the Work, express, implied, statutory or otherwise,
+including without limitation warranties of title, merchantability, fitness for a
+particular purpose, non infringement, or the absence of latent or other defects,
+accuracy, or the present or absence of errors, whether or not discoverable, all
+to the greatest extent permissible under applicable law.
+
+c. Affirmer disclaims responsibility for clearing rights of other persons that
+may apply to the Work or any use thereof, including without limitation any
+person's Copyright and Related Rights in the Work. Further, Affirmer disclaims
+responsibility for obtaining any necessary consents, permissions or other rights
+required for any use of the Work.
+
+d. Affirmer understands and acknowledges that Creative Commons is not a party to
+this document and has no duty or obligation with respect to this CC0 or use of
+the Work.
+
+For more information, please see
+<http://creativecommons.org/publicdomain/zero/1.0/>
diff --git a/third_party/testdata/wesaturate/README.md b/third_party/testdata/wesaturate/README.md
new file mode 100644
index 000000000..c0968bab9
--- /dev/null
+++ b/third_party/testdata/wesaturate/README.md
@@ -0,0 +1,8 @@
+Images taken from https://www.wesaturate.com/
+
+The original pictures were released under a Creative Commons Zero license. See
+[LICENSE.md](LICENSE.md) in this directory for details.
+
+64px/ directory contains 64x64 cropped images.
+500px/ directory contains 500x500 resized and then cropped images. Sample images
+for a greyscale and alpha channel are also included here as derived work.
diff --git a/third_party/testdata/wide-gamut-tests/LICENSE b/third_party/testdata/wide-gamut-tests/LICENSE
new file mode 100644
index 000000000..261eeb9e9
--- /dev/null
+++ b/third_party/testdata/wide-gamut-tests/LICENSE
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/third_party/testdata/wide-gamut-tests/P3-sRGB-blue.png b/third_party/testdata/wide-gamut-tests/P3-sRGB-blue.png
new file mode 100644
index 000000000..ae0c8e6ff
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/P3-sRGB-blue.png differ
diff --git a/third_party/testdata/wide-gamut-tests/P3-sRGB-color-bars.png b/third_party/testdata/wide-gamut-tests/P3-sRGB-color-bars.png
new file mode 100644
index 000000000..d6687d726
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/P3-sRGB-color-bars.png differ
diff --git a/third_party/testdata/wide-gamut-tests/P3-sRGB-color-ring.png b/third_party/testdata/wide-gamut-tests/P3-sRGB-color-ring.png
new file mode 100644
index 000000000..bc3918642
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/P3-sRGB-color-ring.png differ
diff --git a/third_party/testdata/wide-gamut-tests/P3-sRGB-green.png b/third_party/testdata/wide-gamut-tests/P3-sRGB-green.png
new file mode 100644
index 000000000..f6b91784b
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/P3-sRGB-green.png differ
diff --git a/third_party/testdata/wide-gamut-tests/P3-sRGB-red.png b/third_party/testdata/wide-gamut-tests/P3-sRGB-red.png
new file mode 100644
index 000000000..e841dccf8
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/P3-sRGB-red.png differ
diff --git a/third_party/testdata/wide-gamut-tests/R2020-sRGB-blue.png b/third_party/testdata/wide-gamut-tests/R2020-sRGB-blue.png
new file mode 100644
index 000000000..a0b858b23
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/R2020-sRGB-blue.png differ
diff --git a/third_party/testdata/wide-gamut-tests/R2020-sRGB-color-bars.png b/third_party/testdata/wide-gamut-tests/R2020-sRGB-color-bars.png
new file mode 100644
index 000000000..00a272341
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/R2020-sRGB-color-bars.png differ
diff --git a/third_party/testdata/wide-gamut-tests/R2020-sRGB-color-ring.png b/third_party/testdata/wide-gamut-tests/R2020-sRGB-color-ring.png
new file mode 100644
index 000000000..fff201456
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/R2020-sRGB-color-ring.png differ
diff --git a/third_party/testdata/wide-gamut-tests/R2020-sRGB-green.png b/third_party/testdata/wide-gamut-tests/R2020-sRGB-green.png
new file mode 100644
index 000000000..86cfd6e5a
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/R2020-sRGB-green.png differ
diff --git a/third_party/testdata/wide-gamut-tests/R2020-sRGB-red.png b/third_party/testdata/wide-gamut-tests/R2020-sRGB-red.png
new file mode 100644
index 000000000..f7c4ed7a2
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/R2020-sRGB-red.png differ
diff --git a/third_party/testdata/wide-gamut-tests/README.txt b/third_party/testdata/wide-gamut-tests/README.txt
new file mode 100644
index 000000000..597f59f85
--- /dev/null
+++ b/third_party/testdata/wide-gamut-tests/README.txt
@@ -0,0 +1,5 @@
+A series of test images for color profiles and wide gamut support.
+
+A subset of images taken from https://github.com/codelogic/wide-gamut-tests
+
+See the LICENSE file in this directory for license details.
diff --git a/third_party/testdata/wide-gamut-tests/rgb-to-gbr-test.png b/third_party/testdata/wide-gamut-tests/rgb-to-gbr-test.png
new file mode 100644
index 000000000..7b039ba2d
Binary files /dev/null and b/third_party/testdata/wide-gamut-tests/rgb-to-gbr-test.png differ
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
new file mode 100644
index 000000000..a2c9e69cf
--- /dev/null
+++ b/tools/CMakeLists.txt
@@ -0,0 +1,215 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+cmake_minimum_required(VERSION 3.1)
+
+# X11 ICC library used by the comparison and viewer tools.
+if(JPEGXL_ENABLE_VIEWERS)
+find_package(ECM NO_MODULE)
+if (NOT ECM_FOUND)
+  message(WARNING "extra-cmake-modules were not found.")
+else()
+  set(CMAKE_MODULE_PATH ${ECM_FIND_MODULE_DIR})
+  find_package(XCB COMPONENTS XCB)
+  if (XCB_FOUND)
+    add_library(x11_icc STATIC
+      x11/icc.cc
+      x11/icc.h
+    )
+    target_link_libraries(x11_icc PUBLIC jpegxl-static XCB::XCB)
+  endif ()
+endif()
+endif()  # JPEGXL_ENABLE_VIEWERS
+
+# The JPEGXL_VERSION is set from the builders.
+if(NOT DEFINED JPEGXL_VERSION OR JPEGXL_VERSION STREQUAL "")
+  set(JPEGXL_VERSION "(unknown)")
+endif()
+
+set(TOOL_BINARIES
+  cbrunsli_tool
+  cjpegxl
+  djpegxl
+
+  butteraugli_main
+  decode_and_encode
+  epf_main
+  ssimulacra_main
+
+  xyb_range
+
+  # Other tools added conditionally below:
+  #  benchmark_xl
+)
+
+
+add_library(jpegxl_tool STATIC
+  butteraugli_pnorm.cc
+  cmdline.cc
+)
+target_compile_options(jpegxl_tool PUBLIC "${JPEGXL_INTERNAL_FLAGS}")
+target_link_libraries(jpegxl_tool jpegxl-static)
+
+target_include_directories(jpegxl_tool
+  PUBLIC "${CMAKE_SOURCE_DIR}")
+
+add_library(epf epf.cc epf.h)
+target_link_libraries(epf jpegxl-static)
+
+# Main compressors.
+add_library(cjpegxltool STATIC
+  cjpegxl.cc
+  cjxl.cc
+  cbrunsli.cc
+  speed_stats.cc
+)
+target_compile_definitions(cjpegxltool
+  PRIVATE "-DJPEGXL_VERSION=\"${JPEGXL_VERSION}\"")
+target_include_directories(cjpegxltool
+  PUBLIC "${CMAKE_SOURCE_DIR}")
+target_include_directories(cjpegxltool PRIVATE
+  $<TARGET_PROPERTY:brunslicommon-static,INCLUDE_DIRECTORIES>
+  ${JPEG_INCLUDE_DIR}
+)
+target_link_libraries(cjpegxltool
+    ${JPEG_LIBRARIES}
+    jpegxl-static
+    jpegxl_extras-static
+    jpegxl_threads
+)
+
+add_executable(cbrunsli_tool cbrunsli_main.cc)
+add_executable(cjpegxl cjpegxl_main.cc)
+
+# TODO: make brunsli own targets less "public"?
+set_target_properties (cbrunsli_tool PROPERTIES OUTPUT_NAME "cbrunsli" )
+
+target_link_libraries(cbrunsli_tool cjpegxltool)
+target_link_libraries(cjpegxl cjpegxltool)
+
+# Main decompressor.
+add_executable(djpegxl djxl.cc djpegxl.cc djpegxl_main.cc speed_stats.cc)
+target_compile_definitions(djpegxl
+  PRIVATE "-DJPEGXL_VERSION=\"${JPEGXL_VERSION}\"")
+
+# Other tools.
+add_executable(ssimulacra_main ssimulacra_main.cc ssimulacra.cc)
+add_executable(butteraugli_main butteraugli_main.cc)
+add_executable(decode_and_encode decode_and_encode.cc)
+add_executable(epf_main epf_main.cc)
+add_executable(xyb_range xyb_range.cc)
+
+target_link_libraries(epf_main epf)
+
+# Benchmark tools.
+if(${JPEGXL_ENABLE_BENCHMARK})
+  list(APPEND TOOL_BINARIES
+    benchmark_xl
+  )
+
+  add_executable(benchmark_xl
+    benchmark/benchmark_xl.cc
+    benchmark/benchmark_args.cc
+    benchmark/benchmark_codec.cc
+    benchmark/benchmark_file_io.cc
+    benchmark/benchmark_stats.cc
+    benchmark/benchmark_codec_jxl.cc
+    benchmark/benchmark_codec_jxl.h
+    benchmark/benchmark_codec_jpeg.cc
+    benchmark/benchmark_codec_jpeg.h
+    benchmark/benchmark_codec_png.cc
+    benchmark/benchmark_codec_png.h
+    speed_stats.cc
+    speed_stats.h
+    ../third_party/dirent.cc
+  )
+  target_include_directories(benchmark_xl PRIVATE
+    ${JPEG_INCLUDE_DIR}
+    $<TARGET_PROPERTY:brunslicommon-static,INCLUDE_DIRECTORIES>
+  )
+  target_link_libraries(benchmark_xl
+    ${JPEG_LIBRARIES}
+    Threads::Threads
+  )
+
+  find_package(PkgConfig)
+  pkg_check_modules(WebP IMPORTED_TARGET libwebp)
+  if(WebP_FOUND)
+    target_sources(benchmark_xl PRIVATE
+      "${CMAKE_CURRENT_LIST_DIR}/benchmark/benchmark_codec_webp.cc"
+      "${CMAKE_CURRENT_LIST_DIR}/benchmark/benchmark_codec_webp.h"
+    )
+    target_compile_definitions(benchmark_xl PRIVATE -DBENCHMARK_WEBP)
+
+    # Use the static version of webp if available.
+    find_library(WebP_STATIC_LINK_LIBRARY NAMES libwebp.a
+        PATHS "${WebP_LIBDIR}")
+    if("${WebP_STATIC_LINK_LIBRARY}" STREQUAL "WebP_STATIC_LINK_LIBRARY-NOTFOUND")
+      message(WARNING "Using dynamic libwebp")
+      target_link_libraries(benchmark_xl PkgConfig::WebP)
+    else()
+      target_link_libraries(benchmark_xl "${WebP_STATIC_LINK_LIBRARY}")
+      target_include_directories(benchmark_xl
+          PRIVATE ${WebP_STATIC_INCLUDE_DIRS})
+      target_compile_options(benchmark_xl PRIVATE ${WebP_STATIC_CFLAGS_OTHER})
+    endif()
+  endif()
+endif()  # JPEGXL_ENABLE_BENCHMARK
+
+# All tool binaries depend on "jpegxl" library and the tool helpers.
+foreach(BINARY IN LISTS TOOL_BINARIES)
+  target_include_directories("${BINARY}" PRIVATE "${CMAKE_SOURCE_DIR}")
+  target_link_libraries("${BINARY}" jpegxl-static jpegxl_extras-static jpegxl_threads jpegxl_tool)
+endforeach()
+install(TARGETS ${TOOL_BINARIES} RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}")
+
+# Fuzzers.
+if(${JPEGXL_ENABLE_FUZZERS})
+  set(FUZZER_BINARIES
+    color_encoding_fuzzer
+    djxl_fuzzer
+    icc_codec_fuzzer
+    fields_fuzzer
+    rans_fuzzer
+    set_from_bytes_fuzzer
+  )
+  foreach(BINARY IN LISTS FUZZER_BINARIES)
+    add_executable("${BINARY}" "${BINARY}.cc")
+    target_include_directories("${BINARY}" PRIVATE "${CMAKE_SOURCE_DIR}")
+    target_link_libraries("${BINARY}" jpegxl-static jpegxl_extras-static jpegxl_threads jpegxl_tool)
+    target_compile_options("${BINARY}" PRIVATE "-fsanitize=fuzzer-no-link")
+    target_link_libraries("${BINARY}" -fsanitize=fuzzer)
+  endforeach()
+endif()  # JPEGXL_ENABLE_FUZZERS
+
+# Library API test. This test is only to check that we can link against the
+# shared library from C99 file and don't need to use internal symbols.
+file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/tests)
+add_executable(libjpegxl_test libjpegxl_test.c)
+set_property(TARGET libjpegxl_test PROPERTY C_STANDARD 99)
+set_target_properties(libjpegxl_test PROPERTIES PREFIX "tests/")
+target_link_libraries(libjpegxl_test jpegxl)
+target_compile_options(libjpegxl_test PRIVATE -Wall -Wextra -Werror)
+if(NOT WIN32)
+  target_compile_options(libjpegxl_test PRIVATE -pedantic)
+endif()
+
+add_test(NAME LibraryCLinkageTest COMMAND libjpegxl_test)
+
+# Tools defined in subdirectories.
+if(${JPEGXL_ENABLE_VIEWERS})
+add_subdirectory(comparison_viewer)
+add_subdirectory(flicker_test)
+add_subdirectory(viewer)
+endif()
diff --git a/tools/args.h b/tools/args.h
new file mode 100644
index 000000000..d5b59637a
--- /dev/null
+++ b/tools/args.h
@@ -0,0 +1,166 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_ARGS_H_
+#define TOOLS_ARGS_H_
+
+// Helpers for parsing command line arguments. No include guard needed.
+
+#include <stdio.h>
+#include <string.h>
+
+#include <string>
+#include <vector>
+
+#include "jxl/base/override.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"  // DecoderHints
+#include "jxl/gaborish.h"
+
+namespace jpegxl {
+namespace tools {
+
+static inline bool ParseOverride(const char* arg, jxl::Override* out) {
+  const std::string s_arg(arg);
+  if (s_arg == "1") {
+    *out = jxl::Override::kOn;
+    return true;
+  }
+  if (s_arg == "0") {
+    *out = jxl::Override::kOff;
+    return true;
+  }
+  fprintf(stderr, "Invalid flag, %s must be 0 or 1\n", arg);
+  return JXL_FAILURE("Args");
+}
+
+static inline bool ParseUnsigned(const char* arg, size_t* out) {
+  char* end;
+  *out = static_cast<size_t>(strtoull(arg, &end, 0));
+  if (end[0] != '\0') {
+    fprintf(stderr, "Unable to interpret as unsigned integer: %s.\n", arg);
+    return JXL_FAILURE("Args");
+  }
+  return true;
+}
+
+static inline bool ParseUint32(const char* arg, uint32_t* out) {
+  size_t value = 0;
+  bool ret = ParseUnsigned(arg, &value);
+  if (ret) *out = value;
+  return ret;
+}
+
+static inline bool ParseSigned(const char* arg, int* out) {
+  char* end;
+  *out = static_cast<int>(strtol(arg, &end, 0));
+  if (end[0] != '\0') {
+    fprintf(stderr, "Unable to interpret as signed integer: %s.\n", arg);
+    return JXL_FAILURE("Args");
+  }
+  return true;
+}
+
+static inline bool ParseFloat(const char* arg, float* out) {
+  char* end;
+  *out = static_cast<float>(strtod(arg, &end));
+  if (end[0] != '\0') {
+    fprintf(stderr, "Unable to interpret as float: %s.\n", arg);
+    return JXL_FAILURE("Args");
+  }
+  return true;
+}
+
+static inline bool ParseFloatPair(const char* arg,
+                                  std::pair<float, float>* out) {
+  int parsed = sscanf(arg, "%f,%f", &out->first, &out->second);
+  if (parsed == 1) {
+    out->second = out->first;
+  } else if (parsed != 2) {
+    fprintf(stderr,
+            "Unable to interpret as float pair separated by a comma: %s.\n",
+            arg);
+    return JXL_FAILURE("Args");
+  }
+  return true;
+}
+
+static inline bool ParseDouble(const char* arg, double* out) {
+  char* end;
+  *out = static_cast<double>(strtod(arg, &end));
+  if (end[0] != '\0') {
+    fprintf(stderr, "Unable to interpret as double: %s.\n", arg);
+    return JXL_FAILURE("Args");
+  }
+  return true;
+}
+
+static inline bool ParseAndAppendKeyValue(const char* arg,
+                                          jxl::DecoderHints* out) {
+  const char* eq = strchr(arg, '=');
+  if (!eq) {
+    fprintf(stderr, "Expected argument as 'key=value' but received '%s'\n",
+            arg);
+    return false;
+  }
+  std::string key(arg, eq);
+  out->Add(key, std::string(eq + 1));
+  return true;
+}
+
+static inline bool ParsePredictorsVector(const char* arg,
+                                         std::vector<int>* out) {
+  for (; *arg; arg++) {
+    if (*arg == '?') {
+      out->push_back(-1);
+    } else if (*arg >= '0' && *arg <= '9') {
+      out->push_back(*arg - '0');
+    } else {
+      fprintf(stderr, "Invalid predictor value '%c', must be a ? or a digit.",
+              *arg);
+      return JXL_FAILURE("Args");
+    }
+  }
+  return true;
+}
+
+static inline bool ParseString(const char* arg, std::string* out) {
+  out->assign(arg);
+  return true;
+}
+
+static inline bool ParseCString(const char* arg, const char** out) {
+  *out = arg;
+  return true;
+}
+
+static inline bool SetBooleanTrue(bool* out) {
+  *out = true;
+  return true;
+}
+
+static inline bool SetBooleanFalse(bool* out) {
+  *out = false;
+  return true;
+}
+
+static inline bool IncrementUnsigned(size_t* out) {
+  (*out)++;
+  return true;
+}
+
+}  // namespace tools
+}  // namespace jpegxl
+
+#endif  // TOOLS_ARGS_H_
diff --git a/tools/benchmark/benchmark_args.cc b/tools/benchmark/benchmark_args.cc
new file mode 100644
index 000000000..8349a5ac3
--- /dev/null
+++ b/tools/benchmark/benchmark_args.cc
@@ -0,0 +1,256 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/benchmark/benchmark_args.h"
+
+#include <stddef.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include "jxl/base/status.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/extras/codec.h"
+#include "tools/benchmark/benchmark_codec_jpeg.h"  // for AddCommand..
+#include "tools/benchmark/benchmark_codec_jxl.h"
+#include "tools/benchmark/benchmark_codec_png.h"
+
+#ifdef BENCHMARK_WEBP
+#include "tools/benchmark/benchmark_codec_webp.h"
+#endif  // BENCHMARK_WEBP
+
+namespace jxl {
+
+std::vector<std::string> SplitString(const std::string& s, char c) {
+  std::vector<std::string> result;
+  size_t pos = 0;
+  for (size_t i = 0; i <= s.size(); i++) {
+    if (i == s.size() || s[i] == c) {
+      result.push_back(s.substr(pos, i - pos));
+      pos = i + 1;
+    }
+  }
+  return result;
+}
+
+int ParseIntParam(const std::string& param, int lower_bound, int upper_bound) {
+  int val = strtol(param.substr(1).c_str(), nullptr, 10);
+  JXL_CHECK(val >= lower_bound && val <= upper_bound);
+  return val;
+}
+
+BenchmarkArgs* Args() {
+  static BenchmarkArgs args;
+  return &args;
+}
+
+Status BenchmarkArgs::AddCommandLineOptions() {
+  AddString(&input, "input", "File or file pattern matching input files.");
+  AddString(&codec, "codec",
+            "Comma separated list of image codec descriptions to benchmark.",
+            "jxl");
+  AddFlag(&print_details, "print_details",
+          "Prints size and distortion for each image. Not safe for "
+          "concurrent benchmark runs.",
+          false);
+  AddFlag(&print_details_csv, "print_details_csv",
+          "When print_details is used, print as CSV.", false);
+  AddFlag(
+      &print_more_stats, "print_more_stats",
+      "Prints codec-specific stats. Not safe for concurrent benchmark runs.",
+      false);
+  AddFlag(&print_distance_percentiles, "print_distance_percentiles",
+          "Prints distance percentiles for the corpus. Not safe for "
+          "concurrent benchmark runs.",
+          false);
+  AddFlag(&silent_errors, "silent_errors",
+          "If true, doesn't print error messages on compression or"
+          " decompression errors. Errors counts are still visible in the"
+          " 'Errors' column of the result table. Please note that depending"
+          " depending on the JXL build settings, error messages and asserts"
+          " from within the codec may be printed irrespective of this flag"
+          " anyway, use release build to ensure no messages.",
+          false);
+  AddFlag(&save_compressed, "save_compressed",
+          "Saves the compressed files for each input image and each codec.",
+          false);
+  AddFlag(&save_decompressed, "save_decompressed",
+          "Saves the decompressed files as PNG for each input image "
+          "and each codec.",
+          false);
+  AddString(&output_extension, "output_extension",
+            "Extension (starting with dot) to use for saving output images.",
+            ".png");
+  AddString(&output_description, "output_description",
+            "Color encoding (see ParseDescription; e.g. RGB_D65_SRG_Rel_709) "
+            "for saving output images, "
+            " defaults to sRGB.");
+
+  AddString(&dec_hints_string, "dec-hints",
+            "Decoder hints for the input images to encoder. Comma separated "
+            "key=value pairs. The key color_space indicates ColorEncoding (see "
+            "ParseDescription; e.g. RGB_D65_SRG_Rel_709) for input images "
+            "without color encoding (such as PNM)");
+
+  AddUnsigned(
+      &override_bitdepth, "override_bitdepth",
+      "If nonzero, store the given bit depth in the JPEG XL file metadata"
+      " (1-32), instead of using the bit depth from the original input"
+      " image.",
+      0);
+
+  AddDouble(&mul_output, "mul_output",
+            "If nonzero, multiplies linear sRGB by this and clamps to 255",
+            0.0);
+  AddDouble(&heatmap_good, "heatmap_good",
+            "If greater than zero, use this as the good "
+            "threshold for creating heatmap images.",
+            0.0);
+  AddDouble(&heatmap_bad, "heatmap_bad",
+            "If greater than zero, use this as the bad "
+            "threshold for creating heatmap images.",
+            0.0);
+
+  AddFlag(&write_html_report, "write_html_report",
+          "Creates an html report with original and compressed images.", false);
+  AddFlag(&html_report_self_contained, "html_report_self_contained",
+          "Base64-encode the images in the HTML report rather than use "
+          "external file names. May cause very large HTML data size.",
+          false);
+
+  AddFlag(
+      &markdown, "markdown",
+      "Adds formatting around ASCII table to render correctly in Markdown based"
+      " interfaces",
+      true);
+
+  AddFlag(&more_columns, "more_columns", "Print extra columns in the table",
+          false);
+
+  AddString(&originals_url, "originals_url",
+            "Url prefix to serve original images from in the html report.");
+  AddString(&output_dir, "output_dir",
+            "If not empty, save compressed and decompressed "
+            "images here.");
+
+  AddSigned(&num_threads, "num_threads",
+            "The number of threads for concurrent benchmarking. Defaults to "
+            "1 thread per CPU core (if negative).",
+            -1);
+  AddSigned(&inner_threads, "inner_threads",
+            "The number of extra threads per task. "
+            "Defaults to occupy cores (if negative).",
+            -1);
+  AddUnsigned(&encode_reps, "encode_reps",
+              "How many times to encode (>1 for more precise measurements). "
+              "Defaults to 1.",
+              1);
+  AddUnsigned(&decode_reps, "decode_reps",
+              "How many times to decode (>1 for more precise measurements). "
+              "Defaults to 1.",
+              1);
+
+  AddString(&sample_tmp_dir, "sample_tmp_dir",
+            "Directory to put samples from input images.");
+
+  AddSigned(&num_samples, "num_samples", "How many sample areas to take.", 0);
+  AddSigned(&sample_dimensions, "sample_dimensions",
+            "How big areas to sample from the input.", 64);
+
+  AddDouble(&error_pnorm, "error_pnorm",
+            "smallest p norm for pooling butteraugli values", 3.0);
+
+  AddDouble(&hf_asymmetry, "hf_asymmetry",
+            "Multiplier for weighting HF artefacts more than features "
+            "being smoothed out. 1.0 means no HF asymmetry. 0.3 is "
+            "a good value to start exploring for asymmetry.",
+            0.8);
+  AddFlag(&profiler, "profiler", "If true, print profiler results.", false);
+
+  AddFlag(&show_progress, "show_progress",
+          "Show activity dots per completed file during benchmark.", false);
+
+  AddFlag(
+      &skip_butteraugli, "skip_butteraugli",
+      "If true, doesn't compute distance metrics, only compression and"
+      " decompression speed and size. Distance numbers shown in the"
+      " table are invalid.",
+      false);
+
+  AddFlag(
+      &decode_only, "decode_only",
+      "If true, only decodes, and the input files must be compressed with a "
+      "compatible format for the given codec(s). Only measures decompression "
+      "speed and sizes, and can only use a single set of compatible decoders. "
+      "Distance numbers and compression speeds shown in the table are invalid.",
+      false);
+
+  if (!AddCommandLineOptionsJxlCodec(this)) return false;
+  if (!AddCommandLineOptionsJPEGCodec(this)) return false;
+  if (!AddCommandLineOptionsPNGCodec(this)) return false;
+#ifdef BENCHMARK_WEBP
+  if (!AddCommandLineOptionsWebPCodec(this)) return false;
+#endif  // BENCHMARK_WEBP
+
+  return true;
+}
+
+Status BenchmarkArgs::ValidateArgs() {
+  size_t bits_per_sample = 0;  // unused
+  if (CodecFromExtension(output_extension, &bits_per_sample) ==
+      Codec::kUnknown) {
+    JXL_WARNING("Unrecognized output_extension %s, try .png",
+                output_extension.c_str());
+    return false;  // already warned
+  }
+
+  // If empty, don't do anything; callers must only use output_encoding if
+  // output_description is not empty.
+  if (!output_description.empty()) {
+    // Validate, but also create the profile (only needs to happen once).
+    if (!ParseDescription(output_description, &output_encoding)) {
+      JXL_WARNING("Unrecognized output_description %s, try RGB_D65_SRG_Rel_Lin",
+                  output_description.c_str());
+      return false;  // already warned
+    }
+    JXL_RETURN_IF_ERROR(ColorManagement::CreateProfile(&output_encoding));
+  }
+
+  JXL_RETURN_IF_ERROR(ValidateArgsJxlCodec(this));
+
+  if (print_details_csv) print_details = true;
+
+  if (override_bitdepth > 32) {
+    return JXL_FAILURE("override_bitdepth must be <= 32");
+  }
+
+  if (!dec_hints_string.empty()) {
+    std::vector<std::string> hints = SplitString(dec_hints_string, ',');
+    for (const auto& hint : hints) {
+      std::vector<std::string> kv = SplitString(hint, '=');
+      if (kv.size() != 2) {
+        return JXL_FAILURE(
+            "dec-hints key value pairs must have the form 'key=value'");
+      }
+      dec_hints.Add(kv[0], kv[1]);
+    }
+  }
+
+  return true;
+}
+
+}  // namespace jxl
diff --git a/tools/benchmark/benchmark_args.h b/tools/benchmark/benchmark_args.h
new file mode 100644
index 000000000..0d0b08172
--- /dev/null
+++ b/tools/benchmark/benchmark_args.h
@@ -0,0 +1,169 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_BENCHMARK_BENCHMARK_ARGS_H_
+#define TOOLS_BENCHMARK_BENCHMARK_ARGS_H_
+
+// Command line parsing and arguments for benchmark_xl
+
+#include <stddef.h>
+
+#include <algorithm>
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "jxl/base/override.h"
+#include "jxl/base/status.h"
+#include "jxl/color_encoding.h"
+#include "tools/args.h"
+#include "tools/cmdline.h"
+
+namespace jxl {
+
+std::vector<std::string> SplitString(const std::string& s, char c);
+
+int ParseIntParam(const std::string& param, int lower_bound, int upper_bound);
+
+struct BenchmarkArgs {
+  using OptionId = jpegxl::tools::CommandLineParser::OptionId;
+
+  void AddFlag(bool* field, const char* longName, const char* help,
+               bool defaultValue) {
+    const char* noName = RememberString_(std::string("no") + longName);
+    cmdline.AddOptionFlag('\0', longName, nullptr, field,
+                          &jpegxl::tools::SetBooleanTrue);
+    cmdline.AddOptionFlag('\0', noName, help, field,
+                          &jpegxl::tools::SetBooleanFalse);
+    *field = defaultValue;
+  }
+
+  OptionId AddOverride(Override* field, const char* longName,
+                       const char* help) {
+    OptionId result = cmdline.AddOptionValue('\0', longName, "0|1", help, field,
+                                             &jpegxl::tools::ParseOverride);
+    *field = Override::kDefault;
+    return result;
+  }
+
+  OptionId AddString(std::string* field, const char* longName, const char* help,
+                     const std::string& defaultValue = "") {
+    OptionId result = cmdline.AddOptionValue(
+        '\0', longName, "<string>", help, field, &jpegxl::tools::ParseString);
+    *field = defaultValue;
+    return result;
+  }
+
+  OptionId AddDouble(double* field, const char* longName, const char* help,
+                     double defaultValue) {
+    OptionId result = cmdline.AddOptionValue(
+        '\0', longName, "<scalar>", help, field, &jpegxl::tools::ParseDouble);
+    *field = defaultValue;
+    return result;
+  }
+
+  OptionId AddSigned(int* field, const char* longName, const char* help,
+                     int defaultValue) {
+    OptionId result = cmdline.AddOptionValue(
+        '\0', longName, "<integer>", help, field, &jpegxl::tools::ParseSigned);
+    *field = defaultValue;
+    return result;
+  }
+
+  OptionId AddUnsigned(size_t* field, const char* longName, const char* help,
+                       size_t defaultValue) {
+    OptionId result =
+        cmdline.AddOptionValue('\0', longName, "<unsigned>", help, field,
+                               &jpegxl::tools::ParseUnsigned);
+    *field = defaultValue;
+    return result;
+  }
+
+  Status AddCommandLineOptions();
+
+  Status ValidateArgs();
+
+  bool Parse(int argc, const char** argv) { return cmdline.Parse(argc, argv); }
+
+  void PrintHelp() const { cmdline.PrintHelp(); }
+
+  std::string input;
+  std::string codec;
+  bool print_details;
+  bool print_details_csv;
+  bool print_more_stats;
+  bool print_distance_percentiles;
+  bool silent_errors;
+  bool save_compressed;
+  bool save_decompressed;
+  std::string output_extension;    // see CodecFromExtension
+  std::string output_description;  // see ParseDescription
+  ColorEncoding output_encoding;   // determined by output_description
+
+  bool decode_only;
+  bool skip_butteraugli;
+
+  std::string dec_hints_string;
+  jxl::DecoderHints dec_hints;
+
+  size_t override_bitdepth;
+
+  double mul_output;
+  double heatmap_good;
+  double heatmap_bad;
+
+  bool write_html_report;
+  bool html_report_self_contained;
+  bool markdown;
+  bool more_columns;
+
+  std::string originals_url;
+  std::string output_dir;
+
+  int num_threads;
+  int inner_threads;
+  size_t decode_reps;
+  size_t encode_reps;
+
+  std::string sample_tmp_dir;
+
+  int num_samples;
+  int sample_dimensions;
+  double hf_asymmetry;
+
+  bool profiler;
+  double error_pnorm;
+  bool show_progress;
+
+  jpegxl::tools::CommandLineParser cmdline;
+
+ private:
+  const char* RememberString_(const std::string& text) {
+    const char* data = text.c_str();
+    std::vector<char> copy(data, data + text.size() + 1);
+    string_pool_.push_back(copy);
+    return string_pool_.back().data();
+  }
+
+  // A memory pool with stable addresses for strings to provide stable
+  // const char pointers to cmdline.h for dynamic help/name strings.
+  std::deque<std::vector<char>> string_pool_;
+};
+
+// Returns singleton
+BenchmarkArgs* Args();
+
+}  // namespace jxl
+
+#endif  // TOOLS_BENCHMARK_BENCHMARK_ARGS_H_
diff --git a/tools/benchmark/benchmark_codec.cc b/tools/benchmark/benchmark_codec.cc
new file mode 100644
index 000000000..dcc1ae679
--- /dev/null
+++ b/tools/benchmark/benchmark_codec.cc
@@ -0,0 +1,174 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/benchmark/benchmark_codec.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "tools/benchmark/benchmark_args.h"
+#include "tools/benchmark/benchmark_codec_jpeg.h"
+#include "tools/benchmark/benchmark_codec_jxl.h"
+#include "tools/benchmark/benchmark_codec_png.h"
+#include "tools/benchmark/benchmark_stats.h"
+
+#ifdef BENCHMARK_WEBP
+#include "tools/benchmark/benchmark_codec_webp.h"
+#endif  // BENCHMARK_WEBP
+
+namespace jxl {
+
+void ImageCodec::ParseParameters(const std::string& parameters) {
+  params_ = parameters;
+  std::vector<std::string> parts = SplitString(parameters, ':');
+  for (size_t i = 0; i < parts.size(); ++i) {
+    if (!ParseParam(parts[i])) {
+      JXL_ABORT("Invalid parameter %s", parts[i].c_str());
+    }
+  }
+}
+
+Status ImageCodec::ParseParam(const std::string& param) {
+  if (param[0] ==
+      'q') {  // libjpeg-style quality, [0,100]  (or in case of
+              // modular, below 0 is also allowed if you like cubism)
+    const std::string quality_param = param.substr(1);
+    char* end;
+    const float q_target = strtof(quality_param.c_str(), &end);
+    if (end == quality_param.c_str() ||
+        end != quality_param.c_str() + quality_param.size()) {
+      return false;
+    }
+    q_target_ = q_target;
+    return true;
+  }
+  if (param[0] == 'd') {  // butteraugli distance
+    const std::string distance_param = param.substr(1);
+    char* end;
+    const float butteraugli_target = strtof(distance_param.c_str(), &end);
+    if (end == distance_param.c_str() ||
+        end != distance_param.c_str() + distance_param.size()) {
+      return false;
+    }
+    butteraugli_target_ = butteraugli_target;
+
+    // full hf asymmetry at high distance
+    static const double kHighDistance = 2.5;
+
+    // no hf asymmetry at low distance
+    static const double kLowDistance = 0.6;
+
+    if (butteraugli_target_ >= kHighDistance) {
+      hf_asymmetry_ = args_.hf_asymmetry;
+    } else if (butteraugli_target_ >= kLowDistance) {
+      float w =
+          (butteraugli_target_ - kLowDistance) / (kHighDistance - kLowDistance);
+      hf_asymmetry_ = args_.hf_asymmetry * w + 1.0f * (1.0f - w);
+    } else {
+      hf_asymmetry_ = 1.0f;
+    }
+    return true;
+  } else if (param[0] == 'r') {
+    butteraugli_target_ = -1.0;
+    hf_asymmetry_ = args_.hf_asymmetry;
+    bitrate_target_ = strtof(param.substr(1).c_str(), nullptr);
+    return true;
+  }
+  return false;
+}
+
+// Low-overhead "codec" for measuring benchmark overhead.
+class NoneCodec : public ImageCodec {
+ public:
+  explicit NoneCodec(const BenchmarkArgs& args) : ImageCodec(args) {}
+  Status ParseParam(const std::string& param) override { return true; }
+
+  Status Compress(const std::string& filename, const CodecInOut* io,
+                  ThreadPool* pool, PaddedBytes* compressed) override {
+    PROFILER_ZONE("NoneCompress");
+    // Encode image size so we "decompress" something of the same size, as
+    // required by butteraugli.
+    const uint32_t xsize = io->xsize();
+    const uint32_t ysize = io->ysize();
+    compressed->resize(8);
+    memcpy(compressed->data(), &xsize, 4);
+    memcpy(compressed->data() + 4, &ysize, 4);
+    return true;
+  }
+
+  Status Decompress(const std::string& filename,
+                    const Span<const uint8_t> compressed, ThreadPool* pool,
+                    CodecInOut* io) override {
+    PROFILER_ZONE("NoneDecompress");
+    JXL_ASSERT(compressed.size() == 8);
+    uint32_t xsize, ysize;
+    memcpy(&xsize, compressed.data(), 4);
+    memcpy(&ysize, compressed.data() + 4, 4);
+    Image3F image(xsize, ysize);
+    ZeroFillImage(&image);
+    io->metadata.bits_per_sample = 32;
+    io->metadata.color_encoding = ColorManagement::SRGB();
+    io->SetFromImage(std::move(image), io->metadata.color_encoding);
+    return true;
+  }
+
+  void GetMoreStats(BenchmarkStats* stats) override {}
+};
+
+ImageCodecPtr CreateImageCodec(const std::string& description) {
+  std::string name = description;
+  std::string parameters = "";
+  size_t colon = description.find(':');
+  if (colon < description.size()) {
+    name = description.substr(0, colon);
+    parameters = description.substr(colon + 1);
+  }
+  ImageCodecPtr result;
+  if (name == "jxl") {
+    result.reset(CreateNewJxlCodec(*Args()));
+  } else if (name == "jpeg") {
+    result.reset(CreateNewJPEGCodec(*Args()));
+  } else if (name == "png") {
+    result.reset(CreateNewPNGCodec(*Args()));
+  } else if (name == "none") {
+    result.reset(new NoneCodec(*Args()));
+#ifdef BENCHMARK_WEBP
+  } else if (name == "webp") {
+    result.reset(CreateNewWebPCodec(*Args()));
+#endif  // BENCHMARK_WEBP
+  } else {
+    JXL_ABORT("Unknown image codec: %s", name.c_str());
+  }
+  result->set_description(description);
+  if (!parameters.empty()) result->ParseParameters(parameters);
+  return result;
+}
+
+}  // namespace jxl
diff --git a/tools/benchmark/benchmark_codec.h b/tools/benchmark/benchmark_codec.h
new file mode 100644
index 000000000..657103b5f
--- /dev/null
+++ b/tools/benchmark/benchmark_codec.h
@@ -0,0 +1,106 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_BENCHMARK_BENCHMARK_CODEC_H_
+#define TOOLS_BENCHMARK_BENCHMARK_CODEC_H_
+
+#include <stdint.h>
+
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/image.h"
+#include "tools/args.h"
+#include "tools/benchmark/benchmark_args.h"
+#include "tools/benchmark/benchmark_stats.h"
+#include "tools/cmdline.h"
+
+namespace jxl {
+
+// Thread-compatible.
+class ImageCodec {
+ public:
+  explicit ImageCodec(const BenchmarkArgs& args)
+      : args_(args),
+        butteraugli_target_(1.0f),
+        q_target_(100.0f),
+        bitrate_target_(0.0f),
+        hf_asymmetry_(1.0f) {}
+
+  virtual ~ImageCodec() = default;
+
+  void set_description(const std::string& desc) { description_ = desc; }
+  const std::string& description() const { return description_; }
+
+  float hf_asymmetry() const { return hf_asymmetry_; }
+
+  virtual void ParseParameters(const std::string& parameters);
+
+  virtual Status ParseParam(const std::string& param);
+
+  // Returns true iff the codec instance (including parameters) can tolerate
+  // ImageBundle c_current() != metadata()->color_encoding, and the possibility
+  // of negative (out of gamut) pixel values.
+  virtual bool IsColorAware() const { return false; }
+
+  // Returns true iff the codec instance (including parameters) will operate
+  // only with quantized DCT (JPEG) coefficients in input.
+  virtual bool IsJpegTranscoder() const { return false; }
+
+  virtual Status Compress(const std::string& filename, const CodecInOut* io,
+                          ThreadPool* pool, PaddedBytes* compressed) = 0;
+
+  virtual Status Decompress(const std::string& filename,
+                            const Span<const uint8_t> compressed,
+                            ThreadPool* pool, CodecInOut* io) = 0;
+
+  virtual void GetMoreStats(BenchmarkStats* stats) {}
+
+  virtual Status CanRecompressJpeg() const { return false; }
+  virtual Status RecompressJpeg(const std::string& filename,
+                                const std::string& data,
+                                PaddedBytes* compressed) {
+    return false;
+  }
+
+  virtual std::string GetErrorMessage() const { return error_message_; }
+
+ protected:
+  const BenchmarkArgs& args_;
+  std::string params_;
+  std::string description_;
+  float butteraugli_target_;
+  float q_target_;
+  float bitrate_target_;
+  float hf_asymmetry_;
+  std::string error_message_;
+};
+
+using ImageCodecPtr = std::unique_ptr<ImageCodec>;
+
+// Creates an image codec by name, e.g. "jxl" to get a new instance of the
+// jxl codec. Optionally, behind a colon, parameters can be specified,
+// then ParseParameters of the codec gets called with the part behind the colon.
+ImageCodecPtr CreateImageCodec(const std::string& description);
+
+}  // namespace jxl
+
+#endif  // TOOLS_BENCHMARK_BENCHMARK_CODEC_H_
diff --git a/tools/benchmark/benchmark_codec_jpeg.cc b/tools/benchmark/benchmark_codec_jpeg.cc
new file mode 100644
index 000000000..48263bfe7
--- /dev/null
+++ b/tools/benchmark/benchmark_codec_jpeg.cc
@@ -0,0 +1,144 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "tools/benchmark/benchmark_codec_jpeg.h"
+
+#include <stddef.h>
+#include <stdio.h>
+// After stddef/stdio
+#include <jpeglib.h>
+
+#include <stdint.h>
+#include <string.h>
+
+#include <numeric>  // partial_sum
+#include <string>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/extras/codec_jpg.h"
+#include "tools/cmdline.h"
+
+#ifdef MEMORY_SANITIZER
+#include "sanitizer/msan_interface.h"
+#endif
+
+namespace jxl {
+
+namespace {
+
+struct JPEGArgs {
+  JpegEncoder encoder = JpegEncoder::kLibJpeg;
+  YCbCrChromaSubsampling chroma_subsampling = YCbCrChromaSubsampling::kAuto;
+};
+
+JPEGArgs* const jpegargs = new JPEGArgs;
+
+bool ParseChromaSubsampling(const char* param,
+                            YCbCrChromaSubsampling* subsampling) {
+  if (strlen(param) != 3) return false;
+  if (param[0] != '4') return false;
+  switch (param[1]) {
+    case '4':
+      if (param[2] != '4') return false;
+      *subsampling = YCbCrChromaSubsampling::k444;
+      return true;
+
+    case '2':
+      switch (param[2]) {
+        case '2':
+          *subsampling = YCbCrChromaSubsampling::k422;
+          return true;
+
+        case '0':
+          *subsampling = YCbCrChromaSubsampling::k420;
+          return true;
+
+        default:
+          return false;
+      }
+
+    case '1':
+      if (param[2] != '1') return false;
+      *subsampling = YCbCrChromaSubsampling::k411;
+      return true;
+
+    default:
+      return false;
+  }
+}
+
+}  // namespace
+
+Status AddCommandLineOptionsJPEGCodec(BenchmarkArgs* args) {
+  args->cmdline.AddOptionValue(
+      '\0', "chroma_subsampling", "444/422/420/411",
+      "default JPEG chroma subsampling (default: 444).",
+      &jpegargs->chroma_subsampling, &ParseChromaSubsampling);
+  return true;
+}
+
+class JPEGCodec : public ImageCodec {
+ public:
+  explicit JPEGCodec(const BenchmarkArgs& args) : ImageCodec(args) {
+    encoder_ = jpegargs->encoder;
+    chroma_subsampling_ = jpegargs->chroma_subsampling;
+  }
+
+  Status ParseParam(const std::string& param) override {
+    if (ImageCodec::ParseParam(param)) {
+      return true;
+    }
+    if (param == "sjpeg") {
+      encoder_ = JpegEncoder::kSJpeg;
+      return true;
+    }
+    if (param.compare(0, 3, "yuv") == 0) {
+      if (param.size() != 6) return false;
+      return ParseChromaSubsampling(param.c_str() + 3, &chroma_subsampling_);
+    }
+    return false;
+  }
+
+  Status Compress(const std::string& filename, const CodecInOut* io,
+                  ThreadPool* pool, PaddedBytes* compressed) override {
+    if (encoder_ == JpegEncoder::kLibJpeg &&
+        chroma_subsampling_ == YCbCrChromaSubsampling::kAuto) {
+      if (jpegargs->chroma_subsampling != YCbCrChromaSubsampling::kAuto) {
+        chroma_subsampling_ = jpegargs->chroma_subsampling;
+      } else {
+        chroma_subsampling_ = YCbCrChromaSubsampling::k444;
+      }
+    }
+    return EncodeImageJPG(io, encoder_, static_cast<int>(std::round(q_target_)),
+                          chroma_subsampling_, pool, compressed);
+  }
+
+  Status Decompress(const std::string& filename,
+                    const Span<const uint8_t> compressed, ThreadPool* pool,
+                    CodecInOut* io) override {
+    return DecodeImageJPG(compressed, io);
+  }
+
+ protected:
+  JpegEncoder encoder_;
+  YCbCrChromaSubsampling chroma_subsampling_;
+};
+
+ImageCodec* CreateNewJPEGCodec(const BenchmarkArgs& args) {
+  return new JPEGCodec(args);
+}
+
+}  // namespace jxl
diff --git a/tools/benchmark/benchmark_codec_jpeg.h b/tools/benchmark/benchmark_codec_jpeg.h
new file mode 100644
index 000000000..9347cb0eb
--- /dev/null
+++ b/tools/benchmark/benchmark_codec_jpeg.h
@@ -0,0 +1,29 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_BENCHMARK_BENCHMARK_CODEC_JPEG_H_
+#define TOOLS_BENCHMARK_BENCHMARK_CODEC_JPEG_H_
+
+#include "jxl/base/status.h"
+#include "tools/benchmark/benchmark_args.h"
+#include "tools/benchmark/benchmark_codec.h"
+
+namespace jxl {
+ImageCodec* CreateNewJPEGCodec(const BenchmarkArgs& args);
+
+// Registers the jpeg-specific command line options.
+Status AddCommandLineOptionsJPEGCodec(BenchmarkArgs* args);
+}  // namespace jxl
+
+#endif  // TOOLS_BENCHMARK_BENCHMARK_CODEC_JPEG_H_
diff --git a/tools/benchmark/benchmark_codec_jxl.cc b/tools/benchmark/benchmark_codec_jxl.cc
new file mode 100644
index 000000000..56f5c969d
--- /dev/null
+++ b/tools/benchmark/benchmark_codec_jxl.cc
@@ -0,0 +1,349 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "tools/benchmark/benchmark_codec_jxl.h"
+
+#include <brunsli/brunsli_decode.h>
+#include <brunsli/brunsli_encode.h>
+#include <brunsli/jpeg_data.h>
+#include <brunsli/jpeg_data_reader.h>
+
+#include <cstdint>
+#include <cstdlib>
+#include <functional>
+#include <sstream>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/override.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/brunsli.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_file.h"
+#include "jxl/enc_params.h"
+#include "jxl/extras/codec.h"
+#include "jxl/image_bundle.h"
+#include "jxl/modular/encoding/encoding.h"
+#include "tools/benchmark/benchmark_file_io.h"
+#include "tools/benchmark/benchmark_stats.h"
+#include "tools/cmdline.h"
+
+namespace jxl {
+
+// Output function for EncodeBrunsli.
+size_t OutputToBytes(void* data, const uint8_t* buf, size_t count) {
+  PaddedBytes* output = reinterpret_cast<PaddedBytes*>(data);
+  output->append(buf, buf + count);
+  return count;
+}
+
+// TODO(lode): This is copied from brunsli/c/enc/encode.cc, use that one, once
+// the latest version of brunsli is checked out as submodule.
+int EncodeBrunsli(size_t insize, const unsigned char* in, void* outdata,
+                  size_t (*outfun)(void* outdata, const unsigned char* buf,
+                                   size_t size)) {
+  std::vector<uint8_t> output;
+  brunsli::JPEGData jpg;
+  if (!brunsli::ReadJpeg(in, insize, brunsli::JPEG_READ_ALL, &jpg)) {
+    return 0;
+  }
+  size_t output_size = brunsli::GetMaximumBrunsliEncodedSize(jpg);
+  output.resize(output_size);
+  if (!brunsli::BrunsliEncodeJpeg(jpg, output.data(), &output_size)) {
+    return 0;
+  }
+  output.resize(output_size);
+  if (!outfun(outdata, reinterpret_cast<const unsigned char*>(output.data()),
+              output.size())) {
+    return 0;
+  }
+  return 1; /* ok */
+}
+
+struct JxlArgs {
+  double quant_bias;
+
+  bool use_ac_strategy;
+  bool qprogressive;  // progressive with shift-quantization.
+  bool progressive;
+  size_t progressive_dc;
+
+  double intensity_target;
+  BenchmarkArgs::OptionId intensity_target_option_id;
+  bool got_intensity_target;
+
+  Override noise;
+  Override adaptive_reconstruction;
+  Override dots;
+  Override patches;
+  Override gaborish;
+
+  std::string debug_image_dir;
+};
+
+static JxlArgs* const jxlargs = new JxlArgs;
+
+Status AddCommandLineOptionsJxlCodec(BenchmarkArgs* args) {
+  args->AddDouble(&jxlargs->quant_bias, "quant_bias",
+                  "Bias border pixels during quantization by this ratio.", 0.0);
+  args->AddFlag(&jxlargs->use_ac_strategy, "use_ac_strategy",
+                "If true, AC strategy will be used.", false);
+  args->AddFlag(&jxlargs->qprogressive, "qprogressive",
+                "Enable quantized progressive mode for AC.", false);
+  args->AddFlag(&jxlargs->progressive, "progressive",
+                "Enable progressive mode for AC.", false);
+  args->AddUnsigned(&jxlargs->progressive_dc, "progressive_dc",
+                    "Enable progressive mode for DC.", 0);
+
+  jxlargs->intensity_target_option_id = args->AddDouble(
+      &jxlargs->intensity_target, "intensity_target",
+      "Intended viewing intensity target in nits. Defaults to 250 for SDR"
+      " images, 4000 for HDR images (when the input image uses PQ or HLG"
+      " transfer function)",
+      jxl::kDefaultIntensityTarget);
+
+  args->AddOverride(&jxlargs->noise, "noise",
+                    "Enable(1)/disable(0) noise generation.");
+  args->AddOverride(
+      &jxlargs->adaptive_reconstruction, "adaptive_reconstruction",
+      "Enable(1)/disable(0) adaptive reconstruction (deringing).");
+  args->AddOverride(&jxlargs->dots, "dots",
+                    "Enable(1)/disable(0) dots generation.");
+  args->AddOverride(&jxlargs->patches, "patches",
+                    "Enable(1)/disable(0) patch dictionary.");
+  args->AddOverride(&jxlargs->gaborish, "gaborish", "Disable gaborish if 0.");
+
+  args->AddString(
+      &jxlargs->debug_image_dir, "debug_image_dir",
+      "If not empty, saves debug images for each "
+      "input image and each codec that provides it to this directory.");
+
+  return true;
+}
+
+Status ValidateArgsJxlCodec(BenchmarkArgs* args) {
+  jxlargs->got_intensity_target =
+      args->cmdline.GetOption(jxlargs->intensity_target_option_id)->matched();
+  return true;
+}
+
+class JxlCodec : public ImageCodec {
+ public:
+  explicit JxlCodec(const BenchmarkArgs& args) : ImageCodec(args) {}
+
+  Status ParseParam(const std::string& param) override {
+    const std::string kMaxPassesPrefix = "max_passes=";
+    const std::string kDownsamplingPrefix = "downsampling=";
+
+    dparams_.noise = jxlargs->noise;
+    dparams_.adaptive_reconstruction = jxlargs->adaptive_reconstruction;
+    cparams_.dots = jxlargs->dots;
+    cparams_.patches = jxlargs->patches;
+
+    cparams_.progressive_mode = jxlargs->progressive;
+    cparams_.qprogressive_mode = jxlargs->qprogressive;
+    cparams_.progressive_dc = jxlargs->progressive_dc;
+
+    if (ImageCodec::ParseParam(param)) {
+      // Nothing to do.
+    } else if (param[0] == 'u') {
+      cparams_.uniform_quant = strtof(param.substr(1).c_str(), nullptr);
+      hf_asymmetry_ = args_.hf_asymmetry;
+    } else if (param[0] == 'Q') {
+      brunsli_params_.quant_scale = strtof(param.substr(1).c_str(), nullptr);
+    } else if (param.substr(0, kMaxPassesPrefix.size()) == kMaxPassesPrefix) {
+      std::istringstream parser(param.substr(kMaxPassesPrefix.size()));
+      parser >> dparams_.max_passes;
+    } else if (param.substr(0, kDownsamplingPrefix.size()) ==
+               kDownsamplingPrefix) {
+      std::istringstream parser(param.substr(kDownsamplingPrefix.size()));
+      parser >> dparams_.max_downsampling;
+    } else if (param == "nl") {
+      cparams_.color_transform = jxl::ColorTransform::kNone;
+      cparams_.options.entropy_coder = 1;
+      cparams_.near_lossless = 3;
+      cparams_.options.max_properties = 0;
+      cparams_.options.nb_repeats = 0;
+      cparams_.responsive = 0;
+    } else if (ParseSpeedTier(param, &cparams_.speed_tier)) {
+      // Nothing to do.
+    } else if (param[0] == 'X') {
+      cparams_.channel_colors_pre_transform_percent =
+          strtol(param.substr(1).c_str(), nullptr, 10);
+    } else if (param[0] == 'Y') {
+      cparams_.channel_colors_percent =
+          strtol(param.substr(1).c_str(), nullptr, 10);
+    } else if (param[0] == 'p') {
+      cparams_.palette_colors = strtol(param.substr(1).c_str(), nullptr, 10);
+    } else if (param[0] == 'N') {
+      cparams_.near_lossless = strtol(param.substr(1).c_str(), nullptr, 10);
+    } else if (param[0] == 'C') {
+      cparams_.colorspace = strtol(param.substr(1).c_str(), nullptr, 10);
+    } else if (param[0] == 'c') {
+      cparams_.color_transform =
+          (jxl::ColorTransform)strtol(param.substr(1).c_str(), nullptr, 10);
+    } else if (param[0] == 'I') {
+      cparams_.options.nb_repeats = strtof(param.substr(1).c_str(), nullptr);
+    } else if (param == "Brotli") {
+      cparams_.options.entropy_coder = 1;
+    } else if (param == "ANS") {
+      cparams_.options.entropy_coder = 2;
+    } else if (param[0] == 'E') {
+      cparams_.options.max_properties =
+          strtof(param.substr(1).c_str(), nullptr);
+    } else if (param[0] == 'P') {
+      cparams_.options.predictor.clear();
+      cparams_.options.predictor.push_back(
+          strtof(param.substr(1).c_str(), nullptr));
+    } else if (param == "slow") {
+      cparams_.options.nb_repeats = 2;
+    } else if (param == "R") {
+      cparams_.responsive = 1;
+    } else if (param == "mg") {
+      cparams_.modular_group_mode = true;
+      cparams_.color_transform = jxl::ColorTransform::kNone;
+    } else if (param == "bg") {
+      cparams_.brunsli_group_mode = true;
+      cparams_.color_transform = jxl::ColorTransform::kYCbCr;
+    } else if (param == "plt") {
+      cparams_.options.entropy_coder = 1;
+      cparams_.options.brotli_effort = 11;
+      cparams_.options.max_properties = 0;
+      cparams_.options.nb_repeats = 0;
+      cparams_.options.predictor.clear();
+      cparams_.options.predictor.push_back(0);
+      cparams_.responsive = 0;
+      cparams_.colorspace = 0;
+      cparams_.channel_colors_pre_transform_percent = 0;
+      cparams_.channel_colors_percent = 0;
+    } else if (param == "b") {
+      brunsli_mode_ = true;
+    } else if (param == "file") {
+      brunsli_file_ = true;  // Use jxl:b:file
+    } else {
+      return JXL_FAILURE("Unrecognized param");
+    }
+    return true;
+  }
+
+  bool IsColorAware() const override {
+    // Can't deal with negative values from color space conversion.
+    if (cparams_.brunsli_group_mode || cparams_.modular_group_mode)
+      return false;
+    // Otherwise, input may be in any color space.
+    return true;
+  }
+
+  bool IsJpegTranscoder() const override {
+    if (cparams_.brunsli_group_mode) return true;
+    return false;
+  }
+
+  Status Compress(const std::string& filename, const CodecInOut* io,
+                  ThreadPool* pool, PaddedBytes* compressed) override {
+    if (brunsli_mode_) {
+      if (brunsli_file_) {
+        // Encode the original JPG file (or get failure if it was not JPG),
+        // rather than the CodecInOut pixels.
+        PaddedBytes bytes;
+        JXL_RETURN_IF_ERROR(ReadFile(filename, &bytes));
+        if (!EncodeBrunsli(bytes.size(), bytes.data(), compressed,
+                           OutputToBytes)) {
+          return JXL_FAILURE("Failed to encode file to brunsli");
+        }
+        return true;
+      } else {
+        if (io->metadata.HasAlpha()) {
+          // Prevent Abort in ImageBundle::VerifyMetadata when decompressing.
+          return JXL_FAILURE("Alpha not supported for brunsli");
+        }
+        return PixelsToBrunsli(io, compressed, brunsli_params_, pool);
+      }
+    }
+    if (!jxlargs->debug_image_dir.empty()) {
+      cinfo_.dump_image = [](const CodecInOut& io, const std::string& path) {
+        return EncodeToFile(io, path);
+      };
+      cinfo_.debug_prefix =
+          JoinPath(jxlargs->debug_image_dir, FileBaseName(filename)) +
+          ".jxl:" + params_ + ".dbg/";
+      JXL_RETURN_IF_ERROR(MakeDir(cinfo_.debug_prefix));
+    }
+    cparams_.butteraugli_distance = butteraugli_target_;
+    cparams_.target_bitrate = bitrate_target_;
+
+    if (jxlargs->got_intensity_target) {
+      cparams_.intensity_target = static_cast<float>(jxlargs->intensity_target);
+    } else {
+      cparams_.intensity_target = ChooseDefaultIntensityTarget(io->metadata);
+    }
+
+    cparams_.noise = jxlargs->noise;
+    cparams_.adaptive_reconstruction = jxlargs->adaptive_reconstruction;
+    cparams_.gaborish = jxlargs->gaborish;
+
+    cparams_.quant_border_bias = static_cast<float>(jxlargs->quant_bias);
+    cparams_.hf_asymmetry = hf_asymmetry_;
+
+    cparams_.quality_pair.first = q_target_;
+    cparams_.quality_pair.second = q_target_;
+
+    PassesEncoderState passes_encoder_state;
+    return EncodeFile(cparams_, io, &passes_encoder_state, compressed, &cinfo_,
+                      pool);
+  }
+
+  Status Decompress(const std::string& filename,
+                    const Span<const uint8_t> compressed, ThreadPool* pool,
+                    CodecInOut* io) override {
+    if (!jxlargs->debug_image_dir.empty()) {
+      dinfo_.dump_image = [](const CodecInOut& io, const std::string& path) {
+        return EncodeToFile(io, path);
+      };
+      dinfo_.debug_prefix =
+          JoinPath(jxlargs->debug_image_dir, FileBaseName(filename)) +
+          ".jxl:" + params_ + ".dbg/";
+      JXL_RETURN_IF_ERROR(MakeDir(dinfo_.debug_prefix));
+    }
+    return DecodeFile(dparams_, compressed, io, &dinfo_, pool);
+  }
+
+  void GetMoreStats(BenchmarkStats* stats) override {
+    JxlStats jxl_stats;
+    jxl_stats.num_inputs = 1;
+    jxl_stats.aux_out = cinfo_;
+    stats->jxl_stats.Assimilate(jxl_stats);
+  }
+
+ protected:
+  AuxOut cinfo_;
+  AuxOut dinfo_;
+  CompressParams cparams_;
+  DecompressParams dparams_;
+  BrunsliEncoderOptions brunsli_params_;
+  bool brunsli_mode_{false};
+  bool brunsli_file_{false};  // Brunsli on original JPG file instead of pixels
+};
+
+ImageCodec* CreateNewJxlCodec(const BenchmarkArgs& args) {
+  return new JxlCodec(args);
+}
+
+}  // namespace jxl
diff --git a/tools/benchmark/benchmark_codec_jxl.h b/tools/benchmark/benchmark_codec_jxl.h
new file mode 100644
index 000000000..74d6a4789
--- /dev/null
+++ b/tools/benchmark/benchmark_codec_jxl.h
@@ -0,0 +1,32 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_BENCHMARK_BENCHMARK_CODEC_JXL_H_
+#define TOOLS_BENCHMARK_BENCHMARK_CODEC_JXL_H_
+
+#include <string>
+
+#include "jxl/base/status.h"
+#include "tools/benchmark/benchmark_args.h"
+#include "tools/benchmark/benchmark_codec.h"
+
+namespace jxl {
+ImageCodec* CreateNewJxlCodec(const BenchmarkArgs& args);
+
+// Registers the jxl-specific command line options.
+Status AddCommandLineOptionsJxlCodec(BenchmarkArgs* args);
+Status ValidateArgsJxlCodec(BenchmarkArgs* args);
+}  // namespace jxl
+
+#endif  // TOOLS_BENCHMARK_BENCHMARK_CODEC_JXL_H_
diff --git a/tools/benchmark/benchmark_codec_png.cc b/tools/benchmark/benchmark_codec_png.cc
new file mode 100644
index 000000000..6f9d0b5fd
--- /dev/null
+++ b/tools/benchmark/benchmark_codec_png.cc
@@ -0,0 +1,62 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "tools/benchmark/benchmark_codec_png.h"
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <string>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/extras/codec_png.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+
+struct PNGArgs {
+  // Empty, no PNG-specific args currently.
+};
+
+static PNGArgs* const pngargs = new PNGArgs;
+
+Status AddCommandLineOptionsPNGCodec(BenchmarkArgs* args) { return true; }
+
+// Lossless.
+class PNGCodec : public ImageCodec {
+ public:
+  explicit PNGCodec(const BenchmarkArgs& args) : ImageCodec(args) {}
+
+  Status ParseParam(const std::string& param) override { return true; }
+
+  Status Compress(const std::string& filename, const CodecInOut* io,
+                  ThreadPool* pool, PaddedBytes* compressed) override {
+    const size_t bits = io->metadata.bits_per_sample;
+    return EncodeImagePNG(io, io->Main().c_current(), bits, pool, compressed);
+  }
+
+  Status Decompress(const std::string& /*filename*/,
+                    const Span<const uint8_t> compressed, ThreadPool* pool,
+                    CodecInOut* io) override {
+    return DecodeImagePNG(compressed, pool, io);
+  }
+};
+
+ImageCodec* CreateNewPNGCodec(const BenchmarkArgs& args) {
+  return new PNGCodec(args);
+}
+
+}  // namespace jxl
diff --git a/tools/benchmark/benchmark_codec_png.h b/tools/benchmark/benchmark_codec_png.h
new file mode 100644
index 000000000..d4949a006
--- /dev/null
+++ b/tools/benchmark/benchmark_codec_png.h
@@ -0,0 +1,31 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_BENCHMARK_BENCHMARK_CODEC_PNG_H_
+#define TOOLS_BENCHMARK_BENCHMARK_CODEC_PNG_H_
+
+#include <string>
+
+#include "jxl/base/status.h"
+#include "tools/benchmark/benchmark_args.h"
+#include "tools/benchmark/benchmark_codec.h"
+
+namespace jxl {
+ImageCodec* CreateNewPNGCodec(const BenchmarkArgs& args);
+
+// Registers the png-specific command line options.
+Status AddCommandLineOptionsPNGCodec(BenchmarkArgs* args);
+}  // namespace jxl
+
+#endif  // TOOLS_BENCHMARK_BENCHMARK_CODEC_PNG_H_
diff --git a/tools/benchmark/benchmark_codec_webp.cc b/tools/benchmark/benchmark_codec_webp.cc
new file mode 100644
index 000000000..81afaccd2
--- /dev/null
+++ b/tools/benchmark/benchmark_codec_webp.cc
@@ -0,0 +1,278 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "tools/benchmark/benchmark_codec_webp.h"
+
+#include <stdint.h>
+#include <string.h>
+#include <webp/decode.h>
+#include <webp/encode.h>
+
+#include <string>
+#include <vector>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+#ifdef MEMORY_SANITIZER
+#include "sanitizer/msan_interface.h"
+#endif
+
+namespace jxl {
+
+struct WebPArgs {
+  // Empty, no WebP-specific args currently.
+};
+
+static WebPArgs* const webpargs = new WebPArgs;
+
+Status AddCommandLineOptionsWebPCodec(BenchmarkArgs* args) { return true; }
+
+class WebPCodec : public ImageCodec {
+ public:
+  explicit WebPCodec(const BenchmarkArgs& args) : ImageCodec(args) {}
+
+  Status ParseParam(const std::string& param) override {
+    // Ensure that the 'q' parameter is not used up by ImageCodec.
+    if (param[0] == 'q') {
+      if (near_lossless_) {
+        near_lossless_quality_ = ParseIntParam(param, 0, 99);
+      } else {
+        quality_ = ParseIntParam(param, 1, 100);
+      }
+      return true;
+    } else if (ImageCodec::ParseParam(param)) {
+      return true;
+    } else if (param == "ll") {
+      lossless_ = true;
+      JXL_CHECK(!near_lossless_);
+      return true;
+    } else if (param == "nl") {
+      near_lossless_ = true;
+      JXL_CHECK(!lossless_);
+      return true;
+    } else if (param[0] == 'm') {
+      method_ = ParseIntParam(param, 1, 6);
+      return true;
+    }
+    return false;
+  }
+
+  Status Compress(const std::string& filename, const CodecInOut* io,
+                  ThreadPool* pool, PaddedBytes* compressed) override {
+    const ImageBundle& ib = io->Main();
+
+    const ImageU* alpha = ib.HasAlpha() ? &ib.alpha() : nullptr;
+    if (ib.HasAlpha() && ib.metadata()->alpha_bits > 8) {
+      return JXL_FAILURE("WebP alpha must be 8-bit");
+    }
+
+    // TODO: ib.CopyToSRGB to Image3B assert-fails if 16-bit alpha. Fix that,
+    // since it may be bug in external_image (alpha isn't requested here).
+    Image3B srgb;
+    JXL_RETURN_IF_ERROR(ib.CopyToSRGB(Rect(ib), &srgb, pool));
+
+    if (lossless_ || near_lossless_) {
+      // The lossless codec does not support 16-bit channels.
+      // Color models are currently not supported here and the sRGB 8-bit
+      // conversion causes loss due to clipping.
+      if (!ib.IsSRGB() || ib.metadata()->bits_per_sample > 8) {
+        return JXL_FAILURE("%s: webp:ll/nl requires 8-bit sRGB",
+                           filename.c_str());
+      }
+      return CompressInternal(srgb, alpha, 100, compressed);
+    } else if (bitrate_target_ > 0.0) {
+      int quality_bad = 100;
+      int quality_good = 92;
+      size_t target_size = srgb.xsize() * srgb.ysize() * bitrate_target_ / 8.0;
+      while (quality_good > 0 &&
+             CompressInternal(srgb, alpha, quality_good, compressed) &&
+             compressed->size() > target_size) {
+        quality_bad = quality_good;
+        quality_good -= 8;
+      }
+      if (quality_good <= 0) quality_good = 1;
+      while (quality_good + 1 < quality_bad) {
+        int quality = (quality_bad + quality_good) / 2;
+        if (!CompressInternal(srgb, alpha, quality, compressed)) {
+          break;
+        }
+        if (compressed->size() <= target_size) {
+          quality_good = quality;
+        } else {
+          quality_bad = quality;
+        }
+      }
+      return CompressInternal(srgb, alpha, quality_good, compressed);
+    } else if (quality_ > 0) {
+      return CompressInternal(srgb, alpha, quality_, compressed);
+    }
+    return false;
+  }
+
+  Status Decompress(const std::string& filename,
+                    const Span<const uint8_t> compressed, ThreadPool* pool,
+                    CodecInOut* io) override {
+    WebPDecoderConfig config;
+#ifdef MEMORY_SANITIZER
+    // config is initialized by libwebp, which we are not instrumenting with
+    // msan, therefore we need to initialize it here.
+    memset(&config, 0, sizeof(config));
+#endif
+    JXL_RETURN_IF_ERROR(WebPInitDecoderConfig(&config) == 1);
+    config.options.use_threads = 0;
+    config.options.dithering_strength = 0;
+    config.options.bypass_filtering = 0;
+    config.options.no_fancy_upsampling = 0;
+    WebPDecBuffer* const buf = &config.output;
+    buf->colorspace = MODE_RGBA;
+    const uint8_t* webp_data = compressed.data();
+    const int webp_size = compressed.size();
+    if (WebPDecode(webp_data, webp_size, &config) != VP8_STATUS_OK) {
+      return JXL_FAILURE("WebPDecode failed");
+    }
+    JXL_CHECK(buf->u.RGBA.stride == buf->width * 4);
+
+    const bool is_gray = false;
+    const bool has_alpha = true;
+    const uint8_t* data_begin = &buf->u.RGBA.rgba[0];
+    const uint8_t* data_end = data_begin + buf->width * buf->height * 4;
+#ifdef MEMORY_SANITIZER
+    // The image data is initialized by libwebp, which we are not instrumenting
+    // with msan.
+    __msan_unpoison(data_begin, data_end - data_begin);
+#endif
+    if (io->metadata.color_encoding.IsGray() != is_gray) {
+      // TODO(lode): either ensure is_gray matches what the color profile says,
+      // or set a correct color profile, e.g.
+      // io->metadata.color_encoding = ColorManagement::SRGB(is_gray);
+      // Return a standard failure becuase SetFromSRGB triggers a fatal assert
+      // for this instead.
+      return JXL_FAILURE("Color profile is-gray mismatch");
+    }
+    const Status ok =
+        io->Main().SetFromSRGB(buf->width, buf->height, is_gray, has_alpha,
+                               data_begin, data_end, pool);
+    WebPFreeDecBuffer(buf);
+    JXL_RETURN_IF_ERROR(ok);
+    io->enc_size = compressed.size();
+    io->dec_pixels = buf->width * buf->height;
+    return true;
+  }
+
+ private:
+  static int WebPStringWrite(const uint8_t* data, size_t data_size,
+                             const WebPPicture* const picture) {
+    if (data_size) {
+      PaddedBytes* const out = static_cast<PaddedBytes*>(picture->custom_ptr);
+      const size_t pos = out->size();
+      out->resize(pos + data_size);
+      memcpy(out->data() + pos, data, data_size);
+    }
+    return 1;
+  }
+
+  static void Import(const Image3B& srgb, WebPPicture* pic) {
+    const size_t xsize = srgb.xsize();
+    const size_t ysize = srgb.ysize();
+    std::vector<uint8_t> rgb(xsize * ysize * 3);
+    for (size_t y = 0; y < ysize; ++y) {
+      const uint8_t* JXL_RESTRICT row0 = srgb.ConstPlaneRow(0, y);
+      const uint8_t* JXL_RESTRICT row1 = srgb.ConstPlaneRow(1, y);
+      const uint8_t* JXL_RESTRICT row2 = srgb.ConstPlaneRow(2, y);
+      uint8_t* const JXL_RESTRICT row_rgb = &rgb[y * xsize * 3];
+      for (size_t x = 0; x < xsize; ++x) {
+        row_rgb[3 * x + 0] = row0[x];
+        row_rgb[3 * x + 1] = row1[x];
+        row_rgb[3 * x + 2] = row2[x];
+      }
+    }
+    WebPPictureImportRGB(pic, &rgb[0], 3 * srgb.xsize());
+  }
+
+  static void Import(const Image3B& srgb, const ImageU& alpha,
+                     WebPPicture* pic) {
+    const size_t xsize = srgb.xsize();
+    const size_t ysize = srgb.ysize();
+    std::vector<uint8_t> rgba(xsize * ysize * 4);
+    for (size_t y = 0; y < ysize; ++y) {
+      const uint8_t* JXL_RESTRICT row0 = srgb.ConstPlaneRow(0, y);
+      const uint8_t* JXL_RESTRICT row1 = srgb.ConstPlaneRow(1, y);
+      const uint8_t* JXL_RESTRICT row2 = srgb.ConstPlaneRow(2, y);
+      const uint16_t* JXL_RESTRICT rowa = alpha.ConstRow(y);
+      uint8_t* const JXL_RESTRICT row_rgba = &rgba[y * xsize * 4];
+      for (size_t x = 0; x < xsize; ++x) {
+        row_rgba[4 * x + 0] = row0[x];
+        row_rgba[4 * x + 1] = row1[x];
+        row_rgba[4 * x + 2] = row2[x];
+        row_rgba[4 * x + 3] = rowa[x] & 0xFF;
+      }
+    }
+    WebPPictureImportRGBA(pic, &rgba[0], 4 * srgb.xsize());
+  }
+
+  Status CompressInternal(const Image3B& srgb, const ImageU* alpha, int quality,
+                          PaddedBytes* compressed) {
+    *compressed = PaddedBytes();
+    WebPConfig config;
+    WebPConfigInit(&config);
+    JXL_ASSERT(!lossless_ || !near_lossless_);  // can't have both
+    config.lossless = lossless_;
+    config.quality = quality;
+    config.method = method_;
+    config.near_lossless = near_lossless_ ? near_lossless_quality_ : 100;
+    JXL_CHECK(WebPValidateConfig(&config));
+
+    WebPPicture pic;
+    WebPPictureInit(&pic);
+    pic.width = static_cast<int>(srgb.xsize());
+    pic.height = static_cast<int>(srgb.ysize());
+    pic.writer = &WebPStringWrite;
+    if (lossless_ || near_lossless_) pic.use_argb = 1;
+    pic.custom_ptr = compressed;
+
+    if (alpha == nullptr) {
+      Import(srgb, &pic);
+    } else {
+      Import(srgb, *alpha, &pic);
+    }
+
+    // WebP encoding may fail, for example, if the image is more than 16384
+    // pixels high or wide.
+    bool ok = WebPEncode(&config, &pic);
+    WebPPictureFree(&pic);
+#ifdef MEMORY_SANITIZER
+    // Compressed image data is initialized by libwebp, which we are not
+    // instrumenting with msan.
+    __msan_unpoison(compressed->data(), compressed->size());
+#endif
+    return ok;
+  }
+
+  int quality_ = 90;
+  bool lossless_ = false;
+  bool near_lossless_ = false;
+  bool near_lossless_quality_ = 40;  // only used if near_lossless_
+  int method_ = 6;                   // smallest, some speed cost
+};
+
+ImageCodec* CreateNewWebPCodec(const BenchmarkArgs& args) {
+  return new WebPCodec(args);
+}
+
+}  // namespace jxl
diff --git a/tools/benchmark/benchmark_codec_webp.h b/tools/benchmark/benchmark_codec_webp.h
new file mode 100644
index 000000000..b13a9f1cc
--- /dev/null
+++ b/tools/benchmark/benchmark_codec_webp.h
@@ -0,0 +1,32 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#ifndef TOOLS_BENCHMARK_BENCHMARK_CODEC_WEBP_H_
+#define TOOLS_BENCHMARK_BENCHMARK_CODEC_WEBP_H_
+
+// To support webp, install libwebp-dev and rerun cmake.
+
+#include <string>
+
+#include "jxl/base/status.h"
+#include "tools/benchmark/benchmark_args.h"
+#include "tools/benchmark/benchmark_codec.h"
+
+namespace jxl {
+ImageCodec* CreateNewWebPCodec(const BenchmarkArgs& args);
+
+// Registers the webp-specific command line options.
+Status AddCommandLineOptionsWebPCodec(BenchmarkArgs* args);
+}  // namespace jxl
+
+#endif  // TOOLS_BENCHMARK_BENCHMARK_CODEC_WEBP_H_
diff --git a/tools/benchmark/benchmark_file_io.cc b/tools/benchmark/benchmark_file_io.cc
new file mode 100644
index 000000000..d3e487eba
--- /dev/null
+++ b/tools/benchmark/benchmark_file_io.cc
@@ -0,0 +1,233 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+#include "tools/benchmark/benchmark_file_io.h"
+
+#include <errno.h>
+#include <sys/stat.h>
+
+#include <cstdio>
+
+#if defined(_WIN32) || defined(_WIN64)
+#include "third_party/dirent.h"
+#else
+#include <dirent.h>
+#include <unistd.h>
+#endif
+
+#define HAS_GLOB 0
+#if defined __has_include
+#if __has_include(<glob.h>)
+#undef HAS_GLOB
+#define HAS_GLOB 1
+#endif
+#endif
+
+#if HAS_GLOB
+#include <glob.h>
+#endif
+
+
+namespace jxl {
+
+const char kPathSeparator = '/';
+
+
+// RAII, ensures dir is closed even when returning early.
+class DirWrapper {
+ public:
+  DirWrapper(const DirWrapper& other) = delete;
+  DirWrapper& operator=(const DirWrapper& other) = delete;
+
+  explicit DirWrapper(const std::string& pathname)
+      : dir_(opendir(pathname.c_str())) {}
+
+  ~DirWrapper() {
+    if (dir_ != nullptr) {
+      const int err = closedir(dir_);
+      JXL_CHECK(err == 0);
+    }
+  }
+
+  // NOLINTNEXTLINE(google-explicit-constructor)
+  operator DIR*() const { return dir_; }
+
+ private:
+  DIR* const dir_;
+};
+
+// Checks if the file exists, either as file or as directory
+bool PathExists(const std::string& fname) {
+  struct stat s;
+  if (stat(fname.c_str(), &s) != 0) return false;
+  return true;
+}
+
+// Checks if the file exists and is a regular file.
+bool IsRegularFile(const std::string& fname) {
+  struct stat s;
+  if (stat(fname.c_str(), &s) != 0) return false;
+  return S_ISREG(s.st_mode);
+}
+
+// Checks if the file exists and is a directory.
+bool IsDirectory(const std::string& fname) {
+  struct stat s;
+  if (stat(fname.c_str(), &s) != 0) return false;
+  return S_ISDIR(s.st_mode);
+}
+
+// Recursively makes dir, or successfully does nothing if it already exists.
+Status MakeDir(const std::string& dirname) {
+  size_t pos = 0;
+  for (pos = dirname.size(); pos > 0; pos--) {
+    if (pos == dirname.size() || dirname[pos] == kPathSeparator) {
+      // Found existing dir or regular file, break and then start creating
+      // from here (in the latter case we'll get error below).
+      if (PathExists(dirname.substr(0, pos + 1))) {
+        pos += 1;  // Skip past this existing path
+        break;
+      }
+    }
+  }
+  for (; pos <= dirname.size(); pos++) {
+    if (pos == dirname.size() || dirname[pos] == kPathSeparator) {
+      std::string subdir = dirname.substr(0, pos + 1);
+      if (mkdir(subdir.c_str(), 0777) && errno != EEXIST) {
+        return JXL_FAILURE("Failed to create directory");
+      }
+    }
+  }
+  if (!IsDirectory(dirname)) return JXL_FAILURE("Failed to create directory");
+  return true;  // success
+}
+
+Status DeleteFile(const std::string& fname) {
+  if (!IsRegularFile(fname)) {
+    return JXL_FAILURE("Trying to delete non-regular file");
+  }
+  if (std::remove(fname.c_str())) return JXL_FAILURE("Failed to delete file");
+  return true;
+}
+
+std::string FileBaseName(const std::string& fname) {
+  size_t pos = fname.rfind('/');
+  if (pos == std::string::npos) return fname;
+  return fname.substr(pos + 1);
+}
+
+std::string FileDirName(const std::string& fname) {
+  size_t pos = fname.rfind('/');
+  if (pos == std::string::npos) return "";
+  return fname.substr(0, pos);
+}
+
+std::string FileExtension(const std::string& fname) {
+  size_t pos = fname.rfind('.');
+  if (pos == std::string::npos) return "";
+  return fname.substr(pos);
+}
+
+std::string JoinPath(const std::string& first, const std::string& second) {
+  JXL_CHECK(second.empty() || second[0] != kPathSeparator);
+  return (!first.empty() && first.back() == kPathSeparator)
+             ? (first + second)
+             : (first + kPathSeparator + second);
+}
+
+// Can match a single file, or multiple files in a directory (non-recursive).
+// With POSIX, supports glob(), otherwise supports a subset.
+Status MatchFiles(const std::string& pattern, std::vector<std::string>* list) {
+#if HAS_GLOB
+  glob_t g;
+  memset(&g, 0, sizeof(g));
+  int error = glob(pattern.c_str(), GLOB_TILDE, NULL, &g);
+  if (!error) {
+    for (size_t i = 0; i < g.gl_pathc; ++i) {
+      list->push_back(g.gl_pathv[i]);
+    }
+  }
+  globfree(&g);
+  if (error) return JXL_FAILURE("glob failed for %s", pattern.c_str());
+  return true;
+#else
+  std::string dirname = FileDirName(pattern);
+  std::string basename = FileBaseName(pattern);
+  size_t pos0 = basename.find('*');
+  size_t pos1 = pos0 == std::string::npos ? pos0 : basename.find('*', pos0 + 1);
+  std::string prefix, middle, suffix;
+  if (pos0 != std::string::npos) {
+    prefix = basename.substr(0, pos0);
+    if (pos1 != std::string::npos) {
+      middle = basename.substr(pos0 + 1, pos1 - pos0 - 1);
+      suffix = basename.substr(pos1 + 1);
+    } else {
+      suffix = basename.substr(pos0 + 1);
+    }
+  }
+
+  if (prefix.find_first_of("*?[") != std::string::npos ||
+      middle.find_first_of("*?[") != std::string::npos ||
+      suffix.find_first_of("*?[") != std::string::npos ||
+      dirname.find_first_of("*?[") != std::string::npos) {
+    return JXL_FAILURE(
+        "Only glob patterns with max two '*' in the basename"
+        " are supported, e.g. directory/path/*.png or"
+        " /directory/path/*heatmap*");
+  }
+
+  if (pos0 != std::string::npos) {
+    DirWrapper dir(dirname);
+    if (!dir) return JXL_FAILURE("directory %s doesn't exist", dirname.c_str());
+    for (;;) {
+      dirent* ent = readdir(dir);
+      if (!ent) break;
+      std::string name = ent->d_name;
+      // If there was a suffix, only add if it matches (e.g. ".png")
+      bool matches =
+          name.size() >= (prefix.size() + middle.size() + suffix.size());
+      if (matches) {
+        if (!prefix.empty() && name.substr(0, prefix.size()) != prefix) {
+          matches = false;
+        }
+        if (!middle.empty()) {
+          size_t pos = name.find(middle, prefix.size());
+          if (pos == std::string::npos ||
+              pos + middle.size() > name.size() - suffix.size()) {
+            matches = false;
+          }
+        }
+        if (!suffix.empty() &&
+            name.substr(name.size() - suffix.size()) != suffix) {
+          matches = false;
+        }
+      }
+      if (matches) {
+        std::string path = JoinPath(dirname, name);
+
+        if (IsRegularFile(path)) {
+          list->push_back(path);
+        }
+      }
+    }
+    return true;
+  }
+  // No *, so a single regular file is intended
+  if (IsRegularFile(pattern)) {
+    list->push_back(pattern);
+  }
+  return true;
+#endif  // HAS_GLOB
+}
+
+}  // namespace jxl
diff --git a/tools/benchmark/benchmark_file_io.h b/tools/benchmark/benchmark_file_io.h
new file mode 100644
index 000000000..91bde06f1
--- /dev/null
+++ b/tools/benchmark/benchmark_file_io.h
@@ -0,0 +1,62 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// File utilities for benchmarking and testing, but which are not needed for
+// main jxl itself.
+
+#ifndef TOOLS_BENCHMARK_BENCHMARK_FILE_IO_H_
+#define TOOLS_BENCHMARK_BENCHMARK_FILE_IO_H_
+
+#include <string>
+#include <vector>
+
+#include "jxl/base/file_io.h"
+#include "jxl/base/status.h"
+
+namespace jxl {
+
+// Checks if the file exists, either as file or as directory
+bool PathExists(const std::string& fname);
+
+// Checks if the file exists and is a regular file.
+bool IsRegularFile(const std::string& fname);
+
+// Checks if the file exists and is a directory.
+bool IsDirectory(const std::string& fname);
+
+// Recursively makes dir, or successfully does nothing if it already exists.
+Status MakeDir(const std::string& dirname);
+
+// Deletes a single regular file.
+Status DeleteFile(const std::string& fname);
+
+// Returns value similar to unix basename, except it returns empty string if
+// fname ends in '/'.
+std::string FileBaseName(const std::string& fname);
+// Returns value similar to unix dirname, except returns up to before the last
+// slash if fname ends in '/'.
+std::string FileDirName(const std::string& fname);
+
+// Returns the part of the filename starting from the last dot, or empty
+// string if there is no dot.
+std::string FileExtension(const std::string& fname);
+
+// Matches one or more files given glob pattern.
+Status MatchFiles(const std::string& pattern, std::vector<std::string>* list);
+
+std::string JoinPath(const std::string& first, const std::string& second);
+
+}  // namespace jxl
+
+#endif  // TOOLS_BENCHMARK_BENCHMARK_FILE_IO_H_
diff --git a/tools/benchmark/benchmark_stats.cc b/tools/benchmark/benchmark_stats.cc
new file mode 100644
index 000000000..7b396f6dd
--- /dev/null
+++ b/tools/benchmark/benchmark_stats.cc
@@ -0,0 +1,353 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/benchmark/benchmark_stats.h"
+
+#include <stdarg.h>
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <algorithm>
+#include <cmath>
+
+#include "jxl/base/status.h"
+#include "tools/benchmark/benchmark_args.h"
+
+namespace jxl {
+namespace {
+
+// Computes longest codec name from Args()->codec, for table alignment.
+uint32_t ComputeLargestCodecName() {
+  std::vector<std::string> methods = SplitString(Args()->codec, ',');
+  size_t max = strlen("Aggregate:");  // Include final row's name
+  for (const auto& method : methods) {
+    max = std::max(max, method.size());
+  }
+  return max;
+}
+
+// The benchmark result is a table of heterogeneous data, the column type
+// specifies its data type. The type affects how it is printed as well as how
+// aggregate values are computed.
+enum ColumnType {
+  // Formatted string
+  TYPE_STRING,
+  // Positive size, prints 0 as "---"
+  TYPE_SIZE,
+  // Floating point value (double precision) which is interpreted as
+  // "not applicable" if <= 0, must be strictly positive to be valid but can be
+  // set to 0 or negative to be printed as "---", for example for a speed that
+  // is not measured.
+  TYPE_POSITIVE_FLOAT,
+  // Counts of some event
+  TYPE_COUNT,
+};
+
+struct ColumnDescriptor {
+  // Column name, printed across two lines.
+  std::string label[2];
+  // Total width to render the values of this column. If t his is a floating
+  // point value, make sure this is large enough to contain a space and the
+  // point, plus precision digits after the point, plus the max amount of
+  // integer digits you expect in front of the point.
+  uint32_t width;
+  // Amount of digits after the point, or 0 if not a floating point value.
+  uint32_t precision;
+  ColumnType type;
+  bool more;  // Whether to print only if more_columns is enabled
+};
+
+// To add or change a column to the benchmark ASCII table output, add/change
+// an entry here with table header line 1, table header line 2, width of the
+// column, precision after the point in case of floating point, and the
+// data type. Then add/change the corresponding formula or formatting in
+// the function ComputeColumns.
+const std::vector<ColumnDescriptor>& GetColumnDescriptors() {
+  // clang-format off
+  static const std::vector<ColumnDescriptor> result = {
+      {{"Compr", "Method"}, ComputeLargestCodecName() + 1, 0, TYPE_STRING, false},
+      {{"Input", "Pixels"},         13,  0, TYPE_SIZE, false},
+      {{"Compr", "Size"},            9,  0, TYPE_SIZE, false},
+      {{"Compr", "BPP"},            17, 11, TYPE_POSITIVE_FLOAT, false},
+      {{"", "#"},                    4,  0, TYPE_STRING, false},
+      {{"Compr", "MP/s"},            8,  3, TYPE_POSITIVE_FLOAT, false},
+      {{"Decomp", "MP/s"},           8,  3, TYPE_POSITIVE_FLOAT, false},
+      {{"Butteraugli", "Distance"}, 13,  8, TYPE_POSITIVE_FLOAT, false},
+      {{"", "Error p norm"},        16, 11, TYPE_POSITIVE_FLOAT, false},
+      {{"", "PSNR"},                 7,  2, TYPE_POSITIVE_FLOAT, true},
+      {{"", "QABPP"},                8,  3, TYPE_POSITIVE_FLOAT, true},
+      {{"", "DCT2"},                 9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "DCT4"},                 9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "DCT4X8"},               9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "AFV"},                  9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "DCT8"},                 9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "DCT8X16"},              9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "DCT8X32"},              9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "DCT16"},                9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "DCT16X32"},             9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "DCT32"},                9,  7, TYPE_POSITIVE_FLOAT, true},
+      {{"", "BPP*pnorm"},           20, 16, TYPE_POSITIVE_FLOAT, false},
+      {{"", "Errors"},               9,  7, TYPE_COUNT, false},
+  };
+  // clang-format on
+
+  return result;
+}
+
+// Computes throughput [megapixels/s] as reported in the report table
+static double ComputeSpeed(size_t pixels, double time_s) {
+  if (time_s == 0.0) return 0;
+  return pixels * 1E-6 / time_s;
+}
+
+static std::string FormatFloat(const ColumnDescriptor& label, double value) {
+  std::string result =
+      StringPrintf("%*.*f", label.width - 1, label.precision, value);
+
+  // Reduce precision if the value is too wide for the column. However, keep
+  // at least one digit to the right of the point, and especially the integer
+  // digits.
+  if (result.size() >= label.width) {
+    size_t point = result.rfind('.');
+    if (point != std::string::npos) {
+      int end = std::max<int>(point + 2, label.width - 1);
+      result = result.substr(0, end);
+    }
+  }
+  return result;
+}
+
+}  // namespace
+
+std::string StringPrintf(const char* format, ...) {
+  char buf[2000];
+  va_list args;
+  va_start(args, format);
+  vsnprintf(buf, sizeof(buf), format, args);
+  va_end(args);
+  return std::string(buf);
+}
+
+void BenchmarkStats::Assimilate(const BenchmarkStats& victim) {
+  total_input_files += victim.total_input_files;
+  total_input_pixels += victim.total_input_pixels;
+  total_compressed_size += victim.total_compressed_size;
+  total_adj_compressed_size += victim.total_adj_compressed_size;
+  total_time_encode += victim.total_time_encode;
+  total_time_decode += victim.total_time_decode;
+  max_distance = std::max(max_distance, victim.max_distance);
+  distance_p_norm += victim.distance_p_norm;
+  distance_2 += victim.distance_2;
+  distances.insert(distances.end(), victim.distances.begin(),
+                   victim.distances.end());
+  total_errors += victim.total_errors;
+  jxl_stats.Assimilate(victim.jxl_stats);
+}
+
+void BenchmarkStats::PrintMoreStats() const {
+  if (Args()->print_more_stats) {
+    jxl_stats.Print();
+  }
+  if (Args()->print_distance_percentiles) {
+    std::vector<float> sorted = distances;
+    std::sort(sorted.begin(), sorted.end());
+    int p50idx = 0.5 * distances.size();
+    int p90idx = 0.9 * distances.size();
+    printf("50th/90th percentile distance: %.8f  %.8f\n", sorted[p50idx],
+           sorted[p90idx]);
+  }
+}
+
+std::vector<ColumnValue> BenchmarkStats::ComputeColumns(
+    const std::string& codec_desc, size_t corpus_size,
+    size_t num_threads) const {
+  JXL_CHECK(total_input_files == corpus_size);
+  const double comp_bpp = total_compressed_size * 8.0 / total_input_pixels;
+  const double adj_comp_bpp =
+      total_adj_compressed_size * 8.0 / total_input_pixels;
+  // Note: this is not affected by alpha nor bit depth.
+  const double compression_speed =
+      ComputeSpeed(total_input_pixels, total_time_encode);
+  const double decompression_speed =
+      ComputeSpeed(total_input_pixels, total_time_decode);
+  // Already weighted, no need to divide by #channels.
+  const double rmse = std::sqrt(distance_2 / total_input_pixels);
+  const double psnr =
+      total_compressed_size == 0
+          ? 0.0
+          : (distance_2 == 0) ? 99.99 : (20 * std::log10(255 / rmse));
+  const double p_norm = distance_p_norm / total_input_pixels;
+  const double bpp_p_norm = p_norm * comp_bpp;
+
+  std::vector<ColumnValue> values(GetColumnDescriptors().size());
+
+  values[0].s = codec_desc;
+  values[1].i = total_input_pixels;
+  values[2].i = total_compressed_size;
+  values[3].f = comp_bpp;
+  values[4].s = StringPrintf("%zu", num_threads);
+  values[5].f = compression_speed;
+  values[6].f = decompression_speed;
+  values[7].f = static_cast<double>(max_distance);
+  values[8].f = p_norm;
+  values[9].f = psnr;
+  values[10].f = adj_comp_bpp;
+  // The DCT2, DCT4, AFV and DCT4X8 are applied to an 8x8 block by having 4x4
+  // DCT2X2s, 2x2 DCT4x4s/AFVs, or 2x1 DCT4X8s, filling the whole 8x8 blocks.
+  // Thus we need to multiply the block count by 8.0 * 8.0 pixels for these
+  // transforms.
+  values[11].f =
+      jxl_stats.aux_out.num_dct2_blocks * 8.0 * 8.0 / total_input_pixels;
+  values[12].f =
+      jxl_stats.aux_out.num_dct4_blocks * 8.0 * 8.0 / total_input_pixels;
+  values[13].f =
+      jxl_stats.aux_out.num_dct4x8_blocks * 8.0 * 8.0 / total_input_pixels;
+  values[14].f =
+      jxl_stats.aux_out.num_afv_blocks * 8.0 * 8.0 / total_input_pixels;
+  values[15].f =
+      jxl_stats.aux_out.num_dct8_blocks * 8.0 * 8.0 / total_input_pixels;
+  values[16].f =
+      jxl_stats.aux_out.num_dct8x16_blocks * 8.0 * 16.0 / total_input_pixels;
+  values[17].f =
+      jxl_stats.aux_out.num_dct8x32_blocks * 8.0 * 32.0 / total_input_pixels;
+  values[18].f =
+      jxl_stats.aux_out.num_dct16_blocks * 16.0 * 16.0 / total_input_pixels;
+  values[19].f =
+      jxl_stats.aux_out.num_dct16x32_blocks * 16.0 * 32.0 / total_input_pixels;
+  values[20].f =
+      jxl_stats.aux_out.num_dct32_blocks * 32.0 * 32.0 / total_input_pixels;
+  values[21].f = bpp_p_norm;
+  values[22].i = total_errors;
+  return values;
+}
+
+static std::string PrintFormattedEntries(
+    const std::vector<ColumnValue>& values) {
+  const auto& descriptors = GetColumnDescriptors();
+
+  std::string out;
+  for (size_t i = 0; i < descriptors.size(); i++) {
+    if (!Args()->more_columns && descriptors[i].more) continue;
+    std::string value;
+    if (descriptors[i].type == TYPE_STRING) {
+      value = values[i].s;
+    } else if (descriptors[i].type == TYPE_SIZE) {
+      value = values[i].i ? StringPrintf("%zd", values[i].i) : "---";
+    } else if (descriptors[i].type == TYPE_POSITIVE_FLOAT) {
+      value = FormatFloat(descriptors[i], values[i].f);
+      value = FormatFloat(descriptors[i], values[i].f);
+    } else if (descriptors[i].type == TYPE_COUNT) {
+      value = StringPrintf("%zd", values[i].i);
+    }
+
+    int numspaces = descriptors[i].width - value.size();
+    if (numspaces < 1) {
+      numspaces = 1;
+    }
+    // All except the first one are right-aligned, the first one is the name,
+    // others are numbers with digits matching from the right.
+    if (i == 0) out += value.c_str();
+    out += std::string(numspaces, ' ');
+    if (i != 0) out += value.c_str();
+  }
+  return out + "\n";
+}
+
+std::string BenchmarkStats::PrintLine(const std::string& codec_desc,
+                                      size_t corpus_size,
+                                      size_t num_threads) const {
+  std::vector<ColumnValue> values =
+      ComputeColumns(codec_desc, corpus_size, num_threads);
+  return PrintFormattedEntries(values);
+}
+
+std::string PrintHeader() {
+  std::string out;
+  const auto& descriptors = GetColumnDescriptors();
+  for (int row = 0; row < 2; row++) {
+    for (size_t i = 0; i < descriptors.size(); i++) {
+      if (!Args()->more_columns && descriptors[i].more) continue;
+      const std::string& label = descriptors[i].label[row];
+      int numspaces = descriptors[i].width - label.size();
+      // All except the first one are right-aligned.
+      if (i == 0) out += label.c_str();
+      out += std::string(numspaces, ' ');
+      if (i != 0) out += label.c_str();
+    }
+    out += '\n';
+  }
+  for (const auto& descriptor : descriptors) {
+    if (!Args()->more_columns && descriptor.more) continue;
+    out += std::string(descriptor.width, '-');
+  }
+  return out + "\n";
+}
+
+std::string PrintAggregate(
+    const std::vector<std::vector<ColumnValue>>& aggregate) {
+  const auto& descriptors = GetColumnDescriptors();
+
+  for (size_t i = 0; i < aggregate.size(); i++) {
+    // Check when statistics has wrong amount of column entries
+    JXL_CHECK(aggregate[i].size() == descriptors.size());
+  }
+
+  std::vector<ColumnValue> result(descriptors.size());
+
+  // Statistics for the aggregate row are combined together with different
+  // formulas than Assimilate uses for combining the statistics of files.
+  for (size_t i = 0; i < descriptors.size(); i++) {
+    if (descriptors[i].type == TYPE_STRING) {
+      // "---" for the Iters column since this does not have meaning for
+      // the aggregate stats.
+      result[i].s = i == 0 ? "Aggregate:" : "---";
+      continue;
+    }
+    if (descriptors[i].type == TYPE_COUNT) {
+      size_t sum = 0;
+      for (size_t j = 0; j < aggregate.size(); j++) {
+        sum += aggregate[j][i].i;
+      }
+      result[i].i = sum;
+      continue;
+    }
+
+    ColumnType type = descriptors[i].type;
+
+    double logsum = 0;
+    size_t numvalid = 0;
+    for (size_t j = 0; j < aggregate.size(); j++) {
+      double value =
+          (type == TYPE_SIZE) ? aggregate[j][i].i : aggregate[j][i].f;
+      if (value > 0) {
+        numvalid++;
+        logsum += std::log2(value);
+      }
+    }
+    double geomean = numvalid ? std::exp2(logsum / numvalid) : 0.0;
+
+    if (type == TYPE_SIZE || type == TYPE_COUNT) {
+      result[i].i = static_cast<size_t>(geomean + 0.5);
+    } else if (type == TYPE_POSITIVE_FLOAT) {
+      result[i].f = geomean;
+    } else {
+      JXL_ABORT("unknown entry type");
+    }
+  }
+
+  return PrintFormattedEntries(result);
+}
+
+}  // namespace jxl
diff --git a/tools/benchmark/benchmark_stats.h b/tools/benchmark/benchmark_stats.h
new file mode 100644
index 000000000..23aef95fd
--- /dev/null
+++ b/tools/benchmark/benchmark_stats.h
@@ -0,0 +1,89 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_BENCHMARK_BENCHMARK_STATS_H_
+#define TOOLS_BENCHMARK_BENCHMARK_STATS_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <string>
+#include <vector>
+
+#include "jxl/aux_out.h"
+
+namespace jxl {
+
+std::string StringPrintf(const char* format, ...);
+
+struct JxlStats {
+  JxlStats() {
+    num_inputs = 0;
+    aux_out = AuxOut();
+  }
+  void Assimilate(const JxlStats& victim) {
+    num_inputs += victim.num_inputs;
+    aux_out.Assimilate(victim.aux_out);
+  }
+  void Print() const { aux_out.Print(num_inputs); }
+
+  size_t num_inputs;
+  AuxOut aux_out;
+};
+
+// The value of an entry in the table. Depending on the ColumnType, the string,
+// size_t or double should be used.
+struct ColumnValue {
+  std::string s;  // for TYPE_STRING
+  size_t i;  // for TYPE_SIZE and TYPE_COUNT
+  double f;  // for TYPE_POSITIVE_FLOAT
+};
+
+struct BenchmarkStats {
+  void Assimilate(const BenchmarkStats& victim);
+
+  std::vector<ColumnValue> ComputeColumns(const std::string& codec_desc,
+                                          size_t corpus_size,
+                                          size_t num_threads) const;
+
+  std::string PrintLine(const std::string& codec_desc, size_t corpus_size,
+                        size_t num_threads) const;
+
+  void PrintMoreStats() const;
+
+  size_t total_input_files = 0;
+  size_t total_input_pixels = 0;
+  size_t total_compressed_size = 0;
+  size_t total_adj_compressed_size = 0;
+  double total_time_encode = 0.0;
+  double total_time_decode = 0.0;
+  float max_distance = -1.0;  // Max butteraugli score
+  // sum of 8th powers of butteraugli distmap pixels.
+  double distance_p_norm = 0.0;
+  // sum of 2nd powers of differences between R, G, B.
+  double distance_2 = 0.0;
+  std::vector<float> distances;
+  size_t total_errors = 0;
+  JxlStats jxl_stats;
+};
+
+std::string PrintHeader();
+
+// Given the rows of all printed statistics, print an aggregate row.
+std::string PrintAggregate(
+    const std::vector<std::vector<ColumnValue>>& aggregate);
+
+}  // namespace jxl
+
+#endif  // TOOLS_BENCHMARK_BENCHMARK_STATS_H_
diff --git a/tools/benchmark/benchmark_xl.cc b/tools/benchmark/benchmark_xl.cc
new file mode 100644
index 000000000..8468745e3
--- /dev/null
+++ b/tools/benchmark/benchmark_xl.cc
@@ -0,0 +1,1123 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <algorithm>
+#include <hwy/runtime_dispatch.h>
+#include <hwy/static_targets.h>
+#include <memory>
+#include <mutex>
+#include <random>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/base/arch_specific.h"
+#include "jxl/base/cache_aligned.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/file_io.h"
+#include "jxl/base/os_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/robust_statistics.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/butteraugli/butteraugli.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/extras/codec.h"
+#include "jxl/extras/codec_png.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "tools/benchmark/benchmark_args.h"
+#include "tools/benchmark/benchmark_codec.h"
+#include "tools/benchmark/benchmark_file_io.h"
+#include "tools/benchmark/benchmark_stats.h"
+#include "tools/butteraugli_pnorm.h"
+#include "tools/speed_stats.h"
+
+namespace jxl {
+namespace {
+
+// TODO(lode): take alpha into account when needed
+HWY_ATTR double ComputeDistance2(const ImageBundle& ib1,
+                                 const ImageBundle& ib2) {
+  PROFILER_FUNC;
+  // Convert to sRGB - closer to perception than linear.
+  const Image3F* srgb1 = &ib1.color();
+  Image3F copy1;
+  if (!ib1.IsSRGB()) {
+    JXL_CHECK(
+        ib1.CopyTo(Rect(ib1), ColorManagement::SRGB(ib1.IsGray()), &copy1));
+    srgb1 = &copy1;
+  }
+  const Image3F* srgb2 = &ib2.color();
+  Image3F copy2;
+  if (!ib2.IsSRGB()) {
+    JXL_CHECK(
+        ib2.CopyTo(Rect(ib2), ColorManagement::SRGB(ib2.IsGray()), &copy2));
+    srgb2 = &copy2;
+  }
+
+  JXL_CHECK(SameSize(*srgb1, *srgb2));
+
+  HWY_FULL(float) d;
+
+  auto sums = Zero(d);
+  double result = 0;
+  // Weighted PSNR as in JPEG-XL: chroma counts 1/8 (they compute on YCbCr).
+  // Avoid squaring the weight - 1/64 is too extreme.
+  const float weights[3] = {1.0f / 8, 6.0f / 8, 1.0f / 8};
+  for (size_t c = 0; c < 3; ++c) {
+    const auto weight = Set(d, weights[c]);
+    for (size_t y = 0; y < srgb1->ysize(); ++y) {
+      const float* JXL_RESTRICT row1 = srgb1->ConstPlaneRow(c, y);
+      const float* JXL_RESTRICT row2 = srgb2->ConstPlaneRow(c, y);
+      size_t x = 0;
+      for (; x + d.N <= srgb1->xsize(); x += d.N) {
+        const auto diff = Load(d, row1 + x) - Load(d, row2 + x);
+        sums += diff * diff * weight;
+      }
+      for (; x < srgb1->xsize(); ++x) {
+        const float diff = row1[x] - row2[x];
+        result += diff * diff * weights[c];
+      }
+    }
+  }
+  const float sum = GetLane(hwy::ext::SumOfLanes(sums));
+  return sum + result;
+}
+
+Status WritePNG(const Image3B& image, ThreadPool* pool,
+                const std::string& filename) {
+  std::vector<uint8_t> rgb(image.xsize() * image.ysize() * 3);
+  CodecInOut io;
+  io.metadata.bits_per_sample = 8;
+  io.metadata.color_encoding = ColorManagement::SRGB();
+  io.SetFromImage(StaticCastImage3<float>(image), io.metadata.color_encoding);
+  PaddedBytes compressed;
+  JXL_CHECK(EncodeImagePNG(&io, io.Main().c_current(), 8, pool, &compressed));
+  return WriteFile(compressed, filename);
+}
+
+Status ReadPNG(const std::string& filename, Image3B* image) {
+  CodecInOut io;
+  JXL_CHECK(SetFromFile(filename, &io));
+  *image = StaticCastImage3<uint8_t>(io.Main().color());
+  return true;
+}
+
+// Side effect: may sort `elapsed`.
+double SummarizeElapsedTimes(std::vector<double>& elapsed) {
+  JXL_CHECK(!elapsed.empty());
+  if (elapsed.size() == 1) return elapsed[0];
+  // Two: skip first (noisier)
+  if (elapsed.size() == 2) return elapsed[1];
+
+  // Prefer geomean unless numerically unreliable (too many reps)
+  if (std::pow(elapsed[0], elapsed.size()) < 1E100) {
+    double product = 1.0;
+    for (size_t i = 1; i < elapsed.size(); ++i) {
+      product *= elapsed[i];
+    }
+    return std::pow(product, 1.0 / (elapsed.size() - 1));
+  }
+
+  // Else: mode
+  std::sort(elapsed.begin(), elapsed.end());
+  return jxl::HalfSampleMode()(elapsed.data(), elapsed.size());
+}
+
+static bool HasValidAlpha(const CodecInOut& io) {
+  // Test whether the image doesn't have invalid alpha channel values in the
+  // image bundle (values larger than the bit depth indicates), otherwise
+  // an assert (which immediately exits the binary) gets triggered in
+  // external_image.cc.
+  const ImageBundle& ib = io.Main();
+  if (ib.HasAlpha()) {
+    size_t actual_max = (1u << ib.metadata()->alpha_bits) - 1;
+    const ImageU& alpha = ib.alpha();
+    for (size_t y = 0; y < alpha.ysize(); ++y) {
+      auto* const JXL_RESTRICT row = alpha.Row(y);
+      for (size_t x = 0; x < alpha.xsize(); ++x) {
+        if (row[x] > actual_max) {
+          if (!Args()->silent_errors) {
+            JXL_WARNING(
+                "alpha value too large for given bit depth:"
+                " depth: %d, value: %d",
+                ib.metadata()->alpha_bits, row[x]);
+          }
+          return false;
+        }
+      }
+    }
+  }
+  return true;
+}
+
+HWY_ATTR void DoCompress(const std::string& filename, const CodecInOut& io,
+                         ImageCodec* codec, ThreadPool* inner_pool,
+                         PaddedBytes* compressed, BenchmarkStats* s) {
+  PROFILER_FUNC;
+  ++s->total_input_files;
+
+  if (io.frames.size() != 1) {
+    // Multiple frames not supported (io.xsize() will checkfail)
+    s->total_errors++;
+    if (!Args()->silent_errors) {
+      JXL_WARNING("multiframe input image not supported %s", filename.c_str());
+    }
+    return;
+  }
+  const size_t xsize = io.xsize();
+  const size_t ysize = io.ysize();
+  const size_t input_pixels = xsize * ysize;
+
+  double start, end;
+  jpegxl::tools::SpeedStats speed_stats;
+  jpegxl::tools::SpeedStats::Summary summary;
+
+  bool valid = true;  // false if roundtrip, encoding or decoding errors occur.
+
+  if (!Args()->decode_only && (io.xsize() == 0 || io.ysize() == 0)) {
+    // This means the benchmark couldn't load the image, e.g. due to invalid
+    // ICC profile. Warning message about that was already printed. Continue
+    // this function to indicate it as error in the stats.
+    valid = false;
+  }
+
+  std::string ext = FileExtension(filename);
+  if (valid && !Args()->decode_only) {
+    for (size_t i = 0; i < Args()->encode_reps; ++i) {
+      if (codec->CanRecompressJpeg() && (ext == ".jpg" || ext == ".jpeg")) {
+        std::string data_in;
+        JXL_CHECK(ReadFile(filename, &data_in));
+        start = Now();
+        JXL_CHECK(codec->RecompressJpeg(filename, data_in, compressed));
+        end = Now();
+      } else {
+        start = Now();
+        Status status = codec->Compress(filename, &io, inner_pool, compressed);
+        if (!status) {
+          valid = false;
+          if (!Args()->silent_errors) {
+            std::string message = codec->GetErrorMessage();
+            if (!message.empty()) {
+              fprintf(stderr, "Error in %s codec: %s\n",
+                      codec->description().c_str(), message.c_str());
+            } else {
+              fprintf(stderr, "Error in %s codec\n",
+                      codec->description().c_str());
+            }
+          }
+        }
+        end = Now();
+      }
+      speed_stats.NotifyElapsed(end - start);
+    }
+    JXL_CHECK(speed_stats.GetSummary(&summary));
+    s->total_time_encode += summary.central_tendency;
+  }
+
+  if (valid && Args()->decode_only) {
+    std::string data_in;
+    JXL_CHECK(ReadFile(filename, &data_in));
+    compressed->append((uint8_t*)data_in.data(),
+                       (uint8_t*)data_in.data() + data_in.size());
+  }
+
+  // Decompress
+  CodecInOut io2;
+  io2.metadata = io.metadata;
+  if (valid) {
+    speed_stats = jpegxl::tools::SpeedStats();
+    for (size_t i = 0; i < Args()->decode_reps; ++i) {
+      start = Now();
+      if (!codec->Decompress(filename, Span<const uint8_t>(*compressed),
+                             inner_pool, &io2)) {
+        if (!Args()->silent_errors) {
+          fprintf(stderr,
+                  "%s failed to decompress encoded image. Original source:"
+                  " %s\n",
+                  codec->description().c_str(), filename.c_str());
+        }
+        valid = false;
+      }
+      end = Now();
+      speed_stats.NotifyElapsed(end - start);
+
+      // io2.dec_pixels increases each time, but the total should be independent
+      // of decode_reps, so only take the value from the first iteration.
+      if (i == 0) s->total_input_pixels += io2.dec_pixels;
+    }
+    JXL_CHECK(speed_stats.GetSummary(&summary));
+    s->total_time_decode += summary.central_tendency;
+  }
+
+  std::string name = FileBaseName(filename);
+  std::string codec_name = codec->description();
+
+  if (valid && !HasValidAlpha(io2)) {
+    valid = false;
+    if (!Args()->silent_errors) {
+      JXL_WARNING("invalid alpha in decoder result: codec: %s, file: %s",
+                  codec_name.c_str(), name.c_str());
+    }
+  }
+  if (!valid) {
+    s->total_errors++;
+  }
+
+  if (io.frames.size() != io2.frames.size()) {
+    if (!Args()->silent_errors) {
+      // Animated gifs not supported yet?
+      fprintf(stderr,
+              "Frame sizes not equal, is this an animated gif? %s %s %zu %zu\n",
+              codec_name.c_str(), name.c_str(), io.frames.size(),
+              io2.frames.size());
+    }
+    valid = false;
+  }
+
+  bool lossless = codec->IsJpegTranscoder();
+  bool skip_butteraugli =
+      Args()->skip_butteraugli || Args()->decode_only || lossless;
+  ImageF distmap;
+  float max_distance = 1.0f;
+
+  if (valid && !skip_butteraugli) {
+    JXL_ASSERT(io.frames.size() == io2.frames.size());
+    for (size_t i = 0; i < io.frames.size(); i++) {
+      const ImageBundle& ib1 = io.frames[i];
+      ImageBundle& ib2 = io2.frames[i];
+
+      // Verify output
+      // TODO(robryk): Reenable once we reproduce the same alpha bitdepth.
+      // Currently alpha equality is sort-of included in Butteraugli distance.
+#if 0
+      JXL_CHECK(ib1.HasAlpha() == ib2.HasAlpha());
+      if (ib1.HasAlpha()) {
+        JXL_CHECK(SamePixels(ib1.alpha(), ib2.alpha()));
+      }
+#endif
+
+      PROFILER_ZONE("Benchmark stats");
+      float distance;
+      if (SameSize(ib1, ib2)) {
+        // This needs to be ib2 because the codec will have set it on the
+        // encoded image if needed, but we currently don't set it on the
+        // reference.
+        const float intensity_multiplier =
+            ib2.metadata()->IntensityTarget() * kIntensityMultiplier;
+        Image3F linear_rgb1, linear_rgb2;
+        JXL_CHECK(ib1.CopyTo(Rect(ib1),
+                             ColorManagement::LinearSRGB(ib1.IsGray()),
+                             &linear_rgb1, inner_pool));
+        JXL_CHECK(ib2.CopyTo(Rect(ib2),
+                             ColorManagement::LinearSRGB(ib2.IsGray()),
+                             &linear_rgb2, inner_pool));
+        if (intensity_multiplier != 1.f) {
+          ScaleImage(intensity_multiplier, &linear_rgb1);
+          ScaleImage(intensity_multiplier, &linear_rgb2);
+        }
+        double distance_double;
+        JXL_CHECK(butteraugli::ButteraugliInterface(linear_rgb1, linear_rgb2,
+                                                    codec->hf_asymmetry(),
+                                                    distmap, distance_double));
+        distance = static_cast<float>(distance_double);
+        // Ensure pixels in range 0-255
+        s->distance_2 += ComputeDistance2(ib1, ib2);
+      } else {
+        // TODO(veluca): re-upsample and compute proper distance.
+        distance = 1e+4f;
+        distmap = ImageF(1, 1);
+        distmap.Row(0)[0] = distance;
+        s->distance_2 += distance;
+      }
+      // Update stats
+      s->distance_p_norm +=
+          ComputeDistanceP(distmap, Args()->error_pnorm) * input_pixels;
+      s->max_distance = std::max(s->max_distance, distance);
+      s->distances.push_back(distance);
+      max_distance = std::max(max_distance, distance);
+    }
+  }
+
+  s->total_compressed_size += compressed->size();
+  s->total_adj_compressed_size += compressed->size() * max_distance;
+  codec->GetMoreStats(s);
+
+  if (io2.frames.size() == 1 &&
+      (Args()->save_compressed || Args()->save_decompressed)) {
+    JXL_ASSERT(io2.frames.size() == 1);
+    ImageBundle& ib2 = io2.Main();
+
+    // By default the benchmark will save the image after roundtrip with the
+    // same color encoding as the image before roundtrip. Not all codecs
+    // necessarily preserve the amount of channels (1 for gray, 3 for RGB)
+    // though, since not all image formats necessarily allow a way to remember
+    // what amount of channels you happened to give the benchmark codec
+    // input (say, an RGB-only format) and that is fine since in the end what
+    // matters is that the pixels look the same on a 3-channel RGB monitor
+    // while using grayscale encoding is an internal compression optimization.
+    // If that is the case, output with the current color model instead,
+    // because CodecInOut does not automatically convert between 1 or 3
+    // channels, and giving a ColorEncoding  with a different amount of
+    // channels is not allowed.
+    const ColorEncoding* c_desired =
+        (ib2.metadata()->color_encoding.Channels() ==
+         ib2.c_current().Channels())
+            ? &ib2.metadata()->color_encoding
+            : &ib2.c_current();
+    // Allow overriding via --output_encoding.
+    if (!Args()->output_description.empty()) {
+      c_desired = &Args()->output_encoding;
+    }
+
+    std::string dir = FileDirName(filename);
+    std::string outdir =
+        Args()->output_dir.empty() ? dir + "/out" : Args()->output_dir;
+    // Make compatible for filename
+    std::replace(codec_name.begin(), codec_name.end(), ':', '_');
+    std::string compressed_fn = outdir + "/" + name + "." + codec_name;
+    std::string decompressed_fn = compressed_fn + Args()->output_extension;
+    std::string heatmap_fn = compressed_fn + ".heatmap.png";
+    JXL_CHECK(MakeDir(outdir));
+    if (Args()->save_compressed) {
+      std::string compressed_str(
+          reinterpret_cast<const char*>(compressed->data()),
+          compressed->size());
+      JXL_CHECK(WriteFile(compressed_str, compressed_fn));
+    }
+    if (Args()->save_decompressed && valid) {
+      // For verifying HDR: scale output.
+      if (Args()->mul_output != 0.0) {
+        fprintf(stderr, "WARNING: scaling outputs by %f\n", Args()->mul_output);
+        JXL_CHECK(ib2.TransformTo(ColorManagement::LinearSRGB(ib2.IsGray()),
+                                  inner_pool));
+        ScaleImage(static_cast<float>(Args()->mul_output),
+                   const_cast<Image3F*>(&ib2.color()));
+      }
+
+      JXL_CHECK(EncodeToFile(io2, *c_desired, ib2.metadata()->bits_per_sample,
+                             decompressed_fn));
+      if (!skip_butteraugli) {
+        float good = Args()->heatmap_good > 0.0f
+                         ? Args()->heatmap_good
+                         : butteraugli::ButteraugliFuzzyInverse(1.5);
+        float bad = Args()->heatmap_bad > 0.0f
+                        ? Args()->heatmap_bad
+                        : butteraugli::ButteraugliFuzzyInverse(0.5);
+        Image3B heatmap = butteraugli::CreateHeatMapImage(distmap, good, bad);
+        JXL_CHECK(WritePNG(heatmap, inner_pool, heatmap_fn));
+      }
+    }
+  }
+  if (Args()->show_progress) {
+    fprintf(stderr, ".");
+    fflush(stderr);
+  }
+}
+
+// Makes a base64 data URI for embedded image in HTML
+std::string Base64Image(const std::string& filename) {
+  PaddedBytes bytes;
+  if (!ReadFile(filename, &bytes)) {
+    return "";
+  }
+  static const char* symbols =
+      "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
+  std::string result;
+  for (size_t i = 0; i < bytes.size(); i += 3) {
+    uint8_t o0 = bytes[i + 0];
+    uint8_t o1 = (i + 1 < bytes.size()) ? bytes[i + 1] : 0;
+    uint8_t o2 = (i + 2 < bytes.size()) ? bytes[i + 2] : 0;
+    uint32_t value = (o0 << 16) | (o1 << 8) | o2;
+    for (size_t j = 0; j < 4; j++) {
+      result += (i + j <= bytes.size()) ? symbols[(value >> (6 * (3 - j))) & 63]
+                                        : '=';
+    }
+  }
+  // NOTE: Chrome supports max 2MB of data this way for URLs, but appears to
+  // support larger images anyway as long as it's embedded in the HTML file
+  // itself. If more data is needed, use createObjectURL.
+  return "data:image;base64," + result;
+}
+
+struct Task {
+  ImageCodecPtr codec;
+  size_t idx_image;
+  size_t idx_method;
+  const CodecInOut* image;
+  BenchmarkStats stats;
+};
+
+void WriteHtmlReport(const std::string& codec_desc,
+                     const std::vector<std::string>& fnames,
+                     const std::vector<const Task*>& tasks,
+                     const std::vector<const CodecInOut*>& images,
+                     bool self_contained) {
+  std::string toggle_js =
+      "<script type=\"text/javascript\">\n"
+      "  var codecname = '" +
+      codec_desc + "';\n";
+  toggle_js += R"(
+  var maintitle = codecname + ' - click images to toggle, press space to' +
+      ' toggle all, h to toggle all heatmaps. Zoom in with CTRL+wheel or' +
+      ' CTRL+plus.';
+  document.title = maintitle;
+  var counter = [];
+  function setState(i, s) {
+    var preview = document.getElementById("preview" + i);
+    var orig = document.getElementById("orig" + i);
+    var hm = document.getElementById("hm" + i);
+    if (s == 0) {
+      preview.style.display = 'none';
+      orig.style.display = 'block';
+      hm.style.display = 'none';
+    } else if (s == 1) {
+      preview.style.display = 'block';
+      orig.style.display = 'none';
+      hm.style.display = 'none';
+    } else if (s == 2) {
+      preview.style.display = 'none';
+      orig.style.display = 'none';
+      hm.style.display = 'block';
+    }
+  }
+  function toggle3(i) {
+    for (index = counter.length; index <= i; index++) {
+      counter.push(1);
+    }
+    setState(i, counter[i]);
+    counter[i] = (counter[i] + 1) % 3;
+    document.title = maintitle;
+  }
+  var toggleall_state = 1;
+  document.body.onkeydown = function(e) {
+    // space (32) to toggle orig/compr, 'h' (72) to toggle heatmap/compr
+    if (e.keyCode == 32 || e.keyCode == 72) {
+      var divs = document.getElementsByTagName('div');
+      var key_state = (e.keyCode == 32) ? 0 : 2;
+      toggleall_state = (toggleall_state == key_state) ? 1 : key_state;
+      document.title = codecname + ' - ' + (toggleall_state == 0 ?
+          'originals' : (toggleall_state == 1 ? 'compressed' : 'heatmaps'));
+      for (var i = 0; i < divs.length; i++) {
+        setState(i, toggleall_state);
+      }
+      return false;
+    }
+  };
+</script>
+)";
+  std::string out_html;
+  std::string outdir;
+  std::string toggleall = "function(e) {if(e.keyCode == 32 { ";
+  out_html += "<body bgcolor=\"#000\">\n";
+  out_html += "<style>img { image-rendering: pixelated; }</style>\n";
+  std::string codec_name = codec_desc;
+  // Make compatible for filename
+  std::replace(codec_name.begin(), codec_name.end(), ':', '_');
+  for (size_t i = 0; i < fnames.size(); ++i) {
+    std::string name = FileBaseName(fnames[i]);
+    std::string dir = FileDirName(fnames[i]);
+    outdir = Args()->output_dir.empty() ? dir + "/out" : Args()->output_dir;
+    std::string name_out = name + "." + codec_name + Args()->output_extension;
+    std::string heatmap_out = name + "." + codec_name + ".heatmap.png";
+
+    std::string fname_orig = fnames[i];
+    std::string fname_out = outdir + "/" + name_out;
+    std::string fname_heatmap = outdir + "/" + heatmap_out;
+    std::string url_orig = Args()->originals_url.empty()
+                               ? ("file://" + fnames[i])
+                               : (Args()->originals_url + "/" + name);
+    std::string url_out = name_out;
+    std::string url_heatmap = heatmap_out;
+    if (self_contained) {
+      url_orig = Base64Image(fname_orig);
+      url_out = Base64Image(fname_out);
+      url_heatmap = Base64Image(fname_heatmap);
+    }
+    std::string number = StringPrintf("%zu", i);
+    const CodecInOut& image = *images[i];
+    size_t xsize = image.frames.size() == 1 ? image.xsize() : 0;
+    size_t ysize = image.frames.size() == 1 ? image.ysize() : 0;
+    std::string html_width = StringPrintf("%zupx", xsize);
+    std::string html_height = StringPrintf("%zupx", ysize);
+    double bpp = tasks[i]->stats.total_compressed_size * 8.0 /
+                 tasks[i]->stats.total_input_pixels;
+    std::string compressed_title = StringPrintf("compressed: %f bpp", bpp);
+    out_html += "<div onclick=\"toggle3(" + number +
+                ");\" style=\"display:inline-block;width:" + html_width +
+                ";height:" + html_height +
+                ";\">\n"
+                "  <img title=\"" +
+                compressed_title + "\" id=\"preview" + number + "\" src=";
+    out_html += "\"" + url_out + "\"";
+    out_html +=
+        " style=\"display:block;\"/>\n"
+        "  <img title=\"original\" id=\"orig" +
+        number + "\" src=";
+    out_html += "\"" + url_orig + "\"";
+    out_html +=
+        " style=\"display:none;\"/>\n"
+        "  <img title=\"heatmap\" id=\"hm" +
+        number + "\" src=";
+    out_html += "\"" + url_heatmap + "\"";
+    out_html += " style=\"display:none;\"/>\n</div>\n";
+  }
+  out_html += "</body>\n";
+  out_html += toggle_js;
+  JXL_CHECK(WriteFile(out_html, outdir + "/index." + codec_name + ".html"));
+}
+
+// Prints the detailed and aggregate statistics, in the correct order but as
+// soon as possible when multithreaded tasks are done.
+struct StatPrinter {
+  StatPrinter(const std::vector<std::string>& methods,
+              const std::vector<std::string>& fnames,
+              const std::vector<Task>& tasks)
+      : methods_(&methods),
+        fnames_(&fnames),
+        tasks_(&tasks),
+        tasks_done_(0),
+        stats_printed_(0),
+        details_printed_(0) {
+    stats_done_.resize(methods.size(), 0);
+    details_done_.resize(tasks.size(), 0);
+    max_fname_width_ = 0;
+    for (const auto& fname : fnames) {
+      max_fname_width_ = std::max(max_fname_width_, FileBaseName(fname).size());
+    }
+    max_method_width_ = 0;
+    for (const auto& method : methods) {
+      max_method_width_ =
+          std::max(max_method_width_, FileBaseName(method).size());
+    }
+  }
+
+  void TaskDone(size_t task_index, const Task& t) {
+    PROFILER_FUNC;
+    std::lock_guard<std::mutex> guard(mutex);
+    tasks_done_++;
+    if (Args()->print_details || Args()->show_progress) {
+      if (Args()->print_details) {
+        // Render individual results as soon as they are ready and all previous
+        // ones in task order are ready.
+        details_done_[task_index] = 1;
+        if (task_index == details_printed_) {
+          while (details_printed_ < tasks_->size() &&
+                 details_done_[details_printed_]) {
+            PrintDetails((*tasks_)[details_printed_]);
+            details_printed_++;
+          }
+        }
+      }
+      // When using "show_progress" or "print_details", the table must be
+      // rendered at the very end, else the details or progress would be
+      // rendered in-between the table rows.
+      if (tasks_done_ == tasks_->size()) {
+        PrintStatsHeader();
+        for (size_t i = 0; i < methods_->size(); i++) {
+          PrintStats((*methods_)[i], i);
+        }
+        PrintStatsFooter();
+      }
+    } else {
+      if (tasks_done_ == 1) {
+        PrintStatsHeader();
+      }
+      // Render lines of the table as soon as it is ready and all previous
+      // lines have been printed.
+      stats_done_[t.idx_method]++;
+      if (stats_done_[t.idx_method] == fnames_->size() &&
+          t.idx_method == stats_printed_) {
+        while (stats_printed_ < stats_done_.size() &&
+               stats_done_[stats_printed_] == fnames_->size()) {
+          PrintStats((*methods_)[stats_printed_], stats_printed_);
+          stats_printed_++;
+        }
+      }
+      if (tasks_done_ == tasks_->size()) {
+        PrintStatsFooter();
+      }
+    }
+  }
+
+  void PrintDetails(const Task& t) {
+    double comp_bpp =
+        t.stats.total_compressed_size * 8.0 / t.stats.total_input_pixels;
+    double p_norm = t.stats.distance_p_norm / t.stats.total_input_pixels;
+    double bpp_p_norm = p_norm * comp_bpp;
+
+    const double adj_comp_bpp =
+        t.stats.total_adj_compressed_size * 8.0 / t.stats.total_input_pixels;
+
+    const double rmse =
+        std::sqrt(t.stats.distance_2 / t.stats.total_input_pixels);
+    const double psnr =
+        t.stats.total_compressed_size == 0
+            ? 0.0
+            : (t.stats.distance_2 == 0) ? 99.99 : (20 * std::log10(255 / rmse));
+    size_t pixels = t.stats.total_input_pixels;
+
+    const double enc_mps =
+        t.stats.total_input_pixels / (1000000.0 * t.stats.total_time_encode);
+    const double dec_mps =
+        t.stats.total_input_pixels / (1000000.0 * t.stats.total_time_decode);
+    if (Args()->print_details_csv) {
+      printf("%s,%s,%zd,%zd,%zd,%.8f,%.8f,%.8f,%.8f,%.8f,%.8f,%.8f,%.8f\n",
+             (*methods_)[t.idx_method].c_str(),
+             FileBaseName((*fnames_)[t.idx_image]).c_str(),
+             t.stats.total_errors, t.stats.total_compressed_size, pixels,
+             enc_mps, dec_mps, comp_bpp, t.stats.max_distance, psnr, p_norm,
+             bpp_p_norm, adj_comp_bpp);
+    } else {
+      printf("%s", (*methods_)[t.idx_method].c_str());
+      for (size_t i = (*methods_)[t.idx_method].size(); i <= max_method_width_;
+           i++) {
+        printf(" ");
+      }
+      printf("%s", FileBaseName((*fnames_)[t.idx_image]).c_str());
+      for (size_t i = FileBaseName((*fnames_)[t.idx_image]).size();
+           i <= max_fname_width_; i++) {
+        printf(" ");
+      }
+      printf(
+          "error:%zd    size:%8zd    pixels:%9zd    enc_speed:%8.8f"
+          "    dec_speed:%8.8f    bpp:%10.8f    dist:%10.8f"
+          "    psnr:%10.8f    p:%10.8f    bppp:%10.8f    qabpp:%10.8f\n",
+          t.stats.total_errors, t.stats.total_compressed_size, pixels, enc_mps,
+          dec_mps, comp_bpp, t.stats.max_distance, psnr, p_norm, bpp_p_norm,
+          adj_comp_bpp);
+    }
+    fflush(stdout);
+  }
+
+  void PrintStats(const std::string& method, size_t idx_method) {
+    PROFILER_FUNC;
+    // Assimilate all tasks with the same idx_method.
+    BenchmarkStats method_stats;
+    std::vector<const CodecInOut*> images;
+    std::vector<const Task*> tasks;
+    for (const Task& t : *tasks_) {
+      if (t.idx_method == idx_method) {
+        method_stats.Assimilate(t.stats);
+        images.push_back(t.image);
+        tasks.push_back(&t);
+      }
+    }
+
+    std::string out;
+
+    method_stats.PrintMoreStats();  // not concurrent
+    out += method_stats.PrintLine(method, fnames_->size(), /*num_threads=*/1);
+
+    if (Args()->write_html_report) {
+      WriteHtmlReport(method, *fnames_, tasks, images,
+                      Args()->html_report_self_contained);
+    }
+
+    stats_aggregate_.push_back(
+        method_stats.ComputeColumns(method, fnames_->size(), 1));
+
+    printf("%s", out.c_str());
+    fflush(stdout);
+  }
+
+  void PrintStatsHeader() {
+    if (Args()->markdown) {
+      if (Args()->show_progress) {
+        fprintf(stderr, "\n");
+        fflush(stderr);
+      }
+      printf("```\n");
+    }
+    if (fnames_->size() == 1) printf("%s\n", (*fnames_)[0].c_str());
+    printf("%s", PrintHeader().c_str());
+    fflush(stdout);
+  }
+
+  void PrintStatsFooter() {
+    printf("%s", PrintAggregate(stats_aggregate_).c_str());
+    if (Args()->markdown) printf("```\n");
+    printf("\n");
+    fflush(stdout);
+  }
+
+  const std::vector<std::string>* methods_;
+  const std::vector<std::string>* fnames_;
+  const std::vector<Task>* tasks_;
+
+  size_t tasks_done_;
+
+  size_t stats_printed_;
+  std::vector<size_t> stats_done_;
+
+  size_t details_printed_;
+  std::vector<size_t> details_done_;
+
+  size_t max_fname_width_;
+  size_t max_method_width_;
+
+  std::vector<std::vector<ColumnValue>> stats_aggregate_;
+
+  std::mutex mutex;
+};
+
+class Benchmark {
+  using StringVec = std::vector<std::string>;
+
+ public:
+  // Return the exit code of the program.
+  static int Run() {
+    int ret = EXIT_SUCCESS;
+    {
+      PROFILER_FUNC;
+
+      const StringVec methods = GetMethods();
+      const StringVec fnames = GetFilenames();
+      bool all_color_aware;
+      bool jpeg_transcoding_requested;
+      // (non-const because Task.stats are updated)
+      std::vector<Task> tasks = CreateTasks(methods, fnames, &all_color_aware,
+                                            &jpeg_transcoding_requested);
+
+      std::unique_ptr<ThreadPoolInternal> pool;
+      std::vector<std::unique_ptr<ThreadPoolInternal>> inner_pools;
+      InitThreads(static_cast<int>(tasks.size()), &pool, &inner_pools);
+
+      const std::vector<CodecInOut> loaded_images = LoadImages(
+          fnames, all_color_aware, jpeg_transcoding_requested, pool.get());
+
+      if (RunTasks(methods, fnames, loaded_images, pool.get(), inner_pools,
+                   &tasks) != 0) {
+        ret = EXIT_FAILURE;
+        if (!Args()->silent_errors) {
+          fprintf(stderr, "There were error(s) in the benchmark.\n");
+        }
+      }
+    }
+
+    // Must have exited profiler zone above before calling.
+    if (Args()->profiler) {
+      PROFILER_PRINT_RESULTS();
+    }
+    CacheAligned::PrintStats();
+    return ret;
+  }
+
+ private:
+  static int NumCores() {
+    ProcessorTopology topology;
+    JXL_CHECK(DetectProcessorTopology(&topology));
+    const int num_cores =
+        static_cast<int>(topology.packages * topology.cores_per_package);
+    JXL_CHECK(num_cores != 0);
+    return num_cores;
+  }
+
+  static int NumOuterThreads(const int num_cores, const int num_tasks) {
+    int num_threads = Args()->num_threads;
+    // Default to #cores
+    if (num_threads < 0) num_threads = num_cores;
+
+    // No point exceeding #cores - we already max out TDP with 1/core.
+    num_threads = std::min(num_threads, num_cores);
+
+    // Don't create more threads than there are tasks (pointless/wasteful).
+    num_threads = std::min(num_threads, num_tasks);
+
+    // Just one thread is counterproductive.
+    if (num_threads == 1) num_threads = 0;
+
+    return num_threads;
+  }
+
+  static int NumInnerThreads(const int num_cores, const int num_threads) {
+    int num_inner = Args()->inner_threads;
+
+    // Default: distribute remaining cores among tasks.
+    if (num_inner < 0) {
+      const int cores_for_outer = num_cores - num_threads;
+      num_inner = num_threads == 0 ? num_cores : cores_for_outer / num_threads;
+    }
+
+    // Just one thread is counterproductive.
+    if (num_inner == 1) num_inner = 0;
+
+    return num_inner;
+  }
+
+  // Pins the first worker thread in pool to cpus[*next_index] etc.
+  // Not thread-safe (non-atomic update of next_index).
+  static void PinThreads(ThreadPoolInternal* pool, const std::vector<int>& cpus,
+                         size_t* next_index) {
+    // No benefit to pinning if no actual worker threads.
+    if (pool->NumWorkerThreads() == 0) return;
+
+    pool->RunOnEachThread([&](int /*task*/, const int thread) {
+      const size_t index = *next_index + static_cast<size_t>(thread);
+      if (index < cpus.size()) {
+        // printf("pin pool %p thread %3d to index %3zu = cpu %3d\n",
+        //        static_cast<void*>(pool), thread, index, cpus[index]);
+        if (!PinThreadToCPU(cpus[index])) {
+          fprintf(stderr, "WARNING: failed to pin thread %d, next %zu.\n",
+                  thread, *next_index);
+        }
+      }
+    });
+    *next_index += pool->NumWorkerThreads();
+  }
+
+  static void InitThreads(
+      const int num_tasks, std::unique_ptr<ThreadPoolInternal>* pool,
+      std::vector<std::unique_ptr<ThreadPoolInternal>>* inner_pools) {
+    const int num_cores = NumCores();
+    const int num_threads = NumOuterThreads(num_cores, num_tasks);
+    const int num_inner = NumInnerThreads(num_cores, num_threads);
+
+    fprintf(stderr, "%d threads, %d inner threads, %d tasks, %d cores\n",
+            num_threads, num_inner, num_tasks, num_cores);
+
+    pool->reset(new ThreadPoolInternal(num_threads));
+    // Main thread OR worker threads in pool each get a possibly empty nested
+    // pool (helps use all available cores when #tasks < #threads)
+    for (size_t i = 0; i < (*pool)->NumThreads(); ++i) {
+      inner_pools->emplace_back(new ThreadPoolInternal(num_inner));
+    }
+
+    // Pin all actual worker threads to available CPUs.
+    const std::vector<int> cpus = AvailableCPUs();
+    size_t next_index = 0;
+    PinThreads(pool->get(), cpus, &next_index);
+    for (std::unique_ptr<ThreadPoolInternal>& inner : *inner_pools) {
+      PinThreads(inner.get(), cpus, &next_index);
+    }
+  }
+
+  static StringVec GetMethods() {
+    StringVec methods = SplitString(Args()->codec, ',');
+    for (auto it = methods.begin(); it != methods.end();) {
+      if (it->empty()) {
+        it = methods.erase(it);
+      } else {
+        ++it;
+      }
+    }
+    return methods;
+  }
+
+  static StringVec SampleFromInput(const StringVec& fnames,
+                                   const std::string& sample_tmp_dir,
+                                   int num_samples, size_t size) {
+    JXL_CHECK(!sample_tmp_dir.empty());
+    fprintf(stderr, "Creating samples of %zux%zu tiles...\n", size, size);
+    StringVec fnames_out;
+    std::vector<Image3B> images;
+    std::vector<size_t> offsets;
+    size_t total_num_tiles = 0;
+    for (const auto& fname : fnames) {
+      Image3B img;
+      JXL_CHECK(ReadPNG(fname, &img));
+      JXL_CHECK(img.xsize() >= size);
+      JXL_CHECK(img.ysize() >= size);
+      total_num_tiles += (img.xsize() - size + 1) * (img.ysize() - size + 1);
+      offsets.push_back(total_num_tiles);
+      images.emplace_back(std::move(img));
+    }
+    JXL_CHECK(MakeDir(sample_tmp_dir));
+    std::mt19937_64 rng;
+    for (int i = 0; i < num_samples; ++i) {
+      int val = std::uniform_int_distribution<>(0, offsets.back())(rng);
+      size_t idx = (std::lower_bound(offsets.begin(), offsets.end(), val) -
+                    offsets.begin());
+      JXL_CHECK(idx < images.size());
+      const Image3B& img = images[idx];
+      int x0 = std::uniform_int_distribution<>(0, img.xsize() - size)(rng);
+      int y0 = std::uniform_int_distribution<>(0, img.ysize() - size)(rng);
+      Image3B sample(size, size);
+      for (size_t c = 0; c < 3; ++c) {
+        for (size_t y = 0; y < size; ++y) {
+          const uint8_t* JXL_RESTRICT row_in = img.PlaneRow(c, y0 + y);
+          uint8_t* JXL_RESTRICT row_out = sample.PlaneRow(c, y);
+          memcpy(row_out, &row_in[x0], size * sizeof(row_out[0]));
+        }
+      }
+      std::string fn_output =
+          StringPrintf("%s/%s.crop_%dx%d+%d+%d.png", sample_tmp_dir.c_str(),
+                       FileBaseName(fnames[idx]).c_str(), size, size, x0, y0);
+      ThreadPool* null_pool = nullptr;
+      JXL_CHECK(WritePNG(sample, null_pool, fn_output));
+      fnames_out.push_back(fn_output);
+    }
+    fprintf(stderr, "Created %d sample tiles\n", num_samples);
+    return fnames_out;
+  }
+
+  static StringVec GetFilenames() {
+    StringVec fnames;
+    JXL_CHECK(MatchFiles(Args()->input, &fnames));
+    if (fnames.empty()) {
+      JXL_ABORT("No input file matches pattern: '%s'", Args()->input.c_str());
+    }
+    if (Args()->print_details) {
+      std::sort(fnames.begin(), fnames.end());
+    }
+
+    if (Args()->num_samples > 0) {
+      fnames = SampleFromInput(fnames, Args()->sample_tmp_dir,
+                               Args()->num_samples, Args()->sample_dimensions);
+    }
+    return fnames;
+  }
+
+  // (Load only once, not for every codec)
+  static std::vector<CodecInOut> LoadImages(
+      const StringVec& fnames, const bool all_color_aware,
+      const bool jpeg_transcoding_requested, ThreadPool* pool) {
+    PROFILER_FUNC;
+    std::vector<CodecInOut> loaded_images;
+    loaded_images.resize(fnames.size());
+    DecodeTarget decode_target = jpeg_transcoding_requested
+                                     ? DecodeTarget::kQuantizedCoeffs
+                                     : DecodeTarget::kPixels;
+    RunOnPool(
+        pool, 0, static_cast<int>(fnames.size()), ThreadPool::SkipInit(),
+        [&](const int task, int /*thread*/) {
+          const size_t i = static_cast<size_t>(task);
+          Status ok = true;
+          loaded_images[i].dec_hints = Args()->dec_hints;
+          if (!Args()->decode_only) {
+            ok = SetFromFile(fnames[i], &loaded_images[i], nullptr,
+                             decode_target);
+          }
+          if (!ok) {
+            if (!Args()->silent_errors) {
+              fprintf(stderr, "Failed to load image %s\n", fnames[i].c_str());
+            }
+            return;
+          }
+
+          if (!Args()->decode_only && all_color_aware) {
+            const bool is_gray = loaded_images[i].Main().IsGray();
+            const ColorEncoding& c_desired =
+                ColorManagement::LinearSRGB(is_gray);
+            if (!loaded_images[i].TransformTo(c_desired, /*pool=*/nullptr)) {
+              JXL_ABORT("Failed to transform to lin. sRGB %s",
+                        fnames[i].c_str());
+            }
+          }
+
+          if (!Args()->decode_only && Args()->override_bitdepth != 0) {
+            loaded_images[i].metadata.bits_per_sample =
+                Args()->override_bitdepth;
+          }
+        },
+        "Load images");
+    return loaded_images;
+  }
+
+  static std::vector<Task> CreateTasks(const StringVec& methods,
+                                       const StringVec& fnames,
+                                       bool* all_color_aware,
+                                       bool* jpeg_transcoding_requested) {
+    std::vector<Task> tasks;
+    tasks.reserve(methods.size() * fnames.size());
+    *all_color_aware = true;
+    *jpeg_transcoding_requested = false;
+    for (size_t idx_image = 0; idx_image < fnames.size(); ++idx_image) {
+      for (size_t idx_method = 0; idx_method < methods.size(); ++idx_method) {
+        tasks.emplace_back();
+        Task& t = tasks.back();
+        t.codec = CreateImageCodec(methods[idx_method]);
+        *all_color_aware &= t.codec->IsColorAware();
+        *jpeg_transcoding_requested |= t.codec->IsJpegTranscoder();
+        t.idx_image = idx_image;
+        t.idx_method = idx_method;
+        // t.stats is default-initialized.
+      }
+    }
+    JXL_ASSERT(tasks.size() == tasks.capacity());
+    return tasks;
+  }
+
+  // Return the total number of errors.
+  static size_t RunTasks(
+      const StringVec& methods, const StringVec& fnames,
+      const std::vector<CodecInOut>& loaded_images, ThreadPoolInternal* pool,
+      const std::vector<std::unique_ptr<ThreadPoolInternal>>& inner_pools,
+      std::vector<Task>* tasks) {
+    PROFILER_FUNC;
+    StatPrinter printer(methods, fnames, *tasks);
+    if (Args()->print_details_csv) {
+      // Print CSV header
+      printf(
+          "method,image,error,size,pixels,enc_speed,dec_speed,"
+          "bpp,dist,psnr,p,bppp,qabpp\n");
+    }
+
+    std::vector<size_t> errors_thread;
+    RunOnPool(
+        pool, 0, tasks->size(),
+        [&](size_t num_threads) {
+          errors_thread.resize(num_threads);
+          return true;
+        },
+        [&](const int i, const int thread) {
+          Task& t = (*tasks)[i];
+          const CodecInOut& image = loaded_images[t.idx_image];
+          t.image = &image;
+          PaddedBytes compressed;
+          DoCompress(fnames[t.idx_image], image, t.codec.get(),
+                     inner_pools[thread].get(), &compressed, &t.stats);
+          printer.TaskDone(i, t);
+          errors_thread[thread] += t.stats.total_errors;
+        },
+        "Benchmark tasks");
+    if (Args()->show_progress) fprintf(stderr, "\n");
+    return std::accumulate(errors_thread.begin(), errors_thread.end(), 0);
+  }
+};
+
+int BenchmarkMain(int argc, char** argv) {
+  const int bits = hwy::TargetBitfield().Bits();
+  if ((bits & HWY_STATIC_TARGETS) != HWY_STATIC_TARGETS) {
+    fprintf(stderr, "Missing CPU support for targets in static_targets.h.\n");
+    return 1;
+  }
+
+  JXL_CHECK(Args()->AddCommandLineOptions());
+  if (!Args()->Parse(argc, const_cast<const char**>(argv)) ||
+      !Args()->ValidateArgs()) {
+    Args()->PrintHelp();
+    return 1;
+  }
+
+  return Benchmark::Run();
+}
+
+}  // namespace
+}  // namespace jxl
+
+int main(int argc, char** argv) { return jxl::BenchmarkMain(argc, argv); }
diff --git a/tools/butteraugli_main.cc b/tools/butteraugli_main.cc
new file mode 100644
index 000000000..99bac0146
--- /dev/null
+++ b/tools/butteraugli_main.cc
@@ -0,0 +1,124 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdio.h>
+
+#include <string>
+#include <vector>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/file_io.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/butteraugli/butteraugli.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/enc_butteraugli_comparator.h"
+#include "jxl/extras/codec.h"
+#include "jxl/extras/codec_png.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "tools/butteraugli_pnorm.h"
+
+namespace jxl {
+namespace {
+
+Status WritePNG(const Image3B& image, const std::string& filename) {
+  ThreadPoolInternal pool(4);
+  std::vector<uint8_t> rgb(image.xsize() * image.ysize() * 3);
+  CodecInOut io;
+  io.metadata.bits_per_sample = 8;
+  io.metadata.color_encoding = ColorManagement::SRGB();
+  io.SetFromImage(StaticCastImage3<float>(image), io.metadata.color_encoding);
+  PaddedBytes compressed;
+  JXL_CHECK(EncodeImagePNG(&io, io.Main().c_current(), 8, &pool, &compressed));
+  return WriteFile(compressed, filename);
+}
+
+Status RunButteraugli(const char* pathname1, const char* pathname2,
+                      const std::string& distmap_filename,
+                      const std::string& colorspace_hint) {
+  CodecInOut io1;
+  if (colorspace_hint != "") io1.dec_hints.Add("color_space", colorspace_hint);
+  ThreadPoolInternal pool(4);
+  if (!SetFromFile(pathname1, &io1, &pool)) {
+    fprintf(stderr, "Failed to read image from %s\n", pathname1);
+    return false;
+  }
+
+  CodecInOut io2;
+  if (colorspace_hint != "") io2.dec_hints.Add("color_space", colorspace_hint);
+  if (!SetFromFile(pathname2, &io2, &pool)) {
+    fprintf(stderr, "Failed to read image from %s\n", pathname2);
+    return false;
+  }
+
+  if (io1.xsize() != io2.xsize()) {
+    fprintf(stderr, "Width mismatch: %zu %zu\n", io1.xsize(), io2.xsize());
+    return false;
+  }
+  if (io1.ysize() != io2.ysize()) {
+    fprintf(stderr, "Height mismatch: %zu %zu\n", io1.ysize(), io2.ysize());
+    return false;
+  }
+
+  ImageF distmap;
+  const float kHfAsymmetry = 0.8;
+  const float distance = ButteraugliDistance(io1.Main(), io2.Main(),
+                                             kHfAsymmetry, &distmap, &pool);
+  printf("%.10f\n", distance);
+
+  double p = 3.0;
+  double pnorm = ComputeDistanceP(distmap, p);
+  printf("%g-norm: %f\n", p, pnorm);
+
+  if (distmap_filename != "") {
+    float good = butteraugli::ButteraugliFuzzyInverse(1.5);
+    float bad = butteraugli::ButteraugliFuzzyInverse(0.5);
+    Image3B heatmap = butteraugli::CreateHeatMapImage(distmap, good, bad);
+    JXL_CHECK(WritePNG(heatmap, distmap_filename));
+  }
+  return true;
+}
+
+}  // namespace
+}  // namespace jxl
+
+int main(int argc, char** argv) {
+  if (argc < 3) {
+    fprintf(stderr,
+            "Usage: %s <reference> <distorted> [--distmap <distmap>] "
+            "[--colorspace <colorspace_hint>]\n"
+            "NOTE: images get converted to linear sRGB for butteraugli. Images"
+            " without attached profiles (such as ppm or pfm) are interpreted"
+            " as nonlinear sRGB. The hint format is RGB_D65_SRG_Rel_Lin for"
+            " linear sRGB\n",
+            argv[0]);
+    return 1;
+  }
+  std::string distmap;
+  std::string colorspace;
+  for (int i = 2; i < argc; i++) {
+    if (std::string(argv[i]) == "--distmap" && i + 1 < argc)
+      distmap = argv[++i];
+    if (std::string(argv[i]) == "--colorspace" && i + 1 < argc)
+      colorspace = argv[++i];
+  }
+
+  return jxl::RunButteraugli(argv[1], argv[2], distmap, colorspace) ? 0 : 1;
+}
diff --git a/tools/butteraugli_pnorm.cc b/tools/butteraugli_pnorm.cc
new file mode 100644
index 000000000..cc5170d54
--- /dev/null
+++ b/tools/butteraugli_pnorm.cc
@@ -0,0 +1,106 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/butteraugli_pnorm.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <atomic>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/profiler.h"
+
+namespace jxl {
+
+HWY_ATTR double ComputeDistanceP(const ImageF& distmap, double p) {
+  PROFILER_FUNC;
+  const double onePerPixels = 1.0 / (distmap.ysize() * distmap.xsize());
+  if (std::abs(p - 3.0) < 1E-6) {
+#if HWY_HAS_DOUBLE
+    const HWY_FULL(double) dd;
+    const HWY_CAPPED(float, dd.N) df;
+    auto sums0 = Zero(dd);
+    auto sums1 = Zero(dd);
+    auto sums2 = Zero(dd);
+#else
+    const HWY_FULL(float) df;
+    auto sums0 = Zero(df);
+    auto sums1 = Zero(df);
+    auto sums2 = Zero(df);
+#endif
+    double sum1[3] = {0.0};
+    for (size_t y = 0; y < distmap.ysize(); ++y) {
+      const float* JXL_RESTRICT row = distmap.ConstRow(y);
+      size_t x = 0;
+      for (; x + df.N <= distmap.xsize(); x += df.N) {
+#if HWY_HAS_DOUBLE
+        const auto d1 = ConvertTo(dd, Load(df, row + x));
+#else
+        const auto d1 = Load(df, row + x);
+#endif
+        const auto d2 = d1 * d1 * d1;
+        sums0 += d2;
+        const auto d3 = d2 * d2;
+        sums1 += d3;
+        const auto d4 = d3 * d3;
+        sums2 += d4;
+      }
+      for (; x < distmap.xsize(); ++x) {
+        const double d1 = row[x];
+        double d2 = d1 * d1 * d1;
+        sum1[0] += d2;
+        d2 *= d2;
+        sum1[1] += d2;
+        d2 *= d2;
+        sum1[2] += d2;
+      }
+    }
+    double v = 0;
+    v += pow(onePerPixels * (sum1[0] + GetLane(hwy::ext::SumOfLanes(sums0))),
+             1.0 / (p * 1.0));
+    v += pow(onePerPixels * (sum1[1] + GetLane(hwy::ext::SumOfLanes(sums1))),
+             1.0 / (p * 2.0));
+    v += pow(onePerPixels * (sum1[2] + GetLane(hwy::ext::SumOfLanes(sums2))),
+             1.0 / (p * 4.0));
+    v /= 3.0;
+    return v;
+  } else {
+    static std::atomic<int> once{0};
+    if (once.fetch_add(1, std::memory_order_relaxed) == 0) {
+      fprintf(stderr, "WARNING: using slow ComputeDistanceP\n");
+    }
+    double sum1[3] = {0.0};
+    for (size_t y = 0; y < distmap.ysize(); ++y) {
+      const float* JXL_RESTRICT row = distmap.ConstRow(y);
+      for (size_t x = 0; x < distmap.xsize(); ++x) {
+        double d2 = std::pow(row[x], p);
+        sum1[0] += d2;
+        d2 *= d2;
+        sum1[1] += d2;
+        d2 *= d2;
+        sum1[2] += d2;
+      }
+    }
+    double v = 0;
+    for (int i = 0; i < 3; ++i) {
+      v += pow(onePerPixels * (sum1[i]), 1.0 / (p * (1 << i)));
+    }
+    v /= 3.0;
+    return v;
+  }
+}
+
+}  // namespace jxl
diff --git a/tools/butteraugli_pnorm.h b/tools/butteraugli_pnorm.h
new file mode 100644
index 000000000..274358c37
--- /dev/null
+++ b/tools/butteraugli_pnorm.h
@@ -0,0 +1,29 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_BUTTERAUGLI_PNORM_H_
+#define TOOLS_BUTTERAUGLI_PNORM_H_
+
+#include <hwy/static_targets.h>
+
+#include "jxl/image.h"
+
+namespace jxl {
+
+// Computes p-norm given the butteraugli distmap.
+HWY_ATTR double ComputeDistanceP(const ImageF& distmap, double p);
+
+}  // namespace jxl
+
+#endif  // TOOLS_BUTTERAUGLI_PNORM_H_
diff --git a/tools/cbrunsli.cc b/tools/cbrunsli.cc
new file mode 100644
index 000000000..c5c56cd8c
--- /dev/null
+++ b/tools/cbrunsli.cc
@@ -0,0 +1,138 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/cbrunsli.h"
+
+#include <brunsli/brunsli_encode.h>
+#include <brunsli/jpeg_data_reader.h>
+
+#include <algorithm>
+#include <cstdint>
+#include <cstdio>
+#include <string>
+#include <vector>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/file_io.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/brunsli.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/extras/codec.h"
+#include "jxl/image_bundle.h"
+#include "tools/args.h"
+#include "tools/cmdline.h"
+
+namespace jpegxl {
+namespace tools {
+
+BrunsliCompressArgs::BrunsliCompressArgs() {
+  // TODO(eustas): find a way to make it more elegant.
+  // Linear transform is important for HDR PFM files to avoid banding.
+  dec_hints.Add("color_space", "RGB_D65_SRG_Rel_Lin");
+}
+
+jxl::Status BrunsliCompressArgs::AddCommandLineOptions(
+    CommandLineParser* cmdline) {
+  cmdline->AddOptionValue('q', "quant", "Q",
+                          "quant scale (0=best quality, 256=worst)",
+                          &quant_scale, &ParseUnsigned);
+
+  cmdline->AddOptionValue(
+      'x', "dec-hints", "key=value",
+      "color_space indicates the ColorEncoding, see Description().", &dec_hints,
+      &ParseAndAppendKeyValue);
+
+  return true;
+}
+
+jxl::Status BrunsliCompressArgs::ValidateArgs(
+    const tools::CommandLineParser& cmdline) {
+  // TODO(deymo): Make this optional for benchmarking.
+  if (file_out == nullptr) {
+    fprintf(stderr, "Missing OUTPUT filename.\n");
+    return false;
+  }
+
+  if (quant_scale < 0 || quant_scale > 1024) {
+    fprintf(stderr, "quant must be in range [0..1024].\n");
+    return false;
+  } else {
+    options.quant_scale = quant_scale / 64.0f;
+  }
+
+  return true;
+}
+
+jxl::Status CompressBrunsli(jxl::ThreadPool* pool,
+                            const BrunsliCompressArgs& args,
+                            jxl::PaddedBytes* compressed) {
+  jxl::CodecInOut io;
+  io.dec_hints = args.dec_hints;
+
+  std::string extension = jxl::Extension(args.file_in);
+  if (extension == ".jpg") {
+    jxl::PaddedBytes jpg_content;
+    if (!jxl::ReadFile(args.file_in, &jpg_content)) {
+      return JXL_FAILURE("Could not read JPEG file");
+    }
+
+    brunsli::JPEGData jpg;
+    const uint8_t* input_data = jpg_content.data();
+    bool ok = brunsli::ReadJpeg(input_data, jpg_content.size(),
+                                brunsli::JPEG_READ_ALL, &jpg);
+    if (!ok) {
+      return JXL_FAILURE("Could not parse JPEG file");
+    }
+
+    size_t output_size = ::brunsli::GetMaximumBrunsliEncodedSize(jpg);
+    std::vector<uint8_t> output(output_size);
+    // TODO(eustas): introduce streaming API?
+    if (!brunsli::BrunsliEncodeJpeg(jpg, output.data(), &output_size)) {
+      return JXL_FAILURE("Could not encode recompressed JPEG file");
+    }
+    compressed->append(jxl::Span<uint8_t>(output.data(), output_size));
+
+    return true;
+  }
+
+  if (!SetFromFile(args.file_in, &io)) {
+    fprintf(stderr, "Failed to read image %s.\n", args.file_in);
+    return false;
+  }
+
+  jxl::BrunsliEncoderOptions options = args.options;
+  if (io.metadata.bits_per_sample > 8) {
+    options.hdr_orig_colorspace = jxl::Description(io.Main().c_current());
+    jxl::ColorEncoding hdr;
+    std::string hdr_description = "RGB_D65_202_Rel_PeQ";
+    JXL_CHECK(jxl::ParseDescription(hdr_description, &hdr));
+    JXL_CHECK(jxl::ColorManagement::CreateProfile(&hdr));
+    JXL_CHECK(io.Main().TransformTo(hdr, pool));
+    io.metadata.color_encoding = hdr;
+  }
+
+  if (!PixelsToBrunsli(&io, compressed, options, pool)) {
+    fprintf(stderr, "Failed to compress.\n");
+    return false;
+  }
+
+  return true;
+}
+
+}  // namespace tools
+}  // namespace jpegxl
diff --git a/tools/cbrunsli.h b/tools/cbrunsli.h
new file mode 100644
index 000000000..4dcf14598
--- /dev/null
+++ b/tools/cbrunsli.h
@@ -0,0 +1,59 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_CBRUNSLI_H_
+#define TOOLS_CBRUNSLI_H_
+
+#include <stddef.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/brunsli.h"
+#include "jxl/codec_in_out.h"
+#include "tools/cmdline.h"
+
+namespace jpegxl {
+namespace tools {
+
+struct BrunsliCompressArgs {
+  // Initialize non-static default options.
+  BrunsliCompressArgs();
+
+  // Add all the command line options to the CommandLineParser. Note that the
+  // options are tied to the instance that this was called on.
+  jxl::Status AddCommandLineOptions(CommandLineParser* cmdline);
+
+  // Validate the passed arguments, checking whether all passed options are
+  // compatible. Returns whether the validation was successful.
+  jxl::Status ValidateArgs(const CommandLineParser& cmdline);
+
+  jxl::BrunsliEncoderOptions options;
+
+  jxl::DecoderHints dec_hints;
+
+  size_t quant_scale = 0;
+
+  const char* file_in = nullptr;
+  const char* file_out = nullptr;
+};
+
+jxl::Status CompressBrunsli(jxl::ThreadPool* pool,
+                            const BrunsliCompressArgs& args,
+                            jxl::PaddedBytes* compressed);
+
+}  // namespace tools
+}  // namespace jpegxl
+
+#endif  // TOOLS_CBRUNSLI_H_
diff --git a/tools/cbrunsli_main.cc b/tools/cbrunsli_main.cc
new file mode 100644
index 000000000..7c5b4e02c
--- /dev/null
+++ b/tools/cbrunsli_main.cc
@@ -0,0 +1,20 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/cjpegxl.h"
+
+int main(int argc, const char** argv) {
+  return jpegxl::tools::CompressJpegXlMain(
+      jpegxl::tools::CompressionMode::kBrunsli, argc, argv);
+}
diff --git a/tools/cjpegxl.cc b/tools/cjpegxl.cc
new file mode 100644
index 000000000..5c1bef838
--- /dev/null
+++ b/tools/cjpegxl.cc
@@ -0,0 +1,134 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/cjpegxl.h"
+
+#include <stdio.h>
+
+#include <algorithm>
+#include <hwy/runtime_dispatch.h>
+#include <hwy/static_targets.h>
+
+#include "jxl/base/cache_aligned.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/file_io.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/status.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/enc_params.h"
+#include "tools/args.h"
+
+namespace jpegxl {
+namespace tools {
+
+void CompressArgs::AddCommandLineOptions(CommandLineParser* cmdline,
+                                         CompressionMode mode) {
+  // Positional arguments.
+  cmdline->AddPositionalOption("INPUT", "the input can be PNG PPM or PFM.",
+                               &file_in);
+  cmdline->AddPositionalOption(
+      "OUTPUT", "the compressed output file (optional)", &file_out);
+
+  // Flags.
+  cmdline->AddOptionFlag('V', "version", "print version number and exit",
+                         &version, &SetBooleanTrue);
+
+  cmdline->AddOptionValue('\0', "print_profile", "0|1",
+                          "print timing information before exiting",
+                          &print_profile, &ParseOverride);
+
+  switch (mode) {
+    case CompressionMode::kJpegXL:
+      JXL_ASSERT(cjxl_args.AddCommandLineOptions(cmdline));
+      break;
+    case CompressionMode::kBrunsli:
+      JXL_ASSERT(cbrunsli_args.AddCommandLineOptions(cmdline));
+      break;
+  }
+}
+
+jxl::Status CompressArgs::ValidateArgs(const CommandLineParser& cmdline,
+                                       CompressionMode mode) {
+  cjxl_args.params.file_in = file_in;
+  cjxl_args.params.file_out = file_out;
+  cbrunsli_args.file_in = file_in;
+  cbrunsli_args.file_out = file_out;
+
+  if (file_in == nullptr) {
+    fprintf(stderr, "Missing INPUT filename.\n");
+    return false;
+  }
+
+  switch (mode) {
+    case CompressionMode::kJpegXL:
+      return cjxl_args.ValidateArgs(cmdline);
+    case CompressionMode::kBrunsli:
+      return cbrunsli_args.ValidateArgs(cmdline);
+  }
+
+  return true;
+}
+
+int CompressJpegXlMain(CompressionMode mode, int argc, const char* argv[]) {
+  CommandLineParser cmdline;
+  CompressArgs args;
+  args.AddCommandLineOptions(&cmdline, mode);
+
+  if (!cmdline.Parse(argc, argv) || !args.ValidateArgs(cmdline, mode)) {
+    cmdline.PrintHelp();
+    return 1;
+  }
+
+  if (args.version) {
+    fprintf(stderr, "cjpegxl - version " JPEGXL_VERSION "\n");
+    fprintf(stderr, "Copyright (c) the JPEG XL Project\n");
+    return 0;
+  }
+
+  const int bits = hwy::TargetBitfield().Bits();
+  if ((bits & HWY_STATIC_TARGETS) != HWY_STATIC_TARGETS) {
+    fprintf(stderr, "CPU does not support all enabled targets => exiting.\n");
+    return 1;
+  }
+
+  jxl::PaddedBytes compressed;
+
+  switch (mode) {
+    case CompressionMode::kJpegXL: {
+      jxl::ThreadPoolInternal pool(args.cjxl_args.num_threads);
+      if (!CompressJxl(&pool, args.cjxl_args, &compressed)) return 1;
+      if (args.file_out) {
+        if (!jxl::WriteFile(compressed, args.file_out)) return 1;
+      }
+    } break;
+    case CompressionMode::kBrunsli: {
+      // TODO(eustas): add num_threads parameter.
+      jxl::ThreadPoolInternal pool(0);
+      if (!CompressBrunsli(&pool, args.cbrunsli_args, &compressed)) return 1;
+      if (args.file_out) {
+        if (!jxl::WriteFile(compressed, args.file_out)) return 1;
+      }
+    } break;
+  }
+
+  if (args.print_profile == jxl::Override::kOn) {
+    PROFILER_PRINT_RESULTS();
+  }
+  jxl::CacheAligned::PrintStats();
+  return 0;
+}
+
+}  // namespace tools
+}  // namespace jpegxl
diff --git a/tools/cjpegxl.h b/tools/cjpegxl.h
new file mode 100644
index 000000000..21df3e6dc
--- /dev/null
+++ b/tools/cjpegxl.h
@@ -0,0 +1,61 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_CJPEGXL_H_
+#define TOOLS_CJPEGXL_H_
+
+#include <stddef.h>
+
+#include "jxl/base/override.h"
+#include "jxl/base/status.h"
+#include "tools/cbrunsli.h"
+#include "tools/cjxl.h"
+#include "tools/cmdline.h"
+
+namespace jpegxl {
+namespace tools {
+
+enum class CompressionMode {
+  kBrunsli,
+  kJpegXL,
+};
+
+struct CompressArgs {
+  // Add all the command line options to the CommandLineParser. Note that the
+  // options are tied to the instance that this was called on.
+  void AddCommandLineOptions(CommandLineParser* cmdline, CompressionMode mode);
+
+  // Validate the passed arguments, checking whether all passed options are
+  // compatible. Returns whether the validation was successful.
+  jxl::Status ValidateArgs(const CommandLineParser& cmdline,
+                           CompressionMode mode);
+
+  // Common flags.
+  // TODO(deymo): Move more flags here.
+  bool version = false;
+  const char* file_in = nullptr;
+  const char* file_out = nullptr;
+  jxl::Override print_profile = jxl::Override::kDefault;
+
+  // Algorithm-specific flags.
+  JxlCompressArgs cjxl_args;
+  BrunsliCompressArgs cbrunsli_args;
+};
+
+int CompressJpegXlMain(CompressionMode mode, int argc, const char* argv[]);
+
+}  // namespace tools
+}  // namespace jpegxl
+
+#endif  // TOOLS_CJPEGXL_H_
diff --git a/tools/cjpegxl_main.cc b/tools/cjpegxl_main.cc
new file mode 100644
index 000000000..279d42f53
--- /dev/null
+++ b/tools/cjpegxl_main.cc
@@ -0,0 +1,26 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/cjpegxl.h"
+
+namespace {
+
+int CompressMain(int argc, const char** argv) {
+  return jpegxl::tools::CompressJpegXlMain(
+      jpegxl::tools::CompressionMode::kJpegXL, argc, argv);
+}
+
+}  // namespace
+
+int main(int argc, const char** argv) { return CompressMain(argc, argv); }
diff --git a/tools/cjxl.cc b/tools/cjxl.cc
new file mode 100644
index 000000000..65b1b0826
--- /dev/null
+++ b/tools/cjxl.cc
@@ -0,0 +1,497 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/cjxl.h"
+
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <algorithm>
+#include <string>
+#include <utility>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/arch_specific.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/os_specific.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/common.h"
+#include "jxl/enc_cache.h"
+#include "jxl/enc_file.h"
+#include "jxl/extras/codec.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/modular/encoding/encoding.h"
+#include "tools/args.h"
+#include "tools/speed_stats.h"
+
+namespace jpegxl {
+namespace tools {
+namespace {
+
+static inline bool ParseSpeedTier(const char* arg, jxl::SpeedTier* out) {
+  return jxl::ParseSpeedTier(arg, out);
+}
+static inline bool ParseColorTransform(const char* arg,
+                                       jxl::ColorTransform* out) {
+  size_t value = 0;
+  bool ret = ParseUnsigned(arg, &value);
+  if (ret && value > 2) ret = false;
+  if (ret) *out = jxl::ColorTransform(value);
+  return ret;
+}
+// Proposes a distance to try for a given bpp target. This could depend
+// on the entropy in the image, too, but let's start with something.
+static double ApproximateDistanceForBPP(double bpp) {
+  return 1.704 * pow(bpp, -0.804);
+}
+
+}  // namespace
+
+JxlCompressArgs::JxlCompressArgs() {
+  jxl::ProcessorTopology topology;
+  JXL_CHECK(jxl::DetectProcessorTopology(&topology));
+  num_threads = topology.packages * topology.cores_per_package;
+}
+
+jxl::Status JxlCompressArgs::AddCommandLineOptions(CommandLineParser* cmdline) {
+  cmdline->AddPositionalOption(
+      "SPOT", "spot color channel (optional, for testing)", &spot_in);
+  // Flags.
+  cmdline->AddOptionFlag('\0', "progressive",
+                         "Use the progressive mode for AC.",
+                         &params.progressive_mode, &SetBooleanTrue);
+  cmdline->AddOptionFlag('\0', "qprogressive",
+                         "Use the progressive mode for AC.",
+                         &params.qprogressive_mode, &SetBooleanTrue);
+  cmdline->AddOptionValue('\0', "progressive_dc", "num_dc_frames",
+                          "Use progressive mode for DC.",
+                          &params.progressive_dc, &ParseUnsigned);
+  cmdline->AddOptionFlag('g', "modular-group",
+                         "Use the modular-group mode (lossy / lossless).",
+                         &params.modular_group_mode, &SetBooleanTrue);
+  cmdline->AddOptionFlag('b', "jpeg-group", "Use the jpeg-group mode.",
+                         &params.brunsli_group_mode, &SetBooleanTrue);
+
+  cmdline->AddOptionFlag(
+      'j', "jpeg_transcode",
+      "Do lossless transcode of input JPEG file (don't decode it to pixels).",
+      &jpeg_transcode, &SetBooleanTrue);
+
+  cmdline->AddOptionFlag('v', "verbose", "Verbose output.", &params.verbose,
+                         &SetBooleanTrue);
+
+  cmdline->AddOptionValue('\0', "num_threads", "N",
+                          "number of worker threads (zero = none).",
+                          &num_threads, &ParseUnsigned);
+  cmdline->AddOptionValue('\0', "num_reps", "N", "how many times to compress.",
+                          &num_reps, &ParseUnsigned);
+
+  cmdline->AddOptionValue('\0', "noise", "0|1",
+                          "force enable/disable noise generation.",
+                          &params.noise, &ParseOverride);
+  cmdline->AddOptionValue('\0', "dots", "0|1",
+                          "force enable/disable dots generation.", &params.dots,
+                          &ParseOverride);
+
+  cmdline->AddOptionValue('\0', "adaptive_reconstruction", "0|1",
+                          "force enable/disable loop filter.",
+                          &params.adaptive_reconstruction, &ParseOverride);
+
+  cmdline->AddOptionValue('\0', "gaborish", "0|1", "force disable gaborish.",
+                          &params.gaborish, &ParseOverride);
+
+  cmdline->AddOptionValue(
+      '\0', "speed", "falcon|cheetah|hare|wombat|squirrel|kitten|tortoise",
+      "Speed tier to use for compression. kitten is the default; values are in "
+      "order from faster to slower.",
+      &params.speed_tier, &ParseSpeedTier);
+
+  // Target distance/size/bpp
+  opt_distance_id = cmdline->AddOptionValue(
+      '\0', "distance", "maxError",
+      ("Max. butteraugli distance, lower = higher quality.\n"
+       "    Good default: 1.0. Supported range: 0.5 .. 3.0."),
+      &params.butteraugli_distance, &ParseFloat);
+  opt_target_size_id = cmdline->AddOptionValue(
+      '\0', "target_size", "N",
+      ("Aim at file size of N bytes.\n"
+       "    Compresses to 1 % of the target size in ideal conditions.\n"
+       "    Runs the same algorithm as --target_bpp"),
+      &params.target_size, &ParseUnsigned);
+  opt_target_bpp_id = cmdline->AddOptionValue(
+      '\0', "target_bpp", "BPP",
+      ("Aim at file size that has N bits per pixel.\n"
+       "    Compresses to 1 % of the target BPP in ideal conditions."),
+      &params.target_bitrate, &ParseFloat);
+
+  opt_intensity_target_id = cmdline->AddOptionValue(
+      '\0', "intensity_target", "N",
+      ("Intensity target of monitor in nits, higher\n"
+       "   results in higher quality image. Supported range: 250..50000,\n"
+       "   default is 250 for standard images, 4000 for input images known"
+       "   to have PQ or HLG transfer function."),
+      &params.intensity_target, &ParseFloat);
+
+  cmdline->AddOptionValue('\0', "saliency_num_progressive_steps", "N", nullptr,
+                          &params.saliency_num_progressive_steps,
+                          &ParseUnsigned);
+  cmdline->AddOptionValue('\0', "saliency_map_filename", "STRING", nullptr,
+                          &params.saliency_map_filename, &ParseString);
+  cmdline->AddOptionValue('\0', "saliency_threshold", "0..1", nullptr,
+                          &params.saliency_threshold, &ParseFloat);
+
+  cmdline->AddOptionValue(
+      'x', "dec-hints", "key=value",
+      "color_space indicates the ColorEncoding, see Description().", &dec_hints,
+      &ParseAndAppendKeyValue);
+
+  cmdline->AddOptionValue(
+      '\0', "override_bitdepth", "0=use from image, 1-32=override",
+      "If nonzero, store the given bit depth in the JPEG XL file metadata"
+      " (1-32), instead of using the bit depth from the original input"
+      " image.",
+      &override_bitdepth, &ParseUnsigned);
+
+  opt_color_id = cmdline->AddOptionValue(
+      'c', "colortransform", "0..2", "0=XYB, 1=None, 2=YCbCr",
+      &params.color_transform, &ParseColorTransform);
+
+  // modular mode options
+  cmdline->AddOptionValue(
+      'Q', "quality", "luma_q[,chroma_q]",
+      "[modular encoding] lossy 'quality' (100=lossless, lower is more lossy)",
+      &params.quality_pair, &ParseFloatPair);
+
+  cmdline->AddOptionValue(
+      'I', "iterations", "F",
+      "[modular encoding] number of mock encodes to learn MABEGABRAC trees "
+      "(default=0.5, try 0 for no MA and fast decode)",
+      &params.options.nb_repeats, &ParseFloat);
+
+  cmdline->AddOptionValue('T', "ctx-threshold", "F",
+                          ("[modular encoding] number of bits hypothetically "
+                           "saved during MABEGABRAC training "
+                           "to justify adding another context (default: 16)"),
+                          &params.options.ctx_threshold, &ParseFloat);
+
+  cmdline->AddOptionValue(
+      'C', "colorspace", "K",
+      ("[modular encoding] color transform: 0=RGB, 1=YCoCg, "
+       "2-37=RCT (default: YCoCg)"),
+      &params.colorspace, &ParseSigned);
+
+  cmdline->AddOptionValue('P', "predictor", "K",
+                          "[modular encoding] predictor(s) to use: 0=zero, "
+                          "1=avgNW, 2=clampedGradient (default), 3=W, 4=N, "
+                          "5=WeightedPredictor (good for photo)",
+                          &params.options.predictor, &ParsePredictorsVector);
+
+  cmdline->AddOptionValue(
+      'E', "extra-properties", "K",
+      "[modular encoding] number of extra MA tree properties to use",
+      &params.options.max_properties, &ParseSigned);
+
+  cmdline->AddOptionValue('N', "near-lossless", "max_d",
+                          "[modular encoding] apply near-lossless "
+                          "preprocessing with maximum delta = max_d",
+                          &params.near_lossless, &ParseSigned);
+
+  cmdline->AddOptionValue('p', "palette", "K",
+                          "[modular encoding] use a palette if image has at "
+                          "most K colors (default: 1024)",
+                          &params.palette_colors, &ParseSigned);
+
+  cmdline->AddOptionValue(
+      'X', "pre-compact", "PERCENT",
+      ("[modular encoding] compact channels (globally) if ratio "
+       "used/range is below this (default: 80%)"),
+      &params.channel_colors_pre_transform_percent, &ParseFloat);
+
+  cmdline->AddOptionValue(
+      'Y', "post-compact", "PERCENT",
+      ("[modular encoding] compact channels (per-group) if ratio "
+       "used/range is below this (default: 80%)"),
+      &params.channel_colors_percent, &ParseFloat);
+
+  opt_brotli_id = cmdline->AddOptionValue(
+      'B', "brotli", "effort",
+      ("[modular encoding] use Brotli instead of MABEGABRAC/MAANS"
+       " (with encode effort=0..11)"),
+      &params.options.brotli_effort, &ParseSigned);
+
+  cmdline->AddOptionFlag('A', "ans",
+                         "[modular encoding] use MAANS instead of MABEGABRAC",
+                         &params.ans, &SetBooleanTrue);
+
+  cmdline->AddOptionValue('R', "responsive", "K",
+                          "[modular encoding] do Squeeze transform, 0=false, "
+                          "1=true (default: true if lossy, false if lossless)",
+                          &params.responsive, &ParseSigned);
+
+  return true;
+}
+
+jxl::Status JxlCompressArgs::ValidateArgs(
+    const tools::CommandLineParser& cmdline) {
+  bool got_distance = cmdline.GetOption(opt_distance_id)->matched();
+  bool got_target_size = cmdline.GetOption(opt_target_size_id)->matched();
+  bool got_target_bpp = cmdline.GetOption(opt_target_bpp_id)->matched();
+
+  got_intensity_target = cmdline.GetOption(opt_intensity_target_id)->matched();
+
+  if (got_distance) {
+    constexpr float butteraugli_min_dist = 0.125f;
+    constexpr float butteraugli_max_dist = 15.0f;
+    if (!(butteraugli_min_dist <= params.butteraugli_distance &&
+          params.butteraugli_distance <= butteraugli_max_dist)) {
+      fprintf(stderr, "Invalid/out of range distance, try %g to %g.\n",
+              butteraugli_min_dist, butteraugli_max_dist);
+      return false;
+    }
+  }
+
+  if (got_target_bpp + got_target_size + got_distance > 1) {
+    fprintf(stderr,
+            "You can specify only one of '--distance', "
+            "'--target_bpp' and '--target_size'. They are all different ways"
+            " to specify the image quality. When in doubt, use --distance."
+            " It gives the most visually consistent results.\n");
+    return false;
+  }
+
+  if (!params.saliency_map_filename.empty()) {
+    if (!params.progressive_mode) {
+      params.saliency_map_filename.clear();
+      fprintf(stderr,
+              "Warning: Specifying --saliency_map_filename only makes sense "
+              "for --progressive mode.\n");
+    }
+  }
+
+  if (!params.file_in) {
+    fprintf(stderr, "Missing input filename.\n");
+    return false;
+  }
+
+  if (!cmdline.GetOption(opt_color_id)->matched()) {
+    // default to RGB for modular
+    if (params.modular_group_mode)
+      params.color_transform = jxl::ColorTransform::kNone;
+    // default to YCrCb for jpeg
+    if (params.brunsli_group_mode)
+      params.color_transform = jxl::ColorTransform::kYCbCr;
+  }
+
+  if (cmdline.GetOption(opt_brotli_id)->matched())
+    params.options.entropy_coder = 1;
+  else if (params.ans)
+    params.options.entropy_coder = 2;
+
+  if (params.near_lossless) {
+    // Near-lossless assumes -R 0
+    params.responsive = 0;
+  }
+
+  if (override_bitdepth > 32) {
+    fprintf(stderr, "override_bitdepth must be <= 32\n");
+    return false;
+  }
+
+  return true;
+}
+
+jxl::Status LoadSaliencyMap(const std::string& filename_heatmap,
+                            const jxl::CodecInOut* io, jxl::ThreadPool* pool,
+                            jxl::ImageF* out_map) {
+  jxl::CodecInOut io_heatmap;
+  if (!SetFromFile(filename_heatmap, &io_heatmap, pool)) {
+    return JXL_FAILURE("Could not load heatmap.");
+  }
+  jxl::ImageF heatmap(io_heatmap.xsize(), io_heatmap.ysize());
+  for (size_t num_row = 0; num_row < io_heatmap.ysize(); num_row++) {
+    const float* JXL_RESTRICT row_src =
+        io_heatmap.Main().color().ConstPlaneRow(0, num_row);
+    float* JXL_RESTRICT row_dst = heatmap.Row(num_row);
+    for (size_t num_col = 0; num_col < io_heatmap.xsize(); num_col++) {
+      row_dst[num_col] = row_src[num_col] / 255.0f;
+    }
+  }
+  *out_map = std::move(heatmap);
+  return true;
+}
+
+jxl::Status CompressJxl(jxl::ThreadPoolInternal* pool, JxlCompressArgs& args,
+                        jxl::PaddedBytes* compressed) {
+  JXL_CHECK(pool);
+  double t0, t1;
+
+  jxl::CodecInOut io;
+  io.dec_hints = args.dec_hints;
+  t0 = jxl::Now();
+  if (!SetFromFile(args.params.file_in, &io, nullptr,
+                   (args.jpeg_transcode ? jxl::DecodeTarget::kQuantizedCoeffs
+                                        : jxl::DecodeTarget::kPixels))) {
+    fprintf(stderr, "Failed to read image %s.\n", args.params.file_in);
+    return false;
+  }
+  if (args.override_bitdepth != 0) {
+    io.metadata.bits_per_sample = args.override_bitdepth;
+  }
+  jxl::ImageF saliency_map;
+  if (!args.params.saliency_map_filename.empty()) {
+    if (!LoadSaliencyMap(args.params.saliency_map_filename, &io, pool,
+                         &saliency_map)) {
+      fprintf(stderr, "Failed to read saliency map %s.\n",
+              args.params.saliency_map_filename.c_str());
+      return false;
+    }
+    args.params.saliency_map = &saliency_map;
+  }
+  if (!args.got_intensity_target) {
+    args.params.intensity_target = ChooseDefaultIntensityTarget(io.metadata);
+  }
+  if (args.spot_in != nullptr) {
+    jxl::CodecInOut spot_io;
+    spot_io.dec_hints = args.dec_hints;
+    if (!SetFromFile(args.spot_in, &spot_io)) {
+      fprintf(stderr, "Failed to read spot image %s.\n", args.spot_in);
+      return false;
+    }
+    io.metadata.m2.num_extra_channels = 1;
+    io.metadata.m2.extra_channel_bits = 8;
+    jxl::ExtraChannelInfo example;
+    example.rendered = 1;
+    example.color[0] = 255.0f;
+    example.color[1] = 0.0f;
+    example.color[2] = 0.0f;
+    example.color[3] = 1.0f;
+    io.metadata.m2.extra_channel_info.push_back(example);
+    jxl::ImageU sc(spot_io.xsize(), spot_io.ysize());
+    for (size_t y = 0; y < spot_io.ysize(); ++y) {
+      const float* JXL_RESTRICT from = spot_io.Main().color().PlaneRow(1, y);
+      uint16_t* JXL_RESTRICT to = sc.Row(y);
+      for (size_t x = 0; x < spot_io.xsize(); ++x) {
+        to[x] = from[x];
+      }
+    }
+    std::vector<jxl::ImageU> scv;
+    scv.push_back(std::move(sc));
+    io.Main().SetExtraChannels(std::move(scv));
+  }
+  t1 = jxl::Now();
+  const double decode_mps =
+      io.xsize() * io.ysize() * io.frames.size() * 1E-6 / (t1 - t0);
+
+  const size_t xsize = io.xsize();
+  const size_t ysize = io.ysize();
+  if (args.params.target_size > 0 || args.params.target_bitrate > 0) {
+    // Search algorithm for target bpp / size.
+    JxlCompressArgs s = args;  // Args for search.
+    if (s.params.target_size > 0) {
+      s.params.target_bitrate =
+          s.params.target_size * 8.0 / (io.xsize() * io.ysize());
+      s.params.target_size = 0;
+    }
+    double dist = ApproximateDistanceForBPP(s.params.target_bitrate);
+    s.params.butteraugli_distance = dist;
+    double target_size =
+        s.params.target_bitrate * (1 / 8.) * io.xsize() * io.ysize();
+    s.params.target_bitrate = 0;
+    double best_dist = 1.0;
+    double best_loss = 1e99;
+    for (int i = 0; i < 7; ++i) {
+      s.params.butteraugli_distance = dist;
+      jxl::PaddedBytes candidate;
+      bool ok = CompressJxl(pool, s, &candidate);
+      if (!ok) {
+        printf(
+            "Compression error occurred during the search for best size."
+            " Trying with butteraugli distance %.15g\n",
+            best_dist);
+        break;
+      }
+      printf("Butteraugli distance %g yields %zu bytes, %g bpp.\n", dist,
+             candidate.size(),
+             candidate.size() * 8.0 / (io.xsize() * io.ysize()));
+      const double ratio = static_cast<double>(candidate.size()) / target_size;
+      const double loss = std::max(ratio, 1.0 / std::max(ratio, 1e-30));
+      if (best_loss > loss) {
+        best_dist = dist;
+        best_loss = loss;
+      }
+      dist *= ratio;
+      if (dist < 0.01) {
+        dist = 0.01;
+      }
+      if (dist >= 16.0) {
+        dist = 16.0;
+      }
+    }
+    printf("Choosing butteraugli distance %.15g\n", best_dist);
+    args.params.butteraugli_distance = best_dist;
+    args.params.target_bitrate = 0;
+    args.params.target_size = 0;
+  }
+  char mode[200];
+  if (args.params.speed_tier != jxl::SpeedTier::kKitten) {
+    snprintf(mode, sizeof(mode),
+             "in %s mode with maximum Butteraugli distance %f",
+             SpeedTierName(args.params.speed_tier),
+             args.params.butteraugli_distance);
+  } else {
+    snprintf(mode, sizeof(mode), "with maximum Butteraugli distance %f",
+             args.params.butteraugli_distance);
+  }
+  fprintf(stderr,
+          "Read %zu bytes (%zux%zu, %.1f MP/s); compressing %s, %zu threads.\n",
+          io.enc_size, xsize, ysize, decode_mps, mode,
+          pool->NumWorkerThreads());
+
+  jxl::AuxOut aux_out;
+  if (args.inspector_image3f) {
+    aux_out.SetInspectorImage3F(args.inspector_image3f);
+  }
+  SpeedStats stats;
+  jxl::PassesEncoderState passes_encoder_state;
+  for (size_t i = 0; i < args.num_reps; ++i) {
+    t0 = jxl::Now();
+    if (!EncodeFile(args.params, &io, &passes_encoder_state, compressed,
+                    &aux_out, pool)) {
+      fprintf(stderr, "Failed to compress.\n");
+      return false;
+    }
+    t1 = jxl::Now();
+    stats.NotifyElapsed(t1 - t0);
+  }
+  const double bpp =
+      static_cast<double>(compressed->size() * jxl::kBitsPerByte) /
+      (xsize * ysize);
+  fprintf(stderr, "Compressed to %zu bytes (%.3f bpp%s).\n", compressed->size(),
+          bpp / io.frames.size(), io.frames.size() == 1 ? "" : "/frame");
+  JXL_CHECK(stats.Print(xsize, ysize, args.num_threads));
+
+  if (args.params.verbose) {
+    aux_out.Print(1);
+  }
+
+  return true;
+}
+
+}  // namespace tools
+}  // namespace jpegxl
diff --git a/tools/cjxl.h b/tools/cjxl.h
new file mode 100644
index 000000000..950081790
--- /dev/null
+++ b/tools/cjxl.h
@@ -0,0 +1,82 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_CJXL_H_
+#define TOOLS_CJXL_H_
+
+#include <stddef.h>
+
+#include <utility>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/status.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/enc_params.h"
+#include "jxl/jxl_inspection.h"
+#include "tools/cmdline.h"
+
+namespace jpegxl {
+namespace tools {
+
+struct JxlCompressArgs {
+  // Initialize non-static default options.
+  JxlCompressArgs();
+
+  void SetInspectorImage3F(const jxl::InspectorImage3F& inspector) {
+    inspector_image3f = inspector;
+  }
+
+  // Add all the command line options to the CommandLineParser. Note that the
+  // options are tied to the instance that this was called on.
+  jxl::Status AddCommandLineOptions(CommandLineParser* cmdline);
+
+  // Post-processes and validates the passed arguments, checking whether all
+  // passed options are compatible. Returns whether the validation was
+  // successful.
+  jxl::Status ValidateArgs(const CommandLineParser& cmdline);
+
+  jxl::DecoderHints dec_hints;
+  size_t override_bitdepth = 0;
+  jxl::CompressParams params;
+  size_t num_threads = 0;
+  size_t num_reps = 1;
+  bool got_intensity_target = false;
+  bool jpeg_transcode = false;
+
+  // Will get passed on to AuxOut.
+  jxl::InspectorImage3F inspector_image3f;
+
+  // References (ids) of specific options to check if they were matched.
+  CommandLineParser::OptionId opt_distance_id = -1;
+  CommandLineParser::OptionId opt_target_size_id = -1;
+  CommandLineParser::OptionId opt_target_bpp_id = -1;
+  CommandLineParser::OptionId opt_near_lossless_id = -1;
+  CommandLineParser::OptionId opt_intensity_target_id = -1;
+
+  CommandLineParser::OptionId opt_brotli_id = -1;
+  CommandLineParser::OptionId opt_color_id = -1;
+
+  // just for testing: add one extra channel which is a spot color (red)
+  const char* spot_in = nullptr;
+};
+
+jxl::Status CompressJxl(jxl::ThreadPoolInternal* pool, JxlCompressArgs& args,
+                        jxl::PaddedBytes* compressed);
+
+}  // namespace tools
+}  // namespace jpegxl
+
+#endif  // TOOLS_CJXL_H_
diff --git a/tools/cmdline.cc b/tools/cmdline.cc
new file mode 100644
index 000000000..8dc26068e
--- /dev/null
+++ b/tools/cmdline.cc
@@ -0,0 +1,67 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/cmdline.h"
+
+#include <memory>
+#include <string>
+
+namespace jpegxl {
+namespace tools {
+
+void CommandLineParser::PrintHelp() const {
+  fprintf(stderr, "Usage: %s [OPTIONS]\n",
+          program_name_ ? program_name_ : "command");
+  for (const auto& option : options_) {
+    fprintf(stderr, " %s\n", option->help_flags().c_str());
+    const char* help_text = option->help_text();
+    if (help_text) {
+      fprintf(stderr, "    %s\n", help_text);
+    }
+  }
+  fprintf(stderr, " --help\n    Prints this help message.\n");
+}
+
+bool CommandLineParser::Parse(int argc, const char* argv[]) {
+  if (argc) program_name_ = argv[0];
+  int i = 1;  // argv[0] is the program name.
+  while (i < argc) {
+    if (!strcmp("--help", argv[i])) {
+      // Returning false on Parse() forces to print the help message.
+      return false;
+    }
+    bool found = false;
+    for (const auto& option : options_) {
+      if (option->Match(argv[i])) {
+        // Parsing advances the value i on success.
+        const char* arg = argv[i];
+        if (!option->Parse(argc, argv, &i)) {
+          fprintf(stderr, "Error parsing flag %s\n", arg);
+          return false;
+        }
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      // No option matched argv[i].
+      fprintf(stderr, "Unknown argument: %s\n", argv[i]);
+      return false;
+    }
+  }
+  return true;
+}
+
+}  // namespace tools
+}  // namespace jpegxl
diff --git a/tools/cmdline.h b/tools/cmdline.h
new file mode 100644
index 000000000..3e9b0c02b
--- /dev/null
+++ b/tools/cmdline.h
@@ -0,0 +1,276 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_CMDLINE_H_
+#define TOOLS_CMDLINE_H_
+
+#include <stdio.h>
+#include <string.h>
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "jxl/base/status.h"
+
+namespace jpegxl {
+namespace tools {
+
+class CommandLineParser {
+ public:
+  typedef size_t OptionId;
+
+  // An abstract class for defining command line options.
+  class CmdOptionInterface {
+   public:
+    CmdOptionInterface() = default;
+    virtual ~CmdOptionInterface() = default;
+
+    // Return a string with the option name or available flags.
+    virtual std::string help_flags() const = 0;
+
+    // Return the help string if any, or nullptr if no help string.
+    virtual const char* help_text() const = 0;
+
+    // Return whether the option was passed.
+    virtual bool matched() const = 0;
+
+    // Returns whether this option matches the passed command line argument.
+    virtual bool Match(const char* arg) const = 0;
+
+    // Parses the option. The passed i points to the argument with the flag
+    // that matches either the short or the long name.
+    virtual bool Parse(int argc, const char* argv[], int* i) = 0;
+  };
+
+  // Add a positional argument. Returns the id of the added option or
+  // kOptionError on error.
+  OptionId AddPositionalOption(const char* name, const char* help_text,
+                               const char** storage) {
+    options_.emplace_back(new CmdOptionPositional(name, help_text, storage));
+    return options_.size() - 1;
+  }
+
+  // Add an option with a value of type T. The option can be passed as
+  // '-s <value>' or '--long value' or '--long=value'. The CommandLineParser
+  // parser will call the function parser with the string pointing to '<value>'
+  // in either case. Returns the id of the added option or kOptionError on
+  // error.
+  template <typename T>
+  OptionId AddOptionValue(char short_name, const char* long_name,
+                          const char* metavar, const char* help_text,
+                          T* storage, bool(parser)(const char*, T*)) {
+    options_.emplace_back(new CmdOptionFlag<T>(short_name, long_name, metavar,
+                                               help_text, storage, parser));
+    return options_.size() - 1;
+  }
+
+  // Add a flag without a value. Returns the id of the added option or
+  // kOptionError on error.
+  template <typename T>
+  OptionId AddOptionFlag(char short_name, const char* long_name,
+                         const char* help_text, T* storage, bool(parser)(T*)) {
+    options_.emplace_back(new CmdOptionFlag<T>(short_name, long_name, help_text,
+                                               storage, parser));
+    return options_.size() - 1;
+  }
+
+  const CmdOptionInterface* GetOption(OptionId id) const {
+    JXL_ASSERT(id < options_.size());
+    return options_[id].get();
+  }
+
+  // Print the help message.
+  void PrintHelp() const;
+
+  // Parse the command line.
+  bool Parse(int argc, const char* argv[]);
+
+  // Return the remaining positional args
+  std::vector<const char*> PositionalArgs() const;
+
+ private:
+  // A positional argument.
+  class CmdOptionPositional : public CmdOptionInterface {
+   public:
+    CmdOptionPositional(const char* name, const char* help_text,
+                        const char** storage)
+        : name_(name), help_text_(help_text), storage_(storage) {}
+
+    std::string help_flags() const override { return name_; }
+    const char* help_text() const override { return help_text_; }
+    bool matched() const override { return matched_; }
+
+    // Only match non-flag values. This means that you can't pass '-foo' as a
+    // positional argument, but it helps with detecting when passed a flag with
+    // a typo.
+    bool Match(const char* arg) const override {
+      return !matched_ && arg[0] != '-';
+    }
+
+    bool Parse(const int argc, const char* argv[], int* i) override {
+      *storage_ = argv[*i];
+      (*i)++;
+      matched_ = true;
+      return true;
+    }
+
+   private:
+    const char* name_;
+    const char* help_text_;
+    const char** storage_;
+
+    bool matched_{false};
+  };
+
+  // A class for handling an option flag like '-v' or '--foo=bar'.
+  template <typename T>
+  class CmdOptionFlag : public CmdOptionInterface {
+   public:
+    // Construct a flag that doesn't take any value, for example '-v' or
+    // '--long'. Passing a value to it raises an error.
+    CmdOptionFlag(char short_name, const char* long_name, const char* help_text,
+                  T* storage, bool(parser)(T*))
+        : short_name_(short_name),
+          long_name_(long_name),
+          long_name_len_(long_name ? strlen(long_name) : 0),
+          metavar_(nullptr),
+          help_text_(help_text),
+          storage_(storage) {
+      parser_.parser_no_value_ = parser;
+    }
+
+    // Construct a flag that expects a value to be passed.
+    CmdOptionFlag(char short_name, const char* long_name, const char* metavar,
+                  const char* help_text, T* storage,
+                  bool(parser)(const char* arg, T*))
+        : short_name_(short_name),
+          long_name_(long_name),
+          long_name_len_(long_name ? strlen(long_name) : 0),
+          metavar_(metavar ? metavar : ""),
+          help_text_(help_text),
+          storage_(storage) {
+      parser_.parser_with_arg_ = parser;
+    }
+
+    std::string help_flags() const override {
+      std::string ret;
+      if (short_name_) {
+        ret += std::string("-") + short_name_;
+        if (metavar_) ret += std::string(" ") + metavar_;
+        if (long_name_) ret += ", ";
+      }
+      if (long_name_) {
+        ret += std::string("--") + long_name_;
+        if (metavar_) ret += std::string("=") + metavar_;
+      }
+      return ret;
+    }
+    const char* help_text() const override { return help_text_; }
+    bool matched() const override { return matched_; }
+
+    bool Match(const char* arg) const override {
+      return MatchShort(arg) || MatchLong(arg);
+    }
+
+    bool Parse(const int argc, const char* argv[], int* i) override {
+      matched_ = true;
+      if (MatchLong(argv[*i])) {
+        const char* arg = argv[*i] + 2 + long_name_len_;
+        if (arg[0] == '=') {
+          if (metavar_) {
+            // Passed '--long_name=...'.
+            (*i)++;
+            // Skip over the '=' on the LongMatch.
+            arg += 1;
+            return (*parser_.parser_with_arg_)(arg, storage_);
+          } else {
+            fprintf(stderr, "--%s didn't expect any argument passed to it.\n",
+                    argv[*i]);
+            return false;
+          }
+        }
+      }
+      // In any other case, it passed a -s or --long_name
+      (*i)++;
+      if (metavar_) {
+        if (argc <= *i) {
+          fprintf(stderr, "--%s expected an argument but none passed.\n",
+                  argv[*i - 1]);
+          return false;
+        }
+        return (*parser_.parser_with_arg_)(argv[(*i)++], storage_);
+      } else {
+        return (*parser_.parser_no_value_)(storage_);
+      }
+    }
+
+   private:
+    // Returns whether arg matches the short_name flag of this option.
+    bool MatchShort(const char* arg) const {
+      if (!short_name_ || arg[0] != '-') return false;
+      return arg[1] == short_name_ && arg[2] == 0;
+    }
+
+    // Returns whether arg matches the long_name flag of this option,
+    // potentially with an argument passed to it.
+    bool MatchLong(const char* arg) const {
+      if (!long_name_ || arg[0] != '-' || arg[1] != '-') return false;
+      arg += 2;  // Skips the '--'
+      if (strncmp(long_name_, arg, long_name_len_) != 0) return false;
+      arg += long_name_len_;
+      // Allow "--long_name=foo" and "--long_name" as long matches.
+      return arg[0] == 0 || arg[0] == '=';
+    }
+
+    // A short option passed as '-X' where X is the char. A value of 0 means
+    // no short option.
+    const char short_name_;
+
+    // A long option name passed as '--long' where 'long' is the name of the
+    // option.
+    const char* long_name_;
+    size_t long_name_len_;
+
+    // The text to display when referring to the value passed to this flag, for
+    // example "N" in the flag '--value N'. If null, this flag accepts no value
+    // and therefor no value must be passed.
+    const char* metavar_;
+
+    // The help string for this flag.
+    const char* help_text_;
+
+    // The pointer to the storage of this flag used when parsing.
+    T* storage_;
+
+    // The function to use to parse the value when matched. The function used is
+    // parser_with_arg_ when metavar_ is not null (and the value string will be
+    // used) or parser_no_value_ when metavar_ is null.
+    union {
+      bool (*parser_with_arg_)(const char*, T*);
+      bool (*parser_no_value_)(T*);
+    } parser_;
+
+    // Whether this flag was matched.
+    bool matched_{false};
+  };
+
+  const char* program_name_{nullptr};
+
+  std::vector<std::unique_ptr<CmdOptionInterface>> options_;
+};
+
+}  // namespace tools
+}  // namespace jpegxl
+
+#endif  // TOOLS_CMDLINE_H_
diff --git a/tools/color_encoding_fuzzer.cc b/tools/color_encoding_fuzzer.cc
new file mode 100644
index 000000000..486393743
--- /dev/null
+++ b/tools/color_encoding_fuzzer.cc
@@ -0,0 +1,33 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <string>
+
+#include "jxl/color_encoding.h"
+
+namespace jxl {
+
+int TestOneInput(const uint8_t* data, size_t size) {
+  std::string description(reinterpret_cast<const char*>(data), size);
+  ColorEncoding c;
+  (void)ParseDescription(description, &c);
+
+  return 0;
+}
+
+}  // namespace jxl
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  return jxl::TestOneInput(data, size);
+}
diff --git a/tools/comparison_viewer/CMakeLists.txt b/tools/comparison_viewer/CMakeLists.txt
new file mode 100644
index 000000000..2d2eb581a
--- /dev/null
+++ b/tools/comparison_viewer/CMakeLists.txt
@@ -0,0 +1,79 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+find_package(Qt5 COMPONENTS Concurrent Widgets X11Extras)
+if (NOT Qt5_FOUND)
+  message(WARNING "Qt5 was not found. The comparison tool will not be built.")
+  return()
+endif ()
+
+if (NOT TARGET x11_icc)
+  message(WARNING "x11_icc not built. The comparison tool will not be built.")
+  return ()
+endif ()
+
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(CMAKE_AUTOMOC ON)
+set(CMAKE_AUTOUIC ON)
+
+add_library(image_loading STATIC
+  image_loading.cc
+  image_loading.h
+)
+target_link_libraries(image_loading PUBLIC
+  Qt5::Widgets
+  jpegxl-static
+  jpegxl_threads
+  jpegxl_extras-static
+)
+
+add_executable(compare_codecs WIN32
+  codec_comparison_window.cc
+  codec_comparison_window.h
+  codec_comparison_window.ui
+  compare_codecs.cc
+  settings.cc
+  settings.h
+  settings.ui
+  split_image_renderer.cc
+  split_image_renderer.h
+  split_image_view.cc
+  split_image_view.h
+  split_image_view.ui
+)
+target_link_libraries(compare_codecs
+  image_loading
+  Qt5::Concurrent
+  Qt5::Widgets
+  Qt5::X11Extras
+  x11_icc
+)
+
+add_executable(compare_images WIN32
+  compare_images.cc
+  settings.cc
+  settings.h
+  settings.ui
+  split_image_renderer.cc
+  split_image_renderer.h
+  split_image_view.cc
+  split_image_view.h
+  split_image_view.ui
+)
+target_link_libraries(compare_images
+  image_loading
+  Qt5::Widgets
+  Qt5::X11Extras
+  x11_icc
+)
diff --git a/tools/comparison_viewer/codec_comparison_window.cc b/tools/comparison_viewer/codec_comparison_window.cc
new file mode 100644
index 000000000..f5d38ce4a
--- /dev/null
+++ b/tools/comparison_viewer/codec_comparison_window.cc
@@ -0,0 +1,327 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/comparison_viewer/codec_comparison_window.h"
+
+#include <stdlib.h>
+
+#include <QCollator>
+#include <QComboBox>
+#include <QDir>
+#include <QFileInfo>
+#include <QFlags>
+#include <QIcon>
+#include <QImage>
+#include <QImageReader>
+#include <QLabel>
+#include <QList>
+#include <QMap>
+#include <QString>
+#include <QStringList>
+#include <QX11Info>
+#include <QtConcurrent>
+#include <algorithm>
+#include <climits>
+#include <functional>
+#include <utility>
+
+#include "jxl/extras/codec.h"
+#include "tools/comparison_viewer/image_loading.h"
+#include "tools/comparison_viewer/split_image_view.h"
+#include "tools/x11/icc.h"
+
+namespace jxl {
+
+static constexpr char kPngSuffix[] = "png";
+
+namespace {
+
+QVector<QPair<QComboBox*, QString>> currentCodecSelection(
+    const Ui::CodecComparisonWindow& ui) {
+  QVector<QPair<QComboBox*, QString>> result;
+  for (QComboBox* const comboBox :
+       {ui.codec1ComboBox, ui.codec2ComboBox, ui.compressionLevel1ComboBox,
+        ui.compressionLevel2ComboBox}) {
+    result << qMakePair(comboBox, comboBox->currentText());
+  }
+  return result;
+}
+
+void restoreCodecSelection(
+    const QVector<QPair<QComboBox*, QString>>& selection) {
+  for (const auto& comboBox : selection) {
+    const int index = comboBox.first->findText(comboBox.second);
+    if (index != -1) {
+      comboBox.first->setCurrentIndex(index);
+    }
+  }
+}
+
+}  // namespace
+
+CodecComparisonWindow::CodecComparisonWindow(QWidget* const parent)
+    : QMainWindow(parent),
+      monitorIccProfile_(
+          GetMonitorIccProfile(QX11Info::connection(), QX11Info::appScreen())) {
+  ui_.setupUi(this);
+
+  connect(ui_.imageSetComboBox, &QComboBox::currentTextChanged, this,
+          &CodecComparisonWindow::handleImageSetSelection);
+  connect(ui_.imageComboBox, &QComboBox::currentTextChanged, this,
+          &CodecComparisonWindow::handleImageSelection);
+
+  connect(ui_.codec1ComboBox, &QComboBox::currentTextChanged,
+          [this]() { handleCodecChange(Side::LEFT); });
+  connect(ui_.codec2ComboBox, &QComboBox::currentTextChanged,
+          [this]() { handleCodecChange(Side::RIGHT); });
+
+  connect(ui_.compressionLevel1ComboBox, &QComboBox::currentTextChanged,
+          [this]() { updateSideImage(Side::LEFT); });
+  connect(ui_.compressionLevel2ComboBox, &QComboBox::currentTextChanged,
+          [this]() { updateSideImage(Side::RIGHT); });
+
+  connect(ui_.match1Label, &QLabel::linkActivated,
+          [this]() { matchSize(Side::LEFT); });
+  connect(ui_.match2Label, &QLabel::linkActivated,
+          [this]() { matchSize(Side::RIGHT); });
+
+  connect(
+      ui_.splitImageView, &SplitImageView::renderingModeChanged,
+      [this](const SplitImageRenderer::RenderingMode newMode) {
+        switch (newMode) {
+          case SplitImageRenderer::RenderingMode::LEFT:
+          case SplitImageRenderer::RenderingMode::RIGHT: {
+            QString codec, compressionLevel;
+            if (newMode == SplitImageRenderer::RenderingMode::LEFT) {
+              codec = ui_.codec1ComboBox->currentText();
+              compressionLevel = ui_.compressionLevel1ComboBox->currentText();
+            } else {
+              codec = ui_.codec2ComboBox->currentText();
+              compressionLevel = ui_.compressionLevel2ComboBox->currentText();
+            }
+            ui_.renderingModeLabel->setText(tr("Currently displaying: %1 @ %2")
+                                                .arg(codec)
+                                                .arg(compressionLevel));
+            break;
+          }
+
+          case SplitImageRenderer::RenderingMode::MIDDLE:
+            ui_.renderingModeLabel->setText(
+                tr("Currently displaying the original image."));
+            break;
+
+          default:
+            ui_.renderingModeLabel->clear();
+            break;
+        }
+      });
+}
+
+CodecComparisonWindow::CodecComparisonWindow(const QString& directory,
+                                             QWidget* const parent)
+    : CodecComparisonWindow(parent) {
+  loadDirectory(directory);
+}
+
+void CodecComparisonWindow::handleImageSetSelection(
+    const QString& imageSetName) {
+  const auto selection = currentCodecSelection(ui_);
+  {
+    const QSignalBlocker blocker(ui_.imageComboBox);
+    ui_.imageComboBox->clear();
+  }
+  const QStringList imageNames = imageSets_.value(imageSetName).keys();
+  const std::function<QIcon(const QString&)> loadIcon =
+      [this, &imageSetName](const QString& imageName) {
+        return QIcon(pathToOriginalImage(imageSetName, imageName));
+      };
+  const QFuture<QIcon> thumbnails = QtConcurrent::mapped(imageNames, loadIcon);
+  int i = 0;
+  for (const QString& imageName : imageNames) {
+    ui_.imageComboBox->addItem(thumbnails.resultAt(i), imageName);
+    ++i;
+  }
+  restoreCodecSelection(selection);
+}
+
+void CodecComparisonWindow::handleImageSelection(const QString& imageName) {
+  const QString imageSetName = ui_.imageSetComboBox->currentText();
+  ui_.splitImageView->setMiddleImage(loadImage(
+      pathToOriginalImage(imageSetName, imageName), monitorIccProfile_));
+
+  const auto selection = currentCodecSelection(ui_);
+  QStringList codecs = imageSets_.value(imageSetName).value(imageName).keys();
+  for (QComboBox* const codecComboBox :
+       {ui_.codec1ComboBox, ui_.codec2ComboBox}) {
+    {
+      const QSignalBlocker blocker(codecComboBox);
+      codecComboBox->clear();
+    }
+    codecComboBox->addItems(codecs);
+  }
+  restoreCodecSelection(selection);
+}
+
+void CodecComparisonWindow::handleCodecChange(const Side side) {
+  const QComboBox* const codecComboBox =
+      side == Side::LEFT ? ui_.codec1ComboBox : ui_.codec2ComboBox;
+  QComboBox* const compressionLevelComboBox =
+      side == Side::LEFT ? ui_.compressionLevel1ComboBox
+                         : ui_.compressionLevel2ComboBox;
+
+  QStringList compressionLevels =
+      imageSets_.value(ui_.imageSetComboBox->currentText())
+          .value(ui_.imageComboBox->currentText())
+          .value(codecComboBox->currentText())
+          .keys();
+  QCollator collator;
+  collator.setNumericMode(true);
+  std::sort(compressionLevels.begin(), compressionLevels.end(), collator);
+
+  {
+    const QSignalBlocker blocker(compressionLevelComboBox);
+    compressionLevelComboBox->clear();
+  }
+  compressionLevelComboBox->addItems(compressionLevels);
+  matchSize(side);
+}
+
+void CodecComparisonWindow::updateSideImage(const Side side) {
+  const ComparableImage& imageInfo = currentlySelectedImage(side);
+  if (imageInfo.decodedImagePath.isEmpty()) return;
+  QImage image = loadImage(imageInfo.decodedImagePath, monitorIccProfile_);
+  const int pixels = image.width() * image.height();
+  QLabel* const sizeInfoLabel =
+      side == Side::LEFT ? ui_.size1Label : ui_.size2Label;
+  if (pixels == 0) {
+    sizeInfoLabel->setText(tr("Empty image."));
+  } else {
+    const double bpp =
+        CHAR_BIT * static_cast<double>(imageInfo.byteSize) / pixels;
+    sizeInfoLabel->setText(tr("%L1bpp").arg(bpp));
+  }
+
+  if (side == Side::LEFT) {
+    ui_.splitImageView->setLeftImage(std::move(image));
+  } else {
+    ui_.splitImageView->setRightImage(std::move(image));
+  }
+}
+
+QString CodecComparisonWindow::pathToOriginalImage(
+    const QString& imageSetName, const QString& imageName) const {
+  return baseDirectory_.absolutePath() + "/" + imageSetName + "/" + imageName +
+         "/original.png";
+}
+
+CodecComparisonWindow::ComparableImage
+CodecComparisonWindow::currentlySelectedImage(const Side side) const {
+  const QComboBox* const codecComboBox =
+      side == Side::LEFT ? ui_.codec1ComboBox : ui_.codec2ComboBox;
+  QComboBox* const compressionLevelComboBox =
+      side == Side::LEFT ? ui_.compressionLevel1ComboBox
+                         : ui_.compressionLevel2ComboBox;
+
+  return imageSets_.value(ui_.imageSetComboBox->currentText())
+      .value(ui_.imageComboBox->currentText())
+      .value(codecComboBox->currentText())
+      .value(compressionLevelComboBox->currentText());
+}
+
+void CodecComparisonWindow::matchSize(const Side side) {
+  const Side otherSide = (side == Side::LEFT ? Side::RIGHT : Side::LEFT);
+  const qint64 otherSideSize = currentlySelectedImage(otherSide).byteSize;
+  if (otherSideSize == 0) return;
+
+  const QComboBox* const codecComboBox =
+      side == Side::LEFT ? ui_.codec1ComboBox : ui_.codec2ComboBox;
+  QComboBox* const compressionLevelComboBox =
+      side == Side::LEFT ? ui_.compressionLevel1ComboBox
+                         : ui_.compressionLevel2ComboBox;
+  const Codec codec = imageSets_.value(ui_.imageSetComboBox->currentText())
+                          .value(ui_.imageComboBox->currentText())
+                          .value(codecComboBox->currentText());
+  if (codec.empty()) return;
+  Codec::ConstIterator bestMatch = codec.begin();
+  for (auto it = codec.begin(); it != codec.end(); ++it) {
+    if (std::abs(it->byteSize - otherSideSize) <
+        std::abs(bestMatch->byteSize - otherSideSize)) {
+      bestMatch = it;
+    }
+  }
+  compressionLevelComboBox->setCurrentText(bestMatch.key());
+}
+
+void CodecComparisonWindow::loadDirectory(const QString& directory) {
+  baseDirectory_.setPath(directory);
+  baseDirectory_.makeAbsolute();
+  imageSets_.clear();
+  visited_.clear();
+
+  browseDirectory(directory);
+
+  {
+    const QSignalBlocker blocker(ui_.imageSetComboBox);
+    ui_.imageSetComboBox->clear();
+  }
+  ui_.imageSetComboBox->addItems(imageSets_.keys());
+}
+
+void CodecComparisonWindow::browseDirectory(const QDir& directory, int depth) {
+  for (const QFileInfo& subdirectory : directory.entryInfoList(
+           QDir::Dirs | QDir::NoDotAndDotDot | QDir::NoSymLinks)) {
+    if (visited_.contains(subdirectory.absoluteFilePath())) continue;
+    visited_.insert(subdirectory.absoluteFilePath());
+    browseDirectory(subdirectory.absoluteFilePath(), depth + 1);
+  }
+
+  // Need at least image_name/codec_name/file.
+  if (depth < 2) return;
+
+  for (const QFileInfo& file :
+       directory.entryInfoList(QDir::Files | QDir::NoSymLinks)) {
+    if (file.suffix() == kPngSuffix) continue;
+    QString decodedImage;
+    if (canLoadImageWithExtension(file.suffix())) {
+      decodedImage = file.absoluteFilePath();
+    } else {
+      QFileInfo png(file.absolutePath() + "/" + file.completeBaseName() + "." +
+                    kPngSuffix);
+      if (png.exists()) {
+        decodedImage = png.absoluteFilePath();
+      }
+    }
+
+    if (decodedImage.isEmpty()) continue;
+
+    const QString codec = file.absoluteDir().dirName();
+    QDir imageDirectory = file.absoluteDir();
+    if (!imageDirectory.cdUp()) return;
+    const QString imageName = imageDirectory.dirName();
+    QDir imageSetDirectory = imageDirectory;
+    if (!imageSetDirectory.cdUp()) return;
+    QString imageSetPath =
+        baseDirectory_.relativeFilePath(imageSetDirectory.absolutePath());
+    if (imageSetPath.isEmpty()) {
+      imageSetPath = ".";
+    }
+
+    ComparableImage& image =
+        imageSets_[imageSetPath][imageName][codec][file.completeBaseName()];
+    image.decodedImagePath = decodedImage;
+    image.byteSize = file.size();
+  }
+}
+
+}  // namespace jxl
diff --git a/tools/comparison_viewer/codec_comparison_window.h b/tools/comparison_viewer/codec_comparison_window.h
new file mode 100644
index 000000000..b77319dc5
--- /dev/null
+++ b/tools/comparison_viewer/codec_comparison_window.h
@@ -0,0 +1,84 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_COMPARISON_VIEWER_CODEC_COMPARISON_WINDOW_H_
+#define TOOLS_COMPARISON_VIEWER_CODEC_COMPARISON_WINDOW_H_
+
+#include <QDir>
+#include <QMainWindow>
+#include <QMap>
+#include <QSet>
+#include <QString>
+
+#include "jxl/base/padded_bytes.h"
+#include "tools/comparison_viewer/ui_codec_comparison_window.h"
+
+namespace jxl {
+
+class CodecComparisonWindow : public QMainWindow {
+  Q_OBJECT
+
+ public:
+  explicit CodecComparisonWindow(QWidget* parent = nullptr);
+  explicit CodecComparisonWindow(const QString& directory,
+                                 QWidget* parent = nullptr);
+  ~CodecComparisonWindow() override = default;
+
+ private slots:
+  void handleImageSetSelection(const QString& imageSetName);
+  void handleImageSelection(const QString& imageName);
+
+ private:
+  struct ComparableImage {
+    // Absolute path to the decoded PNG (or an image that Qt can read).
+    QString decodedImagePath;
+    // Size of the encoded image (*not* the PNG).
+    qint64 byteSize = 0;
+  };
+  // Keys are compression levels.
+  using Codec = QMap<QString, ComparableImage>;
+  // Keys are codec names.
+  using Codecs = QMap<QString, Codec>;
+  // Keys are image names (relative to the image set directory).
+  using ImageSet = QMap<QString, Codecs>;
+  // Keys are paths to image sets (relative to the base directory chosen by the
+  // user).
+  using ImageSets = QMap<QString, ImageSet>;
+
+  enum class Side { LEFT, RIGHT };
+
+  QString pathToOriginalImage(const QString& imageSet,
+                              const QString& imageName) const;
+  ComparableImage currentlySelectedImage(Side side) const;
+
+  void handleCodecChange(Side side);
+  void updateSideImage(Side side);
+  void matchSize(Side side);
+
+  void loadDirectory(const QString& directory);
+  // Recursive, called by loadDirectory.
+  void browseDirectory(const QDir& directory, int depth = 0);
+
+  Ui::CodecComparisonWindow ui_;
+
+  QDir baseDirectory_;
+  ImageSets imageSets_;
+  QSet<QString> visited_;
+
+  const PaddedBytes monitorIccProfile_;
+};
+
+}  // namespace jxl
+
+#endif  // TOOLS_COMPARISON_VIEWER_CODEC_COMPARISON_WINDOW_H_
diff --git a/tools/comparison_viewer/codec_comparison_window.ui b/tools/comparison_viewer/codec_comparison_window.ui
new file mode 100644
index 000000000..409b75df8
--- /dev/null
+++ b/tools/comparison_viewer/codec_comparison_window.ui
@@ -0,0 +1,171 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <comment>
+  Copyright (c) the JPEG XL Project
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+ </comment>
+ <class>CodecComparisonWindow</class>
+ <widget class="QMainWindow" name="CodecComparisonWindow">
+  <property name="windowTitle">
+   <string>Codec Comparison Tool</string>
+  </property>
+  <widget class="QWidget" name="centralwidget">
+   <layout class="QVBoxLayout" name="verticalLayout" stretch="0,0,0,1">
+    <item>
+     <layout class="QHBoxLayout" name="horizontalLayout_5" stretch="1,0,1">
+      <item>
+       <spacer name="horizontalSpacer">
+        <property name="orientation">
+         <enum>Qt::Horizontal</enum>
+        </property>
+       </spacer>
+      </item>
+      <item>
+       <layout class="QFormLayout" name="formLayout">
+        <item row="0" column="1">
+         <widget class="QComboBox" name="imageSetComboBox"/>
+        </item>
+        <item row="0" column="0">
+         <widget class="QLabel" name="imageSetLabel">
+          <property name="text">
+           <string>Image set:</string>
+          </property>
+         </widget>
+        </item>
+        <item row="1" column="0">
+         <widget class="QLabel" name="imageLabel">
+          <property name="text">
+           <string>Image:</string>
+          </property>
+         </widget>
+        </item>
+        <item row="1" column="1">
+         <widget class="QComboBox" name="imageComboBox"/>
+        </item>
+       </layout>
+      </item>
+      <item>
+       <spacer name="horizontalSpacer_2">
+        <property name="orientation">
+         <enum>Qt::Horizontal</enum>
+        </property>
+       </spacer>
+      </item>
+     </layout>
+    </item>
+    <item>
+     <layout class="QHBoxLayout" name="horizontalLayout" stretch="0,1,0">
+      <item>
+       <layout class="QGridLayout" name="gridLayout">
+        <item row="0" column="1">
+         <widget class="QComboBox" name="compressionLevel1ComboBox"/>
+        </item>
+        <item row="0" column="0">
+         <widget class="QComboBox" name="codec1ComboBox"/>
+        </item>
+        <item row="1" column="1">
+         <widget class="QLabel" name="match1Label">
+          <property name="text">
+           <string>&lt;a href=&quot;#match1&quot;&gt;Match →&lt;/a&gt;</string>
+          </property>
+          <property name="alignment">
+           <set>Qt::AlignCenter</set>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item>
+       <spacer name="horizontalSpacer_3">
+        <property name="orientation">
+         <enum>Qt::Horizontal</enum>
+        </property>
+       </spacer>
+      </item>
+      <item>
+       <layout class="QGridLayout" name="gridLayout_2">
+        <item row="1" column="0">
+         <widget class="QLabel" name="match2Label">
+          <property name="text">
+           <string>&lt;a href=&quot;#match2&quot;&gt;Match ←&lt;/a&gt;</string>
+          </property>
+          <property name="alignment">
+           <set>Qt::AlignCenter</set>
+          </property>
+         </widget>
+        </item>
+        <item row="0" column="1">
+         <widget class="QComboBox" name="compressionLevel2ComboBox"/>
+        </item>
+        <item row="0" column="0">
+         <widget class="QComboBox" name="codec2ComboBox"/>
+        </item>
+       </layout>
+      </item>
+     </layout>
+    </item>
+    <item>
+     <layout class="QHBoxLayout" name="horizontalLayout_2" stretch="0,1,0,1,0">
+      <item>
+       <widget class="QLabel" name="size1Label">
+        <property name="text">
+         <string>No image loaded.</string>
+        </property>
+       </widget>
+      </item>
+      <item>
+       <spacer name="horizontalSpacer_4">
+        <property name="orientation">
+         <enum>Qt::Horizontal</enum>
+        </property>
+       </spacer>
+      </item>
+      <item>
+       <widget class="QLabel" name="renderingModeLabel">
+        <property name="text">
+         <string/>
+        </property>
+        <property name="alignment">
+         <set>Qt::AlignCenter</set>
+        </property>
+       </widget>
+      </item>
+      <item>
+       <spacer name="horizontalSpacer_5">
+        <property name="orientation">
+         <enum>Qt::Horizontal</enum>
+        </property>
+       </spacer>
+      </item>
+      <item>
+       <widget class="QLabel" name="size2Label">
+        <property name="text">
+         <string>No image loaded.</string>
+        </property>
+        <property name="alignment">
+         <set>Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter</set>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </item>
+    <item>
+     <widget class="jxl::SplitImageView" name="splitImageView" native="true"/>
+    </item>
+   </layout>
+  </widget>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>jxl::SplitImageView</class>
+   <extends>QWidget</extends>
+   <header>split_image_view.h</header>
+   <container>1</container>
+  </customwidget>
+ </customwidgets>
+ <resources/>
+ <connections/>
+</ui>
diff --git a/tools/comparison_viewer/compare_codecs.cc b/tools/comparison_viewer/compare_codecs.cc
new file mode 100644
index 000000000..97d184550
--- /dev/null
+++ b/tools/comparison_viewer/compare_codecs.cc
@@ -0,0 +1,66 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdlib.h>
+
+#include <QApplication>
+#include <QMessageBox>
+#include <QString>
+#include <QStringList>
+
+#include "tools/comparison_viewer/codec_comparison_window.h"
+
+namespace {
+
+template <typename Window, typename... Arguments>
+void displayNewWindow(Arguments&&... arguments) {
+  Window* const window = new Window(arguments...);
+  window->setAttribute(Qt::WA_DeleteOnClose);
+  window->show();
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  QApplication application(argc, argv);
+
+  QStringList arguments = application.arguments();
+  arguments.removeFirst();  // program name
+
+  if (arguments.empty()) {
+    QMessageBox message;
+    message.setIcon(QMessageBox::Information);
+    message.setWindowTitle(
+        QCoreApplication::translate("CodecComparisonWindow", "Usage"));
+    message.setText(QCoreApplication::translate(
+        "CodecComparisonWindow", "Please specify a directory to use."));
+    message.setDetailedText(QCoreApplication::translate(
+        "CodecComparisonWindow",
+        "That directory should contain images in the following layout:\n"
+        "- .../<image name>/original.png (optional)\n"
+        "- .../<image_name>/<codec_name>/<compression_level>.<ext>\n"
+        "- .../<image_name>/<codec_name>/<compression_level>.png (optional for "
+        "formats that Qt can load)\n"
+        "With arbitrary nesting allowed before that. (The \"...\" part is "
+        "referred to as an \"image set\" by the tool."));
+    message.exec();
+    return EXIT_FAILURE;
+  }
+
+  for (const QString& argument : arguments) {
+    displayNewWindow<jxl::CodecComparisonWindow>(argument);
+  }
+
+  return application.exec();
+}
diff --git a/tools/comparison_viewer/compare_images.cc b/tools/comparison_viewer/compare_images.cc
new file mode 100644
index 000000000..0f8c7b2a8
--- /dev/null
+++ b/tools/comparison_viewer/compare_images.cc
@@ -0,0 +1,124 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdlib.h>
+
+#include <QApplication>
+#include <QCommandLineOption>
+#include <QCommandLineParser>
+#include <QFlags>
+#include <QImage>
+#include <QMessageBox>
+#include <QStringList>
+#include <QX11Info>
+
+#include "tools/comparison_viewer/image_loading.h"
+#include "tools/comparison_viewer/split_image_view.h"
+#include "tools/x11/icc.h"
+
+namespace {
+
+void displayLoadingError(const QString& path) {
+  QMessageBox message;
+  message.setIcon(QMessageBox::Critical);
+  message.setWindowTitle(
+      QCoreApplication::translate("SplitImageView", "Error"));
+  message.setText(QCoreApplication::translate("SplitImageView",
+                                              "Could not load image \"%1\".")
+                      .arg(path));
+  message.exec();
+}
+
+}  // namespace
+
+int main(int argc, char** argv) {
+  QApplication application(argc, argv);
+
+  QCommandLineParser parser;
+  parser.setApplicationDescription(
+      QCoreApplication::translate("compare_images", "Image comparison tool"));
+  parser.addHelpOption();
+  parser.addPositionalArgument(
+      "left-image",
+      QCoreApplication::translate("compare_images",
+                                  "The image to display on the left."),
+      "<left-image>");
+  parser.addPositionalArgument(
+      "right-image",
+      QCoreApplication::translate("compare_images",
+                                  "The image to display on the right."),
+      "<right-image>");
+  parser.addPositionalArgument(
+      "middle-image",
+      QCoreApplication::translate(
+          "compare_images", "The image to display in the middle (optional)."),
+      "[<middle-image>]");
+
+  QCommandLineOption colorSpaceOption(
+      {"color-space", "c"},
+      QCoreApplication::translate(
+          "compare_images",
+          "The color space to use for untagged images (typically PNM)."),
+      QCoreApplication::translate("compare_images", "color-space"));
+  parser.addOption(colorSpaceOption);
+
+  parser.process(application);
+
+  const QString colorSpaceHint = parser.value(colorSpaceOption);
+
+  QStringList arguments = parser.positionalArguments();
+  if (arguments.size() < 2 || arguments.size() > 3) {
+    parser.showHelp(EXIT_FAILURE);
+  }
+
+  jxl::SplitImageView view;
+
+  const jxl::PaddedBytes monitorIccProfile =
+      jxl::GetMonitorIccProfile(QX11Info::connection(), QX11Info::appScreen());
+
+  const QString leftImagePath = arguments.takeFirst();
+  QImage leftImage =
+      jxl::loadImage(leftImagePath, monitorIccProfile, colorSpaceHint);
+  if (leftImage.isNull()) {
+    displayLoadingError(leftImagePath);
+    return EXIT_FAILURE;
+  }
+  view.setLeftImage(std::move(leftImage));
+
+  const QString rightImagePath = arguments.takeFirst();
+  QImage rightImage =
+      jxl::loadImage(rightImagePath, monitorIccProfile, colorSpaceHint);
+  if (rightImage.isNull()) {
+    displayLoadingError(rightImagePath);
+    return EXIT_FAILURE;
+  }
+  view.setRightImage(std::move(rightImage));
+
+  if (!arguments.empty()) {
+    const QString middleImagePath = arguments.takeFirst();
+    QImage middleImage =
+        jxl::loadImage(middleImagePath, monitorIccProfile, colorSpaceHint);
+    if (middleImage.isNull()) {
+      displayLoadingError(middleImagePath);
+      return EXIT_FAILURE;
+    }
+    view.setMiddleImage(std::move(middleImage));
+  }
+
+  view.setWindowFlags(view.windowFlags() | Qt::Window);
+  view.setWindowState(Qt::WindowMaximized);
+  view.show();
+
+  return application.exec();
+}
diff --git a/tools/comparison_viewer/image_loading.cc b/tools/comparison_viewer/image_loading.cc
new file mode 100644
index 000000000..1717abc48
--- /dev/null
+++ b/tools/comparison_viewer/image_loading.cc
@@ -0,0 +1,104 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/comparison_viewer/image_loading.h"
+
+#include <QRgb>
+#include <QThread>
+
+#include "jxl/base/file_io.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/color_management.h"
+#include "jxl/dec_file.h"
+#include "jxl/extras/codec.h"
+
+namespace jxl {
+
+namespace {
+
+Status loadFromFile(const QString& filename, CodecInOut* const decoded,
+                    ThreadPool* const pool) {
+  PaddedBytes compressed;
+  JXL_RETURN_IF_ERROR(ReadFile(filename.toStdString(), &compressed));
+  const Span<const uint8_t> compressed_span(compressed);
+  jxl::DecompressParams dparams;
+  return DecodeFile(dparams, compressed, decoded, /*aux_out=*/nullptr, pool) ||
+         SetFromBytes(compressed_span, decoded, pool);
+}
+
+}  // namespace
+
+bool canLoadImageWithExtension(QString extension) {
+  extension = extension.toLower();
+  size_t bitsPerSampleUnused;
+  return extension == "j" || extension == "brn" ||
+         CodecFromExtension("." + extension.toStdString(),
+                            &bitsPerSampleUnused) != jxl::Codec::kUnknown;
+}
+
+QImage loadImage(const QString& filename, PaddedBytes targetIccProfile,
+                 const QString& sourceColorSpaceHint) {
+  static ThreadPoolInternal pool(QThread::idealThreadCount());
+
+  CodecInOut decoded;
+  if (!sourceColorSpaceHint.isEmpty()) {
+    decoded.dec_hints.Add("color_space", sourceColorSpaceHint.toStdString());
+  }
+  if (!loadFromFile(filename, &decoded, &pool)) {
+    return QImage();
+  }
+  const ImageBundle& ib = decoded.Main();
+
+  ColorEncoding targetColorSpace;
+  if (!ColorManagement::SetProfile(std::move(targetIccProfile),
+                                   &targetColorSpace)) {
+    targetColorSpace = ColorManagement::SRGB(ib.IsGray());
+  }
+  Image3B converted;
+  if (!ib.CopyTo(Rect(ib), targetColorSpace, &converted, &pool)) {
+    return QImage();
+  }
+
+  QImage image(converted.xsize(), converted.ysize(), QImage::Format_ARGB32);
+
+  if (ib.HasAlpha()) {
+    const int alphaRightShiftAmount =
+        static_cast<int>(decoded.metadata.alpha_bits) - 8;
+    for (int y = 0; y < image.height(); ++y) {
+      QRgb* const row = reinterpret_cast<QRgb*>(image.scanLine(y));
+      const uint16_t* const alphaRow = ib.alpha().ConstRow(y);
+      const uint8_t* const redRow = converted.ConstPlaneRow(0, y);
+      const uint8_t* const greenRow = converted.ConstPlaneRow(1, y);
+      const uint8_t* const blueRow = converted.ConstPlaneRow(2, y);
+      for (int x = 0; x < image.width(); ++x) {
+        row[x] = qRgba(redRow[x], greenRow[x], blueRow[x],
+                       alphaRow[x] >> alphaRightShiftAmount);
+      }
+    }
+  } else {
+    for (int y = 0; y < image.height(); ++y) {
+      QRgb* const row = reinterpret_cast<QRgb*>(image.scanLine(y));
+      const uint8_t* const redRow = converted.ConstPlaneRow(0, y);
+      const uint8_t* const greenRow = converted.ConstPlaneRow(1, y);
+      const uint8_t* const blueRow = converted.ConstPlaneRow(2, y);
+      for (int x = 0; x < image.width(); ++x) {
+        row[x] = qRgb(redRow[x], greenRow[x], blueRow[x]);
+      }
+    }
+  }
+
+  return image;
+}
+
+}  // namespace jxl
diff --git a/tools/comparison_viewer/image_loading.h b/tools/comparison_viewer/image_loading.h
new file mode 100644
index 000000000..ab9f6ac2c
--- /dev/null
+++ b/tools/comparison_viewer/image_loading.h
@@ -0,0 +1,36 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_COMPARISON_VIEWER_IMAGE_LOADING_H_
+#define TOOLS_COMPARISON_VIEWER_IMAGE_LOADING_H_
+
+#include <QImage>
+#include <QString>
+
+#include "jxl/base/padded_bytes.h"
+
+namespace jxl {
+
+// `extension` should not include the dot.
+bool canLoadImageWithExtension(QString extension);
+
+// Converts the loaded image to the given display profile, or sRGB if not
+// specified. Thread-hostile.
+QImage loadImage(const QString& filename,
+                 PaddedBytes targetIccProfile = PaddedBytes(),
+                 const QString& sourceColorSpaceHint = QString());
+
+}  // namespace jxl
+
+#endif  // TOOLS_COMPARISON_VIEWER_IMAGE_LOADING_H_
diff --git a/tools/comparison_viewer/settings.cc b/tools/comparison_viewer/settings.cc
new file mode 100644
index 000000000..0341b6a52
--- /dev/null
+++ b/tools/comparison_viewer/settings.cc
@@ -0,0 +1,60 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/comparison_viewer/settings.h"
+
+namespace jxl {
+
+SettingsDialog::SettingsDialog(QWidget* const parent)
+    : QDialog(parent), settings_("JPEG XL project", "Comparison tool") {
+  ui_.setupUi(this);
+
+  settings_.beginGroup("rendering");
+  renderingSettings_.fadingMSecs = settings_.value("fadingMSecs", 300).toInt();
+  settings_.beginGroup("gray");
+  renderingSettings_.gray = settings_.value("enabled", false).toBool();
+  renderingSettings_.grayMSecs = settings_.value("delayMSecs", 300).toInt();
+  settings_.endGroup();
+  settings_.endGroup();
+
+  settingsToUi();
+}
+
+SplitImageRenderingSettings SettingsDialog::renderingSettings() const {
+  return renderingSettings_;
+}
+
+void SettingsDialog::on_SettingsDialog_accepted() {
+  renderingSettings_.fadingMSecs = ui_.fadingTime->value();
+  renderingSettings_.gray = ui_.grayGroup->isChecked();
+  renderingSettings_.grayMSecs = ui_.grayTime->value();
+
+  settings_.beginGroup("rendering");
+  settings_.setValue("fadingMSecs", renderingSettings_.fadingMSecs);
+  settings_.beginGroup("gray");
+  settings_.setValue("enabled", renderingSettings_.gray);
+  settings_.setValue("delayMSecs", renderingSettings_.grayMSecs);
+  settings_.endGroup();
+  settings_.endGroup();
+}
+
+void SettingsDialog::on_SettingsDialog_rejected() { settingsToUi(); }
+
+void SettingsDialog::settingsToUi() {
+  ui_.fadingTime->setValue(renderingSettings_.fadingMSecs);
+  ui_.grayGroup->setChecked(renderingSettings_.gray);
+  ui_.grayTime->setValue(renderingSettings_.grayMSecs);
+}
+
+}  // namespace jxl
diff --git a/tools/comparison_viewer/settings.h b/tools/comparison_viewer/settings.h
new file mode 100644
index 000000000..c3173b9c4
--- /dev/null
+++ b/tools/comparison_viewer/settings.h
@@ -0,0 +1,49 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_COMPARISON_VIEWER_SETTINGS_H_
+#define TOOLS_COMPARISON_VIEWER_SETTINGS_H_
+
+#include <QDialog>
+#include <QSettings>
+
+#include "tools/comparison_viewer/split_image_renderer.h"
+#include "tools/comparison_viewer/ui_settings.h"
+
+namespace jxl {
+
+class SettingsDialog : public QDialog {
+  Q_OBJECT
+
+ public:
+  explicit SettingsDialog(QWidget* parent = nullptr);
+  ~SettingsDialog() override = default;
+
+  SplitImageRenderingSettings renderingSettings() const;
+
+ private slots:
+  void on_SettingsDialog_accepted();
+  void on_SettingsDialog_rejected();
+
+ private:
+  void settingsToUi();
+
+  Ui::SettingsDialog ui_;
+  QSettings settings_;
+  SplitImageRenderingSettings renderingSettings_;
+};
+
+}  // namespace jxl
+
+#endif  // TOOLS_COMPARISON_VIEWER_SETTINGS_H_
diff --git a/tools/comparison_viewer/settings.ui b/tools/comparison_viewer/settings.ui
new file mode 100644
index 000000000..8785b4e1b
--- /dev/null
+++ b/tools/comparison_viewer/settings.ui
@@ -0,0 +1,121 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <comment>
+  Copyright (c) the JPEG XL Project
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+ </comment>
+ <class>SettingsDialog</class>
+ <widget class="QDialog" name="SettingsDialog">
+  <property name="windowTitle">
+   <string>Comparison tool settings</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout" stretch="0,0,1,0">
+   <property name="sizeConstraint">
+    <enum>QLayout::SetFixedSize</enum>
+   </property>
+   <item>
+    <layout class="QFormLayout" name="settingsLayout">
+     <item row="0" column="0">
+      <widget class="QLabel" name="fadingTimePromptLabel">
+       <property name="text">
+        <string>Fading time:</string>
+       </property>
+      </widget>
+     </item>
+     <item row="0" column="1">
+      <widget class="QSpinBox" name="fadingTime">
+       <property name="suffix">
+        <string> ms</string>
+       </property>
+       <property name="maximum">
+        <number>1000</number>
+       </property>
+       <property name="singleStep">
+        <number>50</number>
+       </property>
+       <property name="value">
+        <number>300</number>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+   <item>
+    <widget class="QGroupBox" name="grayGroup">
+     <property name="title">
+      <string>Gray in between</string>
+     </property>
+     <property name="checkable">
+      <bool>true</bool>
+     </property>
+     <property name="checked">
+      <bool>false</bool>
+     </property>
+     <layout class="QFormLayout" name="formLayout">
+      <item row="0" column="1">
+       <widget class="QSpinBox" name="grayTime">
+        <property name="suffix">
+         <string> ms</string>
+        </property>
+        <property name="minimum">
+         <number>0</number>
+        </property>
+        <property name="maximum">
+         <number>1000</number>
+        </property>
+        <property name="singleStep">
+         <number>50</number>
+        </property>
+        <property name="value">
+         <number>300</number>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="0">
+       <widget class="QLabel" name="grayTimePromptLabel">
+        <property name="text">
+         <string>Time on gray:</string>
+        </property>
+       </widget>
+      </item>
+     </layout>
+    </widget>
+   </item>
+   <item>
+    <spacer name="verticalSpacer">
+     <property name="orientation">
+      <enum>Qt::Vertical</enum>
+     </property>
+    </spacer>
+   </item>
+   <item>
+    <widget class="QDialogButtonBox" name="buttonBox">
+     <property name="orientation">
+      <enum>Qt::Horizontal</enum>
+     </property>
+     <property name="standardButtons">
+      <set>QDialogButtonBox::Cancel|QDialogButtonBox::Ok</set>
+     </property>
+    </widget>
+   </item>
+  </layout>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>accepted()</signal>
+   <receiver>SettingsDialog</receiver>
+   <slot>accept()</slot>
+  </connection>
+  <connection>
+   <sender>buttonBox</sender>
+   <signal>rejected()</signal>
+   <receiver>SettingsDialog</receiver>
+   <slot>reject()</slot>
+  </connection>
+ </connections>
+</ui>
diff --git a/tools/comparison_viewer/split_image_renderer.cc b/tools/comparison_viewer/split_image_renderer.cc
new file mode 100644
index 000000000..9041c7689
--- /dev/null
+++ b/tools/comparison_viewer/split_image_renderer.cc
@@ -0,0 +1,248 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/comparison_viewer/split_image_renderer.h"
+
+#include <algorithm>
+#include <cmath>
+#include <utility>
+
+#include <QEvent>
+#include <QGuiApplication>
+#include <QPainter>
+#include <QPalette>
+#include <QPen>
+#include <QPoint>
+#include <QRect>
+
+namespace jxl {
+
+SplitImageRenderer::SplitImageRenderer(QWidget* const parent)
+    : QWidget(parent) {
+  setAttribute(Qt::WA_OpaquePaintEvent);
+  setMouseTracking(true);
+  setFocusPolicy(Qt::WheelFocus);
+  grabKeyboard();
+
+  connect(&fadingPoint_, &QVariantAnimation::valueChanged,
+          [this] { update(); });
+}
+
+void SplitImageRenderer::setLeftImage(QImage image) {
+  leftImage_ = QPixmap::fromImage(std::move(image));
+  updateMinimumSize();
+  update();
+}
+void SplitImageRenderer::setRightImage(QImage image) {
+  rightImage_ = QPixmap::fromImage(std::move(image));
+  updateMinimumSize();
+  update();
+}
+void SplitImageRenderer::setMiddleImage(QImage image) {
+  middleImage_ = QPixmap::fromImage(std::move(image));
+  updateMinimumSize();
+  update();
+}
+
+void SplitImageRenderer::setRenderingSettings(
+    const SplitImageRenderingSettings& settings) {
+  renderingSettings_ = settings;
+}
+
+void SplitImageRenderer::setMiddleWidthPercent(const int percent) {
+  middleWidthPercent_ = percent;
+  update();
+}
+
+void SplitImageRenderer::setZoomLevel(double scale) {
+  scale_ = scale;
+  updateMinimumSize();
+  update();
+}
+
+void SplitImageRenderer::keyPressEvent(QKeyEvent* const event) {
+  switch (event->key()) {
+    case Qt::Key_Left:
+      setRenderingMode(RenderingMode::LEFT);
+      break;
+
+    case Qt::Key_Right:
+      setRenderingMode(RenderingMode::RIGHT);
+      break;
+
+    case Qt::Key_Up:
+    case Qt::Key_Down:
+      setRenderingMode(RenderingMode::MIDDLE);
+      break;
+
+    case Qt::Key_Escape:
+      QCoreApplication::quit();
+      break;
+
+    case Qt::Key_ZoomIn:
+      emit zoomLevelIncreaseRequested();
+      break;
+    case Qt::Key_ZoomOut:
+      emit zoomLevelDecreaseRequested();
+      break;
+
+    default:
+      QWidget::keyPressEvent(event);
+      break;
+  }
+  update();
+}
+
+void SplitImageRenderer::mouseMoveEvent(QMouseEvent* const event) {
+  setRenderingMode(RenderingMode::SPLIT);
+  middleX_ = event->pos().x();
+  update();
+}
+
+void SplitImageRenderer::wheelEvent(QWheelEvent* event) {
+  if (QGuiApplication::keyboardModifiers().testFlag(Qt::ControlModifier)) {
+    if (event->angleDelta().y() > 0) {
+      emit zoomLevelIncreaseRequested();
+      return;
+    } else if (event->angleDelta().y() < 0) {
+      emit zoomLevelDecreaseRequested();
+      return;
+    }
+  }
+
+  event->ignore();
+}
+
+void SplitImageRenderer::paintEvent(QPaintEvent* const event) {
+  QRectF drawingArea(0., 0., minimumWidth(), minimumHeight());
+
+  QPainter painter(this);
+  painter.fillRect(rect(), QColor(119, 119, 119));
+  painter.translate(QRectF(rect()).center() - drawingArea.center());
+  painter.scale(scale_, scale_);
+  if (scale_ < 1.) {
+    painter.setRenderHint(QPainter::SmoothPixmapTransform);
+  }
+
+  const auto drawSingleImage = [&](const RenderingMode mode) {
+    const QPixmap* image = nullptr;
+    switch (mode) {
+      case RenderingMode::LEFT:
+        image = &leftImage_;
+        break;
+      case RenderingMode::RIGHT:
+        image = &rightImage_;
+        break;
+      case RenderingMode::MIDDLE:
+        image = &middleImage_;
+        break;
+
+      default:
+        return;
+    }
+    painter.drawPixmap(QPointF(0., 0.), *image);
+  };
+
+  if (mode_ != RenderingMode::SPLIT) {
+    if (fadingPoint_.state() != QAbstractAnimation::Running) {
+      drawSingleImage(mode_);
+      return;
+    }
+
+    const float fadingPoint = fadingPoint_.currentValue().toFloat();
+    if (renderingSettings_.gray) {
+      if (fadingPoint < renderingSettings_.fadingMSecs) {
+        painter.setOpacity((renderingSettings_.fadingMSecs - fadingPoint) /
+                           renderingSettings_.fadingMSecs);
+        drawSingleImage(previousMode_);
+      } else if (fadingPoint > renderingSettings_.fadingMSecs +
+                                   renderingSettings_.grayMSecs) {
+        painter.setOpacity((fadingPoint - renderingSettings_.fadingMSecs -
+                            renderingSettings_.grayMSecs) /
+                           renderingSettings_.fadingMSecs);
+        drawSingleImage(mode_);
+      }
+    } else {
+      drawSingleImage(previousMode_);
+      painter.setOpacity(fadingPoint / renderingSettings_.fadingMSecs);
+      drawSingleImage(mode_);
+    }
+
+    return;
+  }
+
+  const qreal middleWidth =
+      std::min<qreal>((minimumWidth() / scale_) * middleWidthPercent_ / 100.,
+                      middleImage_.width());
+
+  const double transformedMiddleX =
+      painter.transform().inverted().map(QPointF(middleX_, 0.)).x();
+  QRectF middleRect = middleImage_.rect();
+  middleRect.setWidth(middleWidth);
+  middleRect.moveCenter(QPointF(transformedMiddleX, middleRect.center().y()));
+  middleRect.setLeft(std::round(middleRect.left()));
+  middleRect.setRight(std::round(middleRect.right()));
+
+  QRectF leftRect = leftImage_.rect();
+  leftRect.setRight(middleRect.left());
+
+  QRectF rightRect = rightImage_.rect();
+  rightRect.setLeft(middleRect.right());
+
+  painter.drawPixmap(leftRect, leftImage_, leftRect);
+  painter.drawPixmap(rightRect, rightImage_, rightRect);
+  painter.drawPixmap(middleRect, middleImage_, middleRect);
+
+  QPen middlePen;
+  middlePen.setStyle(Qt::DotLine);
+  painter.setPen(middlePen);
+  painter.drawLine(leftRect.topRight(), leftRect.bottomRight());
+  painter.drawLine(rightRect.topLeft(), rightRect.bottomLeft());
+}
+
+void SplitImageRenderer::updateMinimumSize() {
+  const int imagesWidth = std::max(
+      std::max(leftImage_.width(), rightImage_.width()), middleImage_.width());
+  const int imagesHeight =
+      std::max(std::max(leftImage_.height(), rightImage_.height()),
+               middleImage_.height());
+  setMinimumSize(scale_ * QSize(imagesWidth, imagesHeight));
+}
+
+void SplitImageRenderer::setRenderingMode(const RenderingMode newMode) {
+  if (newMode == mode_) return;
+  previousMode_ = mode_;
+  mode_ = newMode;
+  if (previousMode_ == RenderingMode::SPLIT || mode_ == RenderingMode::SPLIT) {
+    fadingPoint_.stop();
+  } else {
+    const int msecs =
+        renderingSettings_.gray
+            ? 2 * renderingSettings_.fadingMSecs + renderingSettings_.grayMSecs
+            : renderingSettings_.fadingMSecs;
+    const float startValue = fadingPoint_.state() == QAbstractAnimation::Running
+                                 ? fadingPoint_.endValue().toFloat() -
+                                       fadingPoint_.currentValue().toFloat()
+                                 : 0.f;
+    fadingPoint_.stop();
+    fadingPoint_.setStartValue(startValue);
+    fadingPoint_.setEndValue(static_cast<float>(msecs));
+    fadingPoint_.setDuration(fadingPoint_.endValue().toFloat() -
+                             fadingPoint_.startValue().toFloat());
+    fadingPoint_.start();
+  }
+  emit renderingModeChanged(mode_);
+}
+
+}  // namespace jxl
diff --git a/tools/comparison_viewer/split_image_renderer.h b/tools/comparison_viewer/split_image_renderer.h
new file mode 100644
index 000000000..83a7cb332
--- /dev/null
+++ b/tools/comparison_viewer/split_image_renderer.h
@@ -0,0 +1,99 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_COMPARISON_VIEWER_SPLIT_IMAGE_RENDERER_H_
+#define TOOLS_COMPARISON_VIEWER_SPLIT_IMAGE_RENDERER_H_
+
+#include <QImage>
+#include <QKeyEvent>
+#include <QMouseEvent>
+#include <QPaintEvent>
+#include <QPixmap>
+#include <QVariantAnimation>
+#include <QWheelEvent>
+#include <QWidget>
+
+namespace jxl {
+
+struct SplitImageRenderingSettings {
+  int fadingMSecs;
+  bool gray;
+  int grayMSecs;
+};
+
+class SplitImageRenderer : public QWidget {
+  Q_OBJECT
+
+ public:
+  enum class RenderingMode {
+    // The default mode when using the mouse: one (partial) image is shown on
+    // each side of the cursor, with a vertical band of the middle image if
+    // applicable.
+    SPLIT,
+    // Only show the left image (accessed by pressing the left arrow key when
+    // the renderer has focus).
+    LEFT,
+    // Only show the right image (accessed by pressing the right arrow key).
+    RIGHT,
+    // Only show the middle image (accessed by pressing the up or down arrow
+    // key).
+    MIDDLE,
+  };
+  Q_ENUM(RenderingMode);
+
+  explicit SplitImageRenderer(QWidget* parent = nullptr);
+  ~SplitImageRenderer() override = default;
+
+  QSize sizeHint() const override { return minimumSize(); }
+
+  void setLeftImage(QImage image);
+  void setRightImage(QImage image);
+  void setMiddleImage(QImage image);
+
+  void setRenderingSettings(const SplitImageRenderingSettings& settings);
+
+ public slots:
+  void setMiddleWidthPercent(int percent);
+  void setZoomLevel(double scale);
+
+ signals:
+  void zoomLevelIncreaseRequested();
+  void zoomLevelDecreaseRequested();
+
+  void renderingModeChanged(RenderingMode newMode);
+
+ protected:
+  void keyPressEvent(QKeyEvent* event) override;
+  void mouseMoveEvent(QMouseEvent* event) override;
+  void wheelEvent(QWheelEvent* event) override;
+  void paintEvent(QPaintEvent* event) override;
+
+ private:
+  void updateMinimumSize();
+  void setRenderingMode(RenderingMode newMode);
+
+  QPixmap leftImage_, rightImage_, middleImage_;
+  RenderingMode mode_ = RenderingMode::SPLIT;
+  RenderingMode previousMode_ = RenderingMode::SPLIT;
+  SplitImageRenderingSettings renderingSettings_;
+  // Goes from 0 to the animation duration in milliseconds, as a float.
+  QVariantAnimation fadingPoint_;
+  int middleX_ = 0;
+  int middleWidthPercent_ = 10;
+  double scale_ = 1.;
+};
+
+}  // namespace jxl
+
+#endif  // TOOLS_COMPARISON_VIEWER_SPLIT_IMAGE_RENDERER_H_
diff --git a/tools/comparison_viewer/split_image_view.cc b/tools/comparison_viewer/split_image_view.cc
new file mode 100644
index 000000000..80a2cdee4
--- /dev/null
+++ b/tools/comparison_viewer/split_image_view.cc
@@ -0,0 +1,80 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/comparison_viewer/split_image_view.h"
+
+#include <utility>
+
+#include <QLabel>
+
+#include "tools/comparison_viewer/split_image_renderer.h"
+
+namespace jxl {
+
+SplitImageView::SplitImageView(QWidget* const parent) : QWidget(parent) {
+  ui_.setupUi(this);
+
+  ui_.splitImageRenderer->setRenderingSettings(settings_.renderingSettings());
+
+  connect(ui_.middleWidthSlider, &QSlider::valueChanged,
+          [this](const int value) {
+            ui_.middleWidthDisplayLabel->setText(tr("%L1%").arg(value));
+          });
+  connect(ui_.middleWidthSlider, &QSlider::valueChanged, ui_.splitImageRenderer,
+          &SplitImageRenderer::setMiddleWidthPercent);
+
+  connect(ui_.zoomLevelSlider, &QSlider::valueChanged, [this](const int value) {
+    if (value >= 0) {
+      ui_.zoomLevelDisplayLabel->setText(tr("&times;%L1").arg(1 << value));
+      ui_.splitImageRenderer->setZoomLevel(1 << value);
+    } else {
+      ui_.zoomLevelDisplayLabel->setText(tr("&times;1/%L1").arg(1 << -value));
+      ui_.splitImageRenderer->setZoomLevel(1. / (1 << -value));
+    }
+  });
+
+  connect(ui_.splitImageRenderer,
+          &SplitImageRenderer::zoomLevelIncreaseRequested, [this]() {
+            ui_.zoomLevelSlider->triggerAction(
+                QAbstractSlider::SliderSingleStepAdd);
+          });
+  connect(ui_.splitImageRenderer,
+          &SplitImageRenderer::zoomLevelDecreaseRequested, [this]() {
+            ui_.zoomLevelSlider->triggerAction(
+                QAbstractSlider::SliderSingleStepSub);
+          });
+
+  connect(ui_.splitImageRenderer, &SplitImageRenderer::renderingModeChanged,
+          this, &SplitImageView::renderingModeChanged);
+}
+
+void SplitImageView::setLeftImage(QImage image) {
+  ui_.splitImageRenderer->setLeftImage(std::move(image));
+}
+
+void SplitImageView::setRightImage(QImage image) {
+  ui_.splitImageRenderer->setRightImage(std::move(image));
+}
+
+void SplitImageView::setMiddleImage(QImage image) {
+  ui_.splitImageRenderer->setMiddleImage(std::move(image));
+}
+
+void SplitImageView::on_settingsButton_clicked() {
+  if (settings_.exec()) {
+    ui_.splitImageRenderer->setRenderingSettings(settings_.renderingSettings());
+  }
+}
+
+}  // namespace jxl
diff --git a/tools/comparison_viewer/split_image_view.h b/tools/comparison_viewer/split_image_view.h
new file mode 100644
index 000000000..6fdc269a9
--- /dev/null
+++ b/tools/comparison_viewer/split_image_view.h
@@ -0,0 +1,49 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_COMPARISON_VIEWER_SPLIT_IMAGE_VIEW_H_
+#define TOOLS_COMPARISON_VIEWER_SPLIT_IMAGE_VIEW_H_
+
+#include <QWidget>
+
+#include "tools/comparison_viewer/settings.h"
+#include "tools/comparison_viewer/ui_split_image_view.h"
+
+namespace jxl {
+
+class SplitImageView : public QWidget {
+  Q_OBJECT
+
+ public:
+  explicit SplitImageView(QWidget* parent = nullptr);
+  ~SplitImageView() override = default;
+
+  void setLeftImage(QImage image);
+  void setRightImage(QImage image);
+  void setMiddleImage(QImage image);
+
+ signals:
+  void renderingModeChanged(SplitImageRenderer::RenderingMode newMode);
+
+ private slots:
+  void on_settingsButton_clicked();
+
+ private:
+  Ui::SplitImageView ui_;
+  SettingsDialog settings_;
+};
+
+}  // namespace jxl
+
+#endif  // TOOLS_COMPARISON_VIEWER_SPLIT_IMAGE_VIEW_H_
diff --git a/tools/comparison_viewer/split_image_view.ui b/tools/comparison_viewer/split_image_view.ui
new file mode 100644
index 000000000..87947d668
--- /dev/null
+++ b/tools/comparison_viewer/split_image_view.ui
@@ -0,0 +1,142 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <comment>
+  Copyright (c) the JPEG XL Project
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+ </comment>
+ <class>SplitImageView</class>
+ <widget class="QWidget" name="SplitImageView">
+  <property name="windowTitle">
+   <string>Image Comparison Tool</string>
+  </property>
+  <layout class="QVBoxLayout" name="verticalLayout" stretch="1,0">
+   <item>
+    <widget class="QScrollArea" name="scrollArea">
+     <property name="widgetResizable">
+      <bool>true</bool>
+     </property>
+     <widget class="jxl::SplitImageRenderer" name="splitImageRenderer"/>
+    </widget>
+   </item>
+   <item>
+    <layout class="QHBoxLayout" name="horizontalLayout" stretch="0,1,0,0">
+     <item>
+      <layout class="QFormLayout" name="zoomLevelFormLayout">
+       <item row="0" column="0">
+        <widget class="QLabel" name="zoomLevelPromptLabel">
+         <property name="text">
+          <string>Zoom level:</string>
+         </property>
+        </widget>
+       </item>
+       <item row="0" column="1">
+        <layout class="QHBoxLayout" name="horizontalLayout_2">
+         <item>
+          <widget class="QSlider" name="zoomLevelSlider">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Minimum" vsizetype="Fixed">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="minimum">
+            <number>-3</number>
+           </property>
+           <property name="maximum">
+            <number>3</number>
+           </property>
+           <property name="pageStep">
+            <number>2</number>
+           </property>
+           <property name="orientation">
+            <enum>Qt::Horizontal</enum>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QLabel" name="zoomLevelDisplayLabel">
+           <property name="text">
+            <string>×1</string>
+           </property>
+           <property name="textFormat">
+            <enum>Qt::RichText</enum>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+      </layout>
+     </item>
+     <item>
+      <spacer name="horizontalSpacer">
+       <property name="orientation">
+        <enum>Qt::Horizontal</enum>
+       </property>
+      </spacer>
+     </item>
+     <item>
+      <layout class="QFormLayout" name="middleWidthFormLayout">
+       <item row="0" column="0">
+        <widget class="QLabel" name="middleWidthPromptLabel">
+         <property name="text">
+          <string>Width of the central band:</string>
+         </property>
+        </widget>
+       </item>
+       <item row="0" column="1">
+        <layout class="QHBoxLayout" name="horizontalLayout_3">
+         <item>
+          <widget class="QSlider" name="middleWidthSlider">
+           <property name="sizePolicy">
+            <sizepolicy hsizetype="Minimum" vsizetype="Fixed">
+             <horstretch>0</horstretch>
+             <verstretch>0</verstretch>
+            </sizepolicy>
+           </property>
+           <property name="maximum">
+            <number>100</number>
+           </property>
+           <property name="value">
+            <number>10</number>
+           </property>
+           <property name="orientation">
+            <enum>Qt::Horizontal</enum>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QLabel" name="middleWidthDisplayLabel">
+           <property name="text">
+            <string>10%</string>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </item>
+      </layout>
+     </item>
+     <item>
+      <widget class="QToolButton" name="settingsButton">
+       <property name="text">
+        <string>Settings</string>
+       </property>
+      </widget>
+     </item>
+    </layout>
+   </item>
+  </layout>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>jxl::SplitImageRenderer</class>
+   <extends>QWidget</extends>
+   <header>split_image_renderer.h</header>
+   <container>1</container>
+  </customwidget>
+ </customwidgets>
+ <resources/>
+ <connections/>
+</ui>
diff --git a/tools/decode_and_encode.cc b/tools/decode_and_encode.cc
new file mode 100644
index 000000000..c29891e75
--- /dev/null
+++ b/tools/decode_and_encode.cc
@@ -0,0 +1,58 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include <string>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/status.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/extras/codec.h"
+
+namespace jxl {
+namespace {
+
+// Reads an input file (typically PNM) with color_space hint and writes to an
+// output file (typically PNG) which supports all required metadata.
+int Convert(int argc, char** argv) {
+  if (argc != 4 && argc != 5) {
+    fprintf(stderr, "Args: in colorspace_description out [bits]\n");
+    return 1;
+  }
+  const std::string& pathname_in = argv[1];
+  const std::string& desc = argv[2];
+  const std::string& pathname_out = argv[3];
+
+  CodecInOut io;
+  ThreadPoolInternal pool(4);
+  io.dec_hints.Add("color_space", desc);
+  if (!SetFromFile(pathname_in, &io, &pool)) {
+    fprintf(stderr, "Failed to read %s\n", pathname_in.c_str());
+    return 1;
+  }
+
+  if (!EncodeToFile(io, pathname_out, &pool)) {
+    fprintf(stderr, "Failed to write %s\n", pathname_out.c_str());
+    return 1;
+  }
+
+  return 0;
+}
+
+}  // namespace
+}  // namespace jxl
+
+int main(int argc, char** argv) { return jxl::Convert(argc, argv); }
diff --git a/tools/demo_progressive_saliency_encoding.py b/tools/demo_progressive_saliency_encoding.py
new file mode 100755
index 000000000..cdd020b7b
--- /dev/null
+++ b/tools/demo_progressive_saliency_encoding.py
@@ -0,0 +1,185 @@
+#!/usr/bin/env python3
+
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Produces demos for how progressive-saliency encoding would look like.
+
+As long as we do not have a progressive decoder that allows showing images
+generated from partially-available data, we can resort to building
+animated gifs that show how progressive loading would look like.
+
+Method:
+
+1. JPEG-XL encode the image, but stop at the pre-final (2nd) step.
+2. Use separate tool to compute a heatmap which shows where differences between
+   the pre-final and final image are expected to be perceptually worst.
+3. Use this heatmap to JPEG-XL encode the image with the final step split into
+   'salient parts only' and 'non-salient parts'. Generate a sequence of images
+   that stop decoding after the 1st, 2nd, 3rd, 4th step. JPEG-XL decode these
+   truncated images back to PNG.
+4. Measure byte sizes of the truncated-encoded images.
+5. Build an animated GIF with variable delays by calling ImageMagick's
+   `convert` command.
+
+"""
+
+import ast  # For ast.literal_eval() only.
+import os
+import re
+import shlex
+import subprocess
+import sys
+
+_BLOCKSIZE = 8
+
+_CONF_PARSERS = dict(
+    keep_tempfiles=lambda s: bool(ast.literal_eval(s)),
+    heatmap_command=shlex.split,
+    simulated_progressive_loading_time_sec=float,
+    simulated_progressive_loading_delay_until_looparound_sec=float,
+    jpegxl_encoder=shlex.split,
+    jpegxl_decoder=shlex.split,
+    blurring=lambda s: s.split(),
+)
+
+
+def parse_config(config_filename):
+  """Parses the configuration file."""
+  conf = {}
+  re_comment = re.compile(r'^\s*(?:#.*)?$')
+  re_param = re.compile(r'^(?P<option>\w+)\s*:\s*(?P<value>.*?)\s*$')
+  try:
+    with open(config_filename) as h:
+      for line in h:
+        if re_comment.match(line):
+          continue
+        m = re_param.match(line)
+        if not m:
+          raise ValueError('Syntax error')
+        conf[m.group('option')] = (
+            _CONF_PARSERS[m.group('option')](m.group('value')))
+  except Exception as exn:
+    raise ValueError('Bad Configuration line ({}): {}'.format(exn, line))
+  missing_options = set(_CONF_PARSERS) - set(conf)
+  if missing_options:
+    raise ValueError('Missing configuration options: ' + ', '.join(
+        sorted(missing_options)))
+  return conf
+
+
+def generate_demo_image(config, input_filename, output_filename):
+  tempfiles = []
+  #
+  def encode_img(input_filename, output_filename, num_steps,
+                 heatmap_filename=None):
+    replacements = {
+        '${INPUT}': input_filename,
+        '${OUTPUT}': output_filename,
+        '${STEPS}': str(num_steps),
+        # Heatmap argument will be provided in --param=value form.
+        '${HEATMAP_ARG}': ('--saliency_map_filename=' + heatmap_filename
+                           if heatmap_filename is not None else '')
+        }
+    # Remove empty args. This removes the heatmap-argument if no heatmap
+    # is provided..
+    cmd = list(filter(None, [replacements.get(arg, arg)
+                             for arg in config['jpegxl_encoder']]))
+    tempfiles.append(output_filename)
+    subprocess.call(cmd)
+  #
+  def decode_img(input_filename, output_filename):
+    replacements = {'${INPUT}': input_filename, '${OUTPUT}': output_filename}
+    cmd = [replacements.get(arg, arg) for arg in config['jpegxl_decoder']]
+    tempfiles.append(output_filename)
+    subprocess.call(cmd)
+  #
+  def generate_heatmap(orig_image_filename, coarse_grained_filename,
+                       heatmap_filename):
+    cmd = config['heatmap_command'] + [
+        str(_BLOCKSIZE), orig_image_filename, coarse_grained_filename,
+        heatmap_filename]
+    tempfiles.append(heatmap_filename)
+    subprocess.call(cmd)
+  #
+  try:
+    encode_img(input_filename, output_filename + '._step1.pik', 1)
+    decode_img(output_filename + '._step1.pik', output_filename + '._step1.png')
+    encode_img(input_filename, output_filename + '._step2.pik', 2)
+    decode_img(output_filename + '._step2.pik', output_filename + '._step2.png')
+    generate_heatmap(input_filename, output_filename + '._step2.png',
+                     output_filename + '._heatmap.png')
+    encode_img(input_filename,
+               output_filename + '._step3.pik', 3,
+               output_filename + '._heatmap.png')
+    encode_img(input_filename,
+               output_filename + '._step4.pik', 4,
+               output_filename + '._heatmap.png')
+    decode_img(output_filename + '._step3.pik', output_filename + '._step3.png')
+    decode_img(output_filename + '._step4.pik', output_filename + '._step4.png')
+    data_sizes = [
+        os.stat('{}._step{}.pik'.format(output_filename, num_step)).st_size
+        for num_step in (1, 2, 3, 4)]
+    time_offsets = [0] + [
+        # Imagemagick's `convert` accepts delays in units of 1/100 sec.
+        round(100 * config['simulated_progressive_loading_time_sec'] * size /
+              data_sizes[-1]) for size in data_sizes]
+    time_delays = [t_next - t_prev
+                   for t_next, t_prev in zip(time_offsets[1:], time_offsets)]
+    # Add a fake white initial image. As long as no useable image data is
+    # available, the user will see a white background.
+    subprocess.call(['convert',
+                     output_filename + '._step1.png',
+                     '-fill', 'white', '-colorize', '100%',
+                     output_filename + '._step0.png'])
+    tempfiles.append(output_filename + '._step0.png')
+    subprocess.call(
+        ['convert', '-loop', '0', output_filename + '._step0.png'] +
+        [arg for args in [
+            ['-delay', str(time_delays[n - 1]),
+             '-blur', config['blurring'][n - 1],
+             '{}._step{}.png'.format(output_filename, n)]
+            for n in (1, 2, 3, 4)] for arg in args] +
+        ['-delay', str(round(100 * config[
+            'simulated_progressive_loading_delay_until_looparound_sec'])),
+         output_filename + '._step4.png',
+         output_filename])
+  finally:
+    if not config['keep_tempfiles']:
+      for filename in tempfiles:
+        try:
+          os.unlink(filename)
+        except OSError:
+          pass  # May already have been deleted otherwise.
+
+
+def main():
+  if sys.version.startswith('2.'):
+    sys.exit('This is a python3-only script.')
+  if (len(sys.argv) != 4 or not sys.argv[-1].endswith('.gif')
+      or not sys.argv[-2].endswith('.png')):
+    sys.exit(
+        'Usage: {} [config_options_file] [input.png] [output.gif]'.format(
+            sys.argv[0]))
+  try:
+    _, config_filename, input_filename, output_filename = sys.argv
+    config = parse_config(config_filename)
+    generate_demo_image(config, input_filename, output_filename)
+  except ValueError as exn:
+    sys.exit(exn)
+
+
+
+if __name__ == '__main__':
+  main()
diff --git a/tools/djpegxl.cc b/tools/djpegxl.cc
new file mode 100644
index 000000000..69fac02f6
--- /dev/null
+++ b/tools/djpegxl.cc
@@ -0,0 +1,77 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/djpegxl.h"
+
+#include <stdio.h>
+
+#include "jxl/base/arch_specific.h"
+#include "tools/args.h"
+
+namespace jpegxl {
+namespace tools {
+
+DecompressArgs::DecompressArgs() {
+  jxl::ProcessorTopology topology;
+  JXL_CHECK(jxl::DetectProcessorTopology(&topology));
+  num_threads = topology.packages * topology.cores_per_package;
+}
+
+void DecompressArgs::AddCommandLineOptions(CommandLineParser* cmdline) {
+  // Positional arguments.
+  cmdline->AddPositionalOption("INPUT", "the compressed input file", &file_in);
+
+  cmdline->AddPositionalOption(
+      "OUTPUT", "the output can be PNG with ICC, JPG, or PPM/PFM.", &file_out);
+
+  cmdline->AddOptionFlag('V', "version", "print version number and exit",
+                         &version, &SetBooleanTrue);
+
+  cmdline->AddOptionValue('\0', "num_reps", "N", nullptr, &num_reps,
+                          &ParseUnsigned);
+
+#if JPEGXL_ENABLE_SJPEG
+  cmdline->AddOptionFlag('\0', "use_sjpeg",
+                         "use sjpeg instead of libjpeg for JPEG output",
+                         &use_sjpeg, &SetBooleanTrue);
+#endif
+
+  cmdline->AddOptionValue('\0', "jpeg_quality", "N", "JPEG output quality",
+                          &jpeg_quality, &ParseUnsigned);
+
+  cmdline->AddOptionValue('\0', "num_threads", "N",
+                          "The number of threads to use", &num_threads,
+                          &ParseUnsigned);
+
+  cmdline->AddOptionValue('\0', "print_profile", "0|1",
+                          "print timing information before exiting",
+                          &print_profile, &ParseOverride);
+
+  cmdline->AddOptionValue('\0', "print_info", "0|1",
+                          "print AuxOut before exiting", &print_info,
+                          &ParseOverride);
+
+  djxl_args.AddCommandLineOptions(cmdline);
+}
+
+jxl::Status DecompressArgs::ValidateArgs() {
+  if (file_in == nullptr) {
+    fprintf(stderr, "Missing INPUT filename.\n");
+    return false;
+  }
+  return djxl_args.ValidateArgs();
+}
+
+}  // namespace tools
+}  // namespace jpegxl
diff --git a/tools/djpegxl.h b/tools/djpegxl.h
new file mode 100644
index 000000000..9f3f277fc
--- /dev/null
+++ b/tools/djpegxl.h
@@ -0,0 +1,60 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_DJPEGXL_H_
+#define TOOLS_DJPEGXL_H_
+
+#include <stddef.h>
+
+#include "jxl/base/override.h"
+#include "jxl/base/status.h"
+#include "tools/args.h"
+#include "tools/cmdline.h"
+#include "tools/djxl.h"
+
+namespace jpegxl {
+namespace tools {
+
+// Common JPEG XL decompress arguments.
+struct DecompressArgs {
+  // Initialize non-static default options.
+  DecompressArgs();
+
+  // Common djpegxl parameters.
+  const char* file_in = nullptr;
+  const char* file_out = nullptr;
+  size_t num_threads;
+  bool use_sjpeg = false;
+  size_t jpeg_quality = 95;
+  bool version = false;
+  jxl::Override print_profile = jxl::Override::kDefault;
+  jxl::Override print_info = jxl::Override::kDefault;
+
+  size_t num_reps = 1;
+
+  JxlDecompressArgs djxl_args;
+
+  // Add all the command line options to the CommandLineParser. Note that the
+  // options are tied to the instance that this was called on.
+  void AddCommandLineOptions(CommandLineParser* cmdline);
+
+  // Validate the passed arguments, checking whether all passed options are
+  // compatible. Returns whether the validation was successful.
+  jxl::Status ValidateArgs();
+};
+
+}  // namespace tools
+}  // namespace jpegxl
+
+#endif  // TOOLS_DJPEGXL_H_
diff --git a/tools/djpegxl_main.cc b/tools/djpegxl_main.cc
new file mode 100644
index 000000000..ae9b5b1a3
--- /dev/null
+++ b/tools/djpegxl_main.cc
@@ -0,0 +1,132 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <hwy/runtime_dispatch.h>
+#include <hwy/static_targets.h>
+#include <vector>
+
+#include "jpegxl/decode.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/cache_aligned.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/file_io.h"
+#include "jxl/base/os_specific.h"
+#include "jxl/base/override.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/profiler.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/brunsli.h"
+#include "jxl/codec_in_out.h"
+#include "tools/cmdline.h"
+#include "tools/djpegxl.h"
+#include "tools/djxl.h"
+#include "tools/speed_stats.h"
+
+namespace jpegxl {
+namespace tools {
+namespace {
+
+int DecompressMain(int argc, const char *argv[]) {
+  DecompressArgs args;
+  CommandLineParser cmdline;
+  args.AddCommandLineOptions(&cmdline);
+
+  if (!cmdline.Parse(argc, argv) || !args.ValidateArgs()) {
+    cmdline.PrintHelp();
+    return 1;
+  }
+  if (args.version) {
+    fprintf(stderr, "djpegxl - version " JPEGXL_VERSION "\n");
+    fprintf(stderr, "Copyright (c) the JPEG XL Project\n");
+    return 0;
+  }
+
+  const int bits = hwy::TargetBitfield().Bits();
+  if ((bits & HWY_STATIC_TARGETS) != HWY_STATIC_TARGETS) {
+    fprintf(stderr, "CPU does not support all enabled targets => exiting.\n");
+    return 1;
+  }
+
+  jxl::PaddedBytes compressed;
+  if (!jxl::ReadFile(args.file_in, &compressed)) return 1;
+  fprintf(stderr, "Read %zu compressed bytes\n", compressed.size());
+
+  jxl::ThreadPoolInternal pool(args.num_threads);
+  jxl::CodecInOut io;
+  SpeedStats stats;
+
+  // Set JPEG quality.
+  // TODO(veluca): the decoder should set this value, and the argument should be
+  // an override.
+  // TODO(veluca): the decoder should directly produce a JPEG file, and this
+  // should not be necessary.
+  io.use_sjpeg = args.use_sjpeg;
+  io.jpeg_quality = args.jpeg_quality;
+
+  if ((JpegxlSignatureCheck(compressed.data(), compressed.size()) |
+       JPEGXL_SIG_ANY) != 0) {
+    const std::vector<int> cpus = jxl::AvailableCPUs();
+    pool.RunOnEachThread([&cpus](const int task, const size_t thread) {
+      // 1.1-1.2x speedup (36 cores) from pinning.
+      if (thread < cpus.size()) {
+        if (!jxl::PinThreadToCPU(cpus[thread])) {
+          fprintf(stderr, "WARNING: failed to pin thread %zu.\n", thread);
+        }
+      }
+    });
+
+    jxl::AuxOut aux_out;
+    for (size_t i = 0; i < args.num_reps; ++i) {
+      if (!DecompressJxl(jxl::Span<const uint8_t>(compressed),
+                         args.djxl_args.params, &pool, &io, &aux_out, &stats)) {
+        return 1;
+      }
+    }
+
+    if (!WriteJxlOutput(args.djxl_args, args.file_out, io)) return 1;
+
+    if (args.djxl_args.print_read_bytes) {
+      fprintf(stderr, "Decoded bytes: %zu\n", io.Main().decoded_bytes());
+    }
+
+    if (args.print_info == jxl::Override::kOn) {
+      aux_out.Print(args.num_reps);
+    }
+  } else {
+    fprintf(stderr, "Unknown compressed image format\n");
+    return 1;
+  }
+
+  JXL_CHECK(stats.Print(io.xsize(), io.ysize(), pool.NumWorkerThreads()));
+
+  if (args.print_profile == jxl::Override::kOn) {
+    PROFILER_PRINT_RESULTS();
+  }
+  jxl::CacheAligned::PrintStats();
+  return 0;
+}
+
+}  // namespace
+}  // namespace tools
+}  // namespace jpegxl
+
+int main(int argc, const char *argv[]) {
+  return jpegxl::tools::DecompressMain(argc, argv);
+}
diff --git a/tools/djxl.cc b/tools/djxl.cc
new file mode 100644
index 000000000..6911122f0
--- /dev/null
+++ b/tools/djxl.cc
@@ -0,0 +1,281 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/djxl.h"
+
+#include <stddef.h>
+#include <stdio.h>
+#include <string.h>
+
+#include <algorithm>
+#include <vector>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/os_specific.h"
+#include "jxl/base/override.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/dec_file.h"
+#include "jxl/extras/codec.h"
+#include "jxl/frame_header.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+#include "jxl/image_ops.h"
+#include "tools/args.h"
+
+namespace jpegxl {
+namespace tools {
+
+void JxlDecompressArgs::AddCommandLineOptions(
+    tools::CommandLineParser* cmdline) {
+  // Flags.
+  cmdline->AddOptionValue('\0', "bits_per_sample", "N",
+                          "defaults to original (input) bit depth",
+                          &bits_per_sample, &ParseUnsigned);
+
+  cmdline->AddOptionValue('\0', "color_space", "RGB_D65_SRG_Rel_Lin",
+                          "defaults to original (input) color space",
+                          &color_space, &ParseString);
+
+  cmdline->AddOptionValue('\0', "noise", "0", "disables noise generation",
+                          &params.noise, &ParseOverride);
+
+  cmdline->AddOptionValue('\0', "adaptive_reconstruction", "0|1",
+                          "disables/enables extra filtering",
+                          &params.adaptive_reconstruction, &ParseOverride);
+
+  cmdline->AddOptionValue('s', "downsampling", "1,2,4,8,16",
+                          "maximum permissible downsampling factor (values "
+                          "greater than 16 will return the LQIP if available)",
+                          &params.max_downsampling, &ParseUnsigned);
+
+  cmdline->AddOptionFlag('c', "coalesce", "decode coalesced animation frames",
+                         &coalesce, &SetBooleanTrue);
+
+  cmdline->AddOptionFlag('j', "jpeg", "decode losslessly recompressed JPEG",
+                         &params.keep_dct, &SetBooleanTrue);
+
+  cmdline->AddOptionFlag('\0', "print_read_bytes",
+                         "print total number of decoded bytes",
+                         &print_read_bytes, &SetBooleanTrue);
+
+  cmdline->AddOptionFlag(
+      't', "fix_dc_staircase",
+      "Fix DC staircase, for recompressed JPEG1 files (brunsli) only",
+      &brunsli_fix_dc_staircase, &SetBooleanTrue);
+  cmdline->AddOptionFlag(
+      'g', "gaborish",
+      "Gaborish deblocking, for recompressed JPEG1 files (brunsli) only",
+      &brunsli_gaborish, &SetBooleanTrue);
+}
+
+jxl::Status JxlDecompressArgs::ValidateArgs() {
+  if (params.noise == jxl::Override::kOn) {
+    fprintf(stderr, "Noise can only be enabled by the encoder.\n");
+    return JXL_FAILURE("Cannot force noise on");
+  }
+  params.brunsli.fix_dc_staircase = brunsli_fix_dc_staircase;
+  params.brunsli.gaborish = brunsli_gaborish;
+
+  return true;
+}
+
+jxl::Status DecompressJxl(const jxl::Span<const uint8_t> compressed,
+                          const jxl::DecompressParams& params,
+                          jxl::ThreadPool* pool,
+                          jxl::CodecInOut* JXL_RESTRICT io,
+                          jxl::AuxOut* aux_out,
+                          SpeedStats* JXL_RESTRICT stats) {
+  const double t0 = jxl::Now();
+  if (!DecodeFile(params, compressed, io, aux_out, pool)) {
+    fprintf(stderr, "Failed to decompress.\n");
+    return false;
+  }
+  const double t1 = jxl::Now();
+  stats->NotifyElapsed(t1 - t0);
+  return true;
+}
+
+void RenderSpotColor(const jxl::Image3F& img, const jxl::ImageU& sc,
+                     const float color[4], int ec_bit_depth) {
+  float scale = color[3] / ((1 << ec_bit_depth) - 1.0f);
+  for (size_t c = 0; c < 3; c++) {
+    for (size_t y = 0; y < img.ysize(); y++) {
+      float* JXL_RESTRICT p = img.Plane(c).MutableRow(y);
+      const uint16_t* JXL_RESTRICT s = sc.ConstRow(y);
+      for (size_t x = 0; x < img.xsize(); x++) {
+        float mix = scale * s[x];
+        p[x] = mix * color[c] + (1.0 - mix) * p[x];
+      }
+    }
+  }
+}
+
+jxl::Status WriteJxlOutput(const JxlDecompressArgs& args, const char* file_out,
+                           const jxl::CodecInOut& io) {
+  // Can only write if we decoded and have an output filename.
+  // (Writing large PNGs is slow, so allow skipping it for benchmarks.)
+  if (file_out == nullptr) return true;
+
+  for (size_t ec = 0; ec < io.metadata.m2.num_extra_channels; ec++) {
+    if (io.metadata.m2.extra_channel_info[ec].rendered != 0) {
+      if (io.metadata.m2.extra_channel_info[ec].rendered == 1) {
+        for (size_t fr = 0; fr < io.frames.size(); fr++)
+          RenderSpotColor(io.frames[fr].color(),
+                          io.frames[fr].extra_channels()[ec],
+                          io.metadata.m2.extra_channel_info[ec].color,
+                          io.metadata.m2.extra_channel_bits);
+      } else {
+        fprintf(stderr,
+                "Warning: ignoring extra channel which is supposed to be "
+                "rendered but I don't know how.\n");
+      }
+    }
+  }
+
+  // Override original color space with arg if specified.
+  jxl::ColorEncoding c_out = io.metadata.color_encoding;
+  if (!args.color_space.empty()) {
+    if (!jxl::ParseDescription(args.color_space, &c_out) ||
+        !jxl::ColorManagement::CreateProfile(&c_out)) {
+      fprintf(stderr, "Failed to apply color_space.\n");
+      return false;
+    }
+  }
+
+  // Override original #bits with arg if specified.
+  size_t bits_per_sample = io.metadata.bits_per_sample;
+  if (args.bits_per_sample != 0) bits_per_sample = args.bits_per_sample;
+
+  if (!io.metadata.m2.have_animation) {
+    if (!EncodeToFile(io, c_out, bits_per_sample, file_out)) {
+      fprintf(stderr, "Failed to write decoded image.\n");
+      return false;
+    }
+  } else {
+    const char* extension = strrchr(file_out, '.');
+    std::string base = extension == nullptr
+                           ? std::string(file_out)
+                           : std::string(file_out, extension - file_out);
+    if (extension == nullptr) extension = "";
+    const int digits = 1 + static_cast<int>(std::log10(std::max(
+                               1, static_cast<int>(io.frames.size() - 1))));
+    std::vector<char> output_filename;
+    output_filename.resize(base.size() + 1 + digits + strlen(extension) + 1);
+
+    jxl::CodecInOut frame_io;
+    if (args.coalesce) {
+      frame_io.SetFromImage(jxl::CopyImage(io.frames[0].color()),
+                            io.frames[0].c_current());
+      frame_io.metadata = *io.frames[0].metadata();
+      if (io.frames[0].HasAlpha())
+        frame_io.Main().SetAlpha(jxl::CopyImage(io.frames[0].alpha()));
+    }
+
+    // TODO: take NewBase into account
+    for (size_t i = 0; i < io.frames.size(); ++i) {
+      if (args.coalesce) {
+        if (i > 0) {
+          const jxl::AnimationFrame& af = io.animation_frames[i];
+          jxl::Rect cropbox(frame_io.Main().color());
+          if (af.have_crop)
+            cropbox = jxl::Rect(af.x0, af.y0, af.xsize, af.ysize);
+          if (af.blend_mode() == jxl::AnimationFrame::BlendMode::kAdd) {
+            for (int p = 0; p < 3; p++) {
+              jxl::AddTo(jxl::Rect(io.frames[i].color()),
+                         io.frames[i].color().Plane(p), cropbox,
+                         &frame_io.Main().color().Plane(p));
+            }
+            if (frame_io.Main().HasAlpha()) {
+              jxl::AddTo(jxl::Rect(io.frames[i].alpha()), io.frames[i].alpha(),
+                         cropbox, &frame_io.Main().alpha());
+            }
+          } else if (af.blend_mode() ==
+                         jxl::AnimationFrame::BlendMode::kBlend &&
+                     io.frames[i].HasAlpha()) {
+            // blend without alpha is just replace
+            float max_alpha = (1 << io.metadata.alpha_bits) - 1;
+            float rmax_alpha = 1.0f / max_alpha;
+            for (size_t y = 0; y < cropbox.ysize(); y++) {
+              const uint16_t* JXL_RESTRICT a1 = io.frames[i].alpha().Row(y);
+              const float* JXL_RESTRICT r1 =
+                  io.frames[i].color().PlaneRow(0, y);
+              const float* JXL_RESTRICT g1 =
+                  io.frames[i].color().PlaneRow(1, y);
+              const float* JXL_RESTRICT b1 =
+                  io.frames[i].color().PlaneRow(2, y);
+              uint16_t* JXL_RESTRICT a =
+                  cropbox.MutableRow(&frame_io.Main().alpha(), y);
+              float* JXL_RESTRICT r =
+                  cropbox.MutableRow(&frame_io.Main().color().Plane(0), y);
+              float* JXL_RESTRICT g =
+                  cropbox.MutableRow(&frame_io.Main().color().Plane(1), y);
+              float* JXL_RESTRICT b =
+                  cropbox.MutableRow(&frame_io.Main().color().Plane(2), y);
+              for (size_t x = 0; x < cropbox.xsize(); x++) {
+                if (a1[x] == 0) continue;
+                float new_a = a1[x] + (a[x] * (max_alpha - a1[x])) * rmax_alpha;
+                float rnew_a = 1.0f / new_a;
+                r[x] = (r1[x] * a1[x] + r[x] * a[x] * (max_alpha - a1[x])) *
+                       rnew_a;
+                g[x] = (g1[x] * a1[x] + g[x] * a[x] * (max_alpha - a1[x])) *
+                       rnew_a;
+                b[x] = (b1[x] * a1[x] + b[x] * a[x] * (max_alpha - a1[x])) *
+                       rnew_a;
+                a[x] = new_a;
+              }
+            }
+          } else {  // kReplace
+            jxl::CopyImageTo(
+                io.frames[i].color(), cropbox,
+                const_cast<jxl::Image3F*>(&frame_io.Main().color()));
+            if (frame_io.Main().HasAlpha())
+              jxl::CopyImageTo(
+                  io.frames[i].alpha(), cropbox,
+                  const_cast<jxl::ImageU*>(&frame_io.Main().alpha()));
+          }
+        }
+
+        snprintf(output_filename.data(), output_filename.size(), "%s-%0*zu%s",
+                 base.c_str(), digits, i, extension);
+        if (!EncodeToFile(frame_io, c_out, bits_per_sample,
+                          output_filename.data())) {
+          fprintf(stderr, "Failed to write decoded image for frame %zu/%zu.\n",
+                  i + 1, io.frames.size());
+        }
+
+      } else {
+        jxl::CodecInOut frame_io;
+        frame_io.SetFromImage(jxl::CopyImage(io.frames[i].color()),
+                              io.frames[i].c_current());
+        frame_io.metadata = *io.frames[i].metadata();
+        if (io.frames[i].HasAlpha())
+          frame_io.Main().SetAlpha(jxl::CopyImage(io.frames[i].alpha()));
+        snprintf(output_filename.data(), output_filename.size(), "%s-%0*zu%s",
+                 base.c_str(), digits, i, extension);
+        if (!EncodeToFile(frame_io, c_out, bits_per_sample,
+                          output_filename.data())) {
+          fprintf(stderr, "Failed to write decoded image for frame %zu/%zu.\n",
+                  i + 1, io.frames.size());
+        }
+      }
+    }
+  }
+  fprintf(stderr, "Wrote %zu bytes; done.\n", io.enc_size);
+  return true;
+}
+
+}  // namespace tools
+}  // namespace jpegxl
diff --git a/tools/djxl.h b/tools/djxl.h
new file mode 100644
index 000000000..7a43b6f07
--- /dev/null
+++ b/tools/djxl.h
@@ -0,0 +1,75 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_DJXL_H_
+#define TOOLS_DJXL_H_
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include <string>
+
+#include "jxl/aux_out.h"
+#include "jxl/aux_out_fwd.h"
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/padded_bytes.h"
+#include "jxl/base/span.h"
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/dec_params.h"
+#include "tools/cmdline.h"
+#include "tools/speed_stats.h"
+
+namespace jpegxl {
+namespace tools {
+
+struct JxlDecompressArgs {
+  // Add all the command line options to the CommandLineParser. Note that the
+  // options are tied to the instance that this was called on.
+  void AddCommandLineOptions(tools::CommandLineParser* cmdline);
+
+  // Validate the passed arguments, checking whether all passed options are
+  // compatible. Returns whether the validation was successful.
+  jxl::Status ValidateArgs();
+
+  // The parameters.
+  size_t bits_per_sample = 0;
+  std::string color_space;  // description
+
+  bool brunsli_fix_dc_staircase = false;
+  bool brunsli_gaborish = false;
+
+  jxl::DecompressParams params;
+
+  // If true, print the effective amount of bytes read from the bitstream.
+  bool print_read_bytes = false;
+
+  bool coalesce = false;
+};
+
+// Decompresses and notifies SpeedStats of elapsed time.
+jxl::Status DecompressJxl(const jxl::Span<const uint8_t> compressed,
+                          const jxl::DecompressParams& params,
+                          jxl::ThreadPool* pool,
+                          jxl::CodecInOut* JXL_RESTRICT io,
+                          jxl::AuxOut* aux_out, SpeedStats* JXL_RESTRICT stats);
+
+jxl::Status WriteJxlOutput(const JxlDecompressArgs& args, const char* file_out,
+                           const jxl::CodecInOut& io);
+
+}  // namespace tools
+}  // namespace jpegxl
+
+#endif  // TOOLS_DJXL_H_
diff --git a/tools/djxl_fuzzer.cc b/tools/djxl_fuzzer.cc
new file mode 100644
index 000000000..ef5cbd40d
--- /dev/null
+++ b/tools/djxl_fuzzer.cc
@@ -0,0 +1,61 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+#include <stdio.h>
+
+#include <hwy/runtime_dispatch.h>
+#include <hwy/static_targets.h>
+
+#include "jxl/aux_out.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/span.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+
+namespace jxl {
+
+int TestOneInput(const uint8_t* data, size_t size) {
+  // TODO(b/65240090): Remove
+  const int bits = hwy::TargetBitfield().Bits();
+  if ((bits & HWY_STATIC_TARGETS) != HWY_STATIC_TARGETS) {
+    static bool warned = false;
+    if (!warned) {
+      fprintf(stderr, "CPU doesn't support all enabled targets; exiting.\n");
+      warned = true;
+    }
+    return 0;
+  }
+
+  DecompressParams params;
+
+  CodecInOut io;
+  io.dec_max_xsize = 1u << 16;
+  io.dec_max_ysize = 1u << 16;
+  io.dec_max_pixels = 1 << 21;
+
+  ThreadPoolInternal pool(2);
+  AuxOut aux_out;
+  (void)DecodeFile(params, Span<const uint8_t>(data, size), &io, &aux_out,
+                   &pool);
+  return 0;
+}
+
+}  // namespace jxl
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  return jxl::TestOneInput(data, size);
+}
diff --git a/tools/epf.cc b/tools/epf.cc
new file mode 100644
index 000000000..9c4b2f988
--- /dev/null
+++ b/tools/epf.cc
@@ -0,0 +1,188 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/epf.h"
+
+#include "jxl/ar_control_field.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/common.h"
+#include "jxl/enc_adaptive_quantization.h"
+#include "jxl/enc_xyb.h"
+#include "jxl/epf.h"
+
+namespace jpegxl {
+namespace tools {
+
+namespace {
+
+// Requires `opsin` to be padded.
+jxl::Image3F ComputeDC(const jxl::Image3F& opsin) {
+  jxl::Image3F dc(opsin.xsize() / jxl::kBlockDim,
+                  opsin.ysize() / jxl::kBlockDim);
+  for (int c = 0; c < 3; ++c) {
+    for (size_t by = 0; by < dc.ysize(); ++by) {
+      float* const JXL_RESTRICT dc_row = dc.PlaneRow(c, by);
+      for (size_t bx = 0; bx < dc.xsize(); ++bx) {
+        dc_row[bx] = 0;
+        for (size_t y = 0; y < jxl::kBlockDim; ++y) {
+          const float* const JXL_RESTRICT row =
+              opsin.ConstPlaneRow(c, by * jxl::kBlockDim + y) +
+              bx * jxl::kBlockDim;
+          for (size_t x = 0; x < jxl::kBlockDim; ++x) {
+            dc_row[bx] += row[x];
+          }
+        }
+        dc_row[bx] *= 1.f / (jxl::kBlockDim * jxl::kBlockDim);
+      }
+    }
+  }
+  return dc;
+}
+
+// Expects `state` to have `quantizer`, `raw_quant_field` and `dc` usable.
+// TODO(sboukortt): extract this from dec_group.cc (DecodeGroupImpl) instead of
+// copying it.
+jxl::ImageF ComputeSigma(const jxl::Image3F& opsin,
+                         const int sharpness_parameter,
+                         const jxl::FrameDimensions& frame_dim,
+                         jxl::PassesEncoderState* state,
+                         const jxl::LoopFilter& lf) {
+  jxl::ImageF sigma(frame_dim.xsize_blocks + 4, frame_dim.ysize_blocks + 4);
+  const size_t sigma_stride = sigma.PixelsPerRow();
+  const size_t dc_stride = state->shared.dc->PixelsPerRow();
+  const float quant_scale = state->shared.quantizer.Scale();
+  for (size_t by = 0; by < sigma.ysize(); ++by) {
+    float* const JXL_RESTRICT sigma_row = sigma.Row(by);
+    const int* const JXL_RESTRICT row_quant =
+        state->shared.raw_quant_field.ConstRow(by);
+    const float* const JXL_RESTRICT dc_rows[3] = {
+        state->shared.dc->ConstPlaneRow(0, by),
+        state->shared.dc->ConstPlaneRow(1, by),
+        state->shared.dc->ConstPlaneRow(2, by),
+    };
+    for (size_t bx = 0; bx < sigma.xsize(); ++bx) {
+      float quant = 1.0f / (row_quant[bx] * quant_scale);
+      float sigma_quant = quant * lf.epf_quant_mul;
+      // Increase sigma near edges.
+      float dc_range = 0;
+      for (size_t c = 0; c < 3; c++) {
+        const float* JXL_RESTRICT base_dc_ptr = dc_rows[c] + bx;
+        // UBSAN complains about overflowing unsigned addition here,
+        // hence we use a slightly more convoluted syntax than simple
+        // array access to ensure we only ever add or subtract positive
+        // quantities.
+        float dc_ref = *base_dc_ptr;
+        float dc_top = *(base_dc_ptr - (by == 0 ? 0 : dc_stride));
+        float dc_bottom =
+            base_dc_ptr[by + 1 == frame_dim.ysize_blocks ? 0 : dc_stride];
+        float dc_left = *(base_dc_ptr - (bx == 0 ? 0 : 1));
+        float dc_right = base_dc_ptr[bx + 1 == frame_dim.xsize_blocks ? 0 : 1];
+
+        float dc_range_c = std::abs(dc_top - dc_ref);
+        dc_range_c = std::max(dc_range_c, std::abs(dc_bottom - dc_ref));
+        dc_range_c = std::max(dc_range_c, std::abs(dc_left - dc_ref));
+        dc_range_c = std::max(dc_range_c, std::abs(dc_right - dc_ref));
+        dc_range = std::max(dc_range_c * lf.epf_channel_scale[c], dc_range);
+      }
+      float sigma =
+          sigma_quant * (2.0f - 1.0f / (1.0f + lf.epf_dc_range_mul * dc_range));
+      sigma *= lf.epf_sharp_lut[sharpness_parameter];
+      // Avoid infinities.
+      sigma = std::max(1e-4f, sigma);
+      sigma_row[bx + 2 + 2 * sigma_stride] = jxl::kInvSigmaNum / sigma;
+      // Left padding with mirroring.
+      if (bx == 0) {
+        sigma_row[1 + 2 * sigma_stride] = sigma_row[2 + 2 * sigma_stride];
+      }
+      // Right padding with mirroring.
+      if (bx + 1 == frame_dim.xsize_blocks) {
+        sigma_row[2 * sigma_stride + bx + 3] =
+            sigma_row[2 * sigma_stride + bx + 2];
+      }
+      // Offsets for row copying, in blocks.
+      size_t offset_before = bx == 0 ? 1 : bx + 2;
+      size_t offset_after = bx + 1 == frame_dim.xsize_blocks ? bx + 4 : bx + 3;
+      size_t num = offset_after - offset_before;
+      // Above
+      if (by == 0) {
+        memcpy(sigma_row + offset_before + sigma_stride,
+               sigma_row + offset_before + 2 * sigma_stride,
+               num * sizeof(*sigma_row));
+      }
+      // Below
+      if (by + 1 == frame_dim.ysize_blocks) {
+        memcpy(sigma_row + offset_before + sigma_stride * 3,
+               sigma_row + offset_before + sigma_stride * 2,
+               num * sizeof(*sigma_row));
+      }
+    }
+  }
+  return sigma;
+}
+
+}  // namespace
+
+jxl::Status RunEPF(const float distance, const int sharpness_parameter,
+                   jxl::CodecInOut* const io, jxl::ThreadPool* const pool) {
+  const jxl::ColorEncoding original_color_encoding =
+      io->metadata.color_encoding;
+  jxl::Image3F opsin(io->xsize(), io->ysize());
+  ToXYB(io->Main(), /*linear_multiplier=*/1.f, pool, &opsin);
+
+  const size_t original_xsize = opsin.xsize(), original_ysize = opsin.ysize();
+  opsin = PadImageToMultiple(opsin, jxl::kBlockDim);
+
+  jxl::FrameDimensions frame_dim;
+  frame_dim.Set(original_xsize, original_ysize);
+
+  static constexpr float kAcQuant = 0.84f;
+  const float dc_quant =
+      jxl::InitialQuantDC(distance, jxl::kDefaultIntensityTarget);
+  const float ac_quant = kAcQuant / distance;
+  jxl::PassesEncoderState state;
+  jxl::InitializePassesEncoder(opsin, pool, &state, /*aux_out=*/nullptr);
+  state.shared.raw_quant_field =
+      jxl::ImageI(frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+  state.shared.quantizer.SetQuant(dc_quant, ac_quant,
+                                  &state.shared.raw_quant_field);
+  state.shared.dc_storage = ComputeDC(opsin);
+
+  jxl::LoopFilter lf;
+  jxl::Image3F storage1(frame_dim.xsize_padded + 4 * jxl::kBlockDim,
+                        jxl::kEpf1InputRows);
+  jxl::Image3F storage2(frame_dim.xsize_padded + 2 * jxl::kBlockDim,
+                        jxl::kEpf2InputRows);
+  lf.gab = false;
+  const jxl::ImageF sigma =
+      ComputeSigma(opsin, sharpness_parameter, frame_dim, &state, lf);
+  const jxl::Image3F padded = PadImageSymmetric(opsin, 2 * jxl::kBlockDim);
+
+  jxl::Rect image_rect(0, 0, frame_dim.xsize_padded, frame_dim.ysize_padded);
+  jxl::Rect sigma_rect(0, 0, frame_dim.xsize_blocks, frame_dim.ysize_blocks);
+
+  EdgePreservingFilter(lf, image_rect, padded, sigma_rect, sigma, image_rect,
+                       &opsin, &storage1, &storage2);
+
+  opsin.ShrinkTo(original_xsize, original_ysize);
+  jxl::OpsinParams opsin_params;
+  opsin_params.Init();
+  jxl::OpsinToLinear(&opsin, pool, opsin_params);
+  io->Main().SetFromImage(
+      std::move(opsin), jxl::ColorManagement::LinearSRGB(io->Main().IsGray()));
+  JXL_RETURN_IF_ERROR(io->TransformTo(original_color_encoding, pool));
+  return true;
+}
+
+}  // namespace tools
+}  // namespace jpegxl
diff --git a/tools/epf.h b/tools/epf.h
new file mode 100644
index 000000000..e31165e31
--- /dev/null
+++ b/tools/epf.h
@@ -0,0 +1,30 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_EPF_H_
+#define TOOLS_EPF_H_
+
+#include "jxl/base/status.h"
+#include "jxl/codec_in_out.h"
+
+namespace jpegxl {
+namespace tools {
+
+jxl::Status RunEPF(float distance, int sharpness_parameter, jxl::CodecInOut* io,
+                   jxl::ThreadPool* pool);
+
+}
+}  // namespace jpegxl
+
+#endif  // TOOLS_EPF_H_
diff --git a/tools/epf_main.cc b/tools/epf_main.cc
new file mode 100644
index 000000000..7bef9a409
--- /dev/null
+++ b/tools/epf_main.cc
@@ -0,0 +1,62 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/enc_adaptive_quantization.h"
+#include "jxl/extras/codec.h"
+#include "tools/args.h"
+#include "tools/cmdline.h"
+#include "tools/epf.h"
+
+int main(int argc, const char** argv) {
+  float distance = 1.f;
+  size_t sharpness = 1;
+  const char* input_filename = nullptr;
+  const char* output_filename = nullptr;
+
+  jpegxl::tools::CommandLineParser cmdline;
+  cmdline.AddOptionValue(
+      'd', "distance", "1..",
+      "Butteraugli distance to assume for quantization values", &distance,
+      &jpegxl::tools::ParseFloat);
+  cmdline.AddOptionValue('s', "sharpness", "1..7",
+                         "EPF sharpness value from 1 to 7", &sharpness,
+                         &jpegxl::tools::ParseUnsigned);
+  cmdline.AddPositionalOption("INPUT", "Input image", &input_filename);
+  cmdline.AddPositionalOption("OUTPUT", "Output image", &output_filename);
+  if (!cmdline.Parse(argc, argv) || input_filename == nullptr ||
+      output_filename == nullptr) {
+    cmdline.PrintHelp();
+    return EXIT_FAILURE;
+  }
+
+  jxl::ThreadPoolInternal pool;
+  jxl::CodecInOut io;
+  if (!jxl::SetFromFile(input_filename, &io, &pool)) {
+    fprintf(stderr, "Failed to read from \"%s\".\n", input_filename);
+    return EXIT_FAILURE;
+  }
+
+  if (!jpegxl::tools::RunEPF(distance, sharpness, &io, &pool)) {
+    fprintf(stderr, "Failed to run the EPF\n");
+    return EXIT_FAILURE;
+  }
+
+  if (!jxl::EncodeToFile(io, output_filename, &pool)) {
+    fprintf(stderr, "Failed to write the result to \"%s\".\n", output_filename);
+    return EXIT_FAILURE;
+  }
+}
diff --git a/tools/fields_fuzzer.cc b/tools/fields_fuzzer.cc
new file mode 100644
index 000000000..e5187d776
--- /dev/null
+++ b/tools/fields_fuzzer.cc
@@ -0,0 +1,75 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdint.h>
+
+#include "jxl/dec_bit_reader.h"
+#include "jxl/frame_header.h"
+#include "jxl/headers.h"
+#include "jxl/image_bundle.h"
+#include "jxl/loop_filter.h"
+
+namespace jxl {
+
+int TestOneInput(const uint8_t* data, size_t size) {
+  // First byte controls which header to parse.
+  if (size == 0) return 0;
+  BitReader reader(Span<const uint8_t>(data + 1, size - 1));
+  switch (data[0]) {
+    case 0: {
+      SizeHeader size_header;
+      (void)ReadSizeHeader(&reader, &size_header);
+      break;
+    }
+
+    case 1: {
+      PreviewHeader preview;
+      (void)ReadPreviewHeader(&reader, &preview);
+      break;
+    }
+
+    case 2: {
+      AnimationHeader animation;
+      (void)ReadAnimationHeader(&reader, &animation);
+      break;
+    }
+
+    case 3: {
+      ImageMetadata metadata;
+      (void)ReadImageMetadata(&reader, &metadata);
+      break;
+    }
+
+    case 4: {
+      FrameHeader frame;
+      (void)ReadFrameHeader(&reader, &frame);
+      break;
+    }
+
+    default: {
+      LoopFilter loop_filter;
+      (void)ReadLoopFilter(&reader, &loop_filter);
+      break;
+    }
+  }
+  (void)reader.Close();
+
+  return 0;
+}
+
+}  // namespace jxl
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  return jxl::TestOneInput(data, size);
+}
diff --git a/tools/flicker_test/CMakeLists.txt b/tools/flicker_test/CMakeLists.txt
new file mode 100644
index 000000000..eaf3ffed9
--- /dev/null
+++ b/tools/flicker_test/CMakeLists.txt
@@ -0,0 +1,48 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+find_package(Qt5 COMPONENTS Widgets X11Extras)
+if (NOT Qt5_FOUND)
+  message(WARNING "Qt5 was not found. The flicker test tool will not be built.")
+  return()
+endif ()
+
+if (NOT TARGET x11_icc OR NOT TARGET image_loading)
+  message(WARNING "Comparison tool not built. The flicker test tool will not be built.")
+  return()
+endif ()
+
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(CMAKE_AUTOMOC ON)
+set(CMAKE_AUTOUIC ON)
+
+add_executable(flicker_test WIN32
+  main.cc
+  parameters.cc
+  parameters.h
+  setup.cc
+  setup.h
+  setup.ui
+  split_view.cc
+  split_view.h
+  test_window.cc
+  test_window.h
+  test_window.ui)
+
+target_link_libraries(flicker_test PUBLIC
+  Qt5::Widgets
+  Qt5::X11Extras
+  image_loading
+  x11_icc
+)
diff --git a/tools/flicker_test/main.cc b/tools/flicker_test/main.cc
new file mode 100644
index 000000000..cd7d1efa2
--- /dev/null
+++ b/tools/flicker_test/main.cc
@@ -0,0 +1,31 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <QApplication>
+
+#include "tools/flicker_test/setup.h"
+#include "tools/flicker_test/test_window.h"
+
+int main(int argc, char** argv) {
+  QApplication application(argc, argv);
+
+  jxl::FlickerTestWizard wizard;
+  if (wizard.exec()) {
+    jxl::FlickerTestWindow test_window(wizard.parameters());
+    if (test_window.proceedWithTest()) {
+      test_window.showMaximized();
+      return application.exec();
+    }
+  }
+}
diff --git a/tools/flicker_test/parameters.cc b/tools/flicker_test/parameters.cc
new file mode 100644
index 000000000..6b836de23
--- /dev/null
+++ b/tools/flicker_test/parameters.cc
@@ -0,0 +1,92 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/flicker_test/parameters.h"
+
+namespace jxl {
+
+namespace {
+
+constexpr char kPathsGroup[] = "paths";
+constexpr char kOriginalFolderKey[] = "originalFolder";
+constexpr char kAlteredFolderKey[] = "alteredFolder";
+constexpr char kOutputFileKey[] = "outputFile";
+
+constexpr char kTimingGroup[] = "timing";
+constexpr char kAdvanceTimeKey[] = "advanceTimeMSecs";
+constexpr char kViewingTimeKey[] = "viewingTimeSecs";
+constexpr char kBlankingTimeKey[] = "blankingTimeMSecs";
+constexpr char kGrayGroup[] = "gray";
+constexpr char kGrayKey[] = "enabled";
+constexpr char kGrayFadingTimeKey[] = "fadingTimeMSecs";
+constexpr char kGrayTimeKey[] = "timeMSecs";
+
+constexpr char kDisplayGroup[] = "display";
+constexpr char kSpacingKey[] = "spacing";
+
+}  // namespace
+
+FlickerTestParameters FlickerTestParameters::loadFrom(
+    QSettings* const settings) {
+  FlickerTestParameters parameters;
+
+  settings->beginGroup(kPathsGroup);
+  parameters.originalFolder = settings->value(kOriginalFolderKey).toString();
+  parameters.alteredFolder = settings->value(kAlteredFolderKey).toString();
+  parameters.outputFile = settings->value(kOutputFileKey).toString();
+  settings->endGroup();
+
+  settings->beginGroup(kTimingGroup);
+  parameters.advanceTimeMSecs = settings->value(kAdvanceTimeKey, 100).toInt();
+  parameters.viewingTimeSecs = settings->value(kViewingTimeKey, 4).toInt();
+  parameters.blankingTimeMSecs = settings->value(kBlankingTimeKey, 250).toInt();
+  settings->beginGroup(kGrayGroup);
+  parameters.gray = settings->value(kGrayKey, false).toBool();
+  parameters.grayFadingTimeMSecs =
+      settings->value(kGrayFadingTimeKey, 100).toInt();
+  parameters.grayTimeMSecs = settings->value(kGrayTimeKey, 300).toInt();
+  settings->endGroup();
+  settings->endGroup();
+
+  settings->beginGroup(kDisplayGroup);
+  parameters.spacing = settings->value(kSpacingKey, 50).toInt();
+  settings->endGroup();
+
+  return parameters;
+}
+
+void FlickerTestParameters::saveTo(QSettings* const settings) const {
+  settings->beginGroup(kPathsGroup);
+  settings->setValue(kOriginalFolderKey, originalFolder);
+  settings->setValue(kAlteredFolderKey, alteredFolder);
+  settings->setValue(kOutputFileKey, outputFile);
+  settings->endGroup();
+
+  settings->beginGroup(kTimingGroup);
+  settings->setValue(kAdvanceTimeKey, advanceTimeMSecs);
+  settings->setValue(kViewingTimeKey, viewingTimeSecs);
+  settings->setValue(kBlankingTimeKey, blankingTimeMSecs);
+  settings->beginGroup(kGrayGroup);
+  settings->setValue(kGrayKey, gray);
+  settings->setValue(kGrayFadingTimeKey, grayFadingTimeMSecs);
+  settings->setValue(kGrayTimeKey, grayTimeMSecs);
+  settings->endGroup();
+  settings->endGroup();
+
+  settings->beginGroup(kDisplayGroup);
+  settings->setValue(kSpacingKey, spacing);
+  settings->endGroup();
+}
+
+}  // namespace jxl
diff --git a/tools/flicker_test/parameters.h b/tools/flicker_test/parameters.h
new file mode 100644
index 000000000..0075748ab
--- /dev/null
+++ b/tools/flicker_test/parameters.h
@@ -0,0 +1,40 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_FLICKER_TEST_PARAMETERS_H_
+#define TOOLS_FLICKER_TEST_PARAMETERS_H_
+
+#include <QSettings>
+
+namespace jxl {
+
+struct FlickerTestParameters {
+  QString originalFolder;
+  QString alteredFolder;
+  QString outputFile;
+  int advanceTimeMSecs;
+  int viewingTimeSecs;
+  int blankingTimeMSecs;
+  bool gray;
+  int grayFadingTimeMSecs;
+  int grayTimeMSecs;
+  int spacing;
+
+  static FlickerTestParameters loadFrom(QSettings* settings);
+  void saveTo(QSettings* settings) const;
+};
+
+}  // namespace jxl
+
+#endif  // TOOLS_FLICKER_TEST_PARAMETERS_H_
diff --git a/tools/flicker_test/setup.cc b/tools/flicker_test/setup.cc
new file mode 100644
index 000000000..5b05cee6c
--- /dev/null
+++ b/tools/flicker_test/setup.cc
@@ -0,0 +1,158 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/flicker_test/setup.h"
+
+#include <QCompleter>
+#include <QFileDialog>
+#include <QFileSystemModel>
+#include <QMessageBox>
+#include <QPushButton>
+
+namespace jxl {
+
+FlickerTestWizard::FlickerTestWizard(QWidget* const parent)
+    : QWizard(parent), settings_("JPEG XL project", "Flickering test") {
+  ui_.setupUi(this);
+
+  connect(ui_.grayFadingTime, QOverload<int>::of(&QSpinBox::valueChanged), this,
+          &FlickerTestWizard::updateTotalGrayTime);
+  connect(ui_.grayTime, QOverload<int>::of(&QSpinBox::valueChanged), this,
+          &FlickerTestWizard::updateTotalGrayTime);
+
+  ui_.timingButtonBox->button(QDialogButtonBox::RestoreDefaults)
+      ->setText(tr("Restore ISO/IEC 29170-2:2015 parameters"));
+
+  setButtonText(QWizard::FinishButton, tr("Start test"));
+
+  QCompleter* const completer = new QCompleter(this);
+  QFileSystemModel* const model = new QFileSystemModel(completer);
+  model->setRootPath("/");
+  model->setFilter(QDir::Dirs);
+  completer->setModel(model);
+  ui_.originalFolder->setCompleter(completer);
+  ui_.alteredFolder->setCompleter(completer);
+
+  const auto parameters = FlickerTestParameters::loadFrom(&settings_);
+  ui_.originalFolder->setText(parameters.originalFolder);
+  ui_.alteredFolder->setText(parameters.alteredFolder);
+  ui_.outputFile->setText(parameters.outputFile);
+  ui_.advanceTime->setValue(parameters.advanceTimeMSecs);
+  ui_.viewingTime->setValue(parameters.viewingTimeSecs);
+  ui_.blankingTime->setValue(parameters.blankingTimeMSecs);
+  ui_.grayFlickering->setChecked(parameters.gray);
+  ui_.grayFadingTime->setValue(parameters.grayFadingTimeMSecs);
+  ui_.grayTime->setValue(parameters.grayTimeMSecs);
+  ui_.spacing->setValue(parameters.spacing);
+
+  QImage white(256, 256, QImage::Format_RGB32);
+  white.fill(Qt::white);
+  ui_.spacingDemo->setOriginalImage(white);
+  ui_.spacingDemo->setAlteredImage(white);
+
+  connect(this, &QDialog::accepted,
+          [&] { this->parameters().saveTo(&settings_); });
+}
+
+FlickerTestParameters FlickerTestWizard::parameters() const {
+  FlickerTestParameters result;
+  result.originalFolder = ui_.originalFolder->text();
+  result.alteredFolder = ui_.alteredFolder->text();
+  result.outputFile = ui_.outputFile->text();
+  result.advanceTimeMSecs = ui_.advanceTime->value();
+  result.viewingTimeSecs = ui_.viewingTime->value();
+  result.blankingTimeMSecs = ui_.blankingTime->value();
+  result.gray = ui_.grayFlickering->isChecked();
+  result.grayFadingTimeMSecs = ui_.grayFadingTime->value();
+  result.grayTimeMSecs = ui_.grayTime->value();
+  result.spacing = ui_.spacing->value();
+  return result;
+}
+
+void FlickerTestWizard::on_originalFolderBrowseButton_clicked() {
+  const QString path = QFileDialog::getExistingDirectory(
+      this, tr("Folder with original images"), ui_.originalFolder->text());
+  if (!path.isEmpty()) {
+    ui_.originalFolder->setText(path);
+  }
+}
+
+void FlickerTestWizard::on_alteredFolderBrowseButton_clicked() {
+  const QString path = QFileDialog::getExistingDirectory(
+      this, tr("Folder with altered images"), ui_.alteredFolder->text());
+  if (!path.isEmpty()) {
+    ui_.alteredFolder->setText(path);
+  }
+}
+
+void FlickerTestWizard::on_outputFileBrowseButton_clicked() {
+  // The overwrite check is disabled here because it is carried out in
+  // `validateCurrentPage` (called when the user clicks the "Next" button) so
+  // that it also applies to automatically-reloaded settings.
+  const QString path = QFileDialog::getSaveFileName(
+      this, tr("CSV file in which to save the results"), ui_.outputFile->text(),
+      tr("CSV files (*.csv)"), /*selectedFilter=*/nullptr,
+      QFileDialog::DontConfirmOverwrite);
+  if (!path.isEmpty()) {
+    ui_.outputFile->setText(path);
+  }
+}
+
+void FlickerTestWizard::on_timingButtonBox_clicked(
+    QAbstractButton* const button) {
+  if (ui_.timingButtonBox->standardButton(button) ==
+      QDialogButtonBox::RestoreDefaults) {
+    ui_.advanceTime->setValue(100);
+    ui_.viewingTime->setValue(4);
+    ui_.blankingTime->setValue(250);
+    ui_.grayFlickering->setChecked(false);
+  }
+}
+
+void FlickerTestWizard::updateTotalGrayTime() {
+  ui_.totalGrayTimeLabel->setText(
+      tr("Total gray time: %L1&#8239;ms")
+          .arg(2 * ui_.grayFadingTime->value() + ui_.grayTime->value()));
+}
+
+bool FlickerTestWizard::validateCurrentPage() {
+  if (currentPage() == ui_.pathsPage && QFile::exists(ui_.outputFile->text())) {
+    QMessageBox messageBox(this);
+    messageBox.setIcon(QMessageBox::Warning);
+    messageBox.setStandardButtons(QMessageBox::Ok | QMessageBox::Cancel);
+    messageBox.setWindowTitle(tr("Output file already exists"));
+    messageBox.setText(tr("The selected output file \"%1\" already exists.")
+                           .arg(ui_.outputFile->text()));
+    messageBox.setInformativeText(tr("Do you wish to overwrite it?"));
+    if (messageBox.exec() == QMessageBox::Cancel) {
+      return false;
+    }
+  } else if (currentPage() == ui_.timesPage) {
+    if (ui_.grayFlickering->isChecked() &&
+        2 * ui_.grayFadingTime->value() + ui_.grayTime->value() >
+            ui_.advanceTime->value()) {
+      QMessageBox messageBox(this);
+      messageBox.setIcon(QMessageBox::Warning);
+      messageBox.setStandardButtons(QMessageBox::Ok);
+      messageBox.setWindowTitle(tr("Incompatible times selected"));
+      messageBox.setText(
+          tr("The total gray time is greater than the advance time."));
+      messageBox.exec();
+      return false;
+    }
+  }
+  return QWizard::validateCurrentPage();
+}
+
+}  // namespace jxl
diff --git a/tools/flicker_test/setup.h b/tools/flicker_test/setup.h
new file mode 100644
index 000000000..568202fb2
--- /dev/null
+++ b/tools/flicker_test/setup.h
@@ -0,0 +1,53 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_FLICKER_TEST_SETUP_H_
+#define TOOLS_FLICKER_TEST_SETUP_H_
+
+#include <QWizard>
+
+#include "tools/flicker_test/parameters.h"
+#include "tools/flicker_test/ui_setup.h"
+
+namespace jxl {
+
+class FlickerTestWizard : public QWizard {
+  Q_OBJECT
+
+ public:
+  explicit FlickerTestWizard(QWidget* parent = nullptr);
+  ~FlickerTestWizard() override = default;
+
+  FlickerTestParameters parameters() const;
+
+ protected:
+  bool validateCurrentPage() override;
+
+ private slots:
+  void on_originalFolderBrowseButton_clicked();
+  void on_alteredFolderBrowseButton_clicked();
+  void on_outputFileBrowseButton_clicked();
+
+  void on_timingButtonBox_clicked(QAbstractButton* button);
+
+  void updateTotalGrayTime();
+
+ private:
+  Ui::FlickerTestWizard ui_;
+  QSettings settings_;
+};
+
+}  // namespace jxl
+
+#endif
diff --git a/tools/flicker_test/setup.ui b/tools/flicker_test/setup.ui
new file mode 100644
index 000000000..44cc5b0f6
--- /dev/null
+++ b/tools/flicker_test/setup.ui
@@ -0,0 +1,376 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <comment>
+  Copyright (c) the JPEG XL Project
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+ </comment>
+ <class>FlickerTestWizard</class>
+ <widget class="QWizard" name="FlickerTestWizard">
+  <property name="windowTitle">
+   <string>New flicker test</string>
+  </property>
+  <property name="options">
+   <set>QWizard::NoBackButtonOnStartPage</set>
+  </property>
+  <widget class="QWizardPage" name="pathsPage">
+   <layout class="QVBoxLayout" name="verticalLayout">
+    <item>
+     <layout class="QFormLayout" name="formLayout">
+      <item row="0" column="0">
+       <widget class="QLabel" name="originalFolderPromptLabel">
+        <property name="text">
+         <string>Folder with the original images:</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <layout class="QHBoxLayout" name="horizontalLayout" stretch="1,0">
+        <item>
+         <widget class="QLineEdit" name="originalFolder"/>
+        </item>
+        <item>
+         <widget class="QToolButton" name="originalFolderBrowseButton">
+          <property name="text">
+           <string>Browse…</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item row="1" column="0">
+       <widget class="QLabel" name="alteredFolderPromptLabel">
+        <property name="text">
+         <string>Folder with the altered images:</string>
+        </property>
+       </widget>
+      </item>
+      <item row="1" column="1">
+       <layout class="QHBoxLayout" name="horizontalLayout_2" stretch="1,0">
+        <item>
+         <widget class="QLineEdit" name="alteredFolder"/>
+        </item>
+        <item>
+         <widget class="QToolButton" name="alteredFolderBrowseButton">
+          <property name="text">
+           <string>Browse…</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item row="2" column="0">
+       <widget class="QLabel" name="outputFilePromptLabel">
+        <property name="text">
+         <string>CSV file in which to save the results:</string>
+        </property>
+       </widget>
+      </item>
+      <item row="2" column="1">
+       <layout class="QHBoxLayout" name="horizontalLayout_4" stretch="1,0">
+        <item>
+         <widget class="QLineEdit" name="outputFile"/>
+        </item>
+        <item>
+         <widget class="QToolButton" name="outputFileBrowseButton">
+          <property name="text">
+           <string>Browse…</string>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+     </layout>
+    </item>
+   </layout>
+  </widget>
+  <widget class="QWizardPage" name="timesPage">
+   <layout class="QHBoxLayout" name="horizontalLayout_3" stretch="1,0,1">
+    <item>
+     <spacer name="horizontalSpacer">
+      <property name="orientation">
+       <enum>Qt::Horizontal</enum>
+      </property>
+     </spacer>
+    </item>
+    <item>
+     <layout class="QVBoxLayout" name="verticalLayout_2" stretch="0,0,0,1">
+      <item>
+       <layout class="QFormLayout" name="formLayout_2">
+        <item row="0" column="0">
+         <widget class="QLabel" name="advanceTimePromptLabel">
+          <property name="text">
+           <string>Advance time:</string>
+          </property>
+         </widget>
+        </item>
+        <item row="0" column="1">
+         <widget class="QSpinBox" name="advanceTime">
+          <property name="suffix">
+           <string> ms</string>
+          </property>
+          <property name="minimum">
+           <number>100</number>
+          </property>
+          <property name="maximum">
+           <number>3000</number>
+          </property>
+          <property name="singleStep">
+           <number>100</number>
+          </property>
+         </widget>
+        </item>
+        <item row="1" column="0">
+         <widget class="QLabel" name="viewingTimePromptLabel">
+          <property name="text">
+           <string>Viewing time (t&lt;sub&gt;VIEW&lt;/sub&gt;):</string>
+          </property>
+         </widget>
+        </item>
+        <item row="1" column="1">
+         <widget class="QSpinBox" name="viewingTime">
+          <property name="specialValueText">
+           <string>no limit</string>
+          </property>
+          <property name="suffix">
+           <string> s</string>
+          </property>
+          <property name="minimum">
+           <number>0</number>
+          </property>
+          <property name="maximum">
+           <number>30</number>
+          </property>
+          <property name="value">
+           <number>4</number>
+          </property>
+         </widget>
+        </item>
+        <item row="2" column="0">
+         <widget class="QLabel" name="blankingTimePromptLabel">
+          <property name="text">
+           <string>Blanking time (t&lt;sub&gt;BLANK&lt;/sub&gt;):</string>
+          </property>
+         </widget>
+        </item>
+        <item row="2" column="1">
+         <widget class="QSpinBox" name="blankingTime">
+          <property name="suffix">
+           <string> ms</string>
+          </property>
+          <property name="minimum">
+           <number>50</number>
+          </property>
+          <property name="maximum">
+           <number>1000</number>
+          </property>
+          <property name="singleStep">
+           <number>50</number>
+          </property>
+          <property name="value">
+           <number>250</number>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+      <item>
+       <widget class="QGroupBox" name="grayFlickering">
+        <property name="title">
+         <string>Gray flickering</string>
+        </property>
+        <property name="checkable">
+         <bool>true</bool>
+        </property>
+        <property name="checked">
+         <bool>false</bool>
+        </property>
+        <layout class="QVBoxLayout" name="verticalLayout_4">
+         <item>
+          <layout class="QFormLayout" name="formLayout_4">
+           <item row="0" column="0">
+            <widget class="QLabel" name="grayFadingTimePromptLabel">
+             <property name="text">
+              <string>Fading time to and from gray:</string>
+             </property>
+            </widget>
+           </item>
+           <item row="0" column="1">
+            <widget class="QSpinBox" name="grayFadingTime">
+             <property name="suffix">
+              <string> ms</string>
+             </property>
+             <property name="maximum">
+              <number>1000</number>
+             </property>
+             <property name="singleStep">
+              <number>100</number>
+             </property>
+             <property name="value">
+              <number>100</number>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="0">
+            <widget class="QLabel" name="grayTimePromptLabel">
+             <property name="text">
+              <string>Time on gray:</string>
+             </property>
+            </widget>
+           </item>
+           <item row="1" column="1">
+            <widget class="QSpinBox" name="grayTime">
+             <property name="suffix">
+              <string> ms</string>
+             </property>
+             <property name="maximum">
+              <number>1000</number>
+             </property>
+             <property name="singleStep">
+              <number>100</number>
+             </property>
+             <property name="value">
+              <number>300</number>
+             </property>
+            </widget>
+           </item>
+          </layout>
+         </item>
+         <item>
+          <widget class="QLabel" name="totalGrayTimeLabel">
+           <property name="text">
+            <string>Total gray time: 500 ms</string>
+           </property>
+           <property name="textFormat">
+            <enum>Qt::RichText</enum>
+           </property>
+          </widget>
+         </item>
+        </layout>
+       </widget>
+      </item>
+      <item>
+       <widget class="QDialogButtonBox" name="timingButtonBox">
+        <property name="standardButtons">
+         <set>QDialogButtonBox::RestoreDefaults</set>
+        </property>
+        <property name="centerButtons">
+         <bool>true</bool>
+        </property>
+       </widget>
+      </item>
+      <item>
+       <spacer name="verticalSpacer">
+        <property name="orientation">
+         <enum>Qt::Vertical</enum>
+        </property>
+       </spacer>
+      </item>
+     </layout>
+    </item>
+    <item>
+     <spacer name="horizontalSpacer_2">
+      <property name="orientation">
+       <enum>Qt::Horizontal</enum>
+      </property>
+     </spacer>
+    </item>
+   </layout>
+  </widget>
+  <widget class="QWizardPage" name="spacingPage">
+   <layout class="QVBoxLayout" name="verticalLayout_3" stretch="1,0,0">
+    <item>
+     <widget class="jxl::SplitView" name="spacingDemo" native="true"/>
+    </item>
+    <item>
+     <spacer name="verticalSpacer_2">
+      <property name="orientation">
+       <enum>Qt::Vertical</enum>
+      </property>
+     </spacer>
+    </item>
+    <item>
+     <layout class="QFormLayout" name="formLayout_3">
+      <item row="0" column="0">
+       <widget class="QLabel" name="spacingPromptLabel">
+        <property name="text">
+         <string>Spacing between the images:</string>
+        </property>
+       </widget>
+      </item>
+      <item row="0" column="1">
+       <layout class="QHBoxLayout" name="horizontalLayout_5" stretch="1,0">
+        <item>
+         <widget class="QSlider" name="spacing">
+          <property name="minimum">
+           <number>1</number>
+          </property>
+          <property name="maximum">
+           <number>1000</number>
+          </property>
+          <property name="value">
+           <number>50</number>
+          </property>
+          <property name="orientation">
+           <enum>Qt::Horizontal</enum>
+          </property>
+         </widget>
+        </item>
+        <item>
+         <widget class="QSpinBox" name="spacingSpinBox">
+          <property name="suffix">
+           <string> px</string>
+          </property>
+          <property name="minimum">
+           <number>1</number>
+          </property>
+          <property name="maximum">
+           <number>1000</number>
+          </property>
+          <property name="value">
+           <number>50</number>
+          </property>
+         </widget>
+        </item>
+       </layout>
+      </item>
+     </layout>
+    </item>
+   </layout>
+  </widget>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>jxl::SplitView</class>
+   <extends>QWidget</extends>
+   <header>tools/flicker_test/split_view.h</header>
+   <container>1</container>
+   <slots>
+    <slot>setSpacing(int)</slot>
+   </slots>
+  </customwidget>
+ </customwidgets>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>spacing</sender>
+   <signal>valueChanged(int)</signal>
+   <receiver>spacingDemo</receiver>
+   <slot>setSpacing(int)</slot>
+  </connection>
+  <connection>
+   <sender>spacing</sender>
+   <signal>valueChanged(int)</signal>
+   <receiver>spacingSpinBox</receiver>
+   <slot>setValue(int)</slot>
+  </connection>
+  <connection>
+   <sender>spacingSpinBox</sender>
+   <signal>valueChanged(int)</signal>
+   <receiver>spacing</receiver>
+   <slot>setValue(int)</slot>
+  </connection>
+ </connections>
+</ui>
diff --git a/tools/flicker_test/split_view.cc b/tools/flicker_test/split_view.cc
new file mode 100644
index 000000000..3e8f4487e
--- /dev/null
+++ b/tools/flicker_test/split_view.cc
@@ -0,0 +1,176 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/flicker_test/split_view.h"
+
+#include <QMouseEvent>
+#include <QPainter>
+
+namespace jxl {
+
+SplitView::SplitView(QWidget* const parent)
+    : QWidget(parent), g_(std::random_device()()) {
+  blankingTimer_.setSingleShot(true);
+  blankingTimer_.setTimerType(Qt::PreciseTimer);
+  viewingTimer_.setSingleShot(true);
+  viewingTimer_.setTimerType(Qt::PreciseTimer);
+  flicker_.setLoopCount(-1);
+  connect(&blankingTimer_, &QTimer::timeout, this, &SplitView::startDisplaying);
+  connect(&flicker_, &QVariantAnimation::valueChanged, this, [&] {
+    if (gray_) {
+      update();
+    }
+  });
+  connect(&flicker_, &QAbstractAnimation::currentLoopChanged, [&] {
+    showingAltered_ = !showingAltered_;
+    update();
+  });
+  connect(&viewingTimer_, &QTimer::timeout, [&] {
+    flicker_.stop();
+    original_.fill(Qt::black);
+    altered_.fill(Qt::black);
+    update();
+  });
+}
+
+void SplitView::setOriginalImage(QImage image) {
+  original_ = QPixmap::fromImage(std::move(image));
+  updateMinimumSize();
+  update();
+}
+
+void SplitView::setAlteredImage(QImage image) {
+  altered_ = QPixmap::fromImage(std::move(image));
+  updateMinimumSize();
+  update();
+}
+
+void SplitView::setSpacing(int spacing) {
+  spacing_ = spacing;
+  updateMinimumSize();
+  update();
+}
+
+void SplitView::startTest(QString imageName, const int blankingTimeMSecs,
+                          const int viewingTimeSecs, const int advanceTimeMSecs,
+                          const bool gray, const int grayFadingTimeMSecs,
+                          const int grayTimeMSecs) {
+  imageName_ = std::move(imageName);
+  std::bernoulli_distribution bernoulli;
+  originalSide_ = bernoulli(g_) ? Side::kLeft : Side::kRight;
+  viewingTimer_.setInterval(1000 * viewingTimeSecs);
+
+  flicker_.setDuration(advanceTimeMSecs);
+  gray_ = gray;
+  QVariantAnimation::KeyValues keyValues;
+  if (gray_) {
+    keyValues << QVariantAnimation::KeyValue(0., 0.f)
+              << QVariantAnimation::KeyValue(
+                     static_cast<float>(grayFadingTimeMSecs) / advanceTimeMSecs,
+                     1.f)
+              << QVariantAnimation::KeyValue(
+                     static_cast<float>(advanceTimeMSecs - grayTimeMSecs -
+                                        grayFadingTimeMSecs) /
+                         advanceTimeMSecs,
+                     1.f)
+              << QVariantAnimation::KeyValue(
+                     static_cast<float>(advanceTimeMSecs - grayTimeMSecs) /
+                         advanceTimeMSecs,
+                     0.f)
+              << QVariantAnimation::KeyValue(1.f, 0.f);
+  } else {
+    keyValues << QVariantAnimation::KeyValue(0., 1.f)
+              << QVariantAnimation::KeyValue(1., 1.f);
+  }
+  flicker_.setKeyValues(keyValues);
+
+  state_ = State::kBlanking;
+  blankingTimer_.start(blankingTimeMSecs);
+}
+
+void SplitView::mousePressEvent(QMouseEvent* const event) {
+  if (state_ != State::kDisplaying) return;
+
+  if (leftRect_.contains(event->pos())) {
+    clicking_ = true;
+    clickedSide_ = Side::kLeft;
+  } else if (rightRect_.contains(event->pos())) {
+    clicking_ = true;
+    clickedSide_ = Side::kRight;
+  }
+}
+
+void SplitView::mouseReleaseEvent(QMouseEvent* const event) {
+  if (!clicking_) return;
+  clicking_ = false;
+
+  const int clickDelayMSecs = viewingStartTime_.elapsed();
+
+  if ((clickedSide_ == Side::kLeft && !leftRect_.contains(event->pos())) ||
+      (clickedSide_ == Side::kRight && !rightRect_.contains(event->pos()))) {
+    return;
+  }
+
+  flicker_.stop();
+  viewingTimer_.stop();
+  state_ = State::kBlanking;
+  update();
+
+  emit testResult(imageName_, originalSide_, clickedSide_, clickDelayMSecs);
+}
+
+void SplitView::paintEvent(QPaintEvent* const event) {
+  QPainter painter(this);
+  painter.fillRect(rect(), QColor(119, 119, 119));
+
+  if (state_ == State::kBlanking) return;
+
+  if (gray_ && flicker_.state() == QAbstractAnimation::Running) {
+    painter.setOpacity(flicker_.currentValue().toFloat());
+  }
+
+  const auto imageForSide = [&](const Side side) {
+    if (side == originalSide_) return &original_;
+    return showingAltered_ ? &altered_ : &original_;
+  };
+
+  QPixmap* const leftImage = imageForSide(Side::kLeft);
+  QPixmap* const rightImage = imageForSide(Side::kRight);
+
+  leftRect_ = leftImage->rect();
+  leftRect_.moveCenter(rect().center());
+  leftRect_.moveRight(rect().center().x() - spacing_ / 2 - spacing_ % 2);
+  painter.drawPixmap(leftRect_, *leftImage);
+
+  rightRect_ = rightImage->rect();
+  rightRect_.moveCenter(rect().center());
+  rightRect_.moveLeft(rect().center().x() + 1 + spacing_ / 2);
+  painter.drawPixmap(rightRect_, *rightImage);
+}
+
+void SplitView::startDisplaying() {
+  state_ = State::kDisplaying;
+  flicker_.start();
+  viewingStartTime_.start();
+  if (viewingTimer_.interval() > 0) {
+    viewingTimer_.start();
+  }
+}
+
+void SplitView::updateMinimumSize() {
+  setMinimumWidth(2 * std::max(original_.width(), altered_.width()) + spacing_);
+  setMinimumHeight(std::max(original_.height(), altered_.height()));
+}
+
+}  // namespace jxl
diff --git a/tools/flicker_test/split_view.h b/tools/flicker_test/split_view.h
new file mode 100644
index 000000000..953761e62
--- /dev/null
+++ b/tools/flicker_test/split_view.h
@@ -0,0 +1,93 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_FLICKER_TEST_SPLIT_VIEW_H_
+#define TOOLS_FLICKER_TEST_SPLIT_VIEW_H_
+
+#include <QImage>
+#include <QPixmap>
+#include <QTime>
+#include <QTimer>
+#include <QVariantAnimation>
+#include <QWidget>
+#include <random>
+
+namespace jxl {
+
+class SplitView : public QWidget {
+  Q_OBJECT
+
+ public:
+  enum class Side {
+    kLeft,
+    kRight,
+  };
+  Q_ENUM(Side)
+
+  explicit SplitView(QWidget* parent = nullptr);
+  ~SplitView() override = default;
+
+  void setOriginalImage(QImage image);
+  void setAlteredImage(QImage image);
+
+ signals:
+  void testResult(const QString& imageName, Side flickeringSide,
+                  Side clickedSide, int clickDelayMSecs);
+
+ public slots:
+  void setSpacing(int spacing);
+  void startTest(QString imageName, int blankingTimeMSecs, int viewingTimeSecs,
+                 int advanceTimeMSecs, bool gray, int grayFadingTimeMSecs,
+                 int grayTimeMSecs);
+
+ protected:
+  void mousePressEvent(QMouseEvent* event) override;
+  void mouseReleaseEvent(QMouseEvent* event) override;
+  void paintEvent(QPaintEvent* event) override;
+
+ private slots:
+  void startDisplaying();
+
+ private:
+  enum class State {
+    kBlanking,
+    kDisplaying,
+  };
+
+  void updateMinimumSize();
+
+  int spacing_ = 50;
+
+  std::mt19937 g_;
+
+  QString imageName_;
+  QPixmap original_, altered_;
+  Side originalSide_;
+  bool clicking_ = false;
+  Side clickedSide_;
+  QRect leftRect_, rightRect_;
+  State state_ = State::kDisplaying;
+  bool gray_ = false;
+  QTimer blankingTimer_;
+  QTimer viewingTimer_;
+  // Throughout each cycle, animates the opacity of the image being displayed
+  // between 0 and 1 if fading to gray is enabled.
+  QVariantAnimation flicker_;
+  bool showingAltered_ = true;
+  QTime viewingStartTime_;
+};
+
+}  // namespace jxl
+
+#endif  // TOOLS_FLICKER_TEST_SPLIT_VIEW_H_
diff --git a/tools/flicker_test/test_window.cc b/tools/flicker_test/test_window.cc
new file mode 100644
index 000000000..bbfc9b54a
--- /dev/null
+++ b/tools/flicker_test/test_window.cc
@@ -0,0 +1,184 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/flicker_test/test_window.h"
+
+#include <algorithm>
+#include <random>
+
+#include <QDir>
+#include <QMessageBox>
+#include <QSet>
+#include <QX11Info>
+
+namespace jxl {
+
+FlickerTestWindow::FlickerTestWindow(FlickerTestParameters parameters,
+                                     QWidget* const parent)
+    : QMainWindow(parent),
+      monitorProfile_(
+          GetMonitorIccProfile(QX11Info::connection(), QX11Info::appScreen())),
+      parameters_(std::move(parameters)),
+      originalFolder_(parameters_.originalFolder, "*.png"),
+      alteredFolder_(parameters_.alteredFolder, "*.png"),
+      outputFile_(parameters_.outputFile) {
+  ui_.setupUi(this);
+  ui_.splitView->setSpacing(parameters_.spacing);
+  ui_.endLabel->setText(
+      tr("The test is complete and the results have been saved to \"%1\".")
+          .arg(parameters_.outputFile));
+  connect(ui_.startButton, &QAbstractButton::clicked, [&] {
+    ui_.stackedView->setCurrentWidget(ui_.splitView);
+    nextImage();
+  });
+  connect(ui_.splitView, &SplitView::testResult, this,
+          &FlickerTestWindow::processTestResult);
+
+  if (!outputFile_.open(QIODevice::WriteOnly)) {
+    QMessageBox messageBox;
+    messageBox.setIcon(QMessageBox::Critical);
+    messageBox.setStandardButtons(QMessageBox::Close);
+    messageBox.setWindowTitle(tr("Failed to open output file"));
+    messageBox.setInformativeText(
+        tr("Could not open \"%1\" for writing.").arg(outputFile_.fileName()));
+    messageBox.exec();
+    proceed_ = false;
+    return;
+  }
+  outputStream_.setDevice(&outputFile_);
+  outputStream_ << "image name,original side,clicked side,click delay (ms)\n";
+
+  if (monitorProfile_.empty()) {
+    QMessageBox messageBox;
+    messageBox.setIcon(QMessageBox::Warning);
+    messageBox.setStandardButtons(QMessageBox::Ok);
+    messageBox.setWindowTitle(tr("No monitor profile found"));
+    messageBox.setText(
+        tr("No ICC profile appears to be associated with the display. It will "
+           "be assumed to match sRGB."));
+    messageBox.exec();
+  }
+
+  originalFolder_.setFilter(QDir::Files);
+  alteredFolder_.setFilter(QDir::Files);
+
+  auto originalImages = QSet<QString>::fromList(originalFolder_.entryList());
+  auto alteredImages = QSet<QString>::fromList(alteredFolder_.entryList());
+
+  auto onlyOriginal = originalImages - alteredImages,
+       onlyAltered = alteredImages - originalImages;
+  if (!onlyOriginal.isEmpty() || !onlyAltered.isEmpty()) {
+    QMessageBox messageBox;
+    messageBox.setIcon(QMessageBox::Warning);
+    messageBox.setStandardButtons(QMessageBox::Ok | QMessageBox::Cancel);
+    messageBox.setWindowTitle(tr("Image set mismatch"));
+    messageBox.setText(
+        tr("A mismatch has been detected between the original and altered "
+           "images."));
+    messageBox.setInformativeText(tr("Proceed with the test?"));
+    QStringList detailedTextParagraphs;
+    const QString itemFormat = tr("— %1\n");
+    if (!onlyOriginal.isEmpty()) {
+      QString originalList;
+      for (const QString& original : onlyOriginal) {
+        originalList += itemFormat.arg(original);
+      }
+      detailedTextParagraphs << tr("The following images were only found in "
+                                   "the originals folder:\n%1")
+                                    .arg(originalList);
+    }
+    if (!onlyAltered.isEmpty()) {
+      QString alteredList;
+      for (const QString& altered : onlyAltered) {
+        alteredList += itemFormat.arg(altered);
+      }
+      detailedTextParagraphs << tr("The following images were only found in "
+                                   "the altered images folder:\n%1")
+                                    .arg(alteredList);
+    }
+    messageBox.setDetailedText(detailedTextParagraphs.join("\n\n"));
+    if (messageBox.exec() == QMessageBox::Cancel) {
+      proceed_ = false;
+      return;
+    }
+  }
+
+  remainingImages_ = originalImages.intersect(alteredImages).toList();
+  std::random_device rd;
+  std::mt19937 g(rd());
+  std::shuffle(remainingImages_.begin(), remainingImages_.end(), g);
+}
+
+void FlickerTestWindow::processTestResult(const QString& imageName,
+                                          const SplitView::Side originalSide,
+                                          const SplitView::Side clickedSide,
+                                          const int clickDelayMSecs) {
+  const auto sideToString = [](const SplitView::Side side) {
+    switch (side) {
+      case SplitView::Side::kLeft:
+        return "left";
+
+      case SplitView::Side::kRight:
+        return "right";
+    }
+    return "unknown";
+  };
+  outputStream_ << imageName << "," << sideToString(originalSide) << ","
+                << sideToString(clickedSide) << "," << clickDelayMSecs << "\n";
+
+  nextImage();
+}
+
+void FlickerTestWindow::nextImage() {
+  if (remainingImages_.empty()) {
+    outputStream_.flush();
+    ui_.stackedView->setCurrentWidget(ui_.finalPage);
+    return;
+  }
+  const QString image = remainingImages_.takeFirst();
+retry:
+  QImage originalImage =
+      loadImage(originalFolder_.absoluteFilePath(image), monitorProfile_);
+  QImage alteredImage =
+      loadImage(alteredFolder_.absoluteFilePath(image), monitorProfile_);
+  if (originalImage.isNull() || alteredImage.isNull()) {
+    QMessageBox messageBox(this);
+    messageBox.setIcon(QMessageBox::Warning);
+    messageBox.setStandardButtons(QMessageBox::Retry | QMessageBox::Ignore |
+                                  QMessageBox::Abort);
+    messageBox.setWindowTitle(tr("Failed to load image"));
+    messageBox.setText(tr("Could not load image \"%1\".").arg(image));
+    switch (messageBox.exec()) {
+      case QMessageBox::Retry:
+        goto retry;
+
+      case QMessageBox::Ignore:
+        outputStream_ << image << ",,,\n";
+        return nextImage();
+
+      case QMessageBox::Abort:
+        ui_.stackedView->setCurrentWidget(ui_.finalPage);
+        return;
+    }
+  }
+
+  ui_.splitView->setOriginalImage(std::move(originalImage));
+  ui_.splitView->setAlteredImage(std::move(alteredImage));
+  ui_.splitView->startTest(
+      image, parameters_.blankingTimeMSecs, parameters_.viewingTimeSecs,
+      parameters_.advanceTimeMSecs, parameters_.gray,
+      parameters_.grayFadingTimeMSecs, parameters_.grayTimeMSecs);
+}
+
+}  // namespace jxl
diff --git a/tools/flicker_test/test_window.h b/tools/flicker_test/test_window.h
new file mode 100644
index 000000000..ee87a0b0c
--- /dev/null
+++ b/tools/flicker_test/test_window.h
@@ -0,0 +1,60 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_FLICKER_TEST_TEST_WINDOW_H_
+#define TOOLS_FLICKER_TEST_TEST_WINDOW_H_
+
+#include <QDir>
+#include <QMainWindow>
+#include <QStringList>
+#include <QTextStream>
+
+#include "jxl/base/padded_bytes.h"
+#include "tools/comparison_viewer/image_loading.h"
+#include "tools/flicker_test/parameters.h"
+#include "tools/flicker_test/ui_test_window.h"
+#include "tools/x11/icc.h"
+
+namespace jxl {
+
+class FlickerTestWindow : public QMainWindow {
+  Q_OBJECT
+
+ public:
+  explicit FlickerTestWindow(FlickerTestParameters parameters,
+                             QWidget* parent = nullptr);
+  ~FlickerTestWindow() override = default;
+
+  bool proceedWithTest() const { return proceed_; }
+
+ private slots:
+  void processTestResult(const QString& imageName, SplitView::Side originalSide,
+                         SplitView::Side clickedSide, int clickDelayMSecs);
+
+ private:
+  void nextImage();
+
+  Ui::FlickerTestWindow ui_;
+  bool proceed_ = true;
+  PaddedBytes monitorProfile_;
+  FlickerTestParameters parameters_;
+  QDir originalFolder_, alteredFolder_;
+  QFile outputFile_;
+  QTextStream outputStream_;
+  QStringList remainingImages_;
+};
+
+}  // namespace jxl
+
+#endif  // TOOLS_FLICKER_TEST_TEST_WINDOW_H_
diff --git a/tools/flicker_test/test_window.ui b/tools/flicker_test/test_window.ui
new file mode 100644
index 000000000..69587164a
--- /dev/null
+++ b/tools/flicker_test/test_window.ui
@@ -0,0 +1,116 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <comment>
+  Copyright (c) the JPEG XL Project
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+ </comment>
+ <class>FlickerTestWindow</class>
+ <widget class="QMainWindow" name="FlickerTestWindow">
+  <property name="windowTitle">
+   <string>Flicker test</string>
+  </property>
+  <widget class="QWidget" name="centralwidget">
+   <layout class="QVBoxLayout" name="verticalLayout_2">
+    <item>
+     <widget class="QStackedWidget" name="stackedView">
+      <widget class="QWidget" name="startPage">
+       <layout class="QVBoxLayout" name="verticalLayout" stretch="1,0,1">
+        <item>
+         <spacer name="verticalSpacer">
+          <property name="orientation">
+           <enum>Qt::Vertical</enum>
+          </property>
+         </spacer>
+        </item>
+        <item>
+         <layout class="QHBoxLayout" name="horizontalLayout" stretch="1,0,1">
+          <item>
+           <spacer name="spacer_2">
+            <property name="orientation">
+             <enum>Qt::Horizontal</enum>
+            </property>
+           </spacer>
+          </item>
+          <item>
+           <widget class="QPushButton" name="startButton">
+            <property name="text">
+             <string>Start</string>
+            </property>
+           </widget>
+          </item>
+          <item>
+           <spacer name="spacer">
+            <property name="orientation">
+             <enum>Qt::Horizontal</enum>
+            </property>
+           </spacer>
+          </item>
+         </layout>
+        </item>
+        <item>
+         <spacer name="verticalSpacer_2">
+          <property name="orientation">
+           <enum>Qt::Vertical</enum>
+          </property>
+          <property name="sizeHint" stdset="0">
+           <size>
+            <width>0</width>
+            <height>0</height>
+           </size>
+          </property>
+         </spacer>
+        </item>
+       </layout>
+      </widget>
+      <widget class="jxl::SplitView" name="splitView"/>
+      <widget class="QWidget" name="finalPage">
+       <layout class="QVBoxLayout" name="verticalLayout_3">
+        <item>
+         <layout class="QHBoxLayout" name="horizontalLayout_2" stretch="1,0,1">
+          <item>
+           <spacer name="horizontalSpacer">
+            <property name="orientation">
+             <enum>Qt::Horizontal</enum>
+            </property>
+           </spacer>
+          </item>
+          <item>
+           <widget class="QLabel" name="endLabel">
+            <property name="text">
+             <string/>
+            </property>
+            <property name="alignment">
+             <set>Qt::AlignJustify|Qt::AlignVCenter</set>
+            </property>
+           </widget>
+          </item>
+          <item>
+           <spacer name="horizontalSpacer_2">
+            <property name="orientation">
+             <enum>Qt::Horizontal</enum>
+            </property>
+           </spacer>
+          </item>
+         </layout>
+        </item>
+       </layout>
+      </widget>
+     </widget>
+    </item>
+   </layout>
+  </widget>
+ </widget>
+ <customwidgets>
+  <customwidget>
+   <class>jxl::SplitView</class>
+   <extends>QWidget</extends>
+   <header>tools/flicker_test/split_view.h</header>
+   <container>1</container>
+  </customwidget>
+ </customwidgets>
+ <resources/>
+ <connections/>
+</ui>
diff --git a/tools/icc_codec_fuzzer.cc b/tools/icc_codec_fuzzer.cc
new file mode 100644
index 000000000..ce61a42f2
--- /dev/null
+++ b/tools/icc_codec_fuzzer.cc
@@ -0,0 +1,67 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "jxl/icc_codec.h"
+
+namespace jxl {
+
+int TestOneInput(const uint8_t* data, size_t size) {
+#if defined(JXL_ICC_FUZZER_ONLY_WRITE)
+  bool read = false;
+#elif defined(JXL_ICC_FUZZER_ONLY_READ)
+  bool read = true;
+#else
+  // Decide whether to test the reader or the writer (both use parsing)
+  if (!size) return 0;
+  bool read = data[0] == 0;
+  data++;
+  size--;
+#endif
+
+#ifdef JXL_ICC_FUZZER_SLOW_TEST
+  // Including Brotli compression. Brotli is already fuzzed separately, so it is
+  // better to disable JXL_ICC_FUZZER_SLOW_TEST to focus on the ICC parsing.
+  if (read) {
+    // Reading parses the compressed format.
+    BitReader br(Span<const uint8_t>(data, size));
+    PaddedBytes result;
+    (void)ReadICC(&br, &result);
+    (void)br.Close();
+  } else {
+    // Writing parses the original ICC profile.
+    PaddedBytes icc;
+    icc.assign(data, data + size);
+    BitWriter writer;
+    AuxOut aux;
+    (void)WriteICC(icc, &writer, 0, &aux);
+  }
+#else  // JXL_ICC_FUZZER_SLOW_TEST
+  if (read) {
+    // Reading (unpredicting) parses the compressed format.
+    PaddedBytes result;
+    (void)UnpredictICC(data, size, &result);
+  } else {
+    // Writing (predicting) parses the original ICC profile.
+    PaddedBytes result;
+    (void)PredictICC(data, size, &result);
+  }
+#endif  // JXL_ICC_FUZZER_SLOW_TEST
+  return 0;
+}
+
+}  // namespace jxl
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  return jxl::TestOneInput(data, size);
+}
diff --git a/tools/libjpegxl_test.c b/tools/libjpegxl_test.c
new file mode 100644
index 000000000..1575169ea
--- /dev/null
+++ b/tools/libjpegxl_test.c
@@ -0,0 +1,26 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Program to test that we can link against the public API of libjpegxl from C.
+// This links against the shared libjpegxl library which doesn't expose any of
+// the internals of the jxl namespace.
+
+#include "jpegxl/decode.h"
+
+int main() {
+  if (!JpegxlDecoderVersion()) return 1;
+  JpegxlDecoder* dec = JpegxlDecoderCreate(NULL);
+  if (!dec) return 1;
+  JpegxlDecoderDestroy(dec);
+}
diff --git a/tools/optimizer/simplex_fork.py b/tools/optimizer/simplex_fork.py
new file mode 100755
index 000000000..6f55d9a8e
--- /dev/null
+++ b/tools/optimizer/simplex_fork.py
@@ -0,0 +1,311 @@
+#!/usr/bin/python
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Implementation of simplex search for an external process.
+
+The external process gets the input vector through environment variables.
+Input of vector as setenv("VAR%dimension", val)
+Getting the optimized function with regexp match from stdout
+of the forked process.
+
+https://en.wikipedia.org/wiki/Nelder%E2%80%93Mead_method
+
+start as ./simplex_fork.py binary dimensions amount
+"""
+
+import copy
+import os
+import random
+import re
+import subprocess
+import sys
+
+
+def Midpoint(simplex):
+  """Nelder-Mead-like simplex midpoint calculation."""
+  simplex.sort()
+  dim = len(simplex) - 1
+  retval = [None] + [0.0] * dim
+  for i in range(1, dim + 1):
+    for k in range(dim):
+      retval[i] += simplex[k][i]
+    retval[i] /= dim
+  return retval
+
+
+def Subtract(a, b):
+  """Vector arithmetic, with [0] being ignored."""
+  return [None if k == 0 else a[k] - b[k] for k in range(len(a))]
+
+def Add(a, b):
+  """Vector arithmetic, with [0] being ignored."""
+  return [None if k == 0 else a[k] + b[k] for k in range(len(a))]
+
+def Average(a, b):
+  """Vector arithmetic, with [0] being ignored."""
+  return [None if k == 0 else 0.5 * (a[k] + b[k]) for k in range(len(a))]
+
+
+eval_hash = {}
+
+def EvalCacheForget():
+  global eval_hash
+  eval_hash = {}
+
+def RandomizedJxlCodecs():
+  retval = []
+  minval = 0.6
+  maxval = 5.0
+  rangeval = maxval/minval
+  steps = 5
+  for i in range(steps):
+    mul = minval * rangeval**(float(i)/(steps - 1))
+    mul *= 0.99 + 0.05 * random.random()
+    retval.append("jxl:fast:d%.3f" % mul)
+  steps = 0
+  for i in range(steps):
+    mul = minval * rangeval**(float(i)/(steps - 1))
+    mul *= 0.99 + 0.05 * random.random()
+    retval.append("jxl:d%.3f" % mul)
+  return ",".join(retval)
+
+g_codecs = RandomizedJxlCodecs()
+
+def Eval(vec, binary_name, cached=True):
+  """Evaluates the objective function by forking a process.
+
+  Args:
+    vec: [0] will be set to the objective function, [1:] will
+      contain the vector position for the objective function.
+    binary_name: the name of the binary that evaluates the value.
+  """
+  global eval_hash
+  global g_codecs
+  key = ""
+  # os.environ["BUTTERAUGLI_OPTIMIZE"] = "1"
+  for i in range(300):
+    os.environ["VAR%d" % i] = "0"
+  for i in range(len(vec) - 1):
+    os.environ["VAR%d" % i] = str(vec[i + 1])
+    key += str(vec[i + 1]) + ":"
+  if cached and (eval_hash.has_key(key)):
+    vec[0] = eval_hash[key]
+    return
+
+  process = subprocess.Popen(
+      (binary_name,
+       '--input',
+       '/usr/local/google/home/jyrki/jpeg_xl/256/*.png',
+       '--error_pnorm=7',
+       '--more_columns',
+       '--adaptive_reconstruction=0',
+       '--dots=0',
+       '--noprofiler',
+       '--codec', g_codecs),
+      stdout=subprocess.PIPE,
+      stderr=subprocess.PIPE,
+      env=dict(os.environ))
+
+  # process.wait()
+  found_score = False
+  vec[0] = 1.0
+  dct2 = 0.0
+  dct4 = 0.0
+  dct16 = 0.0
+  dct32 = 0.0
+  n = 0
+  for line in process.communicate(input=None)[0].splitlines():
+    print "BE", line
+    sys.stdout.flush()
+    if line[0:3] == "jxl":
+      bpp = line.split()[3]
+      dist_pnorm = line.split()[8]
+      dct2str = line.split()[11]
+      dct4str = line.split()[12]
+      dct4x8str = line.split()[13]
+      dct8str = line.split()[14]
+      dct8x16str = line.split()[15]
+      dct16str = line.split()[16]
+      dct16x32str = line.split()[17]
+      dct32str = line.split()[18]
+      vec[0] *= float(dist_pnorm) * float(bpp) / 16.0;
+      dct2 += float(dct2str)
+      dct4 += float(dct4str)
+      dct16 += float(dct16str)
+      dct32 += float(dct32str)
+      n += 1
+      found_score = True
+      distance = float(line.split()[0].split('d')[-1])
+      faultybpp = 1.0 + 0.23 * ((float(bpp) * distance ** 0.673) - 1.8926851723) ** 2
+      # vec[0] *= faultybpp
+
+  """
+  # favor small changes
+  for k in vec[1:]:
+    vec[0] *= 1 + 1e-9 * k * k
+
+  dct2 = dct2 / (n + 1e-9)
+  dct4 = dct4 / (n + 1e-9)
+  dct16 = dct16 / (n + 1e-9)
+  dct32 = dct32 / (n + 1e-9)
+  dct16 += 0.1 * dct32
+  dct4 += 0.1 * dct2
+  print "dct2/4/16/32", dct2, dct4, dct16, dct32, vec[0]
+
+  dct2limit = 0.02
+  if dct2 < dct2limit:
+     print "low dct2", dct2, vec[0]
+     vec[0] *= (1.0 + 50.0 * (dct2limit - dct2) ** 2)**n
+     print "corrected dct2", dct2, vec[0]
+
+  dct4limit = 0.025
+  if dct4 < dct4limit:
+     print "low dct4", dct4, vec[0]
+     vec[0] *= (1.0 + 15.0 * (dct4limit - dct4) ** 2)**n
+     print "corrected dct4", dct4, vec[0]
+
+  dct16limit = 0.02
+  if dct16 < dct16limit:
+     print "low dct16", dct16, vec[0]
+     vec[0] *= (1.0 + 5.0 * (dct16limit - dct16) ** 2)**n
+     print "corrected dct16", dct16, vec[0]
+  dct32limit = 0.02
+  if dct32 < dct32limit:
+     print "low dct32", dct32, vec[0]
+     vec[0] *= (1.0 + 5.0 * (dct32limit - dct32) ** 2)**n
+     print "corrected dct32", dct32, vec[0]
+  """
+  print "eval: ", vec
+  if (vec[0] <= 0.0):
+    vec[0] = 1e30
+  if found_score:
+    eval_hash[key] = vec[0]
+    return
+  vec[0] = 1e33
+  return
+  # sys.exit("awful things happened")
+
+def Reflect(simplex, binary):
+  """Main iteration step of Nelder-Mead optimization. Modifies `simplex`."""
+  simplex.sort()
+  last = simplex[-1]
+  mid = Midpoint(simplex)
+  diff = Subtract(mid, last)
+  mirrored = Add(mid, diff)
+  Eval(mirrored, binary)
+  if mirrored[0] > simplex[-2][0]:
+    print "\nStill worst\n\n"
+    # Still the worst, shrink towards the best.
+    shrinking = Average(simplex[-1], simplex[0])
+    Eval(shrinking, binary)
+    print "\nshrinking...\n\n"
+    simplex[-1] = shrinking
+    return
+  if mirrored[0] < simplex[0][0]:
+    # new best
+    print "\nNew Best\n\n"
+    even_further = Add(mirrored, diff)
+    Eval(even_further, binary)
+    if even_further[0] < mirrored[0]:
+      print "\nEven Further\n\n"
+      mirrored = even_further
+    simplex[-1] = mirrored
+    # try to extend
+    return
+  else:
+    # not a best, not a worst point
+    simplex[-1] = mirrored
+
+
+def OneDimensionalSearch(simplex, shrink, index):
+  # last appended was better than the best so far, try to replace it
+  last_attempt = simplex[-1][:]
+  best = simplex[0]
+  if last_attempt[0] < best[0]:
+    # try expansion of the amount
+    diff = simplex[-1][index] - simplex[0][index]
+    simplex[-1][index] = simplex[0][index] + shrink * diff
+    Eval(simplex[-1], g_binary)
+    if simplex[-1][0] < last_attempt[0]:
+      # it got better
+      return True
+  elif last_attempt[0] >= 0:
+    diff = simplex[-1][index] - simplex[0][index]
+    simplex[-1][index] = simplex[0][index] - diff
+    Eval(simplex[-1], g_binary)
+    if simplex[-1][0] < last_attempt[0]:
+      # it got better
+      return True
+  simplex[-1] = last_attempt
+  return False
+
+def InitialSimplex(vec, dim, amount):
+  """Initialize the simplex at origin."""
+  EvalCacheForget()
+  best = vec[:]
+  Eval(best, g_binary)
+  retval = [best]
+  comp_order = list(range(1, dim + 1))
+  random.shuffle(comp_order)
+
+  for i in range(dim):
+    index = comp_order[i]
+    best = retval[0][:]
+    best[index] += amount
+    Eval(best, g_binary)
+    retval.append(best)
+    do_shrink = True
+    while OneDimensionalSearch(retval, 2.0, index):
+      print "OneDimensionalSearch-Grow"
+    while OneDimensionalSearch(retval, 1.1, index):
+      print "OneDimensionalSearch-SlowGrow"
+      do_shrink = False
+    if do_shrink:
+      while OneDimensionalSearch(retval, 0.9, index):
+        print "OneDimensionalSearch-SlowShrinking"
+    retval.sort()
+  return retval
+
+
+if len(sys.argv) != 4:
+  print "usage: ", sys.argv[0], "binary-name number-of-dimensions simplex-size"
+  exit(1)
+
+g_dim = int(sys.argv[2])
+g_amount = float(sys.argv[3])
+g_binary = sys.argv[1]
+g_simplex = InitialSimplex([None] + [0.0] * g_dim,
+                           g_dim, 7.0 * g_amount)
+best = g_simplex[0][:]
+g_codecs = RandomizedJxlCodecs()
+g_simplex = InitialSimplex(best, g_dim, g_amount * 2.47)
+best = g_simplex[0][:]
+g_simplex = InitialSimplex(best, g_dim, g_amount)
+best = g_simplex[0][:]
+g_simplex = InitialSimplex(best, g_dim, g_amount * 0.33)
+best = g_simplex[0][:]
+
+for restarts in range(99999):
+  for ii in range(g_dim * 2):
+    g_simplex.sort()
+    print "reflect", ii, g_simplex[0]
+    Reflect(g_simplex, g_binary)
+
+  mulli = 0.1 + 15 * random.random()**2.0
+  g_codecs = RandomizedJxlCodecs()
+  print "\n\n\nRestart", restarts, "mulli", mulli
+  g_simplex.sort()
+  best = g_simplex[0][:]
+  g_simplex = InitialSimplex(best, g_dim, g_amount * mulli)
diff --git a/tools/progressive_saliency.conf b/tools/progressive_saliency.conf
new file mode 100644
index 000000000..987651a43
--- /dev/null
+++ b/tools/progressive_saliency.conf
@@ -0,0 +1,32 @@
+# Configuration parameters for progressive-saliency encoding.
+# (They are too many and too complex for command-line arguments.)
+
+# The total number of seconds for the simulated progressive-loading animation.
+simulated_progressive_loading_time_sec: 8.0
+
+# Time delay after the last progressive-loading step before the animation loops.
+simulated_progressive_loading_delay_until_looparound_sec: 10.0
+
+# The JPEG-XL encoding command, as one would pass it to the shell,
+# but with parameters ${HEATMAP_ARG}, ${INPUT}, ${OUTPUT}, ${STEPS}.
+jpegxl_encoder: cjpegxl pik ${INPUT} ${OUTPUT} --progressive --saliency_num_progressive_steps ${STEPS} --fast --saliency_threshold 0.8 ${HEATMAP_ARG}
+
+# The JPEG-XL encoding command, as one would pass it to the shell,
+# but with parameters ${INPUT}, ${OUTPUT}.
+jpegxl_decoder: djpegxl ${INPUT} ${OUTPUT}
+
+# The shell command to use for heatmap-generation.
+# This must adhere the calling conventions stated below.
+#
+# When called as:
+#   {heatmap_command} {blocksize} {input_image_filename} {coarse_grained_input_filename} {output_heatmap_filename}
+# This must produce: {output_heatmap_filename} in a format that is readable by the JPEG-XL encoder, and provides one
+# grayscale value per image-block which encodes saliency - ideally in the form of block-percentiles.
+heatmap_command: ml_get_high_level_saliency
+
+# How much to blur each of the four progressive stages.
+blurring: 16x4 16x1.5 0x0 0x0
+
+# Whether to keep tempfiles.
+# Temporary files will be named by appending suffixes to the desired final output filename.
+keep_tempfiles: True
diff --git a/tools/rans_fuzzer.cc b/tools/rans_fuzzer.cc
new file mode 100644
index 000000000..697fce23d
--- /dev/null
+++ b/tools/rans_fuzzer.cc
@@ -0,0 +1,54 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <hwy/static_targets.h>
+
+#include "jxl/dec_ans.h"
+#include "jxl/entropy_coder.h"
+
+namespace jxl {
+
+HWY_ATTR int TestOneInput(const uint8_t* data, size_t size) {
+  if (size < 2) return 0;
+  size_t numContexts = data[0] * 256 * data[1] + 1;
+  data += 2;
+  size -= 2;
+
+  std::vector<uint8_t> context_map;
+  BitReader br(Span<const uint8_t>(data, size));
+  ANSCode code;
+  JXL_RETURN_IF_ERROR(DecodeHistograms(&br, numContexts, ANS_MAX_ALPHA_SIZE,
+                                       &code, &context_map));
+  ANSSymbolReader ansreader(&code, &br);
+
+  // Limit the maximum amount of reads to avoid (valid) infinite loops.
+  const size_t maxreads = size * 8;
+  size_t numreads = 0;
+  int context = 0;
+  while (DivCeil(br.TotalBitsConsumed(), kBitsPerByte) < size &&
+         numreads <= maxreads) {
+    int code = ReadHybridUint(context, &br, &ansreader, context_map);
+    context = code % numContexts;
+    numreads++;
+  }
+  (void)br.Close();
+
+  return 0;
+}
+
+}  // namespace jxl
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  return jxl::TestOneInput(data, size);
+}
diff --git a/tools/set_from_bytes_fuzzer.cc b/tools/set_from_bytes_fuzzer.cc
new file mode 100644
index 000000000..9b371ec20
--- /dev/null
+++ b/tools/set_from_bytes_fuzzer.cc
@@ -0,0 +1,42 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/span.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/extras/codec.h"
+
+namespace jxl {
+
+int TestOneInput(const uint8_t* data, size_t size) {
+  CodecInOut io;
+  io.dec_max_xsize = 1u << 16;
+  io.dec_max_ysize = 1u << 16;
+  io.dec_max_pixels = 1u << 22;
+  ThreadPoolInternal pool(0);
+
+  (void)SetFromBytes(Span<const uint8_t>(data, size), &io, &pool);
+
+  return 0;
+}
+
+}  // namespace jxl
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) {
+  return jxl::TestOneInput(data, size);
+}
diff --git a/tools/speed_stats.cc b/tools/speed_stats.cc
new file mode 100644
index 000000000..44fbb6777
--- /dev/null
+++ b/tools/speed_stats.cc
@@ -0,0 +1,99 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/speed_stats.h"
+
+#include <math.h>
+#include <stddef.h>
+#include <stdio.h>
+
+#include <algorithm>
+
+#include "jxl/base/robust_statistics.h"
+
+namespace jpegxl {
+namespace tools {
+
+void SpeedStats::NotifyElapsed(double elapsed_seconds) {
+  JXL_ASSERT(elapsed_seconds > 0.0);
+  elapsed_.push_back(elapsed_seconds);
+}
+
+jxl::Status SpeedStats::GetSummary(SpeedStats::Summary* s) {
+  if (elapsed_.empty()) return JXL_FAILURE("Didn't call NotifyElapsed");
+
+  s->min = *std::min_element(elapsed_.begin(), elapsed_.end());
+  s->max = *std::max_element(elapsed_.begin(), elapsed_.end());
+
+  // Single rep
+  if (elapsed_.size() == 1) {
+    s->central_tendency = elapsed_[0];
+    s->variability = 0.0;
+    s->type = "";
+    return true;
+  }
+
+  // Two: skip first (noisier)
+  if (elapsed_.size() == 2) {
+    s->central_tendency = elapsed_[1];
+    s->variability = 0.0;
+    s->type = "second: ";
+    return true;
+  }
+
+  // Prefer geomean unless numerically unreliable (too many reps)
+  if (std::pow(elapsed_[0], elapsed_.size()) < 1E100) {
+    double product = 1.0;
+    for (size_t i = 1; i < elapsed_.size(); ++i) {
+      product *= elapsed_[i];
+    }
+
+    s->central_tendency = std::pow(product, 1.0 / (elapsed_.size() - 1));
+    s->variability = 0.0;
+    s->type = "geomean: ";
+    return true;
+  }
+
+  // Else: mode
+  std::sort(elapsed_.begin(), elapsed_.end());
+  s->central_tendency = jxl::HalfSampleMode()(elapsed_.data(), elapsed_.size());
+  s->variability = jxl::MedianAbsoluteDeviation(elapsed_, s->central_tendency);
+  s->type = "mode: ";
+  return true;
+}
+
+jxl::Status SpeedStats::Print(const size_t xsize, const size_t ysize,
+                              const size_t worker_threads) {
+  Summary s;
+  JXL_RETURN_IF_ERROR(GetSummary(&s));
+  char variability[20] = {'\0'};
+  if (s.variability != 0.0) {
+    snprintf(variability, sizeof(variability), " (var %.2f)", s.variability);
+  }
+
+  const double megapixels = xsize * ysize * 1E-6;
+  const double mpps = megapixels / s.central_tendency;
+  // Note flipped order: higher elapsed = lower mpps.
+  const double mpps_min = megapixels / s.max;
+  const double mpps_max = megapixels / s.min;
+
+  fprintf(stderr,
+          "%zu x %zu, %s%.2f MP/s [%.2f, %.2f]%s, %zu reps, %zu threads.\n",
+          xsize, ysize, s.type, mpps, mpps_min, mpps_max, variability,
+          elapsed_.size(), worker_threads);
+  return true;
+}
+
+}  // namespace tools
+}  // namespace jpegxl
diff --git a/tools/speed_stats.h b/tools/speed_stats.h
new file mode 100644
index 000000000..06251cb06
--- /dev/null
+++ b/tools/speed_stats.h
@@ -0,0 +1,55 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_SPEED_STATS_H_
+#define TOOLS_SPEED_STATS_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <vector>
+
+#include "jxl/base/status.h"
+
+namespace jpegxl {
+namespace tools {
+
+class SpeedStats {
+ public:
+  void NotifyElapsed(double elapsed_seconds);
+
+  struct Summary {
+    // How central_tendency was computed - depends on number of reps.
+    const char* type;
+
+    // Elapsed time
+    double central_tendency;
+    double min;
+    double max;
+    double variability;
+  };
+
+  // Non-const, may sort elapsed_.
+  jxl::Status GetSummary(Summary* summary);
+
+  // Calls GetSummary and prints megapixels/sec.
+  jxl::Status Print(size_t xsize, size_t ysize, size_t worker_threads);
+
+ private:
+  std::vector<double> elapsed_;
+};
+
+}  // namespace tools
+}  // namespace jpegxl
+
+#endif  // TOOLS_SPEED_STATS_H_
diff --git a/tools/ssimulacra.cc b/tools/ssimulacra.cc
new file mode 100644
index 000000000..1def89a57
--- /dev/null
+++ b/tools/ssimulacra.cc
@@ -0,0 +1,330 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Re-implementation of //third_party/ssimulacra/ssimulacra.cpp using jxl's
+// ImageF library instead of opencv.
+
+#include "tools/ssimulacra.h"
+
+#include "jxl/image_ops.h"
+
+namespace ssimulacra {
+namespace {
+
+using jxl::Image3F;
+using jxl::ImageF;
+
+static const double kC1 = 0.0001;
+static const double kC2 = 0.0004;
+static const int kNumScales = 6;
+static const double kScaleWeights[kNumScales][3] = {
+    {0.04480, 0.00300, 0.00300}, {0.28560, 0.00896, 0.00896},
+    {0.30010, 0.05712, 0.05712}, {0.23630, 0.06002, 0.06002},
+    {0.13330, 0.06726, 0.06726}, {0.10000, 0.05000, 0.05000},
+};
+const double kMinScaleWeights[kNumScales][3] = {
+    {0.02000, 0.00005, 0.00005}, {0.03000, 0.00025, 0.00025},
+    {0.02500, 0.00100, 0.00100}, {0.02000, 0.00150, 0.00150},
+    {0.01200, 0.00175, 0.00175}, {0.00500, 0.00175, 0.00175},
+};
+const double kEdgeWeight[3] = {1.5, 0.1, 0.1};
+const double kGridWeight[3] = {1.0, 0.1, 0.1};
+
+inline void Rgb2Lab(float r, float g, float b, float* L, float* A, float* B) {
+  const float epsilon = 0.00885645167903563081f;
+  const float s = 0.13793103448275862068f;
+  const float k = 7.78703703703703703703f;
+  const float scale = 1.0f / 255.0f;
+  r *= scale;
+  g *= scale;
+  b *= scale;
+  float fx = (r * 0.43393624408206207259f + g * 0.37619779063650710152f +
+              b * 0.18983429773803261441f);
+  float fy = (r * 0.2126729f + g * 0.7151522f + b * 0.0721750f);
+  float fz = (r * 0.01775381083562901744f + g * 0.10945087235996326905f +
+              b * 0.87263921028466483011f);
+  const float gamma = 1.0f / 3.0f;
+  float X = (fx > epsilon) ? powf(fx, gamma) - s : k * fx;
+  float Y = (fy > epsilon) ? powf(fy, gamma) - s : k * fy;
+  float Z = (fz > epsilon) ? powf(fz, gamma) - s : k * fz;
+  *L = Y * 1.16f;
+  *A = (0.39181818181818181818f + 2.27272727272727272727f * (X - Y));
+  *B = (0.49045454545454545454f + 0.90909090909090909090f * (Y - Z));
+}
+
+Image3F Rgb2Lab(const Image3F& in) {
+  Image3F out(in.xsize(), in.ysize());
+  for (int y = 0; y < in.ysize(); ++y) {
+    const float* row_in[3];
+    float* row_out[3];
+    for (int c = 0; c < 3; c++) {
+      row_in[c] = in.PlaneRow(c, y);
+      row_out[c] = out.PlaneRow(c, y);
+    }
+    for (int x = 0; x < in.xsize(); ++x) {
+      Rgb2Lab(row_in[0][x], row_in[1][x], row_in[2][x], &row_out[0][x],
+              &row_out[1][x], &row_out[2][x]);
+    }
+  }
+  return out;
+}
+
+Image3F Downsample(const Image3F& in, int fx, int fy) {
+  int out_xsize = (in.xsize() + fx - 1) / fx;
+  int out_ysize = (in.ysize() + fy - 1) / fy;
+  Image3F out(out_xsize, out_ysize);
+  for (int oy = 0; oy < out_ysize; ++oy) {
+    for (int ox = 0; ox < out_xsize; ++ox) {
+      for (int c = 0; c < 3; ++c) {
+        float sum = 0.0;
+        for (int iy = 0; iy < fy; ++iy) {
+          for (int ix = 0; ix < fx; ++ix) {
+            int x = std::min<int>(ox * fx + ix, in.xsize() - 1);
+            int y = std::min<int>(oy * fy + iy, in.ysize() - 1);
+            sum += in.PlaneRow(c, y)[x];
+          }
+        }
+        sum /= (fx * fy);
+        out.PlaneRow(c, oy)[ox] = sum;
+      }
+    }
+  }
+  return out;
+}
+
+Image3F Multiply(const Image3F& a, const Image3F& b) {
+  Image3F out(a.xsize(), a.ysize());
+  for (int y = 0; y < a.ysize(); ++y) {
+    for (int x = 0; x < a.xsize(); ++x) {
+      for (int c = 0; c < 3; ++c) {
+        out.PlaneRow(c, y)[x] = a.PlaneRow(c, y)[x] * b.PlaneRow(c, y)[x];
+      }
+    }
+  }
+  return out;
+}
+
+void RowColAvgP2(const ImageF& in, double* rp2, double* cp2) {
+  std::vector<double> ravg(in.ysize());
+  std::vector<double> cavg(in.xsize());
+  for (int y = 0; y < in.ysize(); ++y) {
+    auto row = in.Row(y);
+    for (int x = 0; x < in.xsize(); ++x) {
+      const float val = row[x];
+      ravg[y] += val;
+      cavg[x] += val;
+    }
+  }
+  std::sort(ravg.begin(), ravg.end());
+  std::sort(cavg.begin(), cavg.end());
+  *rp2 = ravg[ravg.size() / 50] / in.xsize();
+  *cp2 = cavg[cavg.size() / 50] / in.ysize();
+}
+
+Image3F EdgeDiffMap(const Image3F& img1, const Image3F& mu1,
+                    const Image3F& img2, const Image3F& mu2) {
+  Image3F out(img1.xsize(), img1.ysize());
+  for (int y = 0; y < img1.ysize(); ++y) {
+    for (int c = 0; c < 3; ++c) {
+      auto row1 = img1.PlaneRow(c, y);
+      auto row2 = img2.PlaneRow(c, y);
+      auto rowm1 = mu1.PlaneRow(c, y);
+      auto rowm2 = mu2.PlaneRow(c, y);
+      for (int x = 0; x < img1.xsize(); ++x) {
+        float edgediff = std::max(
+            std::abs(row2[x] - rowm2[x]) - std::abs(row1[x] - rowm1[x]), 0.0f);
+        out.PlaneRow(c, y)[x] = 1.0f - edgediff;
+      }
+    }
+  }
+  return out;
+}
+
+std::vector<float> GaussianKernel(int radius, float sigma) {
+  const float scaler = -1.0 / (2 * sigma * sigma);
+  std::vector<float> kernel(2 * radius + 1);
+  for (int i = -radius; i <= radius; ++i) {
+    kernel[i + radius] = std::exp(scaler * i * i);
+  }
+  return kernel;
+}
+
+inline void ExtrapolateBorders(const float* const JXL_RESTRICT row_in,
+                               float* const JXL_RESTRICT row_out,
+                               const int xsize, const int radius) {
+  const int lastcol = xsize - 1;
+  for (int x = 1; x <= radius; ++x) {
+    row_out[-x] = row_in[std::min(x, xsize - 1)];
+  }
+  memcpy(row_out, row_in, xsize * sizeof(row_out[0]));
+  for (int x = 1; x <= radius; ++x) {
+    row_out[lastcol + x] = row_in[std::max(0, lastcol - x)];
+  }
+}
+
+ImageF MirrorConvolveXTranspose(const ImageF& in,
+                                const std::vector<float>& kernel) {
+  JXL_CHECK(kernel.size() % 2 == 1);
+  ImageF out(in.ysize(), in.xsize());
+  float weight = 0.0f;
+  for (int i = 0; i < kernel.size(); ++i) {
+    weight += kernel[i];
+  }
+  float scale = 1.0f / weight;
+  const int r = kernel.size() / 2;
+  std::vector<float> row_tmp(in.xsize() + 2 * r);
+  float* const JXL_RESTRICT rowp = &row_tmp[r];
+  const float* const kernelp = &kernel[r];
+  for (int y = 0; y < in.ysize(); ++y) {
+    ExtrapolateBorders(in.Row(y), rowp, in.xsize(), r);
+    for (int x = 0; x < in.xsize(); ++x) {
+      float sum = 0.0f;
+      for (int i = -r; i <= r; ++i) {
+        sum += rowp[x + i] * kernelp[i];
+      }
+      out.Row(x)[y] = sum * scale;
+    }
+  }
+  return out;
+}
+
+ImageF GaussianBlur(const ImageF& in, int radius, float sigma) {
+  std::vector<float> kernel = GaussianKernel(radius, sigma);
+  return MirrorConvolveXTranspose(MirrorConvolveXTranspose(in, kernel), kernel);
+}
+
+Image3F GaussianBlur(const Image3F& in, int radius, float sigma) {
+  return Image3F(GaussianBlur(in.Plane(0), radius, sigma),
+                 GaussianBlur(in.Plane(1), radius, sigma),
+                 GaussianBlur(in.Plane(2), radius, sigma));
+}
+
+Image3F SSIMMap(const Image3F& m1, const Image3F& m2, const Image3F& s11,
+                const Image3F& s22, const Image3F& s12) {
+  Image3F out(m1.xsize(), m1.ysize());
+  for (int c = 0; c < 3; ++c) {
+    for (int y = 0; y < out.ysize(); ++y) {
+      auto row_m1 = m1.PlaneRow(c, y);
+      auto row_m2 = m2.PlaneRow(c, y);
+      auto row_s11 = s11.PlaneRow(c, y);
+      auto row_s22 = s22.PlaneRow(c, y);
+      auto row_s12 = s12.PlaneRow(c, y);
+      auto row_out = out.PlaneRow(c, y);
+      for (int x = 0; x < out.xsize(); ++x) {
+        float mu1 = row_m1[x];
+        float mu2 = row_m2[x];
+        float mu11 = mu1 * mu1;
+        float mu22 = mu2 * mu2;
+        float mu12 = mu1 * mu2;
+        float nom_m = 2 * mu12 + kC1;
+        float nom_s = 2 * (row_s12[x] - mu12) + kC2;
+        float denom_m = mu11 + mu22 + kC1;
+        float denom_s = (row_s11[x] - mu11) + (row_s22[x] - mu22) + kC2;
+        row_out[x] = (nom_m * nom_s) / (denom_m * denom_s);
+      }
+    }
+  }
+  return out;
+}
+
+}  // namespace
+
+double Ssimulacra::Score() const {
+  double ssim = 0.0;
+  double ssim_max = 0.0;
+  for (int c = 0; c < 3; ++c) {
+    for (int scale = 0; scale < scales.size(); ++scale) {
+      ssim += kScaleWeights[scale][c] * scales[scale].avg_ssim[c];
+      ssim_max += kScaleWeights[scale][c];
+      ssim += kMinScaleWeights[scale][c] * scales[scale].min_ssim[c];
+      ssim_max += kMinScaleWeights[scale][c];
+    }
+    ssim += kEdgeWeight[c] * avg_edgediff[c];
+    ssim_max += kEdgeWeight[c];
+    ssim += kGridWeight[c] *
+            (row_p2[0][c] + row_p2[1][c] + col_p2[0][c] + col_p2[1][c]);
+    ssim_max += 4.0 * kGridWeight[c];
+  }
+  double dssim = ssim_max / ssim - 1.0;
+  return std::min(1.0, std::max(0.0, dssim));
+}
+
+inline void PrintItem(const char* name, int scale, const double* vals,
+                      const double* w) {
+  printf("scale %d %s = [%.10f %.10f %.10f]  w = [%.5f %.5f %.5f]\n", scale,
+         name, vals[0], vals[1], vals[2], w[0], w[1], w[2]);
+}
+
+void Ssimulacra::PrintDetails() const {
+  for (int s = 0; s < scales.size(); ++s) {
+    PrintItem("avg ssim", s, scales[s].avg_ssim, &kScaleWeights[s][0]);
+    PrintItem("min ssim", s, scales[s].min_ssim, &kMinScaleWeights[s][0]);
+    if (s == 0) {
+      PrintItem("avg edif", s, avg_edgediff, kEdgeWeight);
+      PrintItem("rp2 ssim", s, &row_p2[0][0], kGridWeight);
+      PrintItem("cp2 ssim", s, &col_p2[0][0], kGridWeight);
+      PrintItem("rp2 edif", s, &row_p2[1][0], kGridWeight);
+      PrintItem("cp2 edif", s, &col_p2[1][0], kGridWeight);
+    }
+  }
+}
+
+Ssimulacra ComputeDiff(const Image3F& img1_arg, const Image3F& img2_arg) {
+  Ssimulacra ssimulacra;
+  Image3F img1 = Rgb2Lab(img1_arg);
+  Image3F img2 = Rgb2Lab(img2_arg);
+  for (int scale = 0; scale < kNumScales; scale++) {
+    if (img1.xsize() < 8 || img1.ysize() < 8) {
+      break;
+    }
+    if (scale) {
+      img1 = Downsample(img1, 2, 2);
+      img2 = Downsample(img2, 2, 2);
+    }
+    SsimulacraScale sscale;
+    Image3F img1_sq = Multiply(img1, img1);
+    Image3F img2_sq = Multiply(img2, img2);
+    Image3F img1_img2 = Multiply(img1, img2);
+    Image3F mu1 = GaussianBlur(img1, 5, 1.5f);
+    Image3F mu2 = GaussianBlur(img2, 5, 1.5f);
+    Image3F sigma1_sq = GaussianBlur(img1_sq, 5, 1.5f);
+    Image3F sigma2_sq = GaussianBlur(img2_sq, 5, 1.5f);
+    Image3F sigma12 = GaussianBlur(img1_img2, 5, 1.5f);
+    Image3F ssim_map = SSIMMap(mu1, mu2, sigma1_sq, sigma2_sq, sigma12);
+
+    for (unsigned int i = 0; i < 3; i++) {
+      sscale.avg_ssim[i] = jxl::ImageAverage(ssim_map.Plane(i));
+    }
+    ssim_map = Downsample(ssim_map, 4, 4);
+    for (unsigned int c = 0; c < 3; c++) {
+      float minval, maxval;
+      ImageMinMax(ssim_map.Plane(c), &minval, &maxval);
+      sscale.min_ssim[c] = minval;
+    }
+    ssimulacra.scales.push_back(sscale);
+    if (scale == 0) {
+      Image3F edgediff = EdgeDiffMap(img1, mu1, img2, mu2);
+      for (unsigned int c = 0; c < 3; c++) {
+        ssimulacra.avg_edgediff[c] = jxl::ImageAverage(edgediff.Plane(c));
+        RowColAvgP2(ssim_map.Plane(c), &ssimulacra.row_p2[0][c],
+                    &ssimulacra.col_p2[0][c]);
+        RowColAvgP2(edgediff.Plane(c), &ssimulacra.row_p2[1][c],
+                    &ssimulacra.col_p2[1][c]);
+      }
+    }
+  }
+  return ssimulacra;
+}
+
+}  // namespace ssimulacra
diff --git a/tools/ssimulacra.cpp b/tools/ssimulacra.cpp
new file mode 100644
index 000000000..09f1b2f07
--- /dev/null
+++ b/tools/ssimulacra.cpp
@@ -0,0 +1,391 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+/*
+    SSIMULACRA - Structural SIMilarity Unveiling Local And Compression Related Artifacts
+
+    Cloudinary's variant of DSSIM, based on Philipp Klaus Krause's adaptation of Rabah Mehdi's SSIM implementation,
+    using ideas from Kornel Lesinski's DSSIM implementation as well as several new ideas.
+
+
+
+
+    Changes compared to Krause's SSIM implementation:
+    - Use C++ OpenCV API
+    - Convert sRGB to linear RGB and then to L*a*b*, to get a perceptually more accurate color space
+    - Multi-scale (6 scales)
+    - Extra penalty for specific kinds of artifacts:
+        - local artifacts
+        - grid-like artifacts (blockiness)
+        - introducing edges where the original is smooth (blockiness / color banding / ringing / mosquito noise)
+
+    Known limitations:
+    - Color profiles are ignored; input images are assumed to be sRGB.
+    - Both input images need to have the same number of channels (Grayscale / RGB / RGBA)
+*/
+
+/*
+    This DSSIM program has been created by Philipp Klaus Krause based on
+    Rabah Mehdi's C++ implementation of SSIM (http://mehdi.rabah.free.fr/SSIM).
+    Originally it has been created for the VMV '09 paper
+    "ftc - floating precision texture compression" by Philipp Klaus Krause.
+
+    The latest version of this program can probably be found somewhere at
+    http://www.colecovision.eu.
+
+    It can be compiled using g++ -I/usr/include/opencv -lcv -lhighgui dssim.cpp
+    Make sure OpenCV is installed (e.g. for Debian/ubuntu: apt-get install
+    libcv-dev libhighgui-dev).
+
+    DSSIM is described in
+    "Structural Similarity-Based Object Tracking in Video Sequences" by Loza et al.
+    however setting all Ci to 0 as proposed there results in numerical instabilities.
+    Thus this implementation used the Ci from the SSIM implementation.
+    SSIM is described in
+    "Image quality assessment: from error visibility to structural similarity" by Wang et al.
+*/
+
+/*
+    Copyright (c) 2005, Rabah Mehdi <mehdi.rabah@gmail.com>
+
+    Feel free to use it as you want and to drop me a mail
+    if it has been useful to you. Please let me know if you enhance it.
+    I'm not responsible if this program destroy your life & blablabla :)
+
+    Copyright (c) 2009, Philipp Klaus Krause <philipp@colecovision.eu>
+
+    Permission to use, copy, modify, and/or distribute this software for any
+    purpose with or without fee is hereby granted, provided that the above
+    copyright notice and this permission notice appear in all copies.
+
+    THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+    WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+    MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+    ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+    WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+    ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+    OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+*/
+
+#include <cv.hpp>
+#include <highgui.h>
+#include <stdio.h>
+#include <set>
+
+// comment this in to produce debug images that show the differences at each scale
+#define DEBUG_IMAGES 1
+using namespace std;
+using namespace cv;
+
+// All of the constants below are more or less arbitrary.
+// Some amount of tweaking/calibration was done, but there is certainly room for improvement.
+
+// SSIM constants. Original C2 was 0.0009, but a smaller value seems to work slightly better.
+const double C1 = 0.0001, C2 = 0.0004;
+
+// Weight of each scale. Somewhat arbitrary.
+// These are based on the values used in IW-SSIM and Kornel's DSSIM.
+// It seems weird to give so little weight to the full-size scale, but then again,
+// differences in more zoomed-out scales have more visual impact.
+// Anyway, these weights seem to work.
+// Added one more scale compared to IW-SSIM and Kornel's DSSIM.
+// Weights for chroma are modified to give more weight to larger scales (similar to Kornel's subsampled chroma)
+const float scale_weights[4][6] = {
+    // 1:1   1:2     1:4     1:8     1:16    1:32
+    {0.0448, 0.2856, 0.3001, 0.2363, 0.1333, 0.1  },
+    {0.015,  0.0448, 0.2856, 0.3001, 0.3363, 0.25 },
+    {0.015,  0.0448, 0.2856, 0.3001, 0.3363, 0.25 },
+    {0.0448, 0.2856, 0.3001, 0.2363, 0.1333, 0.1  },
+    };
+
+// higher value means more importance to chroma (weights above are multiplied by this factor for chroma and alpha)
+const double chroma_weight = 0.2;
+
+// Weights for the worst-case (minimum) score at each scale.
+// Higher value means more importance to worst artifacts, lower value means more importance to average artifacts.
+const float mscale_weights[4][6] = {
+    // 1:4   1:8     1:16    1:32   1:64   1:128
+    {0.2,    0.3,    0.25,   0.2,   0.12,  0.05},
+    {0.01,   0.05,   0.2,    0.3,   0.35,  0.35},
+    {0.01,   0.05,   0.2,    0.3,   0.35,  0.35},
+    {0.2,    0.3,    0.25,   0.2,   0.12,  0.05},
+    };
+
+
+// higher value means more importance to worst local artifacts
+const double min_weight[4] = {0.1,0.005,0.005,0.005};
+
+// higher value means more importance to artifact-edges (edges where original is smooth)
+const double extra_edges_weight[4] = {1.5, 0.1, 0.1, 0.5};
+
+// higher value means more importance to grid-like artifacts (blockiness)
+const double worst_grid_weight[2][4] = 
+    { {1.0, 0.1, 0.1, 0.5},             // on ssim heatmap
+      {1.0, 0.1, 0.1, 0.5} };           // on extra_edges heatmap
+
+
+// Convert linear RGB to L*a*b* (all in 0..1 range)
+inline void rgb2lab(Vec3f &p) __attribute__ ((hot));
+inline void rgb2lab(Vec3f &p) {
+    const float epsilon = 0.00885645167903563081f;
+    const float s = 0.13793103448275862068f;
+    const float k = 7.78703703703703703703f;
+
+    // D65 adjustment included
+    float fx = (p[2] * 0.43393624408206207259f + p[1] * 0.37619779063650710152f + p[0] * .18983429773803261441f) ;
+    float fy = (p[2] * 0.2126729f + p[1] * 0.7151522f + p[0] * 0.0721750f);
+    float fz = (p[2] * 0.01775381083562901744f + p[1] * 0.10945087235996326905f + p[0] * 0.87263921028466483011f) ;
+
+    float X = (fx > epsilon) ? powf(fx,1.0f/3.0f) - s : k * fx;
+    float Y = (fy > epsilon) ? powf(fy,1.0f/3.0f) - s : k * fy;
+    float Z = (fz > epsilon) ? powf(fz,1.0f/3.0f) - s : k * fz;
+
+    p[0] = Y * 1.16f;
+    p[1] = (0.39181818181818181818f + 2.27272727272727272727f * (X - Y));
+    p[2] = (0.49045454545454545454f + 0.90909090909090909090f * (Y - Z));
+}
+
+
+int main(int argc, char** argv) {
+
+    if(argc!=3) {
+        fprintf(stderr, "Usage: %s orig_image distorted_image\n", argv[0]);
+        fprintf(stderr, "Returns a value between 0 (images are identical) and 1 (images are very different)\n");
+        fprintf(stderr, "If the value is above 0.1 (or so), the distortion is likely to be perceptible / annoying.\n");
+        fprintf(stderr, "If the value is below 0.01 (or so), the distortion is likely to be imperceptible.\n");
+        return(-1);
+    }
+
+    Scalar sC1 = {C1,C1,C1,C1}, sC2 = {C2,C2,C2,C2};
+
+    Mat img1, img2, img1_img2, img1_temp, img2_temp, img1_sq, img2_sq, mu1, mu2, mu1_sq, mu2_sq, mu1_mu2, sigma1_sq, sigma2_sq, sigma12, ssim_map;
+
+    // read and validate input images
+
+    img1_temp = imread(argv[1],-1);
+    img2_temp = imread(argv[2],-1);
+
+    int nChan = img1_temp.channels();
+    if (nChan != img2_temp.channels()) {
+        fprintf(stderr, "Image file %s has %i channels, while\n", argv[1], nChan);
+        fprintf(stderr, "image file %s has %i channels. Can't compare.\n", argv[2], img2_temp.channels());
+        return -1;
+    }
+    if (img1_temp.size() != img2_temp.size()) {
+        fprintf(stderr,  "Image dimensions have to be identical.\n");
+        return -1;
+    }
+    if (img1_temp.cols < 8 || img1_temp.rows < 8) {
+        fprintf(stderr,  "Image is too small; need at least 8 rows and columns.\n");
+        return -1;
+    }
+    int pixels = img1_temp.rows * img1_temp.cols;
+    if (nChan == 4) {
+        // blend to a gray background to have a fair comparison of semi-transparent RGB values
+        for( int i=0 ; i < pixels; i++ ) {
+            Vec4b & p = img1_temp.at<Vec4b>(i);
+            p[0] = (p[3]*p[0] + (255-p[3])*128 ) / 255;
+            p[1] = (p[3]*p[1] + (255-p[3])*128 ) / 255;
+            p[2] = (p[3]*p[2] + (255-p[3])*128 ) / 255;
+        }
+        for( int i=0 ; i < pixels; i++ ) {
+            Vec4b & p = img2_temp.at<Vec4b>(i);
+            p[0] = (p[3]*p[0] + (255-p[3])*128 ) / 255;
+            p[1] = (p[3]*p[1] + (255-p[3])*128 ) / 255;
+            p[2] = (p[3]*p[2] + (255-p[3])*128 ) / 255;
+        }
+    }
+
+
+    if (nChan > 1) {
+    // Create lookup table to convert 8-bit sRGB to linear RGB
+    Mat sRGB_gamma_LUT(1, 256, CV_32FC1);
+    for (int i = 0; i < 256; i++) {
+        float c = i / 255.0;
+        sRGB_gamma_LUT.at<float>(i) = (c <= 0.04045 ? c / 12.92 : pow((c + 0.055) / 1.055, 2.4));
+    }
+
+    // Convert from sRGB to linear RGB
+    LUT(img1_temp, sRGB_gamma_LUT, img1);
+    LUT(img2_temp, sRGB_gamma_LUT, img2);
+    } else {
+        img1 = Mat(img1_temp.rows, img1_temp.cols, CV_32FC1);
+        img2 = Mat(img1_temp.rows, img1_temp.cols, CV_32FC1);
+    }
+
+    // Convert from linear RGB to Lab in a 0..1 range
+    if (nChan == 3) {
+      for( int i=0 ; i < pixels; i++ ) rgb2lab(img1.at<Vec3f>(i));
+      for( int i=0 ; i < pixels; i++ ) rgb2lab(img2.at<Vec3f>(i));
+    } else if (nChan == 4) {
+      for( int i=0 ; i < pixels; i++ ) { Vec3f p = {img1.at<Vec4f>(i)[0],img1.at<Vec4f>(i)[1],img1.at<Vec4f>(i)[2]}; rgb2lab(p); img1.at<Vec4f>(i)[0] = p[0]; img1.at<Vec4f>(i)[1] = p[1]; img1.at<Vec4f>(i)[2] = p[2];}
+      for( int i=0 ; i < pixels; i++ ) { Vec3f p = {img2.at<Vec4f>(i)[0],img2.at<Vec4f>(i)[1],img2.at<Vec4f>(i)[2]}; rgb2lab(p); img2.at<Vec4f>(i)[0] = p[0]; img2.at<Vec4f>(i)[1] = p[1]; img2.at<Vec4f>(i)[2] = p[2];}
+    } else if (nChan == 1) {
+      for( int i=0 ; i < pixels; i++ ) { img1.at<float>(i) = img1_temp.at<uchar>(i)/255.0;}
+      for( int i=0 ; i < pixels; i++ ) { img2.at<float>(i) = img2_temp.at<uchar>(i)/255.0;}
+    } else {
+        fprintf(stderr, "Can only deal with Grayscale, RGB or RGBA input.\n");
+        return(-1);
+    }
+
+
+    double dssim=0, dssim_max=0;
+
+    for (int scale = 0; scale < 6; scale++) {
+
+      if (img1.cols < 8 || img1.rows < 8) break;
+      if (scale) {
+        // scale down 50% in each iteration.
+        resize(img1, img1, Size(), 0.5, 0.5, INTER_AREA);
+        resize(img2, img2, Size(), 0.5, 0.5, INTER_AREA);
+      }
+
+      // Standard SSIM computation
+      cv::pow( img1, 2, img1_sq );
+      cv::pow( img2, 2, img2_sq );
+
+      multiply( img1, img2, img1_img2, 1 );
+
+      GaussianBlur(img1, mu1, Size(11,11), 1.5);
+      GaussianBlur(img2, mu2, Size(11,11), 1.5);
+
+      cv::pow( mu1, 2, mu1_sq );
+      cv::pow( mu2, 2, mu2_sq );
+      multiply( mu1, mu2, mu1_mu2, 1 );
+
+      GaussianBlur(img1_sq, sigma1_sq, Size(11,11), 1.5);
+      addWeighted( sigma1_sq, 1, mu1_sq, -1, 0, sigma1_sq );
+
+      GaussianBlur(img2_sq, sigma2_sq, Size(11,11), 1.5);
+      addWeighted( sigma2_sq, 1, mu2_sq, -1, 0, sigma2_sq );
+
+      GaussianBlur(img1_img2, sigma12, Size(11,11), 1.5);
+      addWeighted( sigma12, 1, mu1_mu2, -1, 0, sigma12 );
+
+      ssim_map = ((2*mu1_mu2 + sC1).mul(2*sigma12 + sC2))/((mu1_sq + mu2_sq + sC1).mul(sigma1_sq + sigma2_sq + sC2));
+
+
+      // optional: write a nice debug image that shows the problematic areas
+#ifdef DEBUG_IMAGES
+      Mat ssim_image;
+      ssim_map.convertTo(ssim_image,CV_8UC3,255);
+        for( int i=0 ; i < ssim_image.rows * ssim_image.cols; i++ ) {
+            Vec3b &p = ssim_image.at<Vec3b>(i);
+            p = {(uchar)(255-p[2]),(uchar)(255-p[0]),(uchar)(255-p[1])};
+        }
+      imwrite("debug-scale"+to_string(scale)+".png",ssim_image);
+#endif
+
+
+      // average ssim over the entire image
+      Scalar avg = mean( ssim_map );
+      for(unsigned int i = 0; i < nChan; i++) {
+        printf("avg: %i  %f\n",i,avg[i]);
+        dssim += (i>0?chroma_weight:1.0) * avg[i] * scale_weights[i][scale];
+        dssim_max += (i>0?chroma_weight:1.0) * scale_weights[i][scale];
+      }
+
+//      resize(ssim_map, ssim_map, Size(), 0.5, 0.5, INTER_AREA);
+
+
+      // the edge/blockiness penalty is only done for the fullsize images
+      if (scale == 0) {
+
+        // asymmetric: penalty for introducing edges where there are none (e.g. blockiness), no penalty for smoothing away edges
+        Mat edgediff = max(abs(img2 - mu2) - abs(img1 - mu1), 0);   // positive if img2 has an edge where img1 is smooth
+
+        // optional: write a nice debug image that shows the artifact edges
+#ifdef DEBUG_IMAGES
+        Mat edgediff_image;
+        edgediff.convertTo(edgediff_image,CV_8UC3,5000); // multiplying by more than 255 to make things easier to see
+        for( int i=0 ; i < pixels; i++ ) {
+            Vec3b &p = edgediff_image.at<Vec3b>(i);
+            p = {(uchar)(p[1]+p[2]),p[0],p[0]};
+        }
+        imwrite("debug-edgediff.png",edgediff_image);
+#endif
+
+        edgediff = Scalar(1.0,1.0,1.0,1.0) - edgediff;
+
+        avg = mean(edgediff);
+        for(unsigned int i = 0; i < nChan; i++) {
+          printf("extra_edges: %i  %f\n",i,avg[i]);
+          dssim +=  extra_edges_weight[i] * avg[i];
+          dssim_max +=  extra_edges_weight[i];
+        }
+
+        // grid-like artifact detection
+        // do the things below twice: once for the SSIM map, once for the artifact-edge map
+        Mat errormap;
+        for(int twice=0; twice < 2; twice++) {
+          if (twice == 0) errormap = ssim_map;
+          else errormap = edgediff;
+
+          // Find the 2nd percentile worst row. If the compression uses blocks, there will be artifacts around the block edges,
+          // so even with 32x32 blocks, the 2nd percentile will likely be one of the rows with block borders
+          multiset<double> row_scores[4];
+          for (int y = 0; y < errormap.rows; y++) {
+            Mat roi = errormap(Rect(0,y,errormap.cols,1));
+            Scalar ravg = mean(roi);
+            for (unsigned int i = 0; i < nChan; i++) row_scores[i].insert(ravg[i]);
+          }
+          for(unsigned int i = 0; i < nChan; i++) {
+            int k=0; for (const double& s : row_scores[i]) { if (k++ >= errormap.rows/50) { dssim += worst_grid_weight[twice][i] * s; 
+          printf("grid row %s %i:  %f\n",(twice?"edgediff":"ssimmap"),i,s);
+
+ break; } }
+            dssim_max += worst_grid_weight[twice][i];
+          }
+          // Find the 2nd percentile worst column. Same concept as above.
+          multiset<double> col_scores[4];
+          for (int x = 0; x < errormap.cols; x++) {
+            Mat roi = errormap(Rect(x,0,1,errormap.rows));
+            Scalar cavg = mean(roi);
+            for (unsigned int i = 0; i < nChan; i++) col_scores[i].insert(cavg[i]);
+          }
+          for(unsigned int i = 0; i < nChan; i++) {
+            int k=0; for (const double& s : col_scores[i]) { if (k++ >= errormap.cols/50) { dssim += worst_grid_weight[twice][i] * s; 
+          printf("grid col %s %i:  %f\n",(twice?"edgediff":"ssimmap"),i,s);
+
+break; } }
+            dssim_max += worst_grid_weight[twice][i];
+          }
+        }
+      }
+
+      // worst ssim in a particular 4x4 block (larger blocks are considered too because of multi-scale)
+      resize(ssim_map, ssim_map, Size(), 0.25, 0.25, INTER_AREA);
+//      resize(ssim_map, ssim_map, Size(), 0.5, 0.5, INTER_AREA);
+
+      Mat ssim_map_c[4];
+      split(ssim_map, ssim_map_c);
+      for (unsigned int i=0; i < nChan; i++) {
+        double minVal;
+        minMaxLoc(ssim_map_c[i], &minVal);
+          printf("worst %i:  %f\n",i,minVal);
+        dssim += min_weight[i]  * minVal * mscale_weights[i][scale];
+        dssim_max += min_weight[i]  * mscale_weights[i][scale];
+      }
+
+    }
+
+
+    dssim = dssim_max / dssim - 1;
+    if (dssim < 0) dssim = 0; // should not happen
+    if (dssim > 1) dssim = 1; // very different images
+
+    printf("%.8f\n", dssim);
+
+    return(0);
+}
diff --git a/tools/ssimulacra.h b/tools/ssimulacra.h
new file mode 100644
index 000000000..cd443459b
--- /dev/null
+++ b/tools/ssimulacra.h
@@ -0,0 +1,43 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_SSIMULACRA_H_
+#define TOOLS_SSIMULACRA_H_
+
+#include <vector>
+
+#include "jxl/image.h"
+
+namespace ssimulacra {
+
+struct SsimulacraScale {
+  double avg_ssim[3];
+  double min_ssim[3];
+};
+
+struct Ssimulacra {
+  std::vector<SsimulacraScale> scales;
+  double avg_edgediff[3];
+  double row_p2[2][3];
+  double col_p2[2][3];
+
+  double Score() const;
+  void PrintDetails() const;
+};
+
+Ssimulacra ComputeDiff(const jxl::Image3F& orig, const jxl::Image3F& distorted);
+
+}  // namespace ssimulacra
+
+#endif  // TOOLS_SSIMULACRA_H_
diff --git a/tools/ssimulacra_main.cc b/tools/ssimulacra_main.cc
new file mode 100644
index 000000000..a8803161c
--- /dev/null
+++ b/tools/ssimulacra_main.cc
@@ -0,0 +1,70 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include "jxl/color_management.h"
+#include "jxl/extras/codec.h"
+#include "tools/ssimulacra.h"
+
+namespace ssimulacra {
+namespace {
+
+int PrintUsage(char** argv) {
+  fprintf(stderr, "Usage: %s [-v] orig.png distorted.png\n", argv[0]);
+  return 1;
+}
+
+int Run(int argc, char** argv) {
+  if (argc < 2) return PrintUsage(argv);
+
+  bool verbose = false;
+  int input_arg = 1;
+  if (argv[1][0] == '-' && argv[1][1] == 'v') {
+    verbose = true;
+    input_arg = 2;
+  }
+  if (argc < input_arg + 2) return PrintUsage(argv);
+
+  jxl::CodecInOut io1;
+  jxl::CodecInOut io2;
+  JXL_CHECK(SetFromFile(argv[input_arg], &io1));
+  JXL_CHECK(SetFromFile(argv[input_arg + 1], &io2));
+  JXL_CHECK(
+      io1.TransformTo(jxl::ColorManagement::LinearSRGB(io1.Main().IsGray())));
+  JXL_CHECK(
+      io2.TransformTo(jxl::ColorManagement::LinearSRGB(io2.Main().IsGray())));
+
+  if (io1.xsize() != io2.xsize() || io1.ysize() != io2.ysize()) {
+    fprintf(stderr, "Image size mismatch\n");
+    return 1;
+  }
+  if (io1.xsize() < 8 || io1.ysize() < 8) {
+    fprintf(stderr, "Minimum image size is 8x8 pixels\n");
+    return 1;
+  }
+
+  Ssimulacra ssimulacra = ComputeDiff(io1.Main().color(), io2.Main().color());
+
+  if (verbose) {
+    ssimulacra.PrintDetails();
+  }
+  printf("%.8f\n", ssimulacra.Score());
+  return 0;
+}
+
+}  // namespace
+}  // namespace ssimulacra
+
+int main(int argc, char** argv) { return ssimulacra::Run(argc, argv); }
diff --git a/tools/viewer/CMakeLists.txt b/tools/viewer/CMakeLists.txt
new file mode 100644
index 000000000..a38abc91d
--- /dev/null
+++ b/tools/viewer/CMakeLists.txt
@@ -0,0 +1,44 @@
+# Copyright (c) the JPEG XL Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+find_package(Qt5 COMPONENTS Widgets X11Extras)
+if (NOT Qt5_FOUND)
+  message(WARNING "Qt5 was not found. The directory viewer will not be built.")
+  return()
+endif ()
+
+if (NOT TARGET x11_icc)
+  message(WARNING "The directory viewer depends on the comparison tool and will also not be built.")
+  return ()
+endif ()
+
+set(CMAKE_INCLUDE_CURRENT_DIR ON)
+set(CMAKE_AUTOMOC ON)
+set(CMAKE_AUTOUIC ON)
+
+add_executable(viewer WIN32
+  load_jpegxl.cc
+  load_jpegxl.h
+  main.cc
+  viewer_window.cc
+  viewer_window.h
+  viewer_window.ui
+)
+target_link_libraries(viewer
+  jpegxl
+  jpegxl_threads
+  Qt5::Widgets
+  Qt5::X11Extras
+  x11_icc
+)
diff --git a/tools/viewer/load_jpegxl.cc b/tools/viewer/load_jpegxl.cc
new file mode 100644
index 000000000..bb50307c1
--- /dev/null
+++ b/tools/viewer/load_jpegxl.cc
@@ -0,0 +1,135 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/viewer/load_jpegxl.h"
+
+#include <stdint.h>
+
+#include <QElapsedTimer>
+#include <utility>
+
+#include "jpegxl/decode.h"
+#include "jxl/aux_out.h"
+#include "jxl/base/arch_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/base/file_io.h"
+#include "jxl/base/status.h"
+#include "jxl/base/thread_pool_internal.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/dec_file.h"
+#include "jxl/dec_params.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+
+QImage loadJpegXlImage(const QString& filename, PaddedBytes targetIccProfile,
+                       qint64* elapsed_ns, bool* usedRequestedProfile) {
+  static ProcessorTopology topology;
+  static ThreadPoolInternal pool(topology.packages *
+                                 topology.cores_per_package);
+
+  PaddedBytes jpegXlData;
+  if (!ReadFile(filename.toStdString(), &jpegXlData)) {
+    return QImage();
+  }
+  if (jpegXlData.size() < 4) {
+    return QImage();
+  }
+  CodecInOut io;
+  QElapsedTimer timer;
+  DecompressParams params;
+  AuxOut localInfo;
+  timer.start();
+  if (!DecodeFile(params, jpegXlData, &io, &localInfo, &pool)) {
+    return QImage();
+  }
+  if (elapsed_ns != nullptr) *elapsed_ns = timer.nsecsElapsed();
+
+  const jxl::ImageBundle& ib = io.Main();
+  ColorEncoding targetColorSpace;
+  const bool profileSet = ColorManagement::SetProfile(
+      std::move(targetIccProfile), &targetColorSpace);
+  if (usedRequestedProfile != nullptr) *usedRequestedProfile = profileSet;
+  if (!profileSet) {
+    targetColorSpace = ColorManagement::SRGB(ib.IsGray());
+  }
+  Image3U decoded;
+  if (!ib.CopyTo(Rect(ib), targetColorSpace, &decoded, &pool)) {
+    return QImage();
+  }
+
+  QImage result(ib.xsize(), ib.ysize(),
+#if QT_VERSION >= QT_VERSION_CHECK(5, 12, 0)
+                QImage::Format_RGBA64
+#else
+                QImage::Format_ARGB32
+#endif
+  );
+
+  if (ib.HasAlpha()) {
+    const int alphaLeftShiftAmount =
+        16 - static_cast<int>(io.metadata.alpha_bits);
+    for (int y = 0; y < result.height(); ++y) {
+      QRgb* const row = reinterpret_cast<QRgb*>(result.scanLine(y));
+      const uint16_t* const alphaRow = ib.alpha().ConstRow(y);
+      const uint16_t* const redRow = decoded.ConstPlaneRow(0, y);
+      const uint16_t* const greenRow = decoded.ConstPlaneRow(1, y);
+      const uint16_t* const blueRow = decoded.ConstPlaneRow(2, y);
+      for (int x = 0; x < result.width(); ++x) {
+#if QT_VERSION >= QT_VERSION_CHECK(5, 6, 0)
+        row[x] = qRgba64(redRow[x], greenRow[x], blueRow[x],
+                         alphaRow[x] << alphaLeftShiftAmount)
+#if QT_VERSION >= QT_VERSION_CHECK(5, 12, 0)
+                     .unpremultiplied()
+#else
+                     .toArgb32()
+#endif
+            ;
+#else
+        // Qt version older than 5.6 doesn't have a qRgba64.
+        row[x] = qRgba(redRow[x] >> 8, greenRow[x] >> 8, blueRow[x] >> 8,
+                       alphaRow[x] << alphaLeftShiftAmount);
+#endif
+      }
+    }
+  } else {
+    for (int y = 0; y < result.height(); ++y) {
+      QRgb* const row = reinterpret_cast<QRgb*>(result.scanLine(y));
+      const uint16_t* const redRow = decoded.ConstPlaneRow(0, y);
+      const uint16_t* const greenRow = decoded.ConstPlaneRow(1, y);
+      const uint16_t* const blueRow = decoded.ConstPlaneRow(2, y);
+      for (int x = 0; x < result.width(); ++x) {
+#if QT_VERSION >= QT_VERSION_CHECK(5, 6, 0)
+        row[x] = qRgba64(redRow[x], greenRow[x], blueRow[x], 65535)
+#if QT_VERSION >= QT_VERSION_CHECK(5, 12, 0)
+                     .unpremultiplied()
+#else
+                     .toArgb32()
+#endif
+            ;
+#else
+        // Qt version older than 5.6 doesn't have a qRgba64.
+        row[x] = qRgb(redRow[x] >> 8, greenRow[x] >> 8, blueRow[x] >> 8);
+#endif
+      }
+    }
+  }
+
+  return result;
+}
+
+}  // namespace jxl
diff --git a/tools/viewer/load_jpegxl.h b/tools/viewer/load_jpegxl.h
new file mode 100644
index 000000000..742042824
--- /dev/null
+++ b/tools/viewer/load_jpegxl.h
@@ -0,0 +1,30 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_VIEWER_LOAD_JPEGXL_H_
+#define TOOLS_VIEWER_LOAD_JPEGXL_H_
+
+#include <QImage>
+#include <QString>
+
+#include "jxl/base/padded_bytes.h"
+
+namespace jxl {
+
+QImage loadJpegXlImage(const QString& filename, PaddedBytes targetIccProfile,
+                       qint64* elapsed, bool* usedRequestedProfile = nullptr);
+
+}  // namespace jxl
+
+#endif  // TOOLS_VIEWER_LOAD_JPEGXL_H_
diff --git a/tools/viewer/main.cc b/tools/viewer/main.cc
new file mode 100644
index 000000000..e7a70215c
--- /dev/null
+++ b/tools/viewer/main.cc
@@ -0,0 +1,32 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <QApplication>
+
+#include "tools/viewer/viewer_window.h"
+
+int main(int argc, char** argv) {
+  QApplication application(argc, argv);
+  QStringList arguments = application.arguments();
+  arguments.removeFirst();
+
+  jxl::ViewerWindow window;
+  window.show();
+
+  if (!arguments.empty()) {
+    window.loadFilesAndDirectories(arguments);
+  }
+
+  return application.exec();
+}
diff --git a/tools/viewer/viewer_window.cc b/tools/viewer/viewer_window.cc
new file mode 100644
index 000000000..67040ee00
--- /dev/null
+++ b/tools/viewer/viewer_window.cc
@@ -0,0 +1,142 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/viewer/viewer_window.h"
+
+#include <QElapsedTimer>
+#include <QFileDialog>
+#include <QFileInfo>
+#include <QKeyEvent>
+#include <QMessageBox>
+#include <QSet>
+#include <QX11Info>
+
+#include "tools/viewer/load_jpegxl.h"
+#include "tools/x11/icc.h"
+
+namespace jxl {
+
+namespace {
+
+template <typename Output>
+void recursivelyAddSubEntries(const QFileInfo& info,
+                              QSet<QString>* const visited,
+                              Output* const output) {
+  if (visited->contains(info.absoluteFilePath())) return;
+  *visited << info.absoluteFilePath();
+  if (info.isDir()) {
+    QDir dir(info.absoluteFilePath());
+    for (const QFileInfo& entry : dir.entryInfoList(
+             QStringList() << "*.jxl",
+             QDir::Files | QDir::AllDirs | QDir::NoDotAndDotDot)) {
+      recursivelyAddSubEntries(entry, visited, output);
+    }
+  } else {
+    *output << info.absoluteFilePath();
+  }
+}
+
+}  // namespace
+
+ViewerWindow::ViewerWindow(QWidget* const parent)
+    : QMainWindow(parent),
+      monitorProfile_(
+          GetMonitorIccProfile(QX11Info::connection(), QX11Info::appScreen())) {
+  ui_.setupUi(this);
+  ui_.actionOpen->setShortcut(QKeySequence::Open);
+  ui_.actionExit->setShortcut(QKeySequence::Quit);
+}
+
+void ViewerWindow::loadFilesAndDirectories(QStringList entries) {
+  filenames_.clear();
+  QSet<QString> visited;
+  for (const QString& entry : entries) {
+    recursivelyAddSubEntries(entry, &visited, &filenames_);
+  }
+
+  const bool several = filenames_.size() > 1;
+  ui_.actionPreviousImage->setEnabled(several);
+  ui_.actionNextImage->setEnabled(several);
+
+  currentFileIndex_ = 0;
+  refreshImage();
+}
+
+void ViewerWindow::on_actionOpen_triggered() {
+  QFileDialog dialog(this, tr("Select JPEG XL files to open…"));
+  dialog.setFileMode(QFileDialog::ExistingFiles);
+  dialog.setNameFilter(tr("JPEG XL images (*.xl);;All files (*)"));
+  if (dialog.exec()) {
+    loadFilesAndDirectories(dialog.selectedFiles());
+  }
+}
+
+void ViewerWindow::on_actionPreviousImage_triggered() {
+  currentFileIndex_ =
+      (currentFileIndex_ - 1 + filenames_.size()) % filenames_.size();
+  refreshImage();
+}
+
+void ViewerWindow::on_actionNextImage_triggered() {
+  currentFileIndex_ = (currentFileIndex_ + 1) % filenames_.size();
+  refreshImage();
+}
+
+void ViewerWindow::refreshImage() {
+  if (currentFileIndex_ < 0 || currentFileIndex_ >= filenames_.size()) {
+    return;
+  }
+
+  qint64 elapsed_ns;
+  bool usedRequestedProfile;
+  const QImage image =
+      loadJpegXlImage(filenames_[currentFileIndex_], monitorProfile_,
+                      &elapsed_ns, &usedRequestedProfile);
+  if (image.isNull()) {
+    const QString message =
+        tr("Failed to load \"%1\".").arg(filenames_[currentFileIndex_]);
+    ui_.image->clear();
+    ui_.statusBar->showMessage(message);
+    QMessageBox errorDialog(this);
+    errorDialog.setIcon(QMessageBox::Critical);
+    errorDialog.setWindowTitle(tr("Failed to load image"));
+    errorDialog.setText(message);
+    errorDialog.exec();
+    return;
+  }
+
+  ui_.image->setPixmap(QPixmap::fromImage(image));
+  ui_.statusBar->showMessage(
+      tr("Loaded image %L1/%L2 (%3, %4×%5) in %L6ms (%L7 fps)")
+          .arg(currentFileIndex_ + 1)
+          .arg(filenames_.size())
+          .arg(filenames_[currentFileIndex_])
+          .arg(ui_.image->pixmap()->width())
+          .arg(ui_.image->pixmap()->height())
+          .arg(elapsed_ns / 1e6)
+          .arg(1e9 / elapsed_ns));
+
+  if (!usedRequestedProfile && !hasWarnedAboutMonitorProfile_) {
+    hasWarnedAboutMonitorProfile_ = true;
+    QMessageBox message(this);
+    message.setIcon(QMessageBox::Warning);
+    message.setWindowTitle(tr("No valid monitor profile found"));
+    message.setText(
+        tr("Failed to find a usable monitor profile. Images will be shown "
+           "assuming that the monitor's colorspace is sRGB."));
+    message.exec();
+  }
+}
+
+}  // namespace jxl
diff --git a/tools/viewer/viewer_window.h b/tools/viewer/viewer_window.h
new file mode 100644
index 000000000..827986a67
--- /dev/null
+++ b/tools/viewer/viewer_window.h
@@ -0,0 +1,50 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_VIEWER_VIEWER_WINDOW_H_
+#define TOOLS_VIEWER_VIEWER_WINDOW_H_
+
+#include <QMainWindow>
+#include <QStringList>
+
+#include "jxl/base/padded_bytes.h"
+#include "tools/viewer/ui_viewer_window.h"
+
+namespace jxl {
+
+class ViewerWindow : public QMainWindow {
+  Q_OBJECT
+ public:
+  explicit ViewerWindow(QWidget* parent = nullptr);
+
+ public slots:
+  void loadFilesAndDirectories(QStringList entries);
+
+ private slots:
+  void on_actionOpen_triggered();
+  void on_actionPreviousImage_triggered();
+  void on_actionNextImage_triggered();
+  void refreshImage();
+
+ private:
+  const PaddedBytes monitorProfile_;
+  Ui::ViewerWindow ui_;
+  QStringList filenames_;
+  int currentFileIndex_ = 0;
+  bool hasWarnedAboutMonitorProfile_ = false;
+};
+
+}  // namespace jxl
+
+#endif  // TOOLS_VIEWER_VIEWER_WINDOW_H_
diff --git a/tools/viewer/viewer_window.ui b/tools/viewer/viewer_window.ui
new file mode 100644
index 000000000..c238ca748
--- /dev/null
+++ b/tools/viewer/viewer_window.ui
@@ -0,0 +1,126 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<ui version="4.0">
+ <comment>
+  Copyright (c) the JPEG XL Project
+
+  Use of this source code is governed by an MIT-style
+  license that can be found in the LICENSE file or at
+  https://opensource.org/licenses/MIT.
+ </comment>
+ <class>ViewerWindow</class>
+ <widget class="QMainWindow" name="ViewerWindow">
+  <property name="windowTitle">
+   <string>JPEG XL Viewer</string>
+  </property>
+  <widget class="QWidget" name="centralwidget">
+   <layout class="QVBoxLayout" name="verticalLayout">
+    <property name="leftMargin">
+     <number>0</number>
+    </property>
+    <property name="topMargin">
+     <number>0</number>
+    </property>
+    <property name="rightMargin">
+     <number>0</number>
+    </property>
+    <property name="bottomMargin">
+     <number>0</number>
+    </property>
+    <item>
+     <widget class="QScrollArea" name="scrollArea">
+      <property name="frameShape">
+       <enum>QFrame::NoFrame</enum>
+      </property>
+      <property name="widgetResizable">
+       <bool>true</bool>
+      </property>
+      <widget class="QLabel" name="image">
+       <property name="alignment">
+        <set>Qt::AlignCenter</set>
+       </property>
+      </widget>
+     </widget>
+    </item>
+   </layout>
+  </widget>
+  <widget class="QMenuBar" name="menuBar">
+   <widget class="QMenu" name="menuFile">
+    <property name="title">
+     <string>&amp;File</string>
+    </property>
+    <addaction name="actionOpen"/>
+    <addaction name="separator"/>
+    <addaction name="actionExit"/>
+   </widget>
+   <addaction name="menuFile"/>
+  </widget>
+  <widget class="QStatusBar" name="statusBar"/>
+  <widget class="QToolBar" name="toolBar">
+   <property name="windowTitle">
+    <string>toolBar</string>
+   </property>
+   <attribute name="toolBarArea">
+    <enum>TopToolBarArea</enum>
+   </attribute>
+   <attribute name="toolBarBreak">
+    <bool>false</bool>
+   </attribute>
+   <addaction name="actionOpen"/>
+   <addaction name="actionPreviousImage"/>
+   <addaction name="actionNextImage"/>
+  </widget>
+  <action name="actionOpen">
+   <property name="icon">
+    <iconset theme="document-open"/>
+   </property>
+   <property name="text">
+    <string>&amp;Open…</string>
+   </property>
+   <property name="menuRole">
+    <enum>QAction::NoRole</enum>
+   </property>
+  </action>
+  <action name="actionExit">
+   <property name="icon">
+    <iconset theme="application-exit"/>
+   </property>
+   <property name="text">
+    <string>E&amp;xit</string>
+   </property>
+   <property name="menuRole">
+    <enum>QAction::QuitRole</enum>
+   </property>
+  </action>
+  <action name="actionPreviousImage">
+   <property name="icon">
+    <iconset theme="go-previous"/>
+   </property>
+   <property name="text">
+    <string>Previous image</string>
+   </property>
+   <property name="shortcut">
+    <string>Left</string>
+   </property>
+  </action>
+  <action name="actionNextImage">
+   <property name="icon">
+    <iconset theme="go-next"/>
+   </property>
+   <property name="text">
+    <string>Next image</string>
+   </property>
+   <property name="shortcut">
+    <string>Right</string>
+   </property>
+  </action>
+ </widget>
+ <resources/>
+ <connections>
+  <connection>
+   <sender>actionExit</sender>
+   <signal>triggered()</signal>
+   <receiver>ViewerWindow</receiver>
+   <slot>close()</slot>
+  </connection>
+ </connections>
+</ui>
diff --git a/tools/x11/icc.cc b/tools/x11/icc.cc
new file mode 100644
index 000000000..b6e1003ad
--- /dev/null
+++ b/tools/x11/icc.cc
@@ -0,0 +1,84 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "tools/x11/icc.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+
+#include <algorithm>
+#include <memory>
+
+namespace jxl {
+
+namespace {
+
+constexpr char kIccProfileAtomName[] = "_ICC_PROFILE";
+constexpr uint32_t kMaxIccProfileSize = 1 << 24;
+
+struct FreeDeleter {
+  void operator()(void* const p) const { std::free(p); }
+};
+template <typename T>
+using XcbUniquePtr = std::unique_ptr<T, FreeDeleter>;
+
+}  // namespace
+
+PaddedBytes GetMonitorIccProfile(xcb_connection_t* const connection,
+                                 const int screen_number) {
+  if (connection == nullptr) {
+    return PaddedBytes();
+  }
+
+  const xcb_intern_atom_cookie_t atomRequest =
+      xcb_intern_atom(connection, /*only_if_exists=*/1,
+                      sizeof kIccProfileAtomName - 1, kIccProfileAtomName);
+  const XcbUniquePtr<xcb_intern_atom_reply_t> atomReply(
+      xcb_intern_atom_reply(connection, atomRequest, nullptr));
+  if (atomReply == nullptr) {
+    return PaddedBytes();
+  }
+  const xcb_atom_t iccProfileAtom = atomReply->atom;
+
+  const xcb_screen_t* screen = nullptr;
+  int i = 0;
+  for (xcb_screen_iterator_t it =
+           xcb_setup_roots_iterator(xcb_get_setup(connection));
+       it.rem; xcb_screen_next(&it)) {
+    if (i == screen_number) {
+      screen = it.data;
+      break;
+    }
+    ++i;
+  }
+  if (screen == nullptr) {
+    return PaddedBytes();
+  }
+  const xcb_get_property_cookie_t profileRequest = xcb_get_property(
+      connection, /*_delete=*/0, screen->root, iccProfileAtom,
+      XCB_GET_PROPERTY_TYPE_ANY, /*long_offset=*/0, kMaxIccProfileSize);
+  const XcbUniquePtr<xcb_get_property_reply_t> profile(
+      xcb_get_property_reply(connection, profileRequest, nullptr));
+  if (profile == nullptr || profile->bytes_after > 0) {
+    return PaddedBytes();
+  }
+
+  PaddedBytes result(xcb_get_property_value_length(profile.get()));
+  std::copy_n(
+      reinterpret_cast<const uint8_t*>(xcb_get_property_value(profile.get())),
+      xcb_get_property_value_length(profile.get()), result.begin());
+  return result;
+}
+
+}  // namespace jxl
diff --git a/tools/x11/icc.h b/tools/x11/icc.h
new file mode 100644
index 000000000..ff2a47348
--- /dev/null
+++ b/tools/x11/icc.h
@@ -0,0 +1,30 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#ifndef TOOLS_X11_ICC_H_
+#define TOOLS_X11_ICC_H_
+
+#include <xcb/xcb.h>
+
+#include "jxl/base/padded_bytes.h"
+
+namespace jxl {
+
+// Should be cached if possible.
+PaddedBytes GetMonitorIccProfile(xcb_connection_t* connection,
+                                 int screen_number = 0);
+
+}  // namespace jxl
+
+#endif  // TOOLS_X11_ICC_H_
diff --git a/tools/xyb_range.cc b/tools/xyb_range.cc
new file mode 100644
index 000000000..838ed97ab
--- /dev/null
+++ b/tools/xyb_range.cc
@@ -0,0 +1,87 @@
+// Copyright (c) the JPEG XL Project
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include <stdio.h>
+
+#include <utility>
+
+#include "jxl/base/compiler_specific.h"
+#include "jxl/base/data_parallel.h"
+#include "jxl/codec_in_out.h"
+#include "jxl/color_encoding.h"
+#include "jxl/color_management.h"
+#include "jxl/enc_xyb.h"
+#include "jxl/image.h"
+#include "jxl/image_bundle.h"
+
+namespace jxl {
+namespace {
+
+void PrintXybRange() {
+  Image3F linear(1u << 16, 257);
+  for (int b = 0; b < 256; ++b) {
+    float* JXL_RESTRICT row0 = linear.PlaneRow(0, b + 1);
+    float* JXL_RESTRICT row1 = linear.PlaneRow(1, b + 1);
+    float* JXL_RESTRICT row2 = linear.PlaneRow(2, b + 1);
+    for (int r = 0; r < 256; ++r) {
+      for (int g = 0; g < 256; ++g) {
+        const int x = (r << 8) + g;
+        row0[x] = r;
+        row1[x] = g;
+        row2[x] = b;
+      }
+    }
+  }
+  CodecInOut io;
+  io.metadata.bits_per_sample = 8;
+  io.metadata.color_encoding = ColorManagement::LinearSRGB();
+  io.SetFromImage(std::move(linear), io.metadata.color_encoding);
+  const ImageBundle& ib = io.Main();
+  ThreadPool* null_pool = nullptr;
+  Image3F opsin(ib.xsize(), ib.ysize());
+  ToXYB(ib, 1.0f, null_pool, &opsin);
+  for (size_t c = 0; c < 3; ++c) {
+    float minval = 1e10f;
+    float maxval = -1e10f;
+    int rgb_min = 0;
+    int rgb_max = 0;
+    for (int b = 0; b < 256; ++b) {
+      const float* JXL_RESTRICT row = opsin.PlaneRow(c, b);
+      for (int r = 0; r < 256; ++r) {
+        for (int g = 0; g < 256; ++g) {
+          float val = row[(r << 8) + g];
+          if (val < minval) {
+            minval = val;
+            rgb_min = (r << 16) + (g << 8) + b;
+          }
+          if (val > maxval) {
+            maxval = val;
+            rgb_max = (r << 16) + (g << 8) + b;
+          }
+        }
+      }
+    }
+    printf(
+        "Opsin image plane %zu range: [%8.4f, %8.4f] "
+        "center: %.12f, range: %.12f (RGBmin=%06x, RGBmax=%06x)\n",
+        c, minval, maxval, 0.5 * (minval + maxval), 0.5 * (maxval - minval),
+        rgb_min, rgb_max);
+    // Ensure our constants are at least as wide as those obtained from sRGB.
+  }
+}
+
+}  // namespace
+}  // namespace jxl
+
+int main() { jxl::PrintXybRange(); }