media-libs/libvpx: backport CVE-2023-5217 fix

Bug: https://bugs.gentoo.org/914871
Bug: https://bugs.gentoo.org/914875
Closes: https://github.com/gentoo/gentoo/pull/33095
Signed-off-by: Sam James <sam@gentoo.org>

3 files changed, 292 insertions, 0 deletions

diff --git a/media-libs/libvpx/files/libvpx-1.13.0-VP8-disallow-thread-count-changes.patch b/media-libs/libvpx/files/libvpx-1.13.0-VP8-disallow-thread-count-changes.patch
new file mode 100644
index 000000000000..bdaea724f817
--- /dev/null
+++ b/media-libs/libvpx/files/libvpx-1.13.0-VP8-disallow-thread-count-changes.patch
@@ -0,0 +1,53 @@
+https://bugs.gentoo.org/914875
+
+From 4e4bbfbd414d738b7114194c5d06b45b415486b8 Mon Sep 17 00:00:00 2001
+From: James Zern <jzern@google.com>
+Date: Mon, 25 Sep 2023 18:55:59 -0700
+Subject: [PATCH 2/2] VP8: disallow thread count changes
+
+Currently allocations are done at encoder creation time. Going from
+threaded to non-threaded would cause a crash.
+
+Bug: chromium:1486441
+Change-Id: Ie301c2a70847dff2f0daae408fbef1e4d42e73d4
+(cherry picked from commit 3fbd1dca6a4d2dad332a2110d646e4ffef36d590)
+(cherry picked from commit 7aaffe2df4c9426ab204a272ca5ca52286ca86d4)
+---
+ test/encode_api_test.cc | 4 ----
+ vp8/encoder/onyx_if.c   | 5 +++++
+ 2 files changed, 5 insertions(+), 4 deletions(-)
+
+diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
+index 02aedc057..e0e793b15 100644
+--- a/test/encode_api_test.cc
++++ b/test/encode_api_test.cc
+@@ -366,10 +366,6 @@ TEST(EncodeAPI, ConfigResizeChangeThreadCount) {
+ 
+   for (const auto *iface : kCodecIfaces) {
+     SCOPED_TRACE(vpx_codec_iface_name(iface));
+-    if (!IsVP9(iface)) {
+-      GTEST_SKIP() << "TODO(https://crbug.com/1486441) remove this condition "
+-                      "after VP8 is fixed.";
+-    }
+     for (int i = 0; i < (IsVP9(iface) ? 2 : 1); ++i) {
+       vpx_codec_enc_cfg_t cfg = {};
+       struct Encoder {
+diff --git a/vp8/encoder/onyx_if.c b/vp8/encoder/onyx_if.c
+index 4bbeadef0..148a16cc4 100644
+--- a/vp8/encoder/onyx_if.c
++++ b/vp8/encoder/onyx_if.c
+@@ -1443,6 +1443,11 @@ void vp8_change_config(VP8_COMP *cpi, VP8_CONFIG *oxcf) {
+   last_h = cpi->oxcf.Height;
+   prev_number_of_layers = cpi->oxcf.number_of_layers;
+ 
++  if (cpi->initial_width) {
++    // TODO(https://crbug.com/1486441): Allow changing thread counts; the
++    // allocation is done once in vp8_create_compressor().
++    oxcf->multi_threaded = cpi->oxcf.multi_threaded;
++  }
+   cpi->oxcf = *oxcf;
+ 
+   switch (cpi->oxcf.Mode) {
+-- 
+2.42.0
+
diff --git a/media-libs/libvpx/files/libvpx-1.13.0-encode_api_test-add-ConfigResizeChangeThreadCount.patch b/media-libs/libvpx/files/libvpx-1.13.0-encode_api_test-add-ConfigResizeChangeThreadCount.patch
new file mode 100644
index 000000000000..17aa33567cc3
--- /dev/null
+++ b/media-libs/libvpx/files/libvpx-1.13.0-encode_api_test-add-ConfigResizeChangeThreadCount.patch
@@ -0,0 +1,94 @@
+https://bugs.gentoo.org/914875
+
+From 2f6c3a33e89e504ab3bef0aa847dc053f3355093 Mon Sep 17 00:00:00 2001
+From: James Zern <jzern@google.com>
+Date: Mon, 25 Sep 2023 18:53:41 -0700
+Subject: [PATCH 1/2] encode_api_test: add ConfigResizeChangeThreadCount
+
+Update thread counts and resolution to ensure allocations are updated
+correctly. VP8 is disabled to avoid a crash.
+
+Bug: chromium:1486441
+Change-Id: Ie89776d9818d27dc351eff298a44c699e850761b
+(cherry picked from commit af6dedd715f4307669366944cca6e0417b290282)
+(cherry picked from commit 972691e9af302f0bc14998e78a6d54f7861c92e5)
+---
+ test/encode_api_test.cc | 50 ++++++++++++++++++++++++++++++++++++++++-
+ 1 file changed, 49 insertions(+), 1 deletion(-)
+
+diff --git a/test/encode_api_test.cc b/test/encode_api_test.cc
+index ecdf92834..02aedc057 100644
+--- a/test/encode_api_test.cc
++++ b/test/encode_api_test.cc
+@@ -304,7 +304,6 @@ TEST(EncodeAPI, SetRoi) {
+ 
+ void InitCodec(const vpx_codec_iface_t &iface, int width, int height,
+                vpx_codec_ctx_t *enc, vpx_codec_enc_cfg_t *cfg) {
+-  ASSERT_EQ(vpx_codec_enc_config_default(&iface, cfg, 0), VPX_CODEC_OK);
+   cfg->g_w = width;
+   cfg->g_h = height;
+   cfg->g_lag_in_frames = 0;
+@@ -342,6 +341,7 @@ TEST(EncodeAPI, ConfigChangeThreadCount) {
+         vpx_codec_ctx_t ctx = {};
+       } enc;
+ 
++      ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK);
+       EXPECT_NO_FATAL_FAILURE(
+           InitCodec(*iface, kWidth, kHeight, &enc.ctx, &cfg));
+       if (IsVP9(iface)) {
+@@ -360,4 +360,52 @@ TEST(EncodeAPI, ConfigChangeThreadCount) {
+   }
+ }
+ 
++TEST(EncodeAPI, ConfigResizeChangeThreadCount) {
++  constexpr int kInitWidth = 1024;
++  constexpr int kInitHeight = 1024;
++
++  for (const auto *iface : kCodecIfaces) {
++    SCOPED_TRACE(vpx_codec_iface_name(iface));
++    if (!IsVP9(iface)) {
++      GTEST_SKIP() << "TODO(https://crbug.com/1486441) remove this condition "
++                      "after VP8 is fixed.";
++    }
++    for (int i = 0; i < (IsVP9(iface) ? 2 : 1); ++i) {
++      vpx_codec_enc_cfg_t cfg = {};
++      struct Encoder {
++        ~Encoder() { EXPECT_EQ(vpx_codec_destroy(&ctx), VPX_CODEC_OK); }
++        vpx_codec_ctx_t ctx = {};
++      } enc;
++
++      ASSERT_EQ(vpx_codec_enc_config_default(iface, &cfg, 0), VPX_CODEC_OK);
++      // Start in threaded mode to ensure resolution and thread related
++      // allocations are updated correctly across changes in resolution and
++      // thread counts. See https://crbug.com/1486441.
++      cfg.g_threads = 4;
++      EXPECT_NO_FATAL_FAILURE(
++          InitCodec(*iface, kInitWidth, kInitHeight, &enc.ctx, &cfg));
++      if (IsVP9(iface)) {
++        EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_TILE_COLUMNS, 6),
++                  VPX_CODEC_OK);
++        EXPECT_EQ(vpx_codec_control_(&enc.ctx, VP9E_SET_ROW_MT, i),
++                  VPX_CODEC_OK);
++      }
++
++      cfg.g_w = 1000;
++      cfg.g_h = 608;
++      EXPECT_EQ(vpx_codec_enc_config_set(&enc.ctx, &cfg), VPX_CODEC_OK)
++          << vpx_codec_error_detail(&enc.ctx);
++
++      cfg.g_w = 16;
++      cfg.g_h = 720;
++
++      for (const auto threads : { 1, 4, 8, 6, 2, 1 }) {
++        cfg.g_threads = threads;
++        EXPECT_NO_FATAL_FAILURE(EncodeWithConfig(cfg, &enc.ctx))
++            << "iteration: " << i << " threads: " << threads;
++      }
++    }
++  }
++}
++
+ }  // namespace
+-- 
+2.42.0
+
diff --git a/media-libs/libvpx/libvpx-1.13.0-r1.ebuild b/media-libs/libvpx/libvpx-1.13.0-r1.ebuild
new file mode 100644
index 000000000000..0fc96f933ed0
--- /dev/null
+++ b/media-libs/libvpx/libvpx-1.13.0-r1.ebuild
@@ -0,0 +1,145 @@
+# Copyright 1999-2023 Gentoo Authors
+# Distributed under the terms of the GNU General Public License v2
+
+EAPI=8
+
+inherit edo toolchain-funcs multilib-minimal
+
+# To create a new testdata tarball:
+# 1. Unpack source tarball or checkout git tag
+# 2. mkdir libvpx-testdata
+# 3. export LIBVPX_TEST_DATA_PATH=libvpx-testdata
+# 4. ./configure --enable-unit-tests --enable-vp9-highbitdepth
+# 5. make testdata
+# 6. tar -caf libvpx-testdata-${MY_PV}.tar.xz libvpx-testdata
+
+LIBVPX_TESTDATA_VER=1.13.0
+
+DESCRIPTION="WebM VP8 and VP9 Codec SDK"
+HOMEPAGE="https://www.webmproject.org"
+SRC_URI="
+	https://github.com/webmproject/${PN}/archive/v${PV}.tar.gz -> ${P}.tar.gz
+	test? ( https://dev.gentoo.org/~sam/distfiles/${CATEGORY}/${PN}/${PN}-testdata-${LIBVPX_TESTDATA_VER}.tar.xz )
+"
+
+LICENSE="BSD"
+SLOT="0/8"
+KEYWORDS="~amd64 ~arm ~arm64 ~ia64 ~loong ~ppc ~ppc64 ~riscv ~s390 ~sparc ~x86 ~amd64-linux ~x86-linux"
+IUSE="cpu_flags_ppc_vsx3 doc +highbitdepth postproc static-libs test +threads"
+REQUIRED_USE="test? ( threads )"
+RESTRICT="!test? ( test )"
+
+BDEPEND="
+	dev-lang/perl
+	abi_x86_32? ( dev-lang/yasm )
+	abi_x86_64? ( dev-lang/yasm )
+	abi_x86_x32? ( dev-lang/yasm )
+	doc? (
+		app-doc/doxygen
+		dev-lang/php
+	)
+"
+
+PATCHES=(
+	# bug #501010
+	"${FILESDIR}/${PN}-1.3.0-sparc-configure.patch"
+
+	# bug #914871
+	"${FILESDIR}/${P}-encode_api_test-add-ConfigResizeChangeThreadCount.patch"
+	"${FILESDIR}/${P}-VP8-disallow-thread-count-changes.patch"
+)
+
+src_configure() {
+	# bug #384585, bug #465988
+	# copied from php-pear-r1.eclass
+	addpredict /usr/share/snmp/mibs/.index
+	addpredict /var/lib/net-snmp/
+	addpredict /var/lib/net-snmp/mib_indexes
+	addpredict /session_mm_cli0.sem
+	multilib-minimal_src_configure
+}
+
+multilib_src_configure() {
+	# bug #357487
+	unset CODECS
+	# bug #905986
+	unset DIST_DIR
+
+	# bug #498364: sse doesn't work without sse2 enabled,
+	local myconfargs=(
+		--prefix="${EPREFIX}"/usr
+		--libdir="${EPREFIX}"/usr/$(get_libdir)
+		--enable-pic
+		--enable-vp8
+		--enable-vp9
+		--enable-shared
+		--disable-optimizations
+		$(use_enable postproc)
+		$(use_enable static-libs static)
+		$(use_enable test unit-tests)
+		$(use_enable threads multithread)
+		$(use_enable highbitdepth vp9-highbitdepth)
+	)
+
+	# let the build system decide which AS to use (it honours $AS but
+	# then feeds it with yasm flags without checking...), bug #345161
+	tc-export AS
+	case "${CHOST}" in
+		i?86*) export AS=yasm;;
+		x86_64*) export AS=yasm;;
+	esac
+
+	# libvpx is fragile: both for tests at runtime.
+	# We force using the generic target unless we know things work to
+	# avoid runtime breakage on exotic arches.
+	if [[ ${ABI} == amd64 ]] ; then
+		myconfargs+=( --force-target=x86_64-linux-gcc )
+	elif [[ ${ABI} == x86 ]] ; then
+		myconfargs+=( --force-target=x86-linux-gcc )
+	elif [[ ${ABI} == arm64 ]] ; then
+		myconfargs+=( --force-target=arm64-linux-gcc )
+	elif [[ ${ABI} == arm ]] && [[ ${CHOST} == *armv7* ]] ; then
+		myconfargs+=( --force-target=armv7-linux-gcc )
+	elif [[ ${ABI} == ppc64 ]] && [[ $(tc-endian) != big ]] && use cpu_flags_ppc_vsx3; then
+		# only enable this target for at least power9 CPU running little-endian
+		myconfargs+=( --force-target=ppc64le-linux-gcc )
+	else
+		myconfargs+=( --force-target=generic-gnu )
+	fi
+
+	# powerpc toolchain is not recognized anymore, bug #694368
+	#[[ ${CHOST} == powerpc-* ]] && myconfargs+=( --force-target=generic-gnu )
+
+	# Build with correct toolchain.
+	tc-export CC CXX AR NM
+	# Link with gcc by default, the build system should override this if needed.
+	export LD="${CC}"
+
+	if multilib_is_native_abi; then
+		myconfargs+=( $(use_enable doc install-docs) $(use_enable doc docs) )
+	else
+		# Not needed for multilib and will be overwritten anyway.
+		myconfargs+=( --disable-examples --disable-install-docs --disable-docs )
+	fi
+
+	edo "${S}"/configure "${myconfargs[@]}"
+}
+
+multilib_src_compile() {
+	# Build verbose by default and do not build examples that will not be installed
+	# Disable stripping of debug info, bug #752057
+	# (only works as long as upstream does not use non-gnu strip)
+	emake verbose=yes GEN_EXAMPLES= HAVE_GNU_STRIP=no
+}
+
+multilib_src_test() {
+	local -x LD_LIBRARY_PATH="${BUILD_DIR}"
+	local -x LIBVPX_TEST_DATA_PATH="${WORKDIR}/${PN}-testdata"
+	emake verbose=yes GEN_EXAMPLES= test
+}
+
+multilib_src_install() {
+	emake verbose=yes GEN_EXAMPLES= DESTDIR="${D}" install
+
+	multilib_is_native_abi && use doc && dodoc -r docs/html
+}