Fix an overflow when calculating something for AVC/HEVC videos:

https://security-tracker.debian.org/tracker/TEMP-0000000-C6AAE1

Patch copied from upstream source repository:

https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/commit/0cfbf7ad91c7f121192c8ce135769f8eb276c41d
From 0cfbf7ad91c7f121192c8ce135769f8eb276c41d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sebastian=20Dr=C3=B6ge?= <sebastian@centricular.com>
Date: Tue, 23 Mar 2021 19:19:14 +0200
Subject: [PATCH] h2645parser: Catch overflows in AVC/HEVC NAL unit length
 calculations

Offset and size are stored as 32 bit guint and might overflow when
adding the nal_length_size, so let's avoid that.

For the size this would happen if the AVC/HEVC NAL unit size happens to
be stored in 4 bytes and is 4294967292 or higher, which is likely
corrupted data anyway.

For the offset this is something for the caller of these functions to
take care of but is unlikely to happen as it would require parsing on a
>4GB buffer.

Allowing these overflows causes all kinds of follow-up bugs in the
h2645parse elements, ranging from infinite loops and memory leaks to
potential memory corruptions.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gst-plugins-bad/-/merge_requests/2107>
---
 gst-libs/gst/codecparsers/gsth264parser.c | 16 +++++-
 gst-libs/gst/codecparsers/gsth265parser.c | 16 +++++-
 tests/check/libs/h264parser.c             | 60 +++++++++++++++++++++++
 tests/check/libs/h265parser.c             | 60 +++++++++++++++++++++++
 4 files changed, 150 insertions(+), 2 deletions(-)

diff --git a/gst-libs/gst/codecparsers/gsth264parser.c b/gst-libs/gst/codecparsers/gsth264parser.c
index 012f1d0d7..68aa25068 100644
--- a/gst-libs/gst/codecparsers/gsth264parser.c
+++ b/gst-libs/gst/codecparsers/gsth264parser.c
@@ -1556,6 +1556,14 @@ gst_h264_parser_identify_nalu_avc (GstH264NalParser * nalparser,
 
   memset (nalu, 0, sizeof (*nalu));
 
+  /* Would overflow guint below otherwise: the callers needs to ensure that
+   * this never happens */
+  if (offset > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("offset + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H264_PARSER_BROKEN_DATA;
+  }
+
   if (size < offset + nal_length_size) {
     GST_DEBUG ("Can't parse, buffer has too small size %" G_GSIZE_FORMAT
         ", offset %u", size, offset);
@@ -1570,7 +1578,13 @@ gst_h264_parser_identify_nalu_avc (GstH264NalParser * nalparser,
   nalu->sc_offset = offset;
   nalu->offset = offset + nal_length_size;
 
-  if (size < nalu->size + nal_length_size) {
+  if (nalu->size > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("NALU size + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H264_PARSER_BROKEN_DATA;
+  }
+
+  if (size < (gsize) nalu->size + nal_length_size) {
     nalu->size = 0;
 
     return GST_H264_PARSER_NO_NAL_END;
diff --git a/gst-libs/gst/codecparsers/gsth265parser.c b/gst-libs/gst/codecparsers/gsth265parser.c
index 26e68b276..dc7f27aa9 100644
--- a/gst-libs/gst/codecparsers/gsth265parser.c
+++ b/gst-libs/gst/codecparsers/gsth265parser.c
@@ -1531,6 +1531,14 @@ gst_h265_parser_identify_nalu_hevc (GstH265Parser * parser,
 
   memset (nalu, 0, sizeof (*nalu));
 
+  /* Would overflow guint below otherwise: the callers needs to ensure that
+   * this never happens */
+  if (offset > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("offset + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H265_PARSER_BROKEN_DATA;
+  }
+
   if (size < offset + nal_length_size) {
     GST_DEBUG ("Can't parse, buffer has too small size %" G_GSIZE_FORMAT
         ", offset %u", size, offset);
@@ -1545,7 +1553,13 @@ gst_h265_parser_identify_nalu_hevc (GstH265Parser * parser,
   nalu->sc_offset = offset;
   nalu->offset = offset + nal_length_size;
 
-  if (size < nalu->size + nal_length_size) {
+  if (nalu->size > G_MAXUINT32 - nal_length_size) {
+    GST_WARNING ("NALU size + nal_length_size overflow");
+    nalu->size = 0;
+    return GST_H265_PARSER_BROKEN_DATA;
+  }
+
+  if (size < (gsize) nalu->size + nal_length_size) {
     nalu->size = 0;
 
     return GST_H265_PARSER_NO_NAL_END;
diff --git a/tests/check/libs/h264parser.c b/tests/check/libs/h264parser.c
index c7c46d9a2..d322dd8db 100644
--- a/tests/check/libs/h264parser.c
+++ b/tests/check/libs/h264parser.c
@@ -229,6 +229,65 @@ GST_START_TEST (test_h264_parse_slice_5bytes)
 
 GST_END_TEST;
 
+GST_START_TEST (test_h264_parse_identify_nalu_avc)
+{
+  GstH264ParserResult res;
+  GstH264NalUnit nalu;
+  GstH264NalParser *const parser = gst_h264_nal_parser_new ();
+  /* Skip 3 bytes for the start code */
+  const gsize nal_size = sizeof (slice_dpa) - 3;
+  const gsize buf_size = 4 + nal_size;
+  guint8 *buf = g_new (guint8, buf_size);
+
+  memcpy (buf + 4, slice_dpa + 3, nal_size);
+
+  GST_WRITE_UINT16_BE (buf + 2, nal_size);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 2, buf_size, 2, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_OK);
+  assert_equals_int (nalu.type, GST_H264_NAL_SLICE_DPA);
+  assert_equals_int (nalu.offset, 4);
+  assert_equals_int (nalu.size, nal_size);
+
+  GST_WRITE_UINT32_BE (buf, nal_size);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_OK);
+  assert_equals_int (nalu.type, GST_H264_NAL_SLICE_DPA);
+  assert_equals_int (nalu.offset, 4);
+  assert_equals_int (nalu.size, nal_size);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 2);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 3);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 4);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_NO_NAL_END);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 6);
+  res = gst_h264_parser_identify_nalu_avc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H264_PARSER_NO_NAL_END);
+
+  g_free (buf);
+  gst_h264_nal_parser_free (parser);
+}
+
+GST_END_TEST;
+
 static guint8 nalu_sps_with_vui[] = {
   0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x28,
   0xac, 0xd9, 0x40, 0x78, 0x04, 0x4f, 0xde, 0x03,
@@ -666,6 +725,7 @@ h264parser_suite (void)
   tcase_add_test (tc_chain, test_h264_parse_slice_dpa);
   tcase_add_test (tc_chain, test_h264_parse_slice_eoseq_slice);
   tcase_add_test (tc_chain, test_h264_parse_slice_5bytes);
+  tcase_add_test (tc_chain, test_h264_parse_identify_nalu_avc);
   tcase_add_test (tc_chain, test_h264_parse_invalid_sei);
   tcase_add_test (tc_chain, test_h264_create_sei);
 
diff --git a/tests/check/libs/h265parser.c b/tests/check/libs/h265parser.c
index 0a0e4db97..5b6a215ec 100644
--- a/tests/check/libs/h265parser.c
+++ b/tests/check/libs/h265parser.c
@@ -255,6 +255,65 @@ GST_START_TEST (test_h265_parse_slice_6bytes)
 
 GST_END_TEST;
 
+GST_START_TEST (test_h265_parse_identify_nalu_hevc)
+{
+  GstH265ParserResult res;
+  GstH265NalUnit nalu;
+  GstH265Parser *parser = gst_h265_parser_new ();
+  /* Skip 4 bytes for the start code */
+  const gsize nal_size = sizeof (slice_eos_slice_eob) - 4;
+  const gsize buf_size = 4 + nal_size;
+  guint8 *buf = g_new (guint8, buf_size);
+
+  memcpy (buf + 4, slice_eos_slice_eob + 4, nal_size);
+
+  GST_WRITE_UINT16_BE (buf + 2, nal_size);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 2, buf_size, 2, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_OK);
+  assert_equals_int (nalu.type, GST_H265_NAL_SLICE_IDR_W_RADL);
+  assert_equals_int (nalu.offset, 4);
+  assert_equals_int (nalu.size, nal_size);
+
+  GST_WRITE_UINT32_BE (buf, nal_size);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_OK);
+  assert_equals_int (nalu.type, GST_H265_NAL_SLICE_IDR_W_RADL);
+  assert_equals_int (nalu.offset, 4);
+  assert_equals_int (nalu.size, nal_size);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 2);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 3);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_BROKEN_DATA);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 4);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_NO_NAL_END);
+
+  GST_WRITE_UINT32_BE (buf, G_MAXUINT32 - 6);
+  res = gst_h265_parser_identify_nalu_hevc (parser, buf, 0, buf_size, 4, &nalu);
+
+  assert_equals_int (res, GST_H265_PARSER_NO_NAL_END);
+
+  g_free (buf);
+  gst_h265_parser_free (parser);
+}
+
+GST_END_TEST;
+
 GST_START_TEST (test_h265_base_profiles)
 {
   GstH265ProfileTierLevel ptl;
@@ -1101,6 +1160,7 @@ h265parser_suite (void)
   tcase_add_test (tc_chain, test_h265_parse_slice_eos_slice_eob);
   tcase_add_test (tc_chain, test_h265_parse_pic_timing);
   tcase_add_test (tc_chain, test_h265_parse_slice_6bytes);
+  tcase_add_test (tc_chain, test_h265_parse_identify_nalu_hevc);
   tcase_add_test (tc_chain, test_h265_base_profiles);
   tcase_add_test (tc_chain, test_h265_base_profiles_compat);
   tcase_add_test (tc_chain, test_h265_format_range_profiles_exact_match);
-- 
2.31.1