swscale/output: Fix integer overflow in yuv2ya16_X_c_template()

Found-by: colod colod <colodcolod7@gmail.com>

Fixes: CVE-2025-63757

Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit 0c6b7f9483)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit ac4caa33ba)
Signed-off-by: Carlos Henrique Lima Melara <charlesmelara@riseup.net>
This commit is contained in:
Michael Niedermayer
2025-10-13 14:32:45 +02:00
committed by James Almer
parent 58882aa298
commit 257ab5a6ac

View File

@@ -909,7 +909,7 @@ yuv2ya16_X_c_template(SwsContext *c, const int16_t *lumFilter,
int A = 0xffff;
for (j = 0; j < lumFilterSize; j++)
Y += lumSrc[j][i] * lumFilter[j];
Y += lumSrc[j][i] * (unsigned)lumFilter[j];
Y >>= 15;
Y += (1<<3) + 0x8000;
@@ -918,7 +918,7 @@ yuv2ya16_X_c_template(SwsContext *c, const int16_t *lumFilter,
if (hasAlpha) {
A = -0x40000000 + (1<<14);
for (j = 0; j < lumFilterSize; j++)
A += alpSrc[j][i] * lumFilter[j];
A += alpSrc[j][i] * (unsigned)lumFilter[j];
A >>= 15;
A += 0x8000;