Update for 2.2.13

Signed-off-by: Michael Niedermayer <michaelni@gmx.at>

RELEASE

@@ -1 +1 @@
-2.2.12
+2.2.13

doc/Doxyfile

@@ -31,7 +31,7 @@ PROJECT_NAME           = FFmpeg
 # This could be handy for archiving the generated documentation or
 # if some version control system is used.
 
-PROJECT_NUMBER         = 2.2.12
+PROJECT_NUMBER         = 2.2.13
 
 # With the PROJECT_LOGO tag one can specify a logo or icon that is included
 # in the documentation. The maximum height of the logo should not exceed 55

libavcodec/aarch64/h264dsp_init_aarch64.c

@@ -78,6 +78,7 @@ av_cold void ff_h264dsp_init_aarch64(H264DSPContext *c, const int bit_depth,
         c->h264_v_loop_filter_luma   = ff_h264_v_loop_filter_luma_neon;
         c->h264_h_loop_filter_luma   = ff_h264_h_loop_filter_luma_neon;
         c->h264_v_loop_filter_chroma = ff_h264_v_loop_filter_chroma_neon;
+        if (chroma_format_idc <= 1)
         c->h264_h_loop_filter_chroma = ff_h264_h_loop_filter_chroma_neon;
 
         c->weight_h264_pixels_tab[0] = ff_weight_h264_pixels_16_neon;

libavcodec/arm/videodsp_armv5te.S

@@ -23,9 +23,10 @@
 #include "libavutil/arm/asm.S"
 
 function ff_prefetch_arm, export=1
+1:
         subs            r2,  r2,  #1
         pld             [r0]
         add             r0,  r0,  r1
-        bne             X(ff_prefetch_arm)
+        bne             1b
         bx              lr
 endfunc

libavcodec/dxtory.c

@@ -63,7 +63,7 @@ static int dxtory_decode_v1_410(AVCodecContext *avctx, AVFrame *pic,
     uint8_t *Y1, *Y2, *Y3, *Y4, *U, *V;
     int ret;
 
-    if (src_size < avctx->width * avctx->height * 9L / 8) {
+    if (src_size < avctx->width * avctx->height * 9LL / 8) {
         av_log(avctx, AV_LOG_ERROR, "packet too small\n");
         return AVERROR_INVALIDDATA;
     }
@@ -106,7 +106,7 @@ static int dxtory_decode_v1_420(AVCodecContext *avctx, AVFrame *pic,
     uint8_t *Y1, *Y2, *U, *V;
     int ret;
 
-    if (src_size < avctx->width * avctx->height * 3L / 2) {
+    if (src_size < avctx->width * avctx->height * 3LL / 2) {
         av_log(avctx, AV_LOG_ERROR, "packet too small\n");
         return AVERROR_INVALIDDATA;
     }
@@ -143,7 +143,7 @@ static int dxtory_decode_v1_444(AVCodecContext *avctx, AVFrame *pic,
     uint8_t *Y, *U, *V;
     int ret;
 
-    if (src_size < avctx->width * avctx->height * 3L) {
+    if (src_size < avctx->width * avctx->height * 3LL) {
         av_log(avctx, AV_LOG_ERROR, "packet too small\n");
         return AVERROR_INVALIDDATA;
     }

libavcodec/gif.c

@@ -105,7 +105,7 @@ static int gif_image_write_image(AVCodecContext *avctx,
         /* skip common columns */
         while (x_start < x_end) {
             int same_column = 1;
-            for (y = y_start; y < y_end; y++) {
+            for (y = y_start; y <= y_end; y++) {
                 if (ref[y*ref_linesize + x_start] != buf[y*linesize + x_start]) {
                     same_column = 0;
                     break;
@@ -117,7 +117,7 @@ static int gif_image_write_image(AVCodecContext *avctx,
         }
         while (x_end > x_start) {
             int same_column = 1;
-            for (y = y_start; y < y_end; y++) {
+            for (y = y_start; y <= y_end; y++) {
                 if (ref[y*ref_linesize + x_end] != buf[y*linesize + x_end]) {
                     same_column = 0;
                     break;

libavcodec/h264.c

@@ -3506,6 +3506,9 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
     int must_reinit;
     int needs_reinit = 0;
     int field_pic_flag, bottom_field_flag;
+    int first_slice = h == h0 && !h0->current_slice;
+    int frame_num, picture_structure, droppable;
+    PPS *pps;
 
     h->me.qpel_put = h->h264qpel.put_h264_qpel_pixels_tab;
     h->me.qpel_avg = h->h264qpel.avg_h264_qpel_pixels_tab;
@@ -3570,18 +3573,27 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
                h0->au_pps_id, pps_id);
         return AVERROR_INVALIDDATA;
     }
-    h->pps = *h0->pps_buffers[pps_id];
 
-    if (!h0->sps_buffers[h->pps.sps_id]) {
+    pps = h0->pps_buffers[pps_id];
+
+    if (!h0->sps_buffers[pps->sps_id]) {
         av_log(h->avctx, AV_LOG_ERROR,
                "non-existing SPS %u referenced\n",
                h->pps.sps_id);
         return AVERROR_INVALIDDATA;
     }
+    if (first_slice)
+        h->pps = *h0->pps_buffers[pps_id];
 
-    if (h->pps.sps_id != h->sps.sps_id ||
-        h->pps.sps_id != h->current_sps_id ||
-        h0->sps_buffers[h->pps.sps_id]->new) {
+    if (pps->sps_id != h->sps.sps_id ||
+        pps->sps_id != h->current_sps_id ||
+        h0->sps_buffers[pps->sps_id]->new) {
+
+        if (!first_slice) {
+            av_log(h->avctx, AV_LOG_ERROR,
+               "SPS changed in the middle of the frame\n");
+            return AVERROR_INVALIDDATA;
+        }
 
         h->sps = *h0->sps_buffers[h->pps.sps_id];
 
@@ -3611,13 +3623,15 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
                      || 16*h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag) != h->avctx->coded_height
                      || h->avctx->bits_per_raw_sample != h->sps.bit_depth_luma
                      || h->cur_chroma_format_idc != h->sps.chroma_format_idc
-                     || av_cmp_q(h->sps.sar, h->avctx->sample_aspect_ratio)
                      || h->mb_width  != h->sps.mb_width
                      || h->mb_height != h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag)
                     ));
     if (non_j_pixfmt(h0->avctx->pix_fmt) != non_j_pixfmt(get_pixel_format(h0, 0)))
         must_reinit = 1;
 
+    if (first_slice && av_cmp_q(h->sps.sar, h->avctx->sample_aspect_ratio))
+        must_reinit = 1;
+
     h->mb_width  = h->sps.mb_width;
     h->mb_height = h->sps.mb_height * (2 - h->sps.frame_mbs_only_flag);
     h->mb_num    = h->mb_width * h->mb_height;
@@ -3699,39 +3713,43 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
         init_dequant_tables(h);
     }
 
-    h->frame_num = get_bits(&h->gb, h->sps.log2_max_frame_num);
+    frame_num = get_bits(&h->gb, h->sps.log2_max_frame_num);
+    if (!first_slice) {
+        if (h0->frame_num != frame_num) {
+            av_log(h->avctx, AV_LOG_ERROR, "Frame num change from %d to %d\n",
+                   h0->frame_num, frame_num);
+            return AVERROR_INVALIDDATA;
+        }
+    }
 
     h->mb_mbaff        = 0;
     h->mb_aff_frame    = 0;
     last_pic_structure = h0->picture_structure;
     last_pic_droppable = h0->droppable;
-    h->droppable       = h->nal_ref_idc == 0;
+    droppable          = h->nal_ref_idc == 0;
     if (h->sps.frame_mbs_only_flag) {
-        h->picture_structure = PICT_FRAME;
+        picture_structure = PICT_FRAME;
     } else {
         if (!h->sps.direct_8x8_inference_flag && slice_type == AV_PICTURE_TYPE_B) {
             av_log(h->avctx, AV_LOG_ERROR, "This stream was generated by a broken encoder, invalid 8x8 inference\n");
             return -1;
         }
         field_pic_flag = get_bits1(&h->gb);
+
         if (field_pic_flag) {
             bottom_field_flag = get_bits1(&h->gb);
-            h->picture_structure = PICT_TOP_FIELD + bottom_field_flag;
+            picture_structure = PICT_TOP_FIELD + bottom_field_flag;
         } else {
-            h->picture_structure = PICT_FRAME;
+            picture_structure = PICT_FRAME;
             h->mb_aff_frame      = h->sps.mb_aff;
         }
     }
-    h->mb_field_decoding_flag = h->picture_structure != PICT_FRAME;
-
-    if (h0->current_slice != 0) {
-        if (last_pic_structure != h->picture_structure ||
-            last_pic_droppable != h->droppable) {
+    if (h0->current_slice) {
+        if (last_pic_structure != picture_structure ||
+            last_pic_droppable != droppable) {
             av_log(h->avctx, AV_LOG_ERROR,
                    "Changing field mode (%d -> %d) between slices is not allowed\n",
                    last_pic_structure, h->picture_structure);
-            h->picture_structure = last_pic_structure;
-            h->droppable         = last_pic_droppable;
             return AVERROR_INVALIDDATA;
         } else if (!h0->cur_pic_ptr) {
             av_log(h->avctx, AV_LOG_ERROR,
@@ -3739,7 +3757,14 @@ static int decode_slice_header(H264Context *h, H264Context *h0)
                    h0->current_slice + 1);
             return AVERROR_INVALIDDATA;
         }
-    } else {
+    }
+
+    h->picture_structure = picture_structure;
+    h->droppable         = droppable;
+    h->frame_num         = frame_num;
+    h->mb_field_decoding_flag = picture_structure != PICT_FRAME;
+
+    if (h0->current_slice == 0) {
         /* Shorten frame num gaps so we don't have to allocate reference
          * frames just to throw them away */
         if (h->frame_num != h->prev_frame_num) {
@@ -4945,8 +4970,8 @@ static int decode_nal_units(H264Context *h, const uint8_t *buf, int buf_size,
                 continue;
 
 again:
-            if (   !(avctx->active_thread_type & FF_THREAD_FRAME)
-                || nals_needed >= nal_index)
+            if (   (!(avctx->active_thread_type & FF_THREAD_FRAME) || nals_needed >= nal_index)
+                && !h->current_slice)
                 h->au_pps_id = -1;
             /* Ignore per frame NAL unit type during extradata
              * parsing. Decoding slices is not possible in codec init

libavcodec/h264_cabac.c

@@ -1280,7 +1280,7 @@ void ff_h264_init_cabac_states(H264Context *h) {
 }
 
 static int decode_cabac_field_decoding_flag(H264Context *h) {
-    const long mbb_xy = h->mb_xy - 2L*h->mb_stride;
+    const int mbb_xy = h->mb_xy - 2*h->mb_stride;
 
     unsigned long ctx = 0;
 

libavcodec/h264_ps.c

@@ -383,7 +383,8 @@ int ff_h264_decode_seq_parameter_set(H264Context *h)
                                   "Different chroma and luma bit depth");
             goto fail;
         }
-        if (sps->bit_depth_luma > 14U || sps->bit_depth_chroma > 14U) {
+        if (sps->bit_depth_luma   < 8 || sps->bit_depth_luma   > 14 ||
+            sps->bit_depth_chroma < 8 || sps->bit_depth_chroma > 14) {
             av_log(h->avctx, AV_LOG_ERROR, "illegal bit depth value (%d, %d)\n",
                    sps->bit_depth_luma, sps->bit_depth_chroma);
             goto fail;

libavcodec/mjpegdec.c

@@ -495,9 +495,12 @@ unk_pixfmt:
     }
     if (s->ls) {
         s->upscale_h = s->upscale_v = 0;
-        if (s->nb_components > 1)
+        if (s->nb_components == 3) {
             s->avctx->pix_fmt = AV_PIX_FMT_RGB24;
-        else if (s->bits <= 8)
+        } else if (s->nb_components != 1) {
+            av_log(s->avctx, AV_LOG_ERROR, "Unsupported number of components %d\n", s->nb_components);
+            return AVERROR_PATCHWELCOME;
+        } else if (s->bits <= 8)
             s->avctx->pix_fmt = AV_PIX_FMT_GRAY8;
         else
             s->avctx->pix_fmt = AV_PIX_FMT_GRAY16;
@@ -1180,13 +1183,18 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah,
 
                     if (s->interlaced && s->bottom_field)
                         block_offset += linesize[c] >> 1;
-                    ptr = data[c] + block_offset;
+                    if (   8*(h * mb_x + x) < s->width
+                        && 8*(v * mb_y + y) < s->height) {
+                        ptr = data[c] + block_offset;
+                    } else
+                        ptr = NULL;
                     if (!s->progressive) {
-                        if (copy_mb)
-                            mjpeg_copy_block(s, ptr, reference_data[c] + block_offset,
-                                             linesize[c], s->avctx->lowres);
+                        if (copy_mb) {
+                            if (ptr)
+                                mjpeg_copy_block(s, ptr, reference_data[c] + block_offset,
+                                                linesize[c], s->avctx->lowres);
 
-                        else {
+                        } else {
                             s->dsp.clear_block(s->block);
                             if (decode_block(s, s->block, i,
                                              s->dc_index[i], s->ac_index[i],
@@ -1195,9 +1203,11 @@ static int mjpeg_decode_scan(MJpegDecodeContext *s, int nb_components, int Ah,
                                        "error y=%d x=%d\n", mb_y, mb_x);
                                 return AVERROR_INVALIDDATA;
                             }
-                            s->dsp.idct_put(ptr, linesize[c], s->block);
-                            if (s->bits & 7)
-                                shift_output(s, ptr, linesize[c]);
+                            if (ptr) {
+                                s->dsp.idct_put(ptr, linesize[c], s->block);
+                                if (s->bits & 7)
+                                    shift_output(s, ptr, linesize[c]);
+                            }
                         }
                     } else {
                         int block_idx  = s->block_stride[c] * (v * mb_y + y) +
@@ -1835,6 +1845,10 @@ int ff_mjpeg_find_marker(MJpegDecodeContext *s,
             put_bits(&pb, 8, x);
             if (x == 0xFF) {
                 x = src[b++];
+                if (x & 0x80) {
+                    av_log(s->avctx, AV_LOG_WARNING, "Invalid escape sequence\n");
+                    x &= 0x7f;
+                }
                 put_bits(&pb, 7, x);
                 bit_count--;
             }

libavcodec/mpegvideo_enc.c

@@ -348,18 +348,18 @@ av_cold int ff_MPV_encode_init(AVCodecContext *avctx)
         switch(avctx->codec_id) {
         case AV_CODEC_ID_MPEG1VIDEO:
         case AV_CODEC_ID_MPEG2VIDEO:
-            avctx->rc_buffer_size = FFMAX(avctx->rc_max_rate, 15000000) * 112L / 15000000 * 16384;
+            avctx->rc_buffer_size = FFMAX(avctx->rc_max_rate, 15000000) * 112LL / 15000000 * 16384;
             break;
         case AV_CODEC_ID_MPEG4:
         case AV_CODEC_ID_MSMPEG4V1:
         case AV_CODEC_ID_MSMPEG4V2:
         case AV_CODEC_ID_MSMPEG4V3:
             if       (avctx->rc_max_rate >= 15000000) {
-                avctx->rc_buffer_size = 320 + (avctx->rc_max_rate - 15000000L) * (760-320) / (38400000 - 15000000);
+                avctx->rc_buffer_size = 320 + (avctx->rc_max_rate - 15000000LL) * (760-320) / (38400000 - 15000000);
             } else if(avctx->rc_max_rate >=  2000000) {
-                avctx->rc_buffer_size =  80 + (avctx->rc_max_rate -  2000000L) * (320- 80) / (15000000 -  2000000);
+                avctx->rc_buffer_size =  80 + (avctx->rc_max_rate -  2000000LL) * (320- 80) / (15000000 -  2000000);
             } else if(avctx->rc_max_rate >=   384000) {
-                avctx->rc_buffer_size =  40 + (avctx->rc_max_rate -   384000L) * ( 80- 40) / ( 2000000 -   384000);
+                avctx->rc_buffer_size =  40 + (avctx->rc_max_rate -   384000LL) * ( 80- 40) / ( 2000000 -   384000);
             } else
                 avctx->rc_buffer_size = 40;
             avctx->rc_buffer_size *= 16384;

libavformat/matroskadec.c

@@ -1907,8 +1907,8 @@ static int matroska_read_header(AVFormatContext *s)
                 av_reduce(&st->avg_frame_rate.num, &st->avg_frame_rate.den,
                           1000000000, track->default_duration, 30000);
 #if FF_API_R_FRAME_RATE
-                if (   st->avg_frame_rate.num < st->avg_frame_rate.den * 1000L
-                    && st->avg_frame_rate.num > st->avg_frame_rate.den * 5L)
+                if (   st->avg_frame_rate.num < st->avg_frame_rate.den * 1000LL
+                    && st->avg_frame_rate.num > st->avg_frame_rate.den * 5LL)
                     st->r_frame_rate = st->avg_frame_rate;
 #endif
             }
@@ -2822,7 +2822,7 @@ static int matroska_read_seek(AVFormatContext *s, int stream_index,
                               int64_t timestamp, int flags)
 {
     MatroskaDemuxContext *matroska = s->priv_data;
-    MatroskaTrack *tracks = matroska->tracks.elem;
+    MatroskaTrack *tracks = NULL;
     AVStream *st = s->streams[stream_index];
     int i, index, index_sub, index_min;
 
@@ -2851,6 +2851,7 @@ static int matroska_read_seek(AVFormatContext *s, int stream_index,
         goto err;
 
     index_min = index;
+    tracks = matroska->tracks.elem;
     for (i=0; i < matroska->tracks.nb_elem; i++) {
         tracks[i].audio.pkt_cnt = 0;
         tracks[i].audio.sub_packet_cnt = 0;

libavformat/mpc8.c

@@ -57,7 +57,7 @@ typedef struct {
 
 static inline int64_t bs_get_v(const uint8_t **bs)
 {
-    int64_t v = 0;
+    uint64_t v = 0;
     int br = 0;
     int c;
 
@@ -91,7 +91,7 @@ static int mpc8_probe(AVProbeData *p)
         size = bs_get_v(&bs);
         if (size < 2)
             return 0;
-        if (bs + size - 2 >= bs_end)
+        if (size >= bs_end - bs + 2)
             return AVPROBE_SCORE_EXTENSION - 1; // seems to be valid MPC but no header yet
         if (header_found) {
             if (size < 11 || size > 28)
@@ -108,7 +108,7 @@ static int mpc8_probe(AVProbeData *p)
 
 static inline int64_t gb_get_v(GetBitContext *gb)
 {
-    int64_t v = 0;
+    uint64_t v = 0;
     int bits = 0;
     while(get_bits1(gb) && bits < 64-7){
         v <<= 7;
@@ -223,6 +223,10 @@ static int mpc8_read_header(AVFormatContext *s)
     while(!url_feof(pb)){
         pos = avio_tell(pb);
         mpc8_get_chunk_header(pb, &tag, &size);
+        if (size < 0) {
+            av_log(s, AV_LOG_ERROR, "Invalid chunk length\n");
+            return AVERROR_INVALIDDATA;
+        }
         if(tag == TAG_STREAMHDR)
             break;
         mpc8_handle_chunk(s, tag, pos, size);

libavformat/omadec.c

@@ -172,7 +172,7 @@ static int nprobe(AVFormatContext *s, uint8_t *enc_header, unsigned size,
     taglen  = AV_RB32(&enc_header[pos + 32]);
     datalen = AV_RB32(&enc_header[pos + 36]) >> 4;
 
-    pos += 44L + taglen;
+    pos += 44LL + taglen;
 
     if (pos + (((uint64_t)datalen) << 4) > size)
         return -1;

libavformat/smacker.c

@@ -317,7 +317,7 @@ static int smacker_read_packet(AVFormatContext *s, AVPacket *pkt)
                 int err;
 
                 size = avio_rl32(s->pb) - 4;
-                if (!size || size + 4L > frame_size) {
+                if (!size || size + 4LL > frame_size) {
                     av_log(s, AV_LOG_ERROR, "Invalid audio part size\n");
                     return AVERROR_INVALIDDATA;
                 }

libavformat/thp.c

@@ -184,6 +184,8 @@ static int thp_read_packet(AVFormatContext *s,
         pkt->stream_index = thp->video_stream_index;
     } else {
         ret = av_get_packet(pb, pkt, thp->audiosize);
+        if (ret < 0)
+            return ret;
         if (ret != thp->audiosize) {
             av_free_packet(pkt);
             return AVERROR(EIO);

libavformat/tta.c

@@ -118,8 +118,10 @@ static int tta_read_header(AVFormatContext *s)
     ffio_init_checksum(s->pb, tta_check_crc, UINT32_MAX);
     for (i = 0; i < c->totalframes; i++) {
         uint32_t size = avio_rl32(s->pb);
-        av_add_index_entry(st, framepos, i * c->frame_size, size, 0,
-                           AVINDEX_KEYFRAME);
+        int r;
+        if ((r = av_add_index_entry(st, framepos, i * c->frame_size, size, 0,
+                                    AVINDEX_KEYFRAME)) < 0)
+            return r;
         framepos += size;
     }
     crc = ffio_get_checksum(s->pb) ^ UINT32_MAX;
@@ -153,6 +155,11 @@ static int tta_read_packet(AVFormatContext *s, AVPacket *pkt)
     if (c->currentframe >= c->totalframes)
         return AVERROR_EOF;
 
+    if (st->nb_index_entries < c->totalframes) {
+        av_log(s, AV_LOG_ERROR, "Index entry disappeared\n");
+        return AVERROR_INVALIDDATA;
+    }
+
     size = st->index_entries[c->currentframe].size;
 
     ret = av_get_packet(s->pb, pkt, size);

libavformat/utils.c

@@ -2847,8 +2847,8 @@ static int get_std_framerate(int i)
  * And there are "variable" fps files this needs to detect as well. */
 static int tb_unreliable(AVCodecContext *c)
 {
-    if (c->time_base.den >= 101L * c->time_base.num ||
-        c->time_base.den <    5L * c->time_base.num ||
+    if (c->time_base.den >= 101LL * c->time_base.num ||
+        c->time_base.den <    5LL * c->time_base.num ||
         // c->codec_tag == AV_RL32("DIVX") ||
         // c->codec_tag == AV_RL32("XVID") ||
         c->codec_tag == AV_RL32("mp4v") ||

libavutil/opt.c

@@ -75,8 +75,8 @@ static int read_number(const AVOption *o, void *dst, double *num, int *den, int6
 {
     switch (o->type) {
     case AV_OPT_TYPE_FLAGS:     *intnum = *(unsigned int*)dst;return 0;
-    case AV_OPT_TYPE_PIXEL_FMT:
-    case AV_OPT_TYPE_SAMPLE_FMT:
+    case AV_OPT_TYPE_PIXEL_FMT: *intnum = *(enum AVPixelFormat *)dst;return 0;
+    case AV_OPT_TYPE_SAMPLE_FMT:*intnum = *(enum AVSampleFormat*)dst;return 0;
     case AV_OPT_TYPE_INT:       *intnum = *(int         *)dst;return 0;
     case AV_OPT_TYPE_CHANNEL_LAYOUT:
     case AV_OPT_TYPE_DURATION:
@@ -110,9 +110,9 @@ static int write_number(void *obj, const AVOption *o, void *dst, double num, int
     }
 
     switch (o->type) {
+    case AV_OPT_TYPE_PIXEL_FMT: *(enum AVPixelFormat *)dst = llrint(num/den) * intnum; break;
+    case AV_OPT_TYPE_SAMPLE_FMT:*(enum AVSampleFormat*)dst = llrint(num/den) * intnum; break;
     case AV_OPT_TYPE_FLAGS:
-    case AV_OPT_TYPE_PIXEL_FMT:
-    case AV_OPT_TYPE_SAMPLE_FMT:
     case AV_OPT_TYPE_INT:   *(int       *)dst= llrint(num/den)*intnum; break;
     case AV_OPT_TYPE_DURATION:
     case AV_OPT_TYPE_CHANNEL_LAYOUT:

libswresample/dither.c

@@ -84,14 +84,14 @@ int swri_dither_init(SwrContext *s, enum AVSampleFormat out_fmt, enum AVSampleFo
     in_fmt  = av_get_packed_sample_fmt( in_fmt);
 
     if(in_fmt == AV_SAMPLE_FMT_FLT || in_fmt == AV_SAMPLE_FMT_DBL){
-        if(out_fmt == AV_SAMPLE_FMT_S32) scale = 1.0/(1L<<31);
-        if(out_fmt == AV_SAMPLE_FMT_S16) scale = 1.0/(1L<<15);
-        if(out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1.0/(1L<< 7);
+        if(out_fmt == AV_SAMPLE_FMT_S32) scale = 1.0/(1LL<<31);
+        if(out_fmt == AV_SAMPLE_FMT_S16) scale = 1.0/(1LL<<15);
+        if(out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1.0/(1LL<< 7);
     }
     if(in_fmt == AV_SAMPLE_FMT_S32 && out_fmt == AV_SAMPLE_FMT_S32 && (s->dither.output_sample_bits&31)) scale = 1;
-    if(in_fmt == AV_SAMPLE_FMT_S32 && out_fmt == AV_SAMPLE_FMT_S16) scale = 1L<<16;
-    if(in_fmt == AV_SAMPLE_FMT_S32 && out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1L<<24;
-    if(in_fmt == AV_SAMPLE_FMT_S16 && out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1L<<8;
+    if(in_fmt == AV_SAMPLE_FMT_S32 && out_fmt == AV_SAMPLE_FMT_S16) scale = 1<<16;
+    if(in_fmt == AV_SAMPLE_FMT_S32 && out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1<<24;
+    if(in_fmt == AV_SAMPLE_FMT_S16 && out_fmt == AV_SAMPLE_FMT_U8 ) scale = 1<<8;
 
     scale *= s->dither.scale;
 

libswscale/utils.c

@@ -618,14 +618,15 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos,
         }
 
         if ((*filterPos)[i] + filterSize > srcW) {
-            int shift = (*filterPos)[i] + filterSize - srcW;
+            int shift = (*filterPos)[i] + FFMIN(filterSize - srcW, 0);
+
             // move filter coefficients right to compensate for filterPos
             for (j = filterSize - 2; j >= 0; j--) {
                 int right = FFMIN(j + shift, filterSize - 1);
                 filter[i * filterSize + right] += filter[i * filterSize + j];
                 filter[i * filterSize + j]      = 0;
             }
-            (*filterPos)[i]= srcW - filterSize;
+            (*filterPos)[i]-= shift;
         }
     }
 

tests/ref/fate/gifenc-bgr8

@@ -35,15 +35,15 @@
 0,        330,        330,        1,     4295, 0xf71b0b38, S=1,     1024, 0xf351799f
 0,        340,        340,        1,     2044, 0x5adcb93b, S=1,     1024, 0xf351799f
 0,        350,        350,        1,     3212, 0xcf79eeed, S=1,     1024, 0xf351799f
-0,        360,        360,        1,     2281, 0x68464d30, S=1,     1024, 0xf351799f
+0,        360,        360,        1,     2292, 0xb4386334, S=1,     1024, 0xf351799f
 0,        370,        370,        1,     3633, 0x0010992f, S=1,     1024, 0xf351799f
 0,        380,        380,        1,     3552, 0x23697490, S=1,     1024, 0xf351799f
 0,        390,        390,        1,     3690, 0x62afdbb8, S=1,     1024, 0xf351799f
-0,        400,        400,        1,     1558, 0x7a13e53b, S=1,     1024, 0xf351799f
-0,        410,        410,        1,      940, 0xb1b6cba2, S=1,     1024, 0xf351799f
+0,        400,        400,        1,     1559, 0x5baef54a, S=1,     1024, 0xf351799f
+0,        410,        410,        1,      954, 0xca75ca79, S=1,     1024, 0xf351799f
 0,        420,        420,        1,      273, 0x3687799b, S=1,     1024, 0xf351799f
 0,        430,        430,        1,      930, 0x29f3b0c4, S=1,     1024, 0xf351799f
-0,        440,        440,        1,      271, 0xe7af807c, S=1,     1024, 0xf351799f
+0,        440,        440,        1,      271, 0x305e8094, S=1,     1024, 0xf351799f
 0,        450,        450,        1,      196, 0xf5ab51ee, S=1,     1024, 0xf351799f
 0,        460,        460,        1,     4299, 0x67ec0d55, S=1,     1024, 0xf351799f
 0,        470,        470,        1,     4895, 0xb394406c, S=1,     1024, 0xf351799f
@@ -56,7 +56,7 @@
 0,        540,        540,        1,     5179, 0x860fc6a1, S=1,     1024, 0xf351799f
 0,        550,        550,        1,     5046, 0xce9183d3, S=1,     1024, 0xf351799f
 0,        560,        560,        1,     5140, 0xa6d7b9af, S=1,     1024, 0xf351799f
-0,        570,        570,        1,     4289, 0xb415f717, S=1,     1024, 0xf351799f
+0,        570,        570,        1,     4301, 0x03b6ef3f, S=1,     1024, 0xf351799f
 0,        580,        580,        1,     5079, 0xa8d59e01, S=1,     1024, 0xf351799f
 0,        590,        590,        1,     5284, 0xea34e3b3, S=1,     1024, 0xf351799f
 0,        600,        600,        1,     5426, 0x556a15cd, S=1,     1024, 0xf351799f

tests/ref/fate/gifenc-rgb8

@@ -35,15 +35,15 @@
 0,        330,        330,        1,     4295, 0xc1850a80, S=1,     1024, 0xcfc8799f
 0,        340,        340,        1,     2044, 0x0440c072, S=1,     1024, 0xcfc8799f
 0,        350,        350,        1,     3212, 0xe91af08f, S=1,     1024, 0xcfc8799f
-0,        360,        360,        1,     2281, 0x6a414aa1, S=1,     1024, 0xcfc8799f
+0,        360,        360,        1,     2292, 0x6765633e, S=1,     1024, 0xcfc8799f
 0,        370,        370,        1,     3633, 0xac779aa3, S=1,     1024, 0xcfc8799f
 0,        380,        380,        1,     3552, 0xed2c75b2, S=1,     1024, 0xcfc8799f
 0,        390,        390,        1,     3690, 0x2020dd0d, S=1,     1024, 0xcfc8799f
-0,        400,        400,        1,     1558, 0x2c14e4b2, S=1,     1024, 0xcfc8799f
-0,        410,        410,        1,      940, 0x4927cd90, S=1,     1024, 0xcfc8799f
+0,        400,        400,        1,     1559, 0x596ef330, S=1,     1024, 0xcfc8799f
+0,        410,        410,        1,      954, 0xac12c9c5, S=1,     1024, 0xcfc8799f
 0,        420,        420,        1,      273, 0x138c7831, S=1,     1024, 0xcfc8799f
 0,        430,        430,        1,      930, 0xf1c3ae3f, S=1,     1024, 0xcfc8799f
-0,        440,        440,        1,      271, 0x6d338044, S=1,     1024, 0xcfc8799f
+0,        440,        440,        1,      271, 0x921a80af, S=1,     1024, 0xcfc8799f
 0,        450,        450,        1,      196, 0xa5de5322, S=1,     1024, 0xcfc8799f
 0,        460,        460,        1,     4299, 0x5bac0d86, S=1,     1024, 0xcfc8799f
 0,        470,        470,        1,     4895, 0xc43639a6, S=1,     1024, 0xcfc8799f
@@ -56,7 +56,7 @@
 0,        540,        540,        1,     5179, 0x97aac3a1, S=1,     1024, 0xcfc8799f
 0,        550,        550,        1,     5046, 0x836a80cd, S=1,     1024, 0xcfc8799f
 0,        560,        560,        1,     5140, 0xa725c1e7, S=1,     1024, 0xcfc8799f
-0,        570,        570,        1,     4289, 0x7b3afbc0, S=1,     1024, 0xcfc8799f
+0,        570,        570,        1,     4301, 0x0203f239, S=1,     1024, 0xcfc8799f
 0,        580,        580,        1,     5079, 0xb2e7a2de, S=1,     1024, 0xcfc8799f
 0,        590,        590,        1,     5284, 0xb757dfe1, S=1,     1024, 0xcfc8799f
 0,        600,        600,        1,     5426, 0xf9f11e57, S=1,     1024, 0xcfc8799f