diff --git a/src/decoder/plugins/FfmpegDecoderPlugin.cxx b/src/decoder/plugins/FfmpegDecoderPlugin.cxx
index 5dfe347f7..b7ba4c00c 100644
--- a/src/decoder/plugins/FfmpegDecoderPlugin.cxx
+++ b/src/decoder/plugins/FfmpegDecoderPlugin.cxx
@@ -370,7 +370,7 @@ ffmpeg_send_packet(Decoder &decoder, InputStream &is,
 					  time_from_ffmpeg(pts, stream.time_base));
 	}
 
-	uint8_t *output_buffer;
+	uint8_t *output_buffer = nullptr;
 
 	DecoderCommand cmd = DecoderCommand::NONE;
 	while (packet.size > 0 && cmd == DecoderCommand::NONE) {
@@ -379,15 +379,6 @@ ffmpeg_send_packet(Decoder &decoder, InputStream &is,
 		int len = avcodec_decode_audio4(&codec_context,
 						frame, &got_frame,
 						&packet);
-		if (len >= 0 && got_frame) {
-			audio_size = copy_interleave_frame(codec_context,
-							   *frame,
-							   &output_buffer,
-							   buffer, buffer_size);
-			if (audio_size < 0)
-				len = audio_size;
-		}
-
 		if (len < 0) {
 			/* if error, we skip the frame */
 			LogDefault(ffmpeg_domain,
@@ -395,6 +386,18 @@ ffmpeg_send_packet(Decoder &decoder, InputStream &is,
 			break;
 		}
 
+		if (got_frame) {
+			audio_size = copy_interleave_frame(codec_context,
+							   *frame,
+							   &output_buffer,
+							   buffer, buffer_size);
+			if (audio_size < 0) {
+				/* this must be a serious error,
+				   e.g. OOM */
+				return DecoderCommand::STOP;
+			}
+		}
+
 		packet.data += len;
 		packet.size -= len;