diff --git a/NEWS b/NEWS
index 145361c35..231ac8bd9 100644
--- a/NEWS
+++ b/NEWS
@@ -14,6 +14,7 @@ ver 0.20 (not yet released)
 * decoder
   - ffmpeg: support ReplayGain and MixRamp
   - ffmpeg: support stream tags
+  - mad: reduce memory usage while scanning tags
   - mpcdec: read the bit rate
 * playlist
   - cue: don't skip pregap
diff --git a/src/decoder/plugins/MadDecoderPlugin.cxx b/src/decoder/plugins/MadDecoderPlugin.cxx
index ea2c9ac6c..9baa6fb9c 100644
--- a/src/decoder/plugins/MadDecoderPlugin.cxx
+++ b/src/decoder/plugins/MadDecoderPlugin.cxx
@@ -167,6 +167,15 @@ struct MadDecoder {
 
 	bool DecodeFirstFrame(Tag **tag);
 
+	void AllocateBuffers() {
+		assert(max_frames > 0);
+		assert(frame_offsets == nullptr);
+		assert(times == nullptr);
+
+		frame_offsets = new long[max_frames];
+		times = new mad_timer_t[max_frames];
+	}
+
 	gcc_pure
 	long TimeToFrame(SongTime t) const;
 
@@ -819,9 +828,6 @@ MadDecoder::DecodeFirstFrame(Tag **tag)
 		return false;
 	}
 
-	frame_offsets = new long[max_frames];
-	times = new mad_timer_t[max_frames];
-
 	return true;
 }
 
@@ -1049,6 +1055,8 @@ mp3_decode(Decoder &decoder, InputStream &input_stream)
 		return;
 	}
 
+	data.AllocateBuffers();
+
 	Error error;
 	AudioFormat audio_format;
 	if (!audio_format_init_checked(audio_format,