diff --git a/dev/demux.html b/dev/demux.html
index ab1d1c6..140d3ca 100644
--- a/dev/demux.html
+++ b/dev/demux.html
@@ -17,11 +17,45 @@
source: new Mediabunny.BlobSource(file),
});
+ const track = await input.getPrimaryVideoTrack();
+ if (!track) {
+ throw new Error('The synthetic file has no video track.');
+ }
+
+ let encodedPackets = 0;
+ for await (const packet of new Mediabunny.EncodedPacketSink(track).packets()) {
+ console.log(packet.timestamp)
+ encodedPackets++;
+ }
+
+ let decodedFrames = 0;
+ let lastTimestamp = null;
+ const sink = new Mediabunny.VideoSampleSink(track, {
+ hardwareAcceleration: 'prefer-software',
+ });
+ for await (const sample of sink.samples()) {
+ console.log(sample.timestamp)
+ decodedFrames++;
+ lastTimestamp = sample.timestamp;
+ sample.close();
+ }
+
+ input.dispose();
+ const reproduced = encodedPackets === 48 && decodedFrames === 47;
+ console.log({
+ decodedFrames,
+ encodedPackets,
+ lastTimestamp,
+ reproduced,
+ });
+
+ /*
const track = await input.getPrimaryVideoTrack();
const sink = new Mediabunny.EncodedPacketSink(track);
const packet = await sink.getFirstPacket();
console.log(packet);
+ */
/*
for await (const packet of packetSink.packets()) {
diff --git a/packages/server/src/video-decoder.ts b/packages/server/src/video-decoder.ts
index a91d0ff..f28d56a 100644
--- a/packages/server/src/video-decoder.ts
+++ b/packages/server/src/video-decoder.ts
@@ -82,7 +82,11 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder {
: null;
codecContext.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den);
- const ret = await codecContext.open2();
+ // Honor inferred H.264 reorder buffering even when bitstream restrictions aren't explicitly signaled
+ // See https://github.com/Vanilagy/mediabunny/issues/488
+ const options = NodeAv.Dictionary.fromObject({ strict: NodeAv.FF_COMPLIANCE_STRICT });
+
+ const ret = await codecContext.open2(codec, options);
NodeAv.FFmpegError.throwIfError(ret, 'Open codec context');
this.codecContext = codecContext;
diff --git a/shared/bitstream.ts b/shared/bitstream.ts
index f2d2590..7baa2b2 100644
--- a/shared/bitstream.ts
+++ b/shared/bitstream.ts
@@ -55,7 +55,19 @@ export class Bitstream {
}
this.pos = end;
- };
+ }
+
+ copyBits(n: number, other: Bitstream) {
+ let i = 0;
+ for (i; i < n - 7; i += 8) {
+ this.writeBits(8, other.readBits(8));
+ }
+
+ const leftover = n - i;
+ if (leftover > 0) {
+ this.writeBits(leftover, other.readBits(leftover));
+ }
+ }
readAlignedByte() {
if (this.pos % 8 !== 0) {
diff --git a/src/codec-data.ts b/src/codec-data.ts
index 096caaa..eb00afd 100644
--- a/src/codec-data.ts
+++ b/src/codec-data.ts
@@ -25,6 +25,7 @@ import {
isChromium,
popcount,
setUint24,
+ writeExpGolomb,
} from './misc';
import { Logging } from './logging';
import { PacketType } from './packet';
@@ -185,6 +186,23 @@ const removeEmulationPreventionBytes = (data: Uint8Array) => {
return new Uint8Array(result);
};
+export const addEmulationPreventionBytes = (data: Uint8Array) => {
+ const result: number[] = [];
+ let zeroCount = 0;
+
+ for (const byte of data) {
+ if (zeroCount === 2 && byte <= 0x03) {
+ result.push(0x03);
+ zeroCount = 0;
+ }
+
+ result.push(byte);
+ zeroCount = byte === 0 ? zeroCount + 1 : 0;
+ }
+
+ return new Uint8Array(result);
+};
+
const ANNEX_B_START_CODE = new Uint8Array([0, 0, 0, 1]);
export const concatNalUnitsInAnnexB = (nalUnits: Uint8Array[]) => {
@@ -363,12 +381,14 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu
}
if (
- record.avcProfileIndication === 100
- || record.avcProfileIndication === 110
- || record.avcProfileIndication === 122
- || record.avcProfileIndication === 144
+ (
+ record.avcProfileIndication === 100
+ || record.avcProfileIndication === 110
+ || record.avcProfileIndication === 122
+ || record.avcProfileIndication === 144
+ )
+ && record.chromaFormat !== null // Can happen if the data was too short
) {
- assert(record.chromaFormat !== null);
assert(record.bitDepthLumaMinus8 !== null);
assert(record.bitDepthChromaMinus8 !== null);
assert(record.sequenceParameterSetExt !== null);
@@ -485,6 +505,7 @@ export const deserializeAvcDecoderConfigurationRecord = (data: Uint8Array): AvcD
};
export type AvcSpsInfo = {
+ emulationUnpreventedBytes: Uint8Array;
profileIdc: number;
constraintFlags: number;
levelIdc: number;
@@ -503,6 +524,9 @@ export type AvcSpsInfo = {
fullRangeFlag: number;
numReorderFrames: number;
maxDecFrameBuffering: number;
+ vuiParametersFlagBitOffset: number;
+ bitstreamRestrictionFlagBitOffset: number | null;
+ bitstreamRestrictionFlag: number | null;
};
const AVC_HEVC_ASPECT_RATIO_IDC_TABLE: Partial> = {
@@ -527,7 +551,8 @@ const AVC_HEVC_ASPECT_RATIO_IDC_TABLE: Partial> = {
/** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */
export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
try {
- const bitstream = new Bitstream(removeEmulationPreventionBytes(sps));
+ const emulationUnpreventedBytes = removeEmulationPreventionBytes(sps);
+ const bitstream = new Bitstream(emulationUnpreventedBytes);
bitstream.skipBits(1); // forbidden_zero_bit
bitstream.skipBits(2); // nal_ref_idc
@@ -660,7 +685,10 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
let numReorderFrames: number | null = null;
let maxDecFrameBuffering: number | null = null;
+ let bitstreamRestrictionFlagBitOffset: number | null = null;
+ let bitstreamRestrictionFlag: number | null = null;
+ const vuiParametersFlagBitOffset = bitstream.pos;
const vuiParametersPresentFlag = bitstream.readBits(1);
if (vuiParametersPresentFlag) {
const aspectRatioInfoPresentFlag = bitstream.readBits(1);
@@ -726,7 +754,8 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
bitstream.skipBits(1); // pic_struct_present_flag
- const bitstreamRestrictionFlag = bitstream.readBits(1);
+ bitstreamRestrictionFlagBitOffset = bitstream.pos;
+ bitstreamRestrictionFlag = bitstream.readBits(1);
if (bitstreamRestrictionFlag) {
bitstream.skipBits(1); // motion_vectors_over_pic_boundaries_flag
readExpGolomb(bitstream); // max_bytes_per_pic_denom
@@ -776,6 +805,7 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
assert(maxDecFrameBuffering !== null);
return {
+ emulationUnpreventedBytes,
profileIdc,
constraintFlags,
levelIdc,
@@ -794,6 +824,9 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => {
fullRangeFlag,
numReorderFrames,
maxDecFrameBuffering,
+ vuiParametersFlagBitOffset,
+ bitstreamRestrictionFlagBitOffset,
+ bitstreamRestrictionFlag,
};
} catch (error) {
Logging._error('Error parsing AVC SPS:', error);
@@ -818,6 +851,57 @@ const skipAvcHrdParameters = (bitstream: Bitstream) => {
bitstream.skipBits(5); // time_offset_length
};
+/**
+ * Adds the missing "bitstream restriction" section within the VUI parameter section. This is done to communicate
+ * frame reorder buffer size to the decoder, which may otherwise, in its absence, assume no B-frames and drop or skip
+ * them.
+ * See https://github.com/Vanilagy/mediabunny/issues/488
+ */
+export const addAvcBitstreamRestriction = (sps: AvcSpsInfo) => {
+ assert(sps.bitstreamRestrictionFlag !== 1);
+
+ const modifiedBytes = new Uint8Array(sps.emulationUnpreventedBytes.byteLength + 64);
+ const oldBitstream = new Bitstream(sps.emulationUnpreventedBytes);
+ const newBitstream = new Bitstream(modifiedBytes);
+
+ if (sps.bitstreamRestrictionFlag === null) {
+ // No VUI at all; let's write a minimal one
+ newBitstream.copyBits(sps.vuiParametersFlagBitOffset, oldBitstream);
+ newBitstream.writeBits(1, 1); // vui_parameters_present_flag
+ newBitstream.writeBits(1, 0); // aspect_ratio_info_present_flag
+ newBitstream.writeBits(1, 0); // overscan_info_present_flag
+ newBitstream.writeBits(1, 0); // video_signal_type_present_flag
+ newBitstream.writeBits(1, 0); // chroma_loc_info_present_flag
+ newBitstream.writeBits(1, 0); // timing_info_present_flag
+ newBitstream.writeBits(1, 0); // nal_hrd_parameters_present_flag
+ newBitstream.writeBits(1, 0); // vcl_hrd_parameters_present_flag
+ newBitstream.writeBits(1, 0); // pic_struct_present_flag
+ } else {
+ // We have a VUI but no bitstream restriction info
+ assert(sps.bitstreamRestrictionFlagBitOffset !== null);
+ newBitstream.copyBits(sps.bitstreamRestrictionFlagBitOffset, oldBitstream);
+ }
+ newBitstream.writeBits(1, 1); // bitstream_restriction_flag
+
+ // Defaults from the H.264 spec:
+ newBitstream.writeBits(1, 1); // motion_vectors_over_pic_boundaries_flag
+ writeExpGolomb(newBitstream, 2); // max_bytes_per_pic_denom
+ writeExpGolomb(newBitstream, 1); // max_bits_per_mb_denom
+ writeExpGolomb(newBitstream, 16); // log2_max_mv_length_horizontal
+ writeExpGolomb(newBitstream, 16); // log2_max_mv_length_vertical
+ writeExpGolomb(newBitstream, sps.numReorderFrames);
+ writeExpGolomb(newBitstream, sps.maxDecFrameBuffering);
+
+ // There's nothing after this (VUI is at the end of SPS)
+ newBitstream.writeBits(1, 1); // rbsp_stop_one_bit
+ newBitstream.writeBits((8 - newBitstream.pos % 8) % 8, 0);
+
+ const byteLength = newBitstream.pos / 8;
+ assert(Number.isInteger(byteLength));
+
+ return addEmulationPreventionBytes(modifiedBytes.subarray(0, byteLength));
+};
+
// Data specified in ISO 14496-15
export type HevcDecoderConfigurationRecord = {
configurationVersion: number;
diff --git a/src/media-sink.ts b/src/media-sink.ts
index 7d70b62..2a6e74e 100644
--- a/src/media-sink.ts
+++ b/src/media-sink.ts
@@ -8,6 +8,7 @@
import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
import {
+ addAvcBitstreamRestriction,
AvcNalUnitType,
concatAvcNalUnits,
deserializeAvcDecoderConfigurationRecord,
@@ -19,6 +20,7 @@ import {
iterateHevcNalUnits,
parseAvcSps,
sanitizeHevcPacketForChromium,
+ serializeAvcDecoderConfigurationRecord,
} from './codec-data';
import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
import { InputDisposedError } from './input';
@@ -959,19 +961,31 @@ class VideoDecoderWrapper extends DecoderWrapper {
if (isChromium()) {
if (codec === 'avc' && this.decoderConfig.description) {
- // Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096)
- // which can be worked around by requesting that software decoding be used. So, here we peek into
- // the AVC description, if present, and switch to software decoding if we find interlaced content.
const record = deserializeAvcDecoderConfigurationRecord(
toUint8Array(this.decoderConfig.description),
);
if (record && record.sequenceParameterSets.length > 0) {
const sps = parseAvcSps(record.sequenceParameterSets[0]!);
- if (sps && sps.frameMbsOnlyFlag === 0) {
- this.decoderConfig = {
- ...this.decoderConfig,
- hardwareAcceleration: 'prefer-software',
- };
+ if (sps) {
+ if (sps.frameMbsOnlyFlag === 0) {
+ // Chromium has/had a bug with playing interlaced AVC
+ // (https://issues.chromium.org/issues/456919096) which can be worked around by
+ // requesting that software decoding be used. So, here we peek into the AVC description,
+ // if present, and switch to software decoding if we find interlaced content.
+ this.decoderConfig = {
+ ...this.decoderConfig,
+ hardwareAcceleration: 'prefer-software',
+ };
+ }
+
+ if (sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) {
+ // Modify the SPS to fix potential loss of B frames
+ record.sequenceParameterSets[0] = addAvcBitstreamRestriction(sps);
+ this.decoderConfig = {
+ ...this.decoderConfig,
+ description: serializeAvcDecoderConfigurationRecord(record),
+ };
+ }
}
}
}
@@ -1077,6 +1091,23 @@ class VideoDecoderWrapper extends DecoderWrapper {
}
}
+ if (!this.decoderConfig.description) {
+ // Do SPS fixups if necessary
+ for (let i = 0; i < filteredNalUnits.length; i++) {
+ const nalUnit = filteredNalUnits[i]!;
+ if (extractNalUnitTypeForAvc(nalUnit[0]!) !== AvcNalUnitType.SPS) {
+ continue;
+ }
+
+ const sps = parseAvcSps(nalUnit);
+ if (sps && sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) {
+ filteredNalUnits[i] = addAvcBitstreamRestriction(sps);
+ }
+
+ break;
+ }
+ }
+
const newData = concatAvcNalUnits(filteredNalUnits, this.decoderConfig);
packet = new EncodedPacket(newData, packet.type, packet.timestamp, packet.duration);
} else if (this.codec === 'hevc') {
diff --git a/src/misc.ts b/src/misc.ts
index 7d7d1d1..0de8901 100644
--- a/src/misc.ts
+++ b/src/misc.ts
@@ -61,6 +61,14 @@ export const readExpGolomb = (bitstream: Bitstream) => {
return result;
};
+export const writeExpGolomb = (bitstream: Bitstream, value: number) => {
+ const codeNum = value + 1;
+ const leadingZeroBits = Math.floor(Math.log2(codeNum));
+ bitstream.writeBits(leadingZeroBits, 0);
+ bitstream.writeBits(1, 1);
+ bitstream.writeBits(leadingZeroBits, codeNum - 2 ** leadingZeroBits);
+};
+
/** Reads a signed exponential-Golomb universal code from a Bitstream. */
export const readSignedExpGolomb = (bitstream: Bitstream) => {
const codeNum = readExpGolomb(bitstream);
diff --git a/test/browser/media-sinks.test.ts b/test/browser/media-sinks.test.ts
index 3a8840b..a542ffd 100644
--- a/test/browser/media-sinks.test.ts
+++ b/test/browser/media-sinks.test.ts
@@ -1,9 +1,13 @@
import { expect, test } from 'vitest';
import { Input } from '../../src/input.js';
-import { UrlSource } from '../../src/source.js';
+import { BufferSource, UrlSource } from '../../src/source.js';
import { ALL_FORMATS } from '../../src/input-format.js';
import { assert } from '../../src/misc.js';
-import { AudioSampleSink } from '../../src/media-sink.js';
+import { AudioSampleSink, VideoSampleSink } from '../../src/media-sink.js';
+import { Output } from '../../src/output.js';
+import { MpegTsOutputFormat } from '../../src/output-format.js';
+import { BufferTarget } from '../../src/target.js';
+import { Conversion } from '../../src/conversion.js';
// https://github.com/Vanilagy/mediabunny/issues/370
test('Negative audio timestamps are preserved', async () => {
@@ -24,3 +28,61 @@ test('Negative audio timestamps are preserved', async () => {
break;
}
});
+
+test('No B-frames are skipped when software-decoding AVC', async () => {
+ using input = new Input({
+ source: new UrlSource('/missing-reorder-metadata-v1.mp4'),
+ formats: ALL_FORMATS,
+ });
+
+ const track = await input.getPrimaryVideoTrack();
+ assert(track);
+
+ const sink = new VideoSampleSink(track, {
+ hardwareAcceleration: 'prefer-software',
+ });
+ let count = 0;
+
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ for await (using sample of sink.samples()) {
+ count++;
+ }
+
+ expect(count).toBe(48);
+});
+
+test('No B-frames are skipped when software-decoding AVC, Annex B edition', async () => {
+ using input = new Input({
+ source: new UrlSource('/missing-reorder-metadata-v1.mp4'),
+ formats: ALL_FORMATS,
+ });
+
+ // Force Annex B by converting to MPEG-TS
+ const output = new Output({
+ format: new MpegTsOutputFormat(),
+ target: new BufferTarget(),
+ });
+
+ const conversion = await Conversion.init({ input, output, copy: { mode: 'forced' } });
+ await conversion.execute();
+
+ using newInput = new Input({
+ source: new BufferSource(output.target.buffer!),
+ formats: ALL_FORMATS,
+ });
+
+ const track = await newInput.getPrimaryVideoTrack();
+ assert(track);
+
+ const sink = new VideoSampleSink(track, {
+ hardwareAcceleration: 'prefer-software',
+ });
+ let count = 0;
+
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ for await (using sample of sink.samples()) {
+ count++;
+ }
+
+ expect(count).toBe(48);
+});
diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts
index bc28490..1b0cf4a 100644
--- a/test/node/server-extension.test.ts
+++ b/test/node/server-extension.test.ts
@@ -1223,6 +1223,28 @@ describe('Video', async () => {
await decoder.close();
});
+ test('No B-frames are skipped when decoding AVC', async () => {
+ using input = new Input({
+ source: new FilePathSource('./test/public/missing-reorder-metadata-v1.mp4'),
+ formats: ALL_FORMATS,
+ });
+
+ const track = await input.getPrimaryVideoTrack();
+ assert(track);
+
+ const sink = new VideoSampleSink(track, {
+ hardwareAcceleration: 'prefer-software',
+ });
+ let count = 0;
+
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ for await (using sample of sink.samples()) {
+ count++;
+ }
+
+ expect(count).toBe(48);
+ });
+
describe('VideoSample transformation', () => {
// 400x400 image: red everywhere, with a 200x200 blue square filling the bottom-left quadrant.
const TEST_IMAGE = (() => {
diff --git a/test/public/missing-reorder-metadata-v1.mp4 b/test/public/missing-reorder-metadata-v1.mp4
new file mode 100644
index 0000000..71c4d57
Binary files /dev/null and b/test/public/missing-reorder-metadata-v1.mp4 differ