diff --git a/dev/demux.html b/dev/demux.html index ab1d1c6..140d3ca 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -17,11 +17,45 @@ source: new Mediabunny.BlobSource(file), }); + const track = await input.getPrimaryVideoTrack(); + if (!track) { + throw new Error('The synthetic file has no video track.'); + } + + let encodedPackets = 0; + for await (const packet of new Mediabunny.EncodedPacketSink(track).packets()) { + console.log(packet.timestamp) + encodedPackets++; + } + + let decodedFrames = 0; + let lastTimestamp = null; + const sink = new Mediabunny.VideoSampleSink(track, { + hardwareAcceleration: 'prefer-software', + }); + for await (const sample of sink.samples()) { + console.log(sample.timestamp) + decodedFrames++; + lastTimestamp = sample.timestamp; + sample.close(); + } + + input.dispose(); + const reproduced = encodedPackets === 48 && decodedFrames === 47; + console.log({ + decodedFrames, + encodedPackets, + lastTimestamp, + reproduced, + }); + + /* const track = await input.getPrimaryVideoTrack(); const sink = new Mediabunny.EncodedPacketSink(track); const packet = await sink.getFirstPacket(); console.log(packet); + */ /* for await (const packet of packetSink.packets()) { diff --git a/package-lock.json b/package-lock.json index 075ac76..ff78763 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.56.0", + "version": "1.56.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.56.0", + "version": "1.56.1", "license": "MPL-2.0", "workspaces": [ ".", @@ -12958,7 +12958,7 @@ }, "packages/aac-encoder": { "name": "@mediabunny/aac-encoder", - "version": "1.56.0", + "version": "1.56.1", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12973,7 +12973,7 @@ }, "packages/ac3": { "name": "@mediabunny/ac3", - "version": "1.56.0", + "version": "1.56.1", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12988,7 +12988,7 @@ }, "packages/dts": { "name": "@mediabunny/dts", - "version": "1.56.0", + "version": "1.56.1", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -13003,7 +13003,7 @@ }, "packages/flac-encoder": { "name": "@mediabunny/flac-encoder", - "version": "1.56.0", + "version": "1.56.1", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -13018,7 +13018,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.56.0", + "version": "1.56.1", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -13033,7 +13033,7 @@ }, "packages/prores": { "name": "@mediabunny/prores", - "version": "1.56.0", + "version": "1.56.1", "license": "MPL-2.0", "dependencies": { "turbores": "^1.2.2" @@ -13048,10 +13048,10 @@ }, "packages/server": { "name": "@mediabunny/server", - "version": "1.56.0", + "version": "1.56.1", "license": "MPL-2.0", "dependencies": { - "@mediabunny/prores": "^1.56.0", + "@mediabunny/prores": "^1.56.1", "node-av": "^6.0.0" }, "funding": { diff --git a/package.json b/package.json index f5f0aa8..fc85ea0 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.56.0", + "version": "1.56.1", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ diff --git a/packages/aac-encoder/package.json b/packages/aac-encoder/package.json index a333b58..d7af571 100644 --- a/packages/aac-encoder/package.json +++ b/packages/aac-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/aac-encoder", "author": "Vanilagy", - "version": "1.56.0", + "version": "1.56.1", "description": "AAC encoder extension for Mediabunny, based on libavcodec.", "main": "./dist/bundles/mediabunny-aac-encoder.mjs", "module": "./dist/bundles/mediabunny-aac-encoder.mjs", diff --git a/packages/ac3/package.json b/packages/ac3/package.json index 9c1350b..f8a7d30 100644 --- a/packages/ac3/package.json +++ b/packages/ac3/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/ac3", "author": "Vanilagy", - "version": "1.56.0", + "version": "1.56.1", "description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on libavcodec.", "main": "./dist/bundles/mediabunny-ac3.mjs", "module": "./dist/bundles/mediabunny-ac3.mjs", diff --git a/packages/dts/package.json b/packages/dts/package.json index 2b445bb..7389f96 100644 --- a/packages/dts/package.json +++ b/packages/dts/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/dts", "author": "Vanilagy", - "version": "1.56.0", + "version": "1.56.1", "description": "DTS decoder and encoder extension for Mediabunny, based on libavcodec.", "main": "./dist/bundles/mediabunny-dts.mjs", "module": "./dist/bundles/mediabunny-dts.mjs", diff --git a/packages/flac-encoder/package.json b/packages/flac-encoder/package.json index 932fbe9..22d6851 100644 --- a/packages/flac-encoder/package.json +++ b/packages/flac-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/flac-encoder", "author": "Vanilagy", - "version": "1.56.0", + "version": "1.56.1", "description": "FLAC encoder extension for Mediabunny, based on libFLAC.", "main": "./dist/bundles/mediabunny-flac-encoder.mjs", "module": "./dist/bundles/mediabunny-flac-encoder.mjs", diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index f875142..2976dac 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.56.0", + "version": "1.56.1", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", diff --git a/packages/prores/package.json b/packages/prores/package.json index d489a97..eda856f 100644 --- a/packages/prores/package.json +++ b/packages/prores/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/prores", "author": "Vanilagy", - "version": "1.56.0", + "version": "1.56.1", "description": "Apple ProRes decoder extension for Mediabunny, based on TurboRes.", "main": "./dist/bundles/mediabunny-prores.mjs", "module": "./dist/bundles/mediabunny-prores.mjs", diff --git a/packages/server/package.json b/packages/server/package.json index 76e65e4..1165863 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/server", "author": "Vanilagy", - "version": "1.56.0", + "version": "1.56.1", "description": "Adds full video and audio decoder and encoder support to Mediabunny for use in server-side environments (Node, Bun, Deno). Based on NodeAV.", "main": "./dist/bundles/mediabunny-server.cjs", "module": "./dist/bundles/mediabunny-server.mjs", @@ -35,7 +35,7 @@ }, "dependencies": { "node-av": "^6.0.0", - "@mediabunny/prores": "^1.56.0" + "@mediabunny/prores": "^1.56.1" }, "peerDependencies": { "mediabunny": "^1.45.0" diff --git a/packages/server/src/video-decoder.ts b/packages/server/src/video-decoder.ts index a91d0ff..f28d56a 100644 --- a/packages/server/src/video-decoder.ts +++ b/packages/server/src/video-decoder.ts @@ -82,7 +82,11 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { : null; codecContext.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den); - const ret = await codecContext.open2(); + // Honor inferred H.264 reorder buffering even when bitstream restrictions aren't explicitly signaled + // See https://github.com/Vanilagy/mediabunny/issues/488 + const options = NodeAv.Dictionary.fromObject({ strict: NodeAv.FF_COMPLIANCE_STRICT }); + + const ret = await codecContext.open2(codec, options); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); this.codecContext = codecContext; diff --git a/shared/bitstream.ts b/shared/bitstream.ts index f2d2590..7baa2b2 100644 --- a/shared/bitstream.ts +++ b/shared/bitstream.ts @@ -55,7 +55,19 @@ export class Bitstream { } this.pos = end; - }; + } + + copyBits(n: number, other: Bitstream) { + let i = 0; + for (i; i < n - 7; i += 8) { + this.writeBits(8, other.readBits(8)); + } + + const leftover = n - i; + if (leftover > 0) { + this.writeBits(leftover, other.readBits(leftover)); + } + } readAlignedByte() { if (this.pos % 8 !== 0) { diff --git a/src/codec-data.ts b/src/codec-data.ts index acb5988..eb00afd 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -25,6 +25,7 @@ import { isChromium, popcount, setUint24, + writeExpGolomb, } from './misc'; import { Logging } from './logging'; import { PacketType } from './packet'; @@ -185,6 +186,23 @@ const removeEmulationPreventionBytes = (data: Uint8Array) => { return new Uint8Array(result); }; +export const addEmulationPreventionBytes = (data: Uint8Array) => { + const result: number[] = []; + let zeroCount = 0; + + for (const byte of data) { + if (zeroCount === 2 && byte <= 0x03) { + result.push(0x03); + zeroCount = 0; + } + + result.push(byte); + zeroCount = byte === 0 ? zeroCount + 1 : 0; + } + + return new Uint8Array(result); +}; + const ANNEX_B_START_CODE = new Uint8Array([0, 0, 0, 1]); export const concatNalUnitsInAnnexB = (nalUnits: Uint8Array[]) => { @@ -363,12 +381,14 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu } if ( - record.avcProfileIndication === 100 - || record.avcProfileIndication === 110 - || record.avcProfileIndication === 122 - || record.avcProfileIndication === 144 + ( + record.avcProfileIndication === 100 + || record.avcProfileIndication === 110 + || record.avcProfileIndication === 122 + || record.avcProfileIndication === 144 + ) + && record.chromaFormat !== null // Can happen if the data was too short ) { - assert(record.chromaFormat !== null); assert(record.bitDepthLumaMinus8 !== null); assert(record.bitDepthChromaMinus8 !== null); assert(record.sequenceParameterSetExt !== null); @@ -485,6 +505,7 @@ export const deserializeAvcDecoderConfigurationRecord = (data: Uint8Array): AvcD }; export type AvcSpsInfo = { + emulationUnpreventedBytes: Uint8Array; profileIdc: number; constraintFlags: number; levelIdc: number; @@ -503,6 +524,9 @@ export type AvcSpsInfo = { fullRangeFlag: number; numReorderFrames: number; maxDecFrameBuffering: number; + vuiParametersFlagBitOffset: number; + bitstreamRestrictionFlagBitOffset: number | null; + bitstreamRestrictionFlag: number | null; }; const AVC_HEVC_ASPECT_RATIO_IDC_TABLE: Partial> = { @@ -527,7 +551,8 @@ const AVC_HEVC_ASPECT_RATIO_IDC_TABLE: Partial> = { /** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { try { - const bitstream = new Bitstream(removeEmulationPreventionBytes(sps)); + const emulationUnpreventedBytes = removeEmulationPreventionBytes(sps); + const bitstream = new Bitstream(emulationUnpreventedBytes); bitstream.skipBits(1); // forbidden_zero_bit bitstream.skipBits(2); // nal_ref_idc @@ -660,7 +685,10 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { let numReorderFrames: number | null = null; let maxDecFrameBuffering: number | null = null; + let bitstreamRestrictionFlagBitOffset: number | null = null; + let bitstreamRestrictionFlag: number | null = null; + const vuiParametersFlagBitOffset = bitstream.pos; const vuiParametersPresentFlag = bitstream.readBits(1); if (vuiParametersPresentFlag) { const aspectRatioInfoPresentFlag = bitstream.readBits(1); @@ -726,7 +754,8 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { bitstream.skipBits(1); // pic_struct_present_flag - const bitstreamRestrictionFlag = bitstream.readBits(1); + bitstreamRestrictionFlagBitOffset = bitstream.pos; + bitstreamRestrictionFlag = bitstream.readBits(1); if (bitstreamRestrictionFlag) { bitstream.skipBits(1); // motion_vectors_over_pic_boundaries_flag readExpGolomb(bitstream); // max_bytes_per_pic_denom @@ -776,6 +805,7 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { assert(maxDecFrameBuffering !== null); return { + emulationUnpreventedBytes, profileIdc, constraintFlags, levelIdc, @@ -794,6 +824,9 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { fullRangeFlag, numReorderFrames, maxDecFrameBuffering, + vuiParametersFlagBitOffset, + bitstreamRestrictionFlagBitOffset, + bitstreamRestrictionFlag, }; } catch (error) { Logging._error('Error parsing AVC SPS:', error); @@ -818,6 +851,57 @@ const skipAvcHrdParameters = (bitstream: Bitstream) => { bitstream.skipBits(5); // time_offset_length }; +/** + * Adds the missing "bitstream restriction" section within the VUI parameter section. This is done to communicate + * frame reorder buffer size to the decoder, which may otherwise, in its absence, assume no B-frames and drop or skip + * them. + * See https://github.com/Vanilagy/mediabunny/issues/488 + */ +export const addAvcBitstreamRestriction = (sps: AvcSpsInfo) => { + assert(sps.bitstreamRestrictionFlag !== 1); + + const modifiedBytes = new Uint8Array(sps.emulationUnpreventedBytes.byteLength + 64); + const oldBitstream = new Bitstream(sps.emulationUnpreventedBytes); + const newBitstream = new Bitstream(modifiedBytes); + + if (sps.bitstreamRestrictionFlag === null) { + // No VUI at all; let's write a minimal one + newBitstream.copyBits(sps.vuiParametersFlagBitOffset, oldBitstream); + newBitstream.writeBits(1, 1); // vui_parameters_present_flag + newBitstream.writeBits(1, 0); // aspect_ratio_info_present_flag + newBitstream.writeBits(1, 0); // overscan_info_present_flag + newBitstream.writeBits(1, 0); // video_signal_type_present_flag + newBitstream.writeBits(1, 0); // chroma_loc_info_present_flag + newBitstream.writeBits(1, 0); // timing_info_present_flag + newBitstream.writeBits(1, 0); // nal_hrd_parameters_present_flag + newBitstream.writeBits(1, 0); // vcl_hrd_parameters_present_flag + newBitstream.writeBits(1, 0); // pic_struct_present_flag + } else { + // We have a VUI but no bitstream restriction info + assert(sps.bitstreamRestrictionFlagBitOffset !== null); + newBitstream.copyBits(sps.bitstreamRestrictionFlagBitOffset, oldBitstream); + } + newBitstream.writeBits(1, 1); // bitstream_restriction_flag + + // Defaults from the H.264 spec: + newBitstream.writeBits(1, 1); // motion_vectors_over_pic_boundaries_flag + writeExpGolomb(newBitstream, 2); // max_bytes_per_pic_denom + writeExpGolomb(newBitstream, 1); // max_bits_per_mb_denom + writeExpGolomb(newBitstream, 16); // log2_max_mv_length_horizontal + writeExpGolomb(newBitstream, 16); // log2_max_mv_length_vertical + writeExpGolomb(newBitstream, sps.numReorderFrames); + writeExpGolomb(newBitstream, sps.maxDecFrameBuffering); + + // There's nothing after this (VUI is at the end of SPS) + newBitstream.writeBits(1, 1); // rbsp_stop_one_bit + newBitstream.writeBits((8 - newBitstream.pos % 8) % 8, 0); + + const byteLength = newBitstream.pos / 8; + assert(Number.isInteger(byteLength)); + + return addEmulationPreventionBytes(modifiedBytes.subarray(0, byteLength)); +}; + // Data specified in ISO 14496-15 export type HevcDecoderConfigurationRecord = { configurationVersion: number; @@ -2597,7 +2681,13 @@ export const readVorbisComments = (bytes: Uint8Array, metadataTags: MetadataTags const value = string.slice(separatorIndex + 1); metadataTags.raw ??= {}; - metadataTags.raw[key] ??= value; + if (Array.isArray(metadataTags.raw[key])) { + metadataTags.raw[key] = [...metadataTags.raw[key], value]; + } else if (typeof metadataTags.raw[key] === 'string') { + metadataTags.raw[key] = [metadataTags.raw[key], value]; + } else { + metadataTags.raw[key] ??= value; + } switch (key) { case 'TITLE': { @@ -2728,7 +2818,7 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags commentHeaderParts.push(currentBuffer); - const writtenTags = new Set(); + const writtenTags: string[] = []; const addCommentTag = (key: string, value: string) => { const joined = `${key}=${value}`; const encoded = textEncoder.encode(joined); @@ -2740,7 +2830,7 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags currentBuffer.set(encoded, 4); commentHeaderParts.push(currentBuffer); - writtenTags.add(key); + writtenTags.push(key); }; for (const { key, value } of keyValueIterator(tags)) { @@ -2770,12 +2860,7 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags }; break; case 'date': { - const rawVersion = tags.raw?.['DATE'] ?? tags.raw?.['date']; - if (rawVersion && typeof rawVersion === 'string') { - addCommentTag('DATE', rawVersion); - } else { - addCommentTag('DATE', value.toISOString().slice(0, 10)); - } + addCommentTag('DATE', value.toISOString().slice(0, 10)); }; break; case 'comment': { @@ -2803,8 +2888,7 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags }; break; case 'images': { - // For example, in .flac, we put the pictures in a different section, - // not in the Vorbis comment header. + // For example, in .flac, we put the pictures in a different section, not in the Vorbis comment header. if (!writeImages) { break; } @@ -2861,18 +2945,26 @@ export const createVorbisComments = (headerBytes: Uint8Array, tags: MetadataTags if (tags.raw) { for (const key in tags.raw) { const value = tags.raw[key] ?? tags.raw[key.toLowerCase()]; - if (key === 'vendor' || value == null || writtenTags.has(key)) { + if (key === 'vendor' || value == null || writtenTags.includes(key)) { continue; } if (typeof value === 'string') { addCommentTag(key, value); + } else if (Array.isArray(value)) { + // String arrays turn into the tag being repeated for each element + const isOnlyStrings = value.every(x => typeof x === 'string'); + if (isOnlyStrings) { + for (const elem of value) { + addCommentTag(key, elem); + } + } } } } const listLengthBuffer = new Uint8Array(4); - toDataView(listLengthBuffer).setUint32(0, writtenTags.size, true); + toDataView(listLengthBuffer).setUint32(0, writtenTags.length, true); commentHeaderParts.splice(2, 0, listLengthBuffer); // Insert after the header and vendor section // Merge all comment header parts into a single buffer diff --git a/src/conversion.ts b/src/conversion.ts index f3ad24c..fb28d9b 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -1101,7 +1101,7 @@ export class Conversion { const inputAndOutputFormatMatch = inputFormat.mimeType === this.output.format.mimeType; const rawTagsAreUnchanged = inputTags.raw === outputTags.raw; - if (inputTags.raw && rawTagsAreUnchanged && !inputAndOutputFormatMatch) { + if (rawTagsAreUnchanged && !inputAndOutputFormatMatch) { // If the input and output formats aren't the same, copying over raw metadata tags makes no sense and // only results in junk tags, so let's cut them out. delete outputTags.raw; diff --git a/src/media-sink.ts b/src/media-sink.ts index 7d70b62..2a6e74e 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -8,6 +8,7 @@ import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec'; import { + addAvcBitstreamRestriction, AvcNalUnitType, concatAvcNalUnits, deserializeAvcDecoderConfigurationRecord, @@ -19,6 +20,7 @@ import { iterateHevcNalUnits, parseAvcSps, sanitizeHevcPacketForChromium, + serializeAvcDecoderConfigurationRecord, } from './codec-data'; import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder'; import { InputDisposedError } from './input'; @@ -959,19 +961,31 @@ class VideoDecoderWrapper extends DecoderWrapper { if (isChromium()) { if (codec === 'avc' && this.decoderConfig.description) { - // Chromium has/had a bug with playing interlaced AVC (https://issues.chromium.org/issues/456919096) - // which can be worked around by requesting that software decoding be used. So, here we peek into - // the AVC description, if present, and switch to software decoding if we find interlaced content. const record = deserializeAvcDecoderConfigurationRecord( toUint8Array(this.decoderConfig.description), ); if (record && record.sequenceParameterSets.length > 0) { const sps = parseAvcSps(record.sequenceParameterSets[0]!); - if (sps && sps.frameMbsOnlyFlag === 0) { - this.decoderConfig = { - ...this.decoderConfig, - hardwareAcceleration: 'prefer-software', - }; + if (sps) { + if (sps.frameMbsOnlyFlag === 0) { + // Chromium has/had a bug with playing interlaced AVC + // (https://issues.chromium.org/issues/456919096) which can be worked around by + // requesting that software decoding be used. So, here we peek into the AVC description, + // if present, and switch to software decoding if we find interlaced content. + this.decoderConfig = { + ...this.decoderConfig, + hardwareAcceleration: 'prefer-software', + }; + } + + if (sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) { + // Modify the SPS to fix potential loss of B frames + record.sequenceParameterSets[0] = addAvcBitstreamRestriction(sps); + this.decoderConfig = { + ...this.decoderConfig, + description: serializeAvcDecoderConfigurationRecord(record), + }; + } } } } @@ -1077,6 +1091,23 @@ class VideoDecoderWrapper extends DecoderWrapper { } } + if (!this.decoderConfig.description) { + // Do SPS fixups if necessary + for (let i = 0; i < filteredNalUnits.length; i++) { + const nalUnit = filteredNalUnits[i]!; + if (extractNalUnitTypeForAvc(nalUnit[0]!) !== AvcNalUnitType.SPS) { + continue; + } + + const sps = parseAvcSps(nalUnit); + if (sps && sps.maxDecFrameBuffering !== 0 && sps.bitstreamRestrictionFlag !== 1) { + filteredNalUnits[i] = addAvcBitstreamRestriction(sps); + } + + break; + } + } + const newData = concatAvcNalUnits(filteredNalUnits, this.decoderConfig); packet = new EncodedPacket(newData, packet.type, packet.timestamp, packet.duration); } else if (this.codec === 'hevc') { diff --git a/src/metadata.ts b/src/metadata.ts index 73e679d..b07d014 100644 --- a/src/metadata.ts +++ b/src/metadata.ts @@ -76,14 +76,16 @@ export type MetadataTags = { * user-defined text frames are exposed as a `Record`. * - ADTS: The ID3v2 tags, just like in MP3. * - Ogg: The key-value string pairs from the Vorbis-style comment header (see RFC 7845, Section 5.2). - * Additionally, the `'vendor'` key refers to the vendor string within this header. + * Additionally, the `'vendor'` key refers to the vendor string within this header. If a key exists more than once, + * a string array is used instead. * - WAVE: The individual metadata chunks within the RIFF INFO chunk. Values are always ISO 8859-1 strings. * - FLAC: The key-value string pairs from the vorbis metadata block (see RFC 9639, Section D.2.3). - * Additionally, the `'vendor'` key refers to the vendor string within this header. If ID3v2 tags appear at the - * start of the file, their content is stored just like for MP3. + * Additionally, the `'vendor'` key refers to the vendor string within this header. If a key exists more than once, + * a string array is used instead. If ID3v2 tags appear at the start of the file, their content is stored just like + * for MP3. * - MPEG-TS: Not supported. */ - raw?: Record | null>; + raw?: Record | null>; }; /** @@ -238,13 +240,14 @@ export const validateMetadataTags = (tags: MetadataTags) => { if ( value !== null && typeof value !== 'string' + && !(Array.isArray(value) && value.every(x => typeof x === 'string')) && !(value instanceof Uint8Array) && !(value instanceof RichImageData) && !(value instanceof AttachedFile) && !isRecordStringString(value) ) { throw new TypeError( - 'Each value in tags.raw must be a string, Uint8Array, RichImageData, AttachedFile, ' + 'Each value in tags.raw must be a string, string array, Uint8Array, RichImageData, AttachedFile, ' + 'Record, or null.', ); } diff --git a/src/misc.ts b/src/misc.ts index 7d7d1d1..0de8901 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -61,6 +61,14 @@ export const readExpGolomb = (bitstream: Bitstream) => { return result; }; +export const writeExpGolomb = (bitstream: Bitstream, value: number) => { + const codeNum = value + 1; + const leadingZeroBits = Math.floor(Math.log2(codeNum)); + bitstream.writeBits(leadingZeroBits, 0); + bitstream.writeBits(1, 1); + bitstream.writeBits(leadingZeroBits, codeNum - 2 ** leadingZeroBits); +}; + /** Reads a signed exponential-Golomb universal code from a Bitstream. */ export const readSignedExpGolomb = (bitstream: Bitstream) => { const codeNum = readExpGolomb(bitstream); diff --git a/test/browser/media-sinks.test.ts b/test/browser/media-sinks.test.ts index 3a8840b..a542ffd 100644 --- a/test/browser/media-sinks.test.ts +++ b/test/browser/media-sinks.test.ts @@ -1,9 +1,13 @@ import { expect, test } from 'vitest'; import { Input } from '../../src/input.js'; -import { UrlSource } from '../../src/source.js'; +import { BufferSource, UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { assert } from '../../src/misc.js'; -import { AudioSampleSink } from '../../src/media-sink.js'; +import { AudioSampleSink, VideoSampleSink } from '../../src/media-sink.js'; +import { Output } from '../../src/output.js'; +import { MpegTsOutputFormat } from '../../src/output-format.js'; +import { BufferTarget } from '../../src/target.js'; +import { Conversion } from '../../src/conversion.js'; // https://github.com/Vanilagy/mediabunny/issues/370 test('Negative audio timestamps are preserved', async () => { @@ -24,3 +28,61 @@ test('Negative audio timestamps are preserved', async () => { break; } }); + +test('No B-frames are skipped when software-decoding AVC', async () => { + using input = new Input({ + source: new UrlSource('/missing-reorder-metadata-v1.mp4'), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryVideoTrack(); + assert(track); + + const sink = new VideoSampleSink(track, { + hardwareAcceleration: 'prefer-software', + }); + let count = 0; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (using sample of sink.samples()) { + count++; + } + + expect(count).toBe(48); +}); + +test('No B-frames are skipped when software-decoding AVC, Annex B edition', async () => { + using input = new Input({ + source: new UrlSource('/missing-reorder-metadata-v1.mp4'), + formats: ALL_FORMATS, + }); + + // Force Annex B by converting to MPEG-TS + const output = new Output({ + format: new MpegTsOutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ input, output, copy: { mode: 'forced' } }); + await conversion.execute(); + + using newInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const track = await newInput.getPrimaryVideoTrack(); + assert(track); + + const sink = new VideoSampleSink(track, { + hardwareAcceleration: 'prefer-software', + }); + let count = 0; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (using sample of sink.samples()) { + count++; + } + + expect(count).toBe(48); +}); diff --git a/test/node/flac.test.ts b/test/node/flac.test.ts index 861ded3..12eee98 100644 --- a/test/node/flac.test.ts +++ b/test/node/flac.test.ts @@ -205,6 +205,7 @@ test('can re-mux a .flac', async () => { ...otherInputMetadataTags, raw: { ...otherInputMetadataTags.raw, + DATE: '2020-01-01', vendor: 'Mediabunny', }, images: inputImages, diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts index bc28490..1b0cf4a 100644 --- a/test/node/server-extension.test.ts +++ b/test/node/server-extension.test.ts @@ -1223,6 +1223,28 @@ describe('Video', async () => { await decoder.close(); }); + test('No B-frames are skipped when decoding AVC', async () => { + using input = new Input({ + source: new FilePathSource('./test/public/missing-reorder-metadata-v1.mp4'), + formats: ALL_FORMATS, + }); + + const track = await input.getPrimaryVideoTrack(); + assert(track); + + const sink = new VideoSampleSink(track, { + hardwareAcceleration: 'prefer-software', + }); + let count = 0; + + // eslint-disable-next-line @typescript-eslint/no-unused-vars + for await (using sample of sink.samples()) { + count++; + } + + expect(count).toBe(48); + }); + describe('VideoSample transformation', () => { // 400x400 image: red everywhere, with a 200x200 blue square filling the bottom-left quadrant. const TEST_IMAGE = (() => { diff --git a/test/public/missing-reorder-metadata-v1.mp4 b/test/public/missing-reorder-metadata-v1.mp4 new file mode 100644 index 0000000..71c4d57 Binary files /dev/null and b/test/public/missing-reorder-metadata-v1.mp4 differ