From 0f25dcb09667e31ec86c4ff6e97d4db24f3c2aad Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Thu, 2 Jan 2025 19:08:09 +0100 Subject: [PATCH] Add methods that check decodability --- dev/player.html | 11 +++++- src/input-track.ts | 47 +++++++++++++++++------ src/isobmff/isobmff-demuxer.ts | 68 ++++++++++++++++++---------------- src/media-drain.ts | 33 ++++++++++++----- 4 files changed, 105 insertions(+), 54 deletions(-) diff --git a/dev/player.html b/dev/player.html index f3ad168..9c2e4a3 100644 --- a/dev/player.html +++ b/dev/player.html @@ -40,8 +40,15 @@ const input = new Metamuxer.Input({ console.log(await input.getMimeType()) -const videoTrack = await input.getPrimaryVideoTrack(); -const audioTrack = await input.getPrimaryAudioTrack(); +let videoTrack = await input.getPrimaryVideoTrack(); +let audioTrack = await input.getPrimaryAudioTrack(); + +if (!(await videoTrack.canDecode())) { + videoTrack = null; +} +if (!(await audioTrack.canDecode())) { + audioTrack = null; +} const canvas = document.querySelector('canvas'); const context = canvas.getContext('2d'); diff --git a/src/input-track.ts b/src/input-track.ts index 52bfd83..4bd3298 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -3,7 +3,7 @@ import { ChunkRetrievalOptions } from './media-drain'; import { Rotation } from './misc'; export interface InputTrackBacking { - getCodec(): Promise; + getCodec(): Promise; computeDuration(): Promise; } @@ -17,8 +17,9 @@ export abstract class InputTrack { this._backing = backing; } - abstract getCodec(): Promise; - abstract getCodecMimeType(): Promise; + abstract getCodec(): Promise; + abstract getCodecMimeType(): Promise; + abstract canDecode(): Promise; isVideoTrack(): this is InputVideoTrack { return this instanceof InputVideoTrack; @@ -34,11 +35,11 @@ export abstract class InputTrack { } export interface InputVideoTrackBacking extends InputTrackBacking { - getCodec(): Promise; + getCodec(): Promise; getCodedWidth(): Promise; getCodedHeight(): Promise; getRotation(): Promise; - getDecoderConfig(): Promise; + getDecoderConfig(): Promise; getFirstChunk(options: ChunkRetrievalOptions): Promise; getChunk(timestamp: number, options: ChunkRetrievalOptions): Promise; getNextChunk(chunk: EncodedVideoChunk, options: ChunkRetrievalOptions): Promise; @@ -58,7 +59,7 @@ export class InputVideoTrack extends InputTrack { this._backing = backing; } - getCodec(): Promise { + getCodec(): Promise { return this._backing.getCodec(); } @@ -90,15 +91,25 @@ export class InputVideoTrack extends InputTrack { async getCodecMimeType() { const decoderConfig = await this.getDecoderConfig(); - return decoderConfig.codec; + return decoderConfig?.codec ?? null; + } + + async canDecode() { + const decoderConfig = await this._backing.getDecoderConfig(); + if (!decoderConfig) { + return false; + } + + const support = await VideoDecoder.isConfigSupported(decoderConfig); + return support.supported === true; } } export interface InputAudioTrackBacking extends InputTrackBacking { - getCodec(): Promise; + getCodec(): Promise; getNumberOfChannels(): Promise; getSampleRate(): Promise; - getDecoderConfig(): Promise; + getDecoderConfig(): Promise; getFirstChunk(options: ChunkRetrievalOptions): Promise; getChunk(timestamp: number, options: ChunkRetrievalOptions): Promise; getNextChunk(chunk: EncodedAudioChunk, options: ChunkRetrievalOptions): Promise; @@ -118,7 +129,7 @@ export class InputAudioTrack extends InputTrack { this._backing = backing; } - getCodec(): Promise { + getCodec(): Promise { return this._backing.getCodec(); } @@ -136,6 +147,20 @@ export class InputAudioTrack extends InputTrack { async getCodecMimeType() { const decoderConfig = await this.getDecoderConfig(); - return decoderConfig.codec; + return decoderConfig?.codec ?? null; + } + + async canDecode() { + const decoderConfig = await this._backing.getDecoderConfig(); + if (!decoderConfig) { + return false; + } + + if (decoderConfig.codec.startsWith('pcm-')) { + return true; // Since we decode it ourselves + } else { + const support = await AudioDecoder.isConfigSupported(decoderConfig); + return support.supported === true; + } } } diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index 4161791..640e5a6 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -166,6 +166,7 @@ export class IsobmffDemuxer extends Demuxer { metadataPromise: Promise | null = null; movieTimescale = -1; movieDurationInTimescale = -1; + isQuickTime = false; isFragmented = false; fragmentTrackDefaults: FragmentTrackDefaults[] = []; @@ -196,11 +197,11 @@ export class IsobmffDemuxer extends Demuxer { override async getMimeType() { await this.readMetadata(); - let string = 'video/mp4'; + let string = this.isQuickTime ? 'video/quicktime' : 'video/mp4'; if (this.tracks.length > 0) { const codecMimeTypes = await Promise.all(this.tracks.map(x => x.inputTrack!.getCodecMimeType())); - const uniqueCodecMimeTypes = [...new Set(codecMimeTypes)]; + const uniqueCodecMimeTypes = [...new Set(codecMimeTypes.filter(Boolean))]; string += `; codecs="${uniqueCodecMimeTypes.join(', ')}"`; } @@ -217,7 +218,10 @@ export class IsobmffDemuxer extends Demuxer { const startPos = this.isobmffReader.pos; const boxInfo = this.isobmffReader.readBoxHeader(); - if (boxInfo.name === 'moov') { + if (boxInfo.name === 'ftyp') { + const majorBrand = this.isobmffReader.readAscii(4); + this.isQuickTime = majorBrand === 'qt '; + } else if (boxInfo.name === 'moov') { // Found moov, load it await this.isobmffReader.reader.loadRange( this.isobmffReader.pos, @@ -543,26 +547,14 @@ export class IsobmffDemuxer extends Demuxer { this.readContiguousBoxes(boxInfo.contentSize); if (track.id !== -1 && track.timescale !== -1 && track.info !== null) { - if (track.info.type === 'video' && track.info.codec !== null) { + if (track.info.type === 'video' && track.info.width !== -1) { const videoTrack = track as InternalVideoTrack; track.inputTrack = new InputVideoTrack(new IsobmffVideoTrackBacking(videoTrack)); this.tracks.push(track); - } else if (track.info.type === 'audio' && track.info.codec !== null) { + } else if (track.info.type === 'audio' && track.info.numberOfChannels !== 1) { const audioTrack = track as InternalAudioTrack; track.inputTrack = new InputAudioTrack(new IsobmffAudioTrackBacking(audioTrack)); this.tracks.push(track); - - if (track.info.codec === 'aac') { - // Some videos have metadata mismatch, so let's try to deduce more accurate values directly - // from the AudioSpecificConfig: - const audioSpecificConfig = parseAacAudioSpecificConfig(track.info.codecDescription); - if (audioSpecificConfig.numberOfChannels !== null) { - track.info.numberOfChannels = audioSpecificConfig.numberOfChannels; - } - if (audioSpecificConfig.sampleRate !== null) { - track.info.sampleRate = audioSpecificConfig.sampleRate; - } - } } } @@ -699,9 +691,7 @@ export class IsobmffDemuxer extends Demuxer { } else if (lowercaseBoxName === 'av01') { track.info.codec = 'av1'; } else { - const { name } = sampleBoxInfo; - console.warn(`Unsupported video codec (sample entry type '${name}') - discarding track.`); - break; + console.warn(`Unsupported video codec (sample entry type '${sampleBoxInfo.name}').`); } this.isobmffReader.pos += 6 * 1 + 2 + 2 + 2 + 3 * 4; @@ -735,9 +725,7 @@ export class IsobmffDemuxer extends Demuxer { } else if (lowercaseBoxName === 'alaw') { track.info.codec = 'alaw'; } else { - const { name } = sampleBoxInfo; - console.warn(`Unsupported audio codec (sample entry type '${name}') - discarding track.`); - break; + console.warn(`Unsupported audio codec (sample entry type '${sampleBoxInfo.name}').`); } this.isobmffReader.pos += 6 * 1 + 2; @@ -801,7 +789,6 @@ export class IsobmffDemuxer extends Demuxer { if (track.info.codec === null) { console.warn('Unsupportedd PCM format.'); - break; } } } @@ -1005,6 +992,17 @@ export class IsobmffDemuxer extends Demuxer { const decoderSpecificInfoLength = this.isobmffReader.readIsomVariableInteger(); track.info.codecDescription = this.isobmffReader.readBytes(decoderSpecificInfoLength); + + if (track.info.codec === 'aac') { + // Let's try to deduce more accurate values directly from the AudioSpecificConfig: + const audioSpecificConfig = parseAacAudioSpecificConfig(track.info.codecDescription); + if (audioSpecificConfig.numberOfChannels !== null) { + track.info.numberOfChannels = audioSpecificConfig.numberOfChannels; + } + if (audioSpecificConfig.sampleRate !== null) { + track.info.sampleRate = audioSpecificConfig.sampleRate; + } + } } }; break; @@ -1648,7 +1646,7 @@ abstract class IsobmffTrackBacking { + getCodec(): Promise { throw new Error('Not implemented on base class.'); } @@ -2135,8 +2133,8 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking im this.internalTrack = internalTrack; } - override async getCodec(): Promise { - return this.internalTrack.info.codec!; + override async getCodec(): Promise { + return this.internalTrack.info.codec; } async getCodedWidth() { @@ -2151,7 +2149,11 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking im return this.internalTrack.rotation; } - async getDecoderConfig(): Promise { + async getDecoderConfig(): Promise { + if (!this.internalTrack.info.codec) { + return null; + } + return { codec: extractVideoCodecString(this.internalTrack.info), codedWidth: this.internalTrack.info.width, @@ -2179,8 +2181,8 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking im this.internalTrack = internalTrack; } - override async getCodec(): Promise { - return this.internalTrack.info.codec!; + override async getCodec(): Promise { + return this.internalTrack.info.codec; } async getNumberOfChannels() { @@ -2191,7 +2193,11 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking im return this.internalTrack.info.sampleRate; } - async getDecoderConfig(): Promise { + async getDecoderConfig(): Promise { + if (!this.internalTrack.info.codec) { + return null; + } + return { codec: extractAudioCodecString(this.internalTrack.info), numberOfChannels: this.internalTrack.info.numberOfChannels, diff --git a/src/media-drain.ts b/src/media-drain.ts index 9c462f2..184e0d3 100644 --- a/src/media-drain.ts +++ b/src/media-drain.ts @@ -592,8 +592,6 @@ class VideoDecoderWrapper extends DecoderWrapper export class VideoFrameDrain extends BaseMediaFrameDrain { /** @internal */ _videoTrack: InputVideoTrack; - /** @internal */ - _decoderConfig: VideoDecoderConfig | null = null; constructor(videoTrack: InputVideoTrack) { if (!(videoTrack instanceof InputVideoTrack)) { @@ -607,8 +605,17 @@ export class VideoFrameDrain extends BaseMediaFrameDrain unknown, onError: (error: DOMException) => unknown) { - this._decoderConfig ??= await this._videoTrack.getDecoderConfig(); - return new VideoDecoderWrapper(onFrame, onError, this._decoderConfig); + if (!(await this._videoTrack.canDecode())) { + throw new Error( + 'This video track cannot be decoded by this browser. Make sure to check decodability before using' + + ' a track.', + ); + } + + const decoderConfig = await this._videoTrack.getDecoderConfig(); + assert(decoderConfig); + + return new VideoDecoderWrapper(onFrame, onError, decoderConfig); } /** @internal */ @@ -956,8 +963,6 @@ class PcmAudioDecoderWrapper extends DecoderWrapper { /** @internal */ _audioTrack: InputAudioTrack; - /** @internal */ - _decoderConfig: AudioDecoderConfig | null = null; constructor(audioTrack: InputAudioTrack) { if (!(audioTrack instanceof InputAudioTrack)) { @@ -971,12 +976,20 @@ export class AudioDataDrain extends BaseMediaFrameDrain unknown, onError: (error: DOMException) => unknown) { - this._decoderConfig ??= await this._audioTrack.getDecoderConfig(); + if (!(await this._audioTrack.canDecode())) { + throw new Error( + 'This audio track cannot be decoded by this browser. Make sure to check decodability before using' + + ' a track.', + ); + } - if (this._decoderConfig.codec.startsWith('pcm-')) { - return new PcmAudioDecoderWrapper(onData, onError, this._decoderConfig); + const decoderConfig = await this._audioTrack.getDecoderConfig(); + assert(decoderConfig); + + if (decoderConfig.codec.startsWith('pcm-')) { + return new PcmAudioDecoderWrapper(onData, onError, decoderConfig); } else { - return new AudioDecoderWrapper(onData, onError, this._decoderConfig); + return new AudioDecoderWrapper(onData, onError, decoderConfig); } }