Add methods that check decodability

dev/player.html

@@ -40,8 +40,15 @@ const input = new Metamuxer.Input({
 
 console.log(await input.getMimeType())
 
-const videoTrack = await input.getPrimaryVideoTrack(); 
-const audioTrack = await input.getPrimaryAudioTrack(); 
+let videoTrack = await input.getPrimaryVideoTrack(); 
+let audioTrack = await input.getPrimaryAudioTrack(); 
+
+if (!(await videoTrack.canDecode())) {
+	videoTrack = null;
+}
+if (!(await audioTrack.canDecode())) {
+	audioTrack = null;	
+}
 
 const canvas = document.querySelector('canvas');
 const context = canvas.getContext('2d');

src/input-track.ts

@@ -3,7 +3,7 @@ import { ChunkRetrievalOptions } from './media-drain';
 import { Rotation } from './misc';
 
 export interface InputTrackBacking {
-	getCodec(): Promise<MediaCodec>;
+	getCodec(): Promise<MediaCodec | null>;
 	computeDuration(): Promise<number>;
 }
 
@@ -17,8 +17,9 @@ export abstract class InputTrack {
 		this._backing = backing;
 	}
 
-	abstract getCodec(): Promise<MediaCodec>;
-	abstract getCodecMimeType(): Promise<string>;
+	abstract getCodec(): Promise<MediaCodec | null>;
+	abstract getCodecMimeType(): Promise<string | null>;
+	abstract canDecode(): Promise<boolean>;
 
 	isVideoTrack(): this is InputVideoTrack {
 		return this instanceof InputVideoTrack;
@@ -34,11 +35,11 @@ export abstract class InputTrack {
 }
 
 export interface InputVideoTrackBacking extends InputTrackBacking {
-	getCodec(): Promise<VideoCodec>;
+	getCodec(): Promise<VideoCodec | null>;
 	getCodedWidth(): Promise<number>;
 	getCodedHeight(): Promise<number>;
 	getRotation(): Promise<Rotation>;
-	getDecoderConfig(): Promise<VideoDecoderConfig>;
+	getDecoderConfig(): Promise<VideoDecoderConfig | null>;
 	getFirstChunk(options: ChunkRetrievalOptions): Promise<EncodedVideoChunk | null>;
 	getChunk(timestamp: number, options: ChunkRetrievalOptions): Promise<EncodedVideoChunk | null>;
 	getNextChunk(chunk: EncodedVideoChunk, options: ChunkRetrievalOptions): Promise<EncodedVideoChunk | null>;
@@ -58,7 +59,7 @@ export class InputVideoTrack extends InputTrack {
 		this._backing = backing;
 	}
 
-	getCodec(): Promise<VideoCodec> {
+	getCodec(): Promise<VideoCodec | null> {
 		return this._backing.getCodec();
 	}
 
@@ -90,15 +91,25 @@ export class InputVideoTrack extends InputTrack {
 
 	async getCodecMimeType() {
 		const decoderConfig = await this.getDecoderConfig();
-		return decoderConfig.codec;
+		return decoderConfig?.codec ?? null;
+	}
+
+	async canDecode() {
+		const decoderConfig = await this._backing.getDecoderConfig();
+		if (!decoderConfig) {
+			return false;
+		}
+
+		const support = await VideoDecoder.isConfigSupported(decoderConfig);
+		return support.supported === true;
 	}
 }
 
 export interface InputAudioTrackBacking extends InputTrackBacking {
-	getCodec(): Promise<AudioCodec>;
+	getCodec(): Promise<AudioCodec | null>;
 	getNumberOfChannels(): Promise<number>;
 	getSampleRate(): Promise<number>;
-	getDecoderConfig(): Promise<AudioDecoderConfig>;
+	getDecoderConfig(): Promise<AudioDecoderConfig | null>;
 	getFirstChunk(options: ChunkRetrievalOptions): Promise<EncodedAudioChunk | null>;
 	getChunk(timestamp: number, options: ChunkRetrievalOptions): Promise<EncodedAudioChunk | null>;
 	getNextChunk(chunk: EncodedAudioChunk, options: ChunkRetrievalOptions): Promise<EncodedAudioChunk | null>;
@@ -118,7 +129,7 @@ export class InputAudioTrack extends InputTrack {
 		this._backing = backing;
 	}
 
-	getCodec(): Promise<AudioCodec> {
+	getCodec(): Promise<AudioCodec | null> {
 		return this._backing.getCodec();
 	}
 
@@ -136,6 +147,20 @@ export class InputAudioTrack extends InputTrack {
 
 	async getCodecMimeType() {
 		const decoderConfig = await this.getDecoderConfig();
-		return decoderConfig.codec;
+		return decoderConfig?.codec ?? null;
+	}
+
+	async canDecode() {
+		const decoderConfig = await this._backing.getDecoderConfig();
+		if (!decoderConfig) {
+			return false;
+		}
+
+		if (decoderConfig.codec.startsWith('pcm-')) {
+			return true; // Since we decode it ourselves
+		} else {
+			const support = await AudioDecoder.isConfigSupported(decoderConfig);
+			return support.supported === true;
+		}
 	}
 }

src/isobmff/isobmff-demuxer.ts

@@ -166,6 +166,7 @@ export class IsobmffDemuxer extends Demuxer {
 	metadataPromise: Promise<void> | null = null;
 	movieTimescale = -1;
 	movieDurationInTimescale = -1;
+	isQuickTime = false;
 
 	isFragmented = false;
 	fragmentTrackDefaults: FragmentTrackDefaults[] = [];
@@ -196,11 +197,11 @@ export class IsobmffDemuxer extends Demuxer {
 	override async getMimeType() {
 		await this.readMetadata();
 
-		let string = 'video/mp4';
+		let string = this.isQuickTime ? 'video/quicktime' : 'video/mp4';
 
 		if (this.tracks.length > 0) {
 			const codecMimeTypes = await Promise.all(this.tracks.map(x => x.inputTrack!.getCodecMimeType()));
-			const uniqueCodecMimeTypes = [...new Set(codecMimeTypes)];
+			const uniqueCodecMimeTypes = [...new Set(codecMimeTypes.filter(Boolean))];
 
 			string += `; codecs="${uniqueCodecMimeTypes.join(', ')}"`;
 		}
@@ -217,7 +218,10 @@ export class IsobmffDemuxer extends Demuxer {
 				const startPos = this.isobmffReader.pos;
 				const boxInfo = this.isobmffReader.readBoxHeader();
 
-				if (boxInfo.name === 'moov') {
+				if (boxInfo.name === 'ftyp') {
+					const majorBrand = this.isobmffReader.readAscii(4);
+					this.isQuickTime = majorBrand === 'qt  ';
+				} else if (boxInfo.name === 'moov') {
 					// Found moov, load it
 					await this.isobmffReader.reader.loadRange(
 						this.isobmffReader.pos,
@@ -543,26 +547,14 @@ export class IsobmffDemuxer extends Demuxer {
 				this.readContiguousBoxes(boxInfo.contentSize);
 
 				if (track.id !== -1 && track.timescale !== -1 && track.info !== null) {
-					if (track.info.type === 'video' && track.info.codec !== null) {
+					if (track.info.type === 'video' && track.info.width !== -1) {
 						const videoTrack = track as InternalVideoTrack;
 						track.inputTrack = new InputVideoTrack(new IsobmffVideoTrackBacking(videoTrack));
 						this.tracks.push(track);
-					} else if (track.info.type === 'audio' && track.info.codec !== null) {
+					} else if (track.info.type === 'audio' && track.info.numberOfChannels !== 1) {
 						const audioTrack = track as InternalAudioTrack;
 						track.inputTrack = new InputAudioTrack(new IsobmffAudioTrackBacking(audioTrack));
 						this.tracks.push(track);
-
-						if (track.info.codec === 'aac') {
-							// Some videos have metadata mismatch, so let's try to deduce more accurate values directly
-							// from the AudioSpecificConfig:
-							const audioSpecificConfig = parseAacAudioSpecificConfig(track.info.codecDescription);
-							if (audioSpecificConfig.numberOfChannels !== null) {
-								track.info.numberOfChannels = audioSpecificConfig.numberOfChannels;
-							}
-							if (audioSpecificConfig.sampleRate !== null) {
-								track.info.sampleRate = audioSpecificConfig.sampleRate;
-							}
-						}
 					}
 				}
 
@@ -699,9 +691,7 @@ export class IsobmffDemuxer extends Demuxer {
 						} else if (lowercaseBoxName === 'av01') {
 							track.info.codec = 'av1';
 						} else {
-							const { name } = sampleBoxInfo;
-							console.warn(`Unsupported video codec (sample entry type '${name}') - discarding track.`);
-							break;
+							console.warn(`Unsupported video codec (sample entry type '${sampleBoxInfo.name}').`);
 						}
 
 						this.isobmffReader.pos += 6 * 1 + 2 + 2 + 2 + 3 * 4;
@@ -735,9 +725,7 @@ export class IsobmffDemuxer extends Demuxer {
 						} else if (lowercaseBoxName === 'alaw') {
 							track.info.codec = 'alaw';
 						} else {
-							const { name } = sampleBoxInfo;
-							console.warn(`Unsupported audio codec (sample entry type '${name}') - discarding track.`);
-							break;
+							console.warn(`Unsupported audio codec (sample entry type '${sampleBoxInfo.name}').`);
 						}
 
 						this.isobmffReader.pos += 6 * 1 + 2;
@@ -801,7 +789,6 @@ export class IsobmffDemuxer extends Demuxer {
 
 									if (track.info.codec === null) {
 										console.warn('Unsupportedd PCM format.');
-										break;
 									}
 								}
 							}
@@ -1005,6 +992,17 @@ export class IsobmffDemuxer extends Demuxer {
 
 					const decoderSpecificInfoLength = this.isobmffReader.readIsomVariableInteger();
 					track.info.codecDescription = this.isobmffReader.readBytes(decoderSpecificInfoLength);
+
+					if (track.info.codec === 'aac') {
+						// Let's try to deduce more accurate values directly from the AudioSpecificConfig:
+						const audioSpecificConfig = parseAacAudioSpecificConfig(track.info.codecDescription);
+						if (audioSpecificConfig.numberOfChannels !== null) {
+							track.info.numberOfChannels = audioSpecificConfig.numberOfChannels;
+						}
+						if (audioSpecificConfig.sampleRate !== null) {
+							track.info.sampleRate = audioSpecificConfig.sampleRate;
+						}
+					}
 				}
 			}; break;
 
@@ -1648,7 +1646,7 @@ abstract class IsobmffTrackBacking<Chunk extends EncodedVideoChunk | EncodedAudi
 
 	constructor(public internalTrack: InternalTrack) {}
 
-	getCodec(): Promise<MediaCodec> {
+	getCodec(): Promise<MediaCodec | null> {
 		throw new Error('Not implemented on base class.');
 	}
 
@@ -2135,8 +2133,8 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoChunk> im
 		this.internalTrack = internalTrack;
 	}
 
-	override async getCodec(): Promise<VideoCodec> {
-		return this.internalTrack.info.codec!;
+	override async getCodec(): Promise<VideoCodec | null> {
+		return this.internalTrack.info.codec;
 	}
 
 	async getCodedWidth() {
@@ -2151,7 +2149,11 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoChunk> im
 		return this.internalTrack.rotation;
 	}
 
-	async getDecoderConfig(): Promise<VideoDecoderConfig> {
+	async getDecoderConfig(): Promise<VideoDecoderConfig | null> {
+		if (!this.internalTrack.info.codec) {
+			return null;
+		}
+
 		return {
 			codec: extractVideoCodecString(this.internalTrack.info),
 			codedWidth: this.internalTrack.info.width,
@@ -2179,8 +2181,8 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking<EncodedAudioChunk> im
 		this.internalTrack = internalTrack;
 	}
 
-	override async getCodec(): Promise<AudioCodec> {
-		return this.internalTrack.info.codec!;
+	override async getCodec(): Promise<AudioCodec | null> {
+		return this.internalTrack.info.codec;
 	}
 
 	async getNumberOfChannels() {
@@ -2191,7 +2193,11 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking<EncodedAudioChunk> im
 		return this.internalTrack.info.sampleRate;
 	}
 
-	async getDecoderConfig(): Promise<AudioDecoderConfig> {
+	async getDecoderConfig(): Promise<AudioDecoderConfig | null> {
+		if (!this.internalTrack.info.codec) {
+			return null;
+		}
+
 		return {
 			codec: extractAudioCodecString(this.internalTrack.info),
 			numberOfChannels: this.internalTrack.info.numberOfChannels,

src/media-drain.ts

@@ -592,8 +592,6 @@ class VideoDecoderWrapper extends DecoderWrapper<EncodedVideoChunk, VideoFrame>
 export class VideoFrameDrain extends BaseMediaFrameDrain<EncodedVideoChunk, VideoFrame> {
 	/** @internal */
 	_videoTrack: InputVideoTrack;
-	/** @internal */
-	_decoderConfig: VideoDecoderConfig | null = null;
 
 	constructor(videoTrack: InputVideoTrack) {
 		if (!(videoTrack instanceof InputVideoTrack)) {
@@ -607,8 +605,17 @@ export class VideoFrameDrain extends BaseMediaFrameDrain<EncodedVideoChunk, Vide
 
 	/** @internal */
 	async _createDecoder(onFrame: (frame: VideoFrame) => unknown, onError: (error: DOMException) => unknown) {
-		this._decoderConfig ??= await this._videoTrack.getDecoderConfig();
-		return new VideoDecoderWrapper(onFrame, onError, this._decoderConfig);
+		if (!(await this._videoTrack.canDecode())) {
+			throw new Error(
+				'This video track cannot be decoded by this browser. Make sure to check decodability before using'
+				+ ' a track.',
+			);
+		}
+
+		const decoderConfig = await this._videoTrack.getDecoderConfig();
+		assert(decoderConfig);
+
+		return new VideoDecoderWrapper(onFrame, onError, decoderConfig);
 	}
 
 	/** @internal */
@@ -956,8 +963,6 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioChunk, AudioData
 export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioChunk, AudioData> {
 	/** @internal */
 	_audioTrack: InputAudioTrack;
-	/** @internal */
-	_decoderConfig: AudioDecoderConfig | null = null;
 
 	constructor(audioTrack: InputAudioTrack) {
 		if (!(audioTrack instanceof InputAudioTrack)) {
@@ -971,12 +976,20 @@ export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioChunk, Audio
 
 	/** @internal */
 	async _createDecoder(onData: (data: AudioData) => unknown, onError: (error: DOMException) => unknown) {
-		this._decoderConfig ??= await this._audioTrack.getDecoderConfig();
+		if (!(await this._audioTrack.canDecode())) {
+			throw new Error(
+				'This audio track cannot be decoded by this browser. Make sure to check decodability before using'
+				+ ' a track.',
+			);
+		}
 
-		if (this._decoderConfig.codec.startsWith('pcm-')) {
-			return new PcmAudioDecoderWrapper(onData, onError, this._decoderConfig);
+		const decoderConfig = await this._audioTrack.getDecoderConfig();
+		assert(decoderConfig);
+
+		if (decoderConfig.codec.startsWith('pcm-')) {
+			return new PcmAudioDecoderWrapper(onData, onError, decoderConfig);
 		} else {
-			return new AudioDecoderWrapper(onData, onError, this._decoderConfig);
+			return new AudioDecoderWrapper(onData, onError, decoderConfig);
 		}
 	}