arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 02:43:48 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add methods that check decodability

Browse Source
This commit is contained in:
Vanilagy
2025-01-02 19:08:09 +01:00
parent ad653434a6
commit 0f25dcb096
4 changed files with 105 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/player.html
+9 -2
View File
@@ -40,8 +40,15 @@ const input = new Metamuxer.Input({
console.log(await input.getMimeType())
const videoTrack = await input.getPrimaryVideoTrack();
const audioTrack = await input.getPrimaryAudioTrack();
let videoTrack = await input.getPrimaryVideoTrack();
let audioTrack = await input.getPrimaryAudioTrack();
if (!(await videoTrack.canDecode())) {
videoTrack = null;
}
if (!(await audioTrack.canDecode())) {
audioTrack = null;
}
const canvas = document.querySelector('canvas');
const context = canvas.getContext('2d');
src/input-track.ts
+36 -11
View File
@@ -3,7 +3,7 @@ import { ChunkRetrievalOptions } from './media-drain';
import { Rotation } from './misc';
export interface InputTrackBacking {
getCodec(): Promise<MediaCodec>;
getCodec(): Promise<MediaCodec | null>;
computeDuration(): Promise<number>;
}
@@ -17,8 +17,9 @@ export abstract class InputTrack {
this._backing = backing;
}
abstract getCodec(): Promise<MediaCodec>;
abstract getCodecMimeType(): Promise<string>;
abstract getCodec(): Promise<MediaCodec | null>;
abstract getCodecMimeType(): Promise<string | null>;
abstract canDecode(): Promise<boolean>;
isVideoTrack(): this is InputVideoTrack {
return this instanceof InputVideoTrack;
@@ -34,11 +35,11 @@ export abstract class InputTrack {
}
export interface InputVideoTrackBacking extends InputTrackBacking {
getCodec(): Promise<VideoCodec>;
getCodec(): Promise<VideoCodec | null>;
getCodedWidth(): Promise<number>;
getCodedHeight(): Promise<number>;
getRotation(): Promise<Rotation>;
getDecoderConfig(): Promise<VideoDecoderConfig>;
getDecoderConfig(): Promise<VideoDecoderConfig | null>;
getFirstChunk(options: ChunkRetrievalOptions): Promise<EncodedVideoChunk | null>;
getChunk(timestamp: number, options: ChunkRetrievalOptions): Promise<EncodedVideoChunk | null>;
getNextChunk(chunk: EncodedVideoChunk, options: ChunkRetrievalOptions): Promise<EncodedVideoChunk | null>;
@@ -58,7 +59,7 @@ export class InputVideoTrack extends InputTrack {
this._backing = backing;
}
getCodec(): Promise<VideoCodec> {
getCodec(): Promise<VideoCodec | null> {
return this._backing.getCodec();
}
@@ -90,15 +91,25 @@ export class InputVideoTrack extends InputTrack {
async getCodecMimeType() {
const decoderConfig = await this.getDecoderConfig();
return decoderConfig.codec;
return decoderConfig?.codec ?? null;
}
async canDecode() {
const decoderConfig = await this._backing.getDecoderConfig();
if (!decoderConfig) {
return false;
}
const support = await VideoDecoder.isConfigSupported(decoderConfig);
return support.supported === true;
}
}
export interface InputAudioTrackBacking extends InputTrackBacking {
getCodec(): Promise<AudioCodec>;
getCodec(): Promise<AudioCodec | null>;
getNumberOfChannels(): Promise<number>;
getSampleRate(): Promise<number>;
getDecoderConfig(): Promise<AudioDecoderConfig>;
getDecoderConfig(): Promise<AudioDecoderConfig | null>;
getFirstChunk(options: ChunkRetrievalOptions): Promise<EncodedAudioChunk | null>;
getChunk(timestamp: number, options: ChunkRetrievalOptions): Promise<EncodedAudioChunk | null>;
getNextChunk(chunk: EncodedAudioChunk, options: ChunkRetrievalOptions): Promise<EncodedAudioChunk | null>;
@@ -118,7 +129,7 @@ export class InputAudioTrack extends InputTrack {
this._backing = backing;
}
getCodec(): Promise<AudioCodec> {
getCodec(): Promise<AudioCodec | null> {
return this._backing.getCodec();
}
@@ -136,6 +147,20 @@ export class InputAudioTrack extends InputTrack {
async getCodecMimeType() {
const decoderConfig = await this.getDecoderConfig();
return decoderConfig.codec;
return decoderConfig?.codec ?? null;
}
async canDecode() {
const decoderConfig = await this._backing.getDecoderConfig();
if (!decoderConfig) {
return false;
}
if (decoderConfig.codec.startsWith('pcm-')) {
return true; // Since we decode it ourselves
} else {
const support = await AudioDecoder.isConfigSupported(decoderConfig);
return support.supported === true;
}
}
}
src/isobmff/isobmff-demuxer.ts
+37 -31
View File
@@ -166,6 +166,7 @@ export class IsobmffDemuxer extends Demuxer {
metadataPromise: Promise<void> | null = null;
movieTimescale = -1;
movieDurationInTimescale = -1;
isQuickTime = false;
isFragmented = false;
fragmentTrackDefaults: FragmentTrackDefaults[] = [];
@@ -196,11 +197,11 @@ export class IsobmffDemuxer extends Demuxer {
override async getMimeType() {
await this.readMetadata();
let string = 'video/mp4';
let string = this.isQuickTime ? 'video/quicktime' : 'video/mp4';
if (this.tracks.length > 0) {
const codecMimeTypes = await Promise.all(this.tracks.map(x => x.inputTrack!.getCodecMimeType()));
const uniqueCodecMimeTypes = [...new Set(codecMimeTypes)];
const uniqueCodecMimeTypes = [...new Set(codecMimeTypes.filter(Boolean))];
string += `; codecs="${uniqueCodecMimeTypes.join(', ')}"`;
}
@@ -217,7 +218,10 @@ export class IsobmffDemuxer extends Demuxer {
const startPos = this.isobmffReader.pos;
const boxInfo = this.isobmffReader.readBoxHeader();
if (boxInfo.name === 'moov') {
if (boxInfo.name === 'ftyp') {
const majorBrand = this.isobmffReader.readAscii(4);
this.isQuickTime = majorBrand === 'qt ';
} else if (boxInfo.name === 'moov') {
// Found moov, load it
await this.isobmffReader.reader.loadRange(
this.isobmffReader.pos,
@@ -543,26 +547,14 @@ export class IsobmffDemuxer extends Demuxer {
this.readContiguousBoxes(boxInfo.contentSize);
if (track.id !== -1 && track.timescale !== -1 && track.info !== null) {
if (track.info.type === 'video' && track.info.codec !== null) {
if (track.info.type === 'video' && track.info.width !== -1) {
const videoTrack = track as InternalVideoTrack;
track.inputTrack = new InputVideoTrack(new IsobmffVideoTrackBacking(videoTrack));
this.tracks.push(track);
} else if (track.info.type === 'audio' && track.info.codec !== null) {
} else if (track.info.type === 'audio' && track.info.numberOfChannels !== 1) {
const audioTrack = track as InternalAudioTrack;
track.inputTrack = new InputAudioTrack(new IsobmffAudioTrackBacking(audioTrack));
this.tracks.push(track);
if (track.info.codec === 'aac') {
// Some videos have metadata mismatch, so let's try to deduce more accurate values directly
// from the AudioSpecificConfig:
const audioSpecificConfig = parseAacAudioSpecificConfig(track.info.codecDescription);
if (audioSpecificConfig.numberOfChannels !== null) {
track.info.numberOfChannels = audioSpecificConfig.numberOfChannels;
}
if (audioSpecificConfig.sampleRate !== null) {
track.info.sampleRate = audioSpecificConfig.sampleRate;
}
}
}
}
@@ -699,9 +691,7 @@ export class IsobmffDemuxer extends Demuxer {
} else if (lowercaseBoxName === 'av01') {
track.info.codec = 'av1';
} else {
const { name } = sampleBoxInfo;
console.warn(`Unsupported video codec (sample entry type '${name}') - discarding track.`);
break;
console.warn(`Unsupported video codec (sample entry type '${sampleBoxInfo.name}').`);
}
this.isobmffReader.pos += 6 * 1 + 2 + 2 + 2 + 3 * 4;
@@ -735,9 +725,7 @@ export class IsobmffDemuxer extends Demuxer {
} else if (lowercaseBoxName === 'alaw') {
track.info.codec = 'alaw';
} else {
const { name } = sampleBoxInfo;
console.warn(`Unsupported audio codec (sample entry type '${name}') - discarding track.`);
break;
console.warn(`Unsupported audio codec (sample entry type '${sampleBoxInfo.name}').`);
}
this.isobmffReader.pos += 6 * 1 + 2;
@@ -801,7 +789,6 @@ export class IsobmffDemuxer extends Demuxer {
if (track.info.codec === null) {
console.warn('Unsupportedd PCM format.');
break;
}
}
}
@@ -1005,6 +992,17 @@ export class IsobmffDemuxer extends Demuxer {
const decoderSpecificInfoLength = this.isobmffReader.readIsomVariableInteger();
track.info.codecDescription = this.isobmffReader.readBytes(decoderSpecificInfoLength);
if (track.info.codec === 'aac') {
// Let's try to deduce more accurate values directly from the AudioSpecificConfig:
const audioSpecificConfig = parseAacAudioSpecificConfig(track.info.codecDescription);
if (audioSpecificConfig.numberOfChannels !== null) {
track.info.numberOfChannels = audioSpecificConfig.numberOfChannels;
}
if (audioSpecificConfig.sampleRate !== null) {
track.info.sampleRate = audioSpecificConfig.sampleRate;
}
}
}
}; break;
@@ -1648,7 +1646,7 @@ abstract class IsobmffTrackBacking<Chunk extends EncodedVideoChunk | EncodedAudi
constructor(public internalTrack: InternalTrack) {}
getCodec(): Promise<MediaCodec> {
getCodec(): Promise<MediaCodec | null> {
throw new Error('Not implemented on base class.');
}
@@ -2135,8 +2133,8 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoChunk> im
this.internalTrack = internalTrack;
}
override async getCodec(): Promise<VideoCodec> {
return this.internalTrack.info.codec!;
override async getCodec(): Promise<VideoCodec | null> {
return this.internalTrack.info.codec;
}
async getCodedWidth() {
@@ -2151,7 +2149,11 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoChunk> im
return this.internalTrack.rotation;
}
async getDecoderConfig(): Promise<VideoDecoderConfig> {
async getDecoderConfig(): Promise<VideoDecoderConfig | null> {
if (!this.internalTrack.info.codec) {
return null;
}
return {
codec: extractVideoCodecString(this.internalTrack.info),
codedWidth: this.internalTrack.info.width,
@@ -2179,8 +2181,8 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking<EncodedAudioChunk> im
this.internalTrack = internalTrack;
}
override async getCodec(): Promise<AudioCodec> {
return this.internalTrack.info.codec!;
override async getCodec(): Promise<AudioCodec | null> {
return this.internalTrack.info.codec;
}
async getNumberOfChannels() {
@@ -2191,7 +2193,11 @@ class IsobmffAudioTrackBacking extends IsobmffTrackBacking<EncodedAudioChunk> im
return this.internalTrack.info.sampleRate;
}
async getDecoderConfig(): Promise<AudioDecoderConfig> {
async getDecoderConfig(): Promise<AudioDecoderConfig | null> {
if (!this.internalTrack.info.codec) {
return null;
}
return {
codec: extractAudioCodecString(this.internalTrack.info),
numberOfChannels: this.internalTrack.info.numberOfChannels,
src/media-drain.ts
+23 -10
View File
@@ -592,8 +592,6 @@ class VideoDecoderWrapper extends DecoderWrapper<EncodedVideoChunk, VideoFrame>
export class VideoFrameDrain extends BaseMediaFrameDrain<EncodedVideoChunk, VideoFrame> {
/** @internal */
_videoTrack: InputVideoTrack;
/** @internal */
_decoderConfig: VideoDecoderConfig | null = null;
constructor(videoTrack: InputVideoTrack) {
if (!(videoTrack instanceof InputVideoTrack)) {
@@ -607,8 +605,17 @@ export class VideoFrameDrain extends BaseMediaFrameDrain<EncodedVideoChunk, Vide
/** @internal */
async _createDecoder(onFrame: (frame: VideoFrame) => unknown, onError: (error: DOMException) => unknown) {
this._decoderConfig ??= await this._videoTrack.getDecoderConfig();
return new VideoDecoderWrapper(onFrame, onError, this._decoderConfig);
if (!(await this._videoTrack.canDecode())) {
throw new Error(
'This video track cannot be decoded by this browser. Make sure to check decodability before using'
+ ' a track.',
);
}
const decoderConfig = await this._videoTrack.getDecoderConfig();
assert(decoderConfig);
return new VideoDecoderWrapper(onFrame, onError, decoderConfig);
}
/** @internal */
@@ -956,8 +963,6 @@ class PcmAudioDecoderWrapper extends DecoderWrapper<EncodedAudioChunk, AudioData
export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioChunk, AudioData> {
/** @internal */
_audioTrack: InputAudioTrack;
/** @internal */
_decoderConfig: AudioDecoderConfig | null = null;
constructor(audioTrack: InputAudioTrack) {
if (!(audioTrack instanceof InputAudioTrack)) {
@@ -971,12 +976,20 @@ export class AudioDataDrain extends BaseMediaFrameDrain<EncodedAudioChunk, Audio
/** @internal */
async _createDecoder(onData: (data: AudioData) => unknown, onError: (error: DOMException) => unknown) {
this._decoderConfig ??= await this._audioTrack.getDecoderConfig();
if (!(await this._audioTrack.canDecode())) {
throw new Error(
'This audio track cannot be decoded by this browser. Make sure to check decodability before using'
+ ' a track.',
);
}
if (this._decoderConfig.codec.startsWith('pcm-')) {
return new PcmAudioDecoderWrapper(onData, onError, this._decoderConfig);
const decoderConfig = await this._audioTrack.getDecoderConfig();
assert(decoderConfig);
if (decoderConfig.codec.startsWith('pcm-')) {
return new PcmAudioDecoderWrapper(onData, onError, decoderConfig);
} else {
return new AudioDecoderWrapper(onData, onError, this._decoderConfig);
return new AudioDecoderWrapper(onData, onError, decoderConfig);
}
}