From c600b60ee02273c5e78b49cea4d87d9ca30b4564 Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Thu, 5 Jun 2025 15:31:38 +0200 Subject: [PATCH] Provide "in-band" encoding errors --- dev/convert.html | 30 ++++++++------- dev/mux.html | 4 +- docs/guide/media-sources.md | 8 ---- src/codec.ts | 2 +- src/conversion.ts | 17 ++++----- src/media-source.ts | 74 ++++++++++++++++++++++++++++++------- 6 files changed, 87 insertions(+), 48 deletions(-) diff --git a/dev/convert.html b/dev/convert.html index 80190bb..94c7224 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -7,9 +7,9 @@ fileInput.type = 'file'; document.body.append(fileInput); - const progress = document.createElement('progress'); - progress.max = 1; - document.body.append(progress); + const progressElement = document.createElement('progress'); + progressElement.max = 1; + document.body.append(progressElement); fileInput.addEventListener('change', async () => { const file = fileInput.files[0]; @@ -21,7 +21,7 @@ chunked: true, chunkSize: 2**20 }); - const outputFormat = new Metamuxer.WavOutputFormat(); + const outputFormat = new Metamuxer.Mp4OutputFormat(); const button = document.createElement('button'); button.textContent = 'Cancel'; @@ -38,10 +38,10 @@ target }), audio: { - numberOfChannels: 1, - sampleRate: 4000 + //numberOfChannels: 1, + //sampleRate: 4000 //discard: true - //forceReencode: true, + forceReencode: true, }, /* video: { @@ -68,11 +68,11 @@ }, */ video: { - discard: true, - width: 1280, + //discard: true, + //width: 1280, //discard: true, //width: 640 - //forceReencode: true, + forceReencode: true, //rotate: 90 //width: 720 ?? 2160, //height: 1280 ?? 3840, @@ -87,14 +87,16 @@ start: 0, end: 40 }, - computeProgress: true }); console.log(conversion); - function updateProgress() { - progress.value = conversion.progress; + let progress = 0; + conversion.onProgress = newProgress => progress = newProgress; - if (conversion.progress === 1) { + function updateProgress() { + progressElement.value = progress; + + if (progress === 1) { return; } diff --git a/dev/mux.html b/dev/mux.html index c7bc618..10ceb03 100644 --- a/dev/mux.html +++ b/dev/mux.html @@ -115,7 +115,7 @@ */ let videoSource = new Metamuxer.CanvasSource(canvas, { - codec: 'hevc', + codec: 'avc', //fullCodecString: 'avc1.42001f', bitrate: 1e6 }); @@ -206,5 +206,5 @@ Testing... <00:17.350>One... <00:18.125>Two... await output.finalize(); console.log(target); - download(new Blob([target.buffer]), 'test' + format.fileExtension); + //download(new Blob([target.buffer]), 'test' + format.fileExtension); \ No newline at end of file diff --git a/docs/guide/media-sources.md b/docs/guide/media-sources.md index 43e6941..2cd5462 100644 --- a/docs/guide/media-sources.md +++ b/docs/guide/media-sources.md @@ -59,9 +59,6 @@ type VideoEncodingConfig = { packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined ) => unknown; - onEncoderError?: ( - error: Error - ) => unknown; onEncoderConfig?: ( config: AudioEncoderConfig ) => unknown; @@ -73,7 +70,6 @@ type VideoEncodingConfig = { - `keyFrameInterval`: The maximum interval in seconds between two adjacent key frames. Defaults to 5 seconds. More frequent key frames improve seeking behavior but increase file size. When using multiple video tracks, this value should be set to the same value for all tracks. - `fullCodecString`: Allows you to optionally specify the full codec string used by the video encoder, as specified in the [WebCodecs Codec Registry](https://www.w3.org/TR/webcodecs-codec-registry/). For example, you may set it to `'avc1.42001f'` when using AVC. Keep in mind that the codec string must still match the codec specified in `codec`. If you don't set this field, a codec string will be generated automatically. - `onEncodedPacket`: Called for each successfully encoded packet. Useful for determining encoding progress. -- `onEncoderError`: Called when an error occurs within [VideoEncoder](https://developer.mozilla.org/en-US/docs/Web/API/VideoEncoder). - `onEncoderConfig`: Called when the internal encoder config, as used by the WebCodecs API, is created. You can use this to introspect the full codec string. ### Audio encoding config @@ -89,9 +85,6 @@ type AudioEncodingConfig = { packet: EncodedPacket, meta: EncodedAudioChunkMetadata | undefined ) => unknown; - onEncoderError?: ( - error: Error - ) => unknown; onEncoderConfig?: ( config: AudioEncoderConfig ) => unknown; @@ -101,7 +94,6 @@ type AudioEncodingConfig = { - `bitrate`: The target number of bits per second. Alternatively, this can be a [subjective quality](#subjective-qualities). - `fullCodecString`: Allows you to optionally specify the full codec string used by the audio encoder, as specified in the [WebCodecs Codec Registry](https://www.w3.org/TR/webcodecs-codec-registry/). For example, you may set it to `'mp4a.40.2'` when using AAC. Keep in mind that the codec string must still match the codec specified in `codec`. If you don't set this field, a codec string will be generated automatically. - `onEncodedPacket`: Called for each successfully encoded packet. Useful for determining encoding progress. -- `onEncoderError`: Called when an error occurs within [AudioEncoder](https://developer.mozilla.org/en-US/docs/Web/API/AudioEncoder). - `onEncoderConfig`: Called when the internal encoder config, as used by the WebCodecs API, is created. You can use this to introspect the full codec string. ### Subjective qualities diff --git a/src/codec.ts b/src/codec.ts index b73a1b5..564d797 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -756,7 +756,7 @@ export class Quality { /** @internal */ _toAudioBitrate(codec: AudioCodec) { if ((PCM_AUDIO_CODECS as readonly string[]).includes(codec) || codec === 'flac') { - return -1; + return undefined; } const baseRates = { diff --git a/src/conversion.ts b/src/conversion.ts index 3fb7608..addaa1c 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -1,8 +1,8 @@ import { AUDIO_CODECS, AudioCodec, + getFirstEncodableVideoCodec, getEncodableAudioCodecs, - getEncodableVideoCodecs, NON_PCM_AUDIO_CODECS, Quality, QUALITY_HIGH, @@ -520,8 +520,8 @@ export class Conversion { const bitrate = this._options.video?.bitrate ?? QUALITY_HIGH; - const encodableCodecs = await getEncodableVideoCodecs(videoCodecs, { width, height, bitrate }); - if (encodableCodecs.length === 0) { + const encodableCodec = await getFirstEncodableVideoCodec(videoCodecs, { width, height, bitrate }); + if (!encodableCodec) { this.discardedTracks.push({ track, reason: 'no_encodable_target_codec', @@ -530,7 +530,7 @@ export class Conversion { } const encodingConfig: VideoEncodingConfig = { - codec: encodableCodecs[0]!, + codec: encodableCodec, bitrate, onEncodedPacket: sample => this._reportProgress(track.id, sample.timestamp + sample.duration), }; @@ -714,13 +714,12 @@ export class Conversion { bitrate, }); - if ( - encodableCodecsWithDefaultParams - .some(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec)) - ) { + const nonPcmCodec = encodableCodecsWithDefaultParams + .find(codec => (NON_PCM_AUDIO_CODECS as readonly string[]).includes(codec)); + if (nonPcmCodec) { // We are able to encode using a non-PCM codec, but it'll require resampling needsResample = true; - codecOfChoice = encodableCodecsWithDefaultParams[0]!; + codecOfChoice = nonPcmCodec; numberOfChannels = FALLBACK_NUMBER_OF_CHANNELS; sampleRate = FALLBACK_SAMPLE_RATE; } diff --git a/src/media-source.ts b/src/media-source.ts index be0998e..49fdc67 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -201,8 +201,6 @@ export type VideoEncodingConfig = { /** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */ onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown; - /** Called when an error occurs within VideoEncoder. */ - onEncoderError?: (error: Error) => unknown; /** Called when the internal encoder config, as used by the WebCodecs API, is created. */ onEncoderConfig?: (config: VideoEncoderConfig) => unknown; }; @@ -238,9 +236,6 @@ const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') { throw new TypeError('config.onEncodedChunk, when provided, must be a function.'); } - if (config.onEncoderError !== undefined && typeof config.onEncoderError !== 'function') { - throw new TypeError('config.onEncodingError, when provided, must be a function.'); - } if (config.onEncoderConfig !== undefined && typeof config.onEncoderConfig !== 'function') { throw new TypeError('config.onEncoderConfig, when provided, must be a function.'); } @@ -259,9 +254,17 @@ class VideoEncoderWrapper { private customEncoderCallSerializer = new CallSerializer(); private customEncoderQueueSize = 0; + /** + * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context. + * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. + * So, we keep track of the encoder error and throw it as soon as we get the chance. + */ + private encoderError: Error | null = null; + constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {} async add(videoSample: VideoSample, shouldClose: boolean, encodeOptions?: VideoEncoderEncodeOptions) { + this.checkForEncoderError(); this.source._ensureValidAdd(); // Ensure video sample size remains constant @@ -316,6 +319,9 @@ class VideoEncoderWrapper { if (shouldClose) { videoSample.close(); } + }) + .catch((error: Error) => { + this.encoderError ??= error; }); if (this.customEncoderQueueSize >= 4) { @@ -411,7 +417,9 @@ class VideoEncoderWrapper { this.encodingConfig.onEncodedPacket?.(packet, meta); void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta); }, - error: this.encodingConfig.onEncoderError ?? (error => console.error('VideoEncoder error:', error)), + error: (error) => { + this.encoderError ??= error; + }, }); this.encoder.configure(encoderConfig); } @@ -424,6 +432,8 @@ class VideoEncoderWrapper { } async flushAndClose() { + this.checkForEncoderError(); + if (this.customEncoder) { void this.customEncoderCallSerializer.call(() => this.customEncoder!.flush()); await this.customEncoderCallSerializer.call(() => this.customEncoder!.close()); @@ -431,6 +441,8 @@ class VideoEncoderWrapper { await this.encoder.flush(); this.encoder.close(); } + + this.checkForEncoderError(); } getQueueSize() { @@ -440,6 +452,13 @@ class VideoEncoderWrapper { return this.encoder?.encodeQueueSize ?? 0; } } + + checkForEncoderError() { + if (this.encoderError) { + this.encoderError.stack = new Error().stack; // Provide a more useful stack trace + throw this.encoderError; + } + } } /** @@ -581,7 +600,11 @@ export class MediaStreamVideoTrackSource extends VideoSource { return; } - void this._encoder.add(new VideoSample(videoFrame), true); + void this._encoder.add(new VideoSample(videoFrame), true) + .catch((error) => { + this._abortController?.abort(); + throw error; + }); }, }); @@ -681,8 +704,6 @@ export type AudioEncodingConfig = { /** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */ onEncodedPacket?: (packet: EncodedPacket, meta: EncodedAudioChunkMetadata | undefined) => unknown; - /** Called when an error occurs within AudioEncoder. */ - onEncoderError?: (error: Error) => unknown; /** Called when the internal encoder config, as used by the WebCodecs API, is created. */ onEncoderConfig?: (config: AudioEncoderConfig) => unknown; }; @@ -719,9 +740,6 @@ const validateAudioEncodingConfig = (config: AudioEncodingConfig) => { if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') { throw new TypeError('config.onEncodedChunk, when provided, must be a function.'); } - if (config.onEncoderError !== undefined && typeof config.onEncoderError !== 'function') { - throw new TypeError('config.onEncodingError, when provided, must be a function.'); - } if (config.onEncoderConfig !== undefined && typeof config.onEncoderConfig !== 'function') { throw new TypeError('config.onEncoderConfig, when provided, must be a function.'); } @@ -743,9 +761,17 @@ class AudioEncoderWrapper { private customEncoderCallSerializer = new CallSerializer(); private customEncoderQueueSize = 0; + /** + * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context. + * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. + * So, we keep track of the encoder error and throw it as soon as we get the chance. + */ + private encoderError: Error | null = null; + constructor(private source: AudioSource, private encodingConfig: AudioEncodingConfig) {} async add(audioSample: AudioSample, shouldClose: boolean) { + this.checkForEncoderError(); this.source._ensureValidAdd(); // Ensure audio parameters remain constant @@ -790,6 +816,9 @@ class AudioEncoderWrapper { if (shouldClose) { audioSample.close(); } + }) + .catch((error: Error) => { + this.encoderError ??= error; }); if (this.customEncoderQueueSize >= 4) { @@ -957,7 +986,9 @@ class AudioEncoderWrapper { this.encodingConfig.onEncodedPacket?.(packet, meta); void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta); }, - error: this.encodingConfig.onEncoderError ?? (error => console.error('AudioEncoder error:', error)), + error: (error) => { + this.encoderError ??= error; + }, }); this.encoder.configure(encoderConfig); } @@ -1051,6 +1082,8 @@ class AudioEncoderWrapper { } async flushAndClose() { + this.checkForEncoderError(); + if (this.customEncoder) { void this.customEncoderCallSerializer.call(() => this.customEncoder!.flush()); await this.customEncoderCallSerializer.call(() => this.customEncoder!.close()); @@ -1058,6 +1091,8 @@ class AudioEncoderWrapper { await this.encoder.flush(); this.encoder.close(); } + + this.checkForEncoderError(); } getQueueSize() { @@ -1069,6 +1104,13 @@ class AudioEncoderWrapper { return this.encoder?.encodeQueueSize ?? 0; } } + + checkForEncoderError() { + if (this.encoderError) { + this.encoderError.stack = new Error().stack; // Provide a more useful stack trace + throw this.encoderError; + } + } } /** @@ -1234,7 +1276,11 @@ export class MediaStreamAudioTrackSource extends AudioSource { return; } - void this._encoder.add(new AudioSample(audioData), true); + void this._encoder.add(new AudioSample(audioData), true) + .catch((error) => { + this._abortController?.abort(); + throw error; + }); }, });