arcodange/mediabunny
1
0
Fork 0
mirror of https://github.com/arcodange-org/mediabunny.git synced 2026-09-27 10:53:50 +02:00
Code Issues Packages Projects Releases Wiki Activity

Add fix to ensure VideoFrames are output in timestamp order, fix atTimestamps iterator logic & leaks

Browse Source
This commit is contained in:
Vanilagy
2025-02-16 18:30:14 +01:00
parent 444770d4f8
commit a4c5ab4a8c
10 changed files with 154 additions and 104 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dev/index.html → dev/mux.html
View File
src/custom-coder.ts
+32 -36
View File
@@ -2,75 +2,71 @@ import { AudioCodec, VideoCodec } from './codec';
import { EncodedAudioSample, EncodedVideoSample } from './sample';
/** @public */
export class CustomVideoDecoder {
constructor(
public codec: VideoCodec,
public config: VideoDecoderConfig,
public onFrame: (frame: VideoFrame) => unknown,
) {}
export abstract class CustomVideoDecoder {
codec!: VideoCodec;
config!: VideoDecoderConfig;
onFrame!: (frame: VideoFrame) => unknown;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
static supports(codec: VideoCodec, config: VideoDecoderConfig): boolean {
return false;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
decode(sample: EncodedVideoSample): Promise<void> | void {}
flush(): Promise<void> | void {}
abstract init(): void;
abstract decode(sample: EncodedVideoSample): Promise<void> | void;
abstract flush(): Promise<void> | void;
abstract close(): Promise<void> | void;
}
/** @public */
export class CustomAudioDecoder {
constructor(
public codec: AudioCodec,
public config: AudioDecoderConfig,
public onData: (data: AudioData) => unknown,
) {}
export abstract class CustomAudioDecoder {
codec!: AudioCodec;
config!: AudioDecoderConfig;
onData!: (data: AudioData) => unknown;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
static supports(codec: AudioCodec, config: AudioDecoderConfig): boolean {
return false;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
decode(sample: EncodedAudioSample): Promise<void> | void {}
flush(): Promise<void> | void {}
abstract init(): void;
abstract decode(sample: EncodedAudioSample): Promise<void> | void;
abstract flush(): Promise<void> | void;
abstract close(): Promise<void> | void;
}
/** @public */
export class CustomVideoEncoder {
constructor(
public codec: VideoCodec,
public config: VideoEncoderConfig,
public onSample: (sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) => unknown,
) {}
export abstract class CustomVideoEncoder {
codec!: VideoCodec;
config!: VideoEncoderConfig;
onSample!: (sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) => unknown;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
static supports(codec: VideoCodec, config: VideoEncoderConfig): boolean {
return false;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
encode(videoFrame: VideoFrame, options: VideoEncoderEncodeOptions): Promise<void> | void {}
flush(): Promise<void> | void {}
abstract init(): void;
abstract encode(videoFrame: VideoFrame, options: VideoEncoderEncodeOptions): Promise<void> | void;
abstract flush(): Promise<void> | void;
abstract close(): Promise<void> | void;
}
/** @public */
export class CustomAudioEncoder {
constructor(
public codec: AudioCodec,
public config: AudioEncoderConfig,
public onSample: (sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) => unknown,
) {}
export abstract class CustomAudioEncoder {
codec!: AudioCodec;
config!: AudioEncoderConfig;
onSample!: (sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) => unknown;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
static supports(codec: AudioCodec, config: AudioEncoderConfig): boolean {
return false;
}
// eslint-disable-next-line @typescript-eslint/no-unused-vars
encode(audioData: AudioData): Promise<void> | void {}
flush(): Promise<void> | void {}
abstract init(): void;
abstract encode(audioData: AudioData): Promise<void> | void;
abstract flush(): Promise<void> | void;
abstract close(): Promise<void> | void;
}
export const customVideoDecoders: typeof CustomVideoDecoder[] = [];
src/isobmff/isobmff-demuxer.ts
+5 -1
View File
@@ -1797,6 +1797,7 @@ abstract class IsobmffTrackBacking<
type: SampleType,
timestamp: number,
duration: number,
sequenceNumber: number
): Sample;
async getFirstSample(options: SampleRetrievalOptions) {
@@ -2067,6 +2068,7 @@ abstract class IsobmffTrackBacking<
sampleInfo.isKeyFrame ? 'key' : 'delta',
timestamp,
duration,
sampleIndex,
);
this.sampleToSampleIndex.set(sample, sampleIndex);
@@ -2103,6 +2105,7 @@ abstract class IsobmffTrackBacking<
fragmentSample.isKeyFrame ? 'key' : 'delta',
timestamp,
duration,
fragment.moofOffset + sampleIndex,
);
this.sampleToFragmentLocation.set(sample, { fragment, sampleIndex });
@@ -2362,8 +2365,9 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking<EncodedVideoSample> i
type: SampleType,
timestamp: number,
duration: number,
sequenceNumber: number,
) {
return new EncodedVideoSample(data, type, timestamp, duration, byteLength);
return new EncodedVideoSample(data, type, timestamp, duration, sequenceNumber, byteLength);
}
}
src/matroska/matroska-demuxer.ts
+4 -1
View File
@@ -1008,6 +1008,7 @@ abstract class MatroskaTrackBacking<
type: SampleType,
timestamp: number,
duration: number,
sequenceNumber: number,
): Sample;
async getFirstSample(options: SampleRetrievalOptions) {
@@ -1223,6 +1224,7 @@ abstract class MatroskaTrackBacking<
block.isKeyFrame ? 'key' : 'delta',
timestamp,
duration,
cluster.dataStartPos + blockIndex,
);
this.sampleToClusterLocation.set(sample, { cluster, blockIndex });
@@ -1506,8 +1508,9 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking<EncodedVideoSample>
type: SampleType,
timestamp: number,
duration: number,
sequenceNumber: number,
) {
return new EncodedVideoSample(data, type, timestamp, duration, byteLength);
return new EncodedVideoSample(data, type, timestamp, duration, sequenceNumber, byteLength);
}
}
src/media-sink.ts
+85 -22
View File
@@ -4,8 +4,10 @@ import { InputAudioTrack, InputVideoTrack } from './input-track';
import {
AnyIterable,
assert,
binarySearchLessOrEqual,
getInt24,
getUint24,
last,
mapAsyncGenerator,
promiseWithResolvers,
toAsyncIterator,
@@ -338,6 +340,8 @@ export abstract class BaseMediaFrameSink<
onQueueDequeue();
onQueueNotEmpty();
lastFrame?.frame.close();
for (const frame of frameQueue) {
frame.frame.close();
}
@@ -443,30 +447,40 @@ export abstract class BaseMediaFrameSink<
continue;
}
timestampsOfInterest.push(targetSample.timestamp);
if (lastSample && targetSample.sequenceNumber < lastSample.sequenceNumber) {
// We're going back in time with this one, let's flush and reset to an clean state
await decoder.flush();
timestampsOfInterest.length = 0;
}
if (
lastKeySample
&& keySample.timestamp === lastKeySample.timestamp
&& keySample.sequenceNumber === lastKeySample.sequenceNumber
&& targetSample.timestamp >= lastSample!.timestamp
) {
assert(lastSample);
if (targetSample.timestamp === lastSample.timestamp && timestampsOfInterest.length === 1) {
if (
targetSample.sequenceNumber === lastSample.sequenceNumber
&& timestampsOfInterest.length === 0
) {
// Special case: We have a repeat sample, but the frame for that sample has already been
// decoded. Therefore, we need to push the frame here instead of in the decoder callback.
if (lastUsedFrame) {
pushToQueue(this._duplicateFrame(lastUsedFrame));
}
timestampsOfInterest.shift();
} else {
timestampsOfInterest.push(targetSample.timestamp);
}
} else {
// The key sample has changed
lastKeySample = keySample;
lastSample = keySample;
decoder.decode(keySample);
timestampsOfInterest.push(targetSample.timestamp);
}
while (lastSample.timestamp !== targetSample.timestamp) {
while (lastSample.sequenceNumber < targetSample.sequenceNumber) {
const nextSample = await sampleSink.getNextSample(lastSample);
assert(nextSample);
@@ -475,7 +489,10 @@ export abstract class BaseMediaFrameSink<
}
}
if (!terminated) await decoder.flush();
if (!terminated) {
await decoder.flush();
lastUsedFrame?.frame.close();
}
decoder.close();
decoderIsFlushed = true;
@@ -584,30 +601,48 @@ class VideoDecoderWrapper extends DecoderWrapper<EncodedVideoSample, VideoFrame>
lastCustomDecoderPromise = Promise.resolve();
customDecoderQueueSize = 0;
frameQueue: VideoFrame[] = [];
constructor(
onFrame: (frame: WrappedMediaFrame<VideoFrame>) => unknown,
onError: (error: DOMException) => unknown,
codec: VideoCodec,
decoderConfig: VideoDecoderConfig,
timeResolution: number,
public timeResolution: number,
) {
super(onFrame, onError);
const frameHandler = (frame: VideoFrame) => {
// Round the microsecond timestamps to the time resolution
const timestamp = Math.round(frame.timestamp / 1e6 * timeResolution) / timeResolution;
const duration = Math.round((frame.duration ?? 0) / 1e6 * timeResolution) / timeResolution;
// For correct B-frame handling, we don't just hand over the frames directly but instead add them to a
// queue, because we want to ensure frames are emitted in presentation order. We flush the queue each time
// we receive a frame with a timestamp larger than the highest we've seen so far, as we can sure that is
// not a B-frame. Typically, WebCodecs automatically guarantees that frames are emitted in presentation
// order, but some browsers (Safari) don't always follow this rule.
if (this.frameQueue.length > 0 && (frame.timestamp >= last(this.frameQueue)!.timestamp)) {
for (const frame of this.frameQueue) {
this.wrapAndEmitFrame(frame);
}
onFrame({
frame,
timestamp,
duration,
});
this.frameQueue.length = 0;
}
const insertionIndex = binarySearchLessOrEqual(
this.frameQueue,
frame.timestamp,
x => x.timestamp,
);
this.frameQueue.splice(insertionIndex + 1, 0, frame);
};
const MatchingCustomDecoder = customVideoDecoders.find(x => x.supports(codec, decoderConfig));
if (MatchingCustomDecoder) {
this.customDecoder = new MatchingCustomDecoder(codec, decoderConfig, frameHandler);
// @ts-expect-error "Can't create instance of abstract class 🤓"
this.customDecoder = new MatchingCustomDecoder() as CustomVideoDecoder;
this.customDecoder.codec = codec;
this.customDecoder.config = decoderConfig;
this.customDecoder.onFrame = frameHandler;
this.customDecoder.init();
} else {
this.decoder = new VideoDecoder({
output: frameHandler,
@@ -617,6 +652,18 @@ class VideoDecoderWrapper extends DecoderWrapper<EncodedVideoSample, VideoFrame>
}
}
wrapAndEmitFrame(frame: VideoFrame) {
// Round the microsecond timestamps to the time resolution
const timestamp = Math.round(frame.timestamp / 1e6 * this.timeResolution) / this.timeResolution;
const duration = Math.round((frame.duration ?? 0) / 1e6 * this.timeResolution) / this.timeResolution;
this.onFrame({
frame,
timestamp,
duration,
});
}
getDecodeQueueSize() {
if (this.customDecoder) {
return this.customDecoderQueueSize;
@@ -640,22 +687,32 @@ class VideoDecoderWrapper extends DecoderWrapper<EncodedVideoSample, VideoFrame>
}
}
flush() {
async flush() {
if (this.customDecoder) {
return this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush());
await this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush());
} else {
assert(this.decoder);
return this.decoder.flush();
await this.decoder.flush();
}
for (const frame of this.frameQueue) {
this.wrapAndEmitFrame(frame);
}
this.frameQueue.length = 0;
}
close() {
if (this.customDecoder) {
void this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush());
void this.lastCustomDecoderPromise.then(() => this.customDecoder!.close());
} else {
assert(this.decoder);
this.decoder.close();
}
for (const frame of this.frameQueue) {
frame.close();
}
this.frameQueue.length = 0;
}
}
@@ -907,7 +964,13 @@ class AudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioData>
const MatchingCustomDecoder = customAudioDecoders.find(x => x.supports(codec, decoderConfig));
if (MatchingCustomDecoder) {
this.customDecoder = new MatchingCustomDecoder(codec, decoderConfig, dataHandler);
// @ts-expect-error "Can't create instance of abstract class 🤓"
this.customDecoder = new MatchingCustomDecoder() as CustomAudioDecoder;
this.customDecoder.codec = codec;
this.customDecoder.config = decoderConfig;
this.customDecoder.onData = dataHandler;
this.customDecoder.init();
} else {
this.decoder = new AudioDecoder({
output: dataHandler,
@@ -951,7 +1014,7 @@ class AudioDecoderWrapper extends DecoderWrapper<EncodedAudioSample, AudioData>
close() {
if (this.customDecoder) {
void this.lastCustomDecoderPromise.then(() => this.customDecoder!.flush());
void this.lastCustomDecoderPromise.then(() => this.customDecoder!.close());
} else {
assert(this.decoder);
this.decoder.close();
src/media-source.ts
+20 -16
View File
@@ -302,14 +302,16 @@ class VideoEncoderWrapper {
));
if (MatchingCustomEncoder) {
this.customEncoder = new MatchingCustomEncoder(
this.encodingConfig.codec,
encoderConfig,
(sample, meta) => {
this.encodingConfig.onEncodedSample?.(sample, meta);
void this.muxer!.addEncodedVideoSample(this.source._connectedTrack!, sample, meta);
},
);
// @ts-expect-error "Can't create instance of abstract class 🤓"
this.customEncoder = new MatchingCustomEncoder() as CustomVideoEncoder;
this.customEncoder.codec = this.encodingConfig.codec;
this.customEncoder.config = encoderConfig;
this.customEncoder.onSample = (sample, meta) => {
this.encodingConfig.onEncodedSample?.(sample, meta);
void this.muxer!.addEncodedVideoSample(this.source._connectedTrack!, sample, meta);
};
this.customEncoder.init();
} else {
if (typeof VideoEncoder === 'undefined') {
throw new Error('VideoEncoder is not supported by this browser.');
@@ -751,14 +753,16 @@ class AudioEncoderWrapper {
));
if (MatchingCustomEncoder) {
this.customEncoder = new MatchingCustomEncoder(
this.encodingConfig.codec,
encoderConfig,
(sample, meta) => {
this.encodingConfig.onEncodedSample?.(sample, meta);
void this.muxer!.addEncodedAudioSample(this.source._connectedTrack!, sample, meta);
},
);
// @ts-expect-error "Can't create instance of abstract class 🤓"
this.customEncoder = new MatchingCustomEncoder() as CustomAudioEncoder;
this.customEncoder.codec = this.encodingConfig.codec;
this.customEncoder.config = encoderConfig;
this.customEncoder.onSample = (sample, meta) => {
this.encodingConfig.onEncodedSample?.(sample, meta);
void this.muxer!.addEncodedAudioSample(this.source._connectedTrack!, sample, meta);
};
this.customEncoder.init();
} else if ((PCM_AUDIO_CODECS as readonly string[]).includes(this.encodingConfig.codec)) {
this.initPcmEncoder();
} else {
src/mp3/mp3-demuxer.ts
+1
View File
@@ -179,6 +179,7 @@ class Mp3AudioTrackBacking implements InputAudioTrackBacking {
'key',
rawSample.timestamp,
rawSample.duration,
sampleIndex,
);
}
src/ogg/ogg-demuxer.ts
+4 -2
View File
@@ -478,6 +478,7 @@ class OggAudioTrackBacking implements InputAudioTrackBacking {
'key',
Math.max(0, additional.timestampInSamples) / this.internalSampleRate,
durationInSamples / this.internalSampleRate,
packet.endPage.headerStartPos + packet.endSegmentIndex,
);
this.sampleToMetadata.set(sample, {
@@ -738,11 +739,12 @@ class OggAudioTrackBacking implements InputAudioTrackBacking {
}
endSegmentIndex = currentSegmentIndex - 1;
const nextPosition = await this.demuxer.findNextPacketStart(reader, {
const pseudopacket: Packet = {
data: PLACEHOLDER_DATA,
endPage,
endSegmentIndex,
});
};
const nextPosition = await this.demuxer.findNextPacketStart(reader, pseudopacket);
if (nextPosition) {
// Let's rewind a single step (packet) - this previous packet ensures that we'll correctly compute the
src/sample.ts
+2 -26
View File
@@ -10,6 +10,7 @@ export class EncodedVideoSample {
public readonly type: SampleType,
public readonly timestamp: number,
public readonly duration: number,
public readonly sequenceNumber = -1,
public readonly byteLength = data.byteLength,
) {
if (!(data instanceof Uint8Array)) {
@@ -58,19 +59,6 @@ export class EncodedVideoSample {
});
}
is(otherSample: EncodedVideoSample) {
if (!(otherSample instanceof EncodedVideoSample)) {
throw new TypeError('otherSample must be an EncodedVideoSample.');
}
return (
this.type === otherSample.type
&& this.timestamp === otherSample.timestamp
&& this.duration === otherSample.duration
&& this.byteLength === otherSample.byteLength
);
}
clone(options?: {
timestamp?: number;
duration?: number;
@@ -118,6 +106,7 @@ export class EncodedAudioSample {
public readonly type: SampleType,
public readonly timestamp: number,
public readonly duration: number,
public readonly sequenceNumber = -1,
public readonly byteLength = data.byteLength,
) {
if (!(data instanceof Uint8Array)) {
@@ -166,19 +155,6 @@ export class EncodedAudioSample {
});
}
is(otherSample: EncodedAudioSample) {
if (!(otherSample instanceof EncodedAudioSample)) {
throw new TypeError('otherSample must be an EncodedAudioSample.');
}
return (
this.type === otherSample.type
&& this.timestamp === otherSample.timestamp
&& this.duration === otherSample.duration
&& this.byteLength === otherSample.byteLength
);
}
clone(options?: {
timestamp?: number;
duration?: number;
src/wave/wave-demuxer.ts
+1
View File
@@ -274,6 +274,7 @@ class WaveAudioTrackBacking implements InputAudioTrackBacking {
'key',
timestamp,
duration,
sampleIndex,
);
}