Use new sample classes for muxers as well

api-sketch.ts

@@ -1,169 +0,0 @@
-abstract class Target {
-
-}
-
-class ArrayBufferTarget extends Target {
-
-}
-
-class StreamTarget extends Target {
-
-}
-
-abstract class OutputFormat {
-	
-}
-
-class Mp4OutputFormat extends OutputFormat {
-	constructor(options: {
-		fastStart: false | 'in-memory' | 'fragmented' | {
-			expectedVideoChunks?: number,
-			expectedAudioChunks?: number
-		},
-	}) {
-		super();
-	}
-}
-
-class WebMOutputFormat extends OutputFormat {
-
-}
-
-class MkvOutputFormat extends OutputFormat {
-
-}
-
-type OutputOptions = {
-	format: OutputFormat,
-	target: Target
-};
-
-class Output {
-	constructor(options: OutputOptions) {
-
-	}
-
-	addVideoTrack(source: VideoSource) {
-
-	}
-
-	addAudioTrack(source: AudioSource) {
-
-	}
-
-	start() {
-		
-	}
-
-	async finalize() {
-
-	}
-}
-
-abstract class InputSource {
-
-}
-
-class InputFormat {
-	constructor(format: string) {}
-}
-
-const MP4_INPUT_FORMAT = new InputFormat('mp4');
-const WEB_M_INPUT_FORMAT = new InputFormat('webm');
-const MKV_INPUT_FORMAT = new InputFormat('mkv');
-
-type InputOptions = {
-	formats: InputFormat[],
-	source: InputSource
-};
-
-class Input {
-	constructor(options: InputOptions) {
-
-	}
-}
-
-abstract class VideoSource {
-
-}
-
-abstract class AudioSource {
-	
-}
-
-type VideoCodecConfig = {
-	codec: 'avc' | 'hevc' | 'vp8' | 'vp9' | 'av1',
-	bitrate: number 
-};
-
-type AudioCodecConfig = {
-	codec: 'aac' | 'opus' | 'vorbis',
-	bitrate: number
-};
-
-class VideoFrameSource extends VideoSource {
-	constructor(codecConfig: VideoCodecConfig) {
-		super();
-	}
-
-	digest(videoFrame: VideoFrame) {
-		
-	}
-}
-
-class CanvasSource extends VideoSource {
-	constructor(canvas: HTMLCanvasElement, codecConfig: VideoCodecConfig) {
-		super();
-	}
-
-	digest(timestamp: number) {
-
-	}
-}
-
-class MediaStreamTrackVideoSource extends VideoSource {
-	constructor(track: MediaStreamTrack, codecConfig: VideoCodecConfig) {
-		super();
-	}
-}
-
-class AudioDataSource extends AudioSource {
-	constructor(audioData: AudioData, codecConfig: AudioCodecConfig) {
-		super();
-	}
-}
-
-class AudioBufferSource extends AudioSource {
-	constructor(audioBuffer: AudioBuffer, codecConfig: AudioCodecConfig) {
-		super();
-	}
-}
-
-class MediaStreamTrackAudioSource extends AudioSource {
-	constructor(track: MediaStreamTrack, codecConfig: AudioCodecConfig) {
-		super();
-	}
-}
-
-let output = new Output({
-	format: new Mp4OutputFormat({
-		fastStart: false
-	}),
-	target: new ArrayBufferTarget()
-});
-
-let source = new CanvasSource(document.createElement('canvas'), {
-	codec: 'avc',
-	bitrate: 1e6
-});
-
-output.addVideoTrack(source);
-
-output.start();
-
-source.digest(0)
-source.digest(1)
-source.digest(2)
-source.digest(3)
-
-output.finalize()

dev/index.html

@@ -88,7 +88,7 @@
 		bitrate: 1e6
 	});
 	let audioSource = new Metamuxer.AudioBufferSource({
-		codec: 'flac',
+		codec: 'opus',
 		bitrate: 128e3,
 	});
 	let subtitleSource = new Metamuxer.TextSubtitleSource('webvtt');

src/index.ts

@@ -13,12 +13,12 @@ export {
 	AudioEncodingConfig,
 	MediaSource,
 	VideoSource,
-	EncodedVideoChunkSource,
+	EncodedVideoSampleSource,
 	VideoFrameSource,
 	CanvasSource,
 	MediaStreamVideoTrackSource,
 	AudioSource,
-	EncodedAudioChunkSource,
+	EncodedAudioSampleSource,
 	AudioDataSource,
 	AudioBufferSource,
 	MediaStreamAudioTrackSource,

src/isobmff/isobmff-muxer.ts

@@ -7,6 +7,7 @@ import { Mp4OutputFormat, Mp4OutputFormatOptions } from '../output-format';
 import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles';
 import { ArrayBufferTarget } from '../target';
 import { validateAudioChunkMetadata, validateSubtitleMetadata, validateVideoChunkMetadata } from '../codec';
+import { EncodedAudioSample, EncodedVideoSample } from '../sample';
 
 export const GLOBAL_TIMESCALE = 1000;
 const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970
@@ -269,58 +270,51 @@ export class IsobmffMuxer extends Muxer {
 		return newTrackData;
 	}
 
-	async addEncodedVideoChunk(track: OutputVideoTrack, chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) {
+	async addEncodedVideoSample(track: OutputVideoTrack, sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) {
 		const release = await this.mutex.acquire();
 
 		try {
 			const trackData = this.getVideoTrackData(track, meta);
 
-			const data = new Uint8Array(chunk.byteLength);
-			chunk.copyTo(data);
-
 			const timestamp = this.validateAndNormalizeTimestamp(
 				trackData.track,
-				chunk.timestamp,
+				sample.timestamp,
-				chunk.type === 'key',
+				sample.type === 'key',
 			);
-			const sample = this.createSampleForTrack(
+			const internalSample = this.createSampleForTrack(
 				trackData,
-				data,
+				sample.data,
 				timestamp,
-				(chunk.duration ?? 0) / 1e6,
+				sample.duration,
-				chunk.type,
+				sample.type,
 			);
 
-			await this.registerSample(trackData, sample);
+			await this.registerSample(trackData, internalSample);
 		} finally {
 			release();
 		}
 	}
 
-	async addEncodedAudioChunk(track: OutputAudioTrack, chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) {
+	async addEncodedAudioSample(track: OutputAudioTrack, sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) {
 		const release = await this.mutex.acquire();
 
 		try {
 			const trackData = this.getAudioTrackData(track, meta);
 
-			const data = new Uint8Array(chunk.byteLength);
-			chunk.copyTo(data);
-
-			const chunkType = chunk.type as 'key' | 'delta'; // Types are weird for EncodedAudioChunk
 			const timestamp = this.validateAndNormalizeTimestamp(
 				trackData.track,
-				chunk.timestamp,
+				sample.timestamp,
-				chunkType === 'key',
+				sample.type === 'key',
 			);
-			const sample = this.createSampleForTrack(
+			const internalSample = this.createSampleForTrack(
 				trackData,
-				data,
+				sample.data,
 				timestamp,
-				(chunk.duration ?? 0) / 1e6,
+				sample.duration,
-				chunkType,
+				sample.type,
 			);
 
-			await this.registerSample(trackData, sample);
+			await this.registerSample(trackData, internalSample);
 		} finally {
 			release();
 		}
@@ -332,7 +326,7 @@ export class IsobmffMuxer extends Muxer {
 		try {
 			const trackData = this.getSubtitleTrackData(track, meta);
 
-			this.validateAndNormalizeTimestamp(trackData.track, 1e6 * cue.timestamp, true);
+			this.validateAndNormalizeTimestamp(trackData.track, cue.timestamp, true);
 
 			if (track.source._codec === 'webvtt') {
 				trackData.cueQueue.push(cue);

src/matroska/matroska-muxer.ts

@@ -40,6 +40,7 @@ import {
 } from '../codec';
 import { Muxer } from '../muxer';
 import { Writer } from '../writer';
+import { EncodedAudioSample, EncodedVideoSample } from '../sample';
 
 const MAX_CHUNK_LENGTH_MS = 2 ** 15;
 const APP_NAME = 'https://github.com/Vanilagy/webm-muxer'; // TODO
@@ -610,18 +611,15 @@ export class MatroskaMuxer extends Muxer {
 		return newTrackData;
 	}
 
-	async addEncodedVideoChunk(track: OutputVideoTrack, chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) {
+	async addEncodedVideoSample(track: OutputVideoTrack, sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) {
 		const release = await this.mutex.acquire();
 
 		try {
 			const trackData = this.getVideoTrackData(track, meta);
 
-			const data = new Uint8Array(chunk.byteLength);
+			const isKeyFrame = sample.type === 'key';
-			chunk.copyTo(data);
+			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, sample.timestamp, isKeyFrame);
-
+			const videoChunk = this.createInternalChunk(sample.data, timestamp, sample.duration, sample.type);
-			const isKeyFrame = chunk.type === 'key';
-			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, chunk.timestamp, isKeyFrame);
-			const videoChunk = this.createInternalChunk(data, timestamp, (chunk.duration ?? 0) / 1e6, chunk.type);
 			if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk);
 
 			trackData.chunkQueue.push(videoChunk);
@@ -631,19 +629,15 @@ export class MatroskaMuxer extends Muxer {
 		}
 	}
 
-	async addEncodedAudioChunk(track: OutputAudioTrack, chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) {
+	async addEncodedAudioSample(track: OutputAudioTrack, sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) {
 		const release = await this.mutex.acquire();
 
 		try {
 			const trackData = this.getAudioTrackData(track, meta);
 
-			const data = new Uint8Array(chunk.byteLength);
+			const isKeyFrame = sample.type === 'key';
-			chunk.copyTo(data);
+			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, sample.timestamp, isKeyFrame);
-
+			const audioChunk = this.createInternalChunk(sample.data, timestamp, sample.duration, sample.type);
-			const chunkType = chunk.type as 'key' | 'delta'; // Types are weird for EncodedAudioChunk
-			const isKeyFrame = chunkType === 'key';
-			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, chunk.timestamp, isKeyFrame);
-			const audioChunk = this.createInternalChunk(data, timestamp, (chunk.duration ?? 0) / 1e6, chunkType);
 
 			trackData.chunkQueue.push(audioChunk);
 			await this.interleaveChunks();
@@ -658,7 +652,7 @@ export class MatroskaMuxer extends Muxer {
 		try {
 			const trackData = this.getSubtitleTrackData(track, meta);
 
-			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, 1e6 * cue.timestamp, true);
+			const timestamp = this.validateAndNormalizeTimestamp(trackData.track, cue.timestamp, true);
 
 			let bodyText = cue.text;
 			const timestampMs = Math.floor(timestamp * 1000);

src/media-source.ts

@@ -14,6 +14,7 @@ import { OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack }
 import { assert } from './misc';
 import { Muxer } from './muxer';
 import { SubtitleParser } from './subtitles';
+import { EncodedAudioSample, EncodedVideoSample } from './sample';
 
 /** @public */
 export abstract class MediaSource {
@@ -100,19 +101,18 @@ export abstract class VideoSource extends MediaSource {
 }
 
 /** @public */
-export class EncodedVideoChunkSource extends VideoSource {
+export class EncodedVideoSampleSource extends VideoSource {
 	constructor(codec: VideoCodec) {
 		super(codec);
 	}
 
-	digest(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) {
+	digest(sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) {
-		if (!(chunk instanceof EncodedVideoChunk)) {
+		if (!(sample instanceof EncodedVideoSample)) {
-			// TODO add polyfill for browsers that don't have this
+			throw new TypeError('sample must be an EncodedVideoSample.');
-			throw new TypeError('chunk must be an EncodedVideoChunk.');
 		}
 
 		this._ensureValidDigest();
-		return this._connectedTrack!.output._muxer.addEncodedVideoChunk(this._connectedTrack!, chunk, meta);
+		return this._connectedTrack!.output._muxer.addEncodedVideoSample(this._connectedTrack!, sample, meta);
 	}
 }
 
@@ -212,7 +212,11 @@ class VideoEncoderWrapper {
 		this.encoder = new VideoEncoder({
 			output: (chunk, meta) => {
 				this.encodingConfig.onEncodedChunk?.(chunk, meta);
-				void this.muxer!.addEncodedVideoChunk(this.source._connectedTrack!, chunk, meta);
+				void this.muxer!.addEncodedVideoSample(
+					this.source._connectedTrack!,
+					EncodedVideoSample.fromEncodedVideoChunk(chunk),
+					meta,
+				);
 			},
 			error: this.encodingConfig.onEncodingError ?? (error => console.error('VideoEncoder error:', error)),
 		});
@@ -391,19 +395,18 @@ export abstract class AudioSource extends MediaSource {
 }
 
 /** @public */
-export class EncodedAudioChunkSource extends AudioSource {
+export class EncodedAudioSampleSource extends AudioSource {
 	constructor(codec: AudioCodec) {
 		super(codec);
 	}
 
-	digest(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) {
+	digest(sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) {
-		if (!(chunk instanceof EncodedAudioChunk)) {
+		if (!(sample instanceof EncodedAudioSample)) {
-			// TODO add polyfill for browsers that don't have this
+			throw new TypeError('chunk must be an EncodedAudioSample.');
-			throw new TypeError('chunk must be an EncodedAudioChunk.');
 		}
 
 		this._ensureValidDigest();
-		return this._connectedTrack!.output._muxer.addEncodedAudioChunk(this._connectedTrack!, chunk, meta);
+		return this._connectedTrack!.output._muxer.addEncodedAudioSample(this._connectedTrack!, sample, meta);
 	}
 }
 /** @public */
@@ -479,7 +482,11 @@ class AudioEncoderWrapper {
 		this.encoder = new AudioEncoder({
 			output: (chunk, meta) => {
 				this.encodingConfig.onEncodedChunk?.(chunk, meta);
-				void this.muxer!.addEncodedAudioChunk(this.source._connectedTrack!, chunk, meta);
+				void this.muxer!.addEncodedAudioSample(
+					this.source._connectedTrack!,
+					EncodedAudioSample.fromEncodedAudioChunk(chunk),
+					meta,
+				);
 			},
 			error: this.encodingConfig.onEncodingError ?? (error => console.error('AudioEncoder error:', error)),
 		});

src/muxer.ts

@@ -1,5 +1,6 @@
 import { AsyncMutex } from './misc';
 import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from './output';
+import { EncodedAudioSample, EncodedVideoSample } from './sample';
 import { SubtitleCue, SubtitleMetadata } from './subtitles';
 
 export abstract class Muxer {
@@ -11,14 +12,14 @@ export abstract class Muxer {
 	}
 
 	abstract start(): Promise<void>;
-	abstract addEncodedVideoChunk(
+	abstract addEncodedVideoSample(
 		track: OutputVideoTrack,
-		chunk: EncodedVideoChunk,
+		chunk: EncodedVideoSample,
 		meta?: EncodedVideoChunkMetadata
 	): Promise<void>;
-	abstract addEncodedAudioChunk(
+	abstract addEncodedAudioSample(
 		track: OutputAudioTrack,
-		chunk: EncodedAudioChunk,
+		chunk: EncodedAudioSample,
 		meta?: EncodedAudioChunkMetadata
 	): Promise<void>;
 	abstract addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata): Promise<void>;
@@ -34,9 +35,7 @@ export abstract class Muxer {
 	}>();
 
 	abstract timestampsMustStartAtZero: boolean;
-	protected validateAndNormalizeTimestamp(track: OutputTrack, rawTimestampInUs: number, isKeyFrame: boolean) {
+	protected validateAndNormalizeTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyFrame: boolean) {
-		let timestampInSeconds = rawTimestampInUs / 1e6;
-
 		let timestampInfo = this.trackTimestampInfo.get(track);
 		if (!timestampInfo) {
 			if (!isKeyFrame) {

src/sample.ts

@@ -41,6 +41,22 @@ export class EncodedVideoSample {
 			duration: this.microsecondDuration,
 		});
 	}
+
+	static fromEncodedVideoChunk(chunk: EncodedVideoChunk) {
+		if (typeof EncodedVideoChunk === 'undefined') {
+			throw new Error('Your browser does not support EncodedVideoChunk.');
+		}
+
+		const data = new Uint8Array(chunk.byteLength);
+		chunk.copyTo(data);
+
+		return new EncodedVideoSample(
+			data,
+			chunk.type,
+			chunk.timestamp / 1e6,
+			(chunk.duration ?? 0) / 1e6,
+		);
+	}
 }
 
 /** @public */
@@ -81,4 +97,20 @@ export class EncodedAudioSample {
 			duration: this.microsecondDuration,
 		});
 	}
+
+	static fromEncodedAudioChunk(chunk: EncodedAudioChunk) {
+		if (typeof EncodedAudioChunk === 'undefined') {
+			throw new Error('Your browser does not support EncodedAudioChunk.');
+		}
+
+		const data = new Uint8Array(chunk.byteLength);
+		chunk.copyTo(data);
+
+		return new EncodedAudioSample(
+			data,
+			chunk.type as SampleType, // Typing is weird
+			chunk.timestamp / 1e6,
+			(chunk.duration ?? 0) / 1e6,
+		);
+	}
 }