Add subtitle tracks and implement subtitle tracks for Matroska muxer

src/index.ts

@@ -1,4 +1,4 @@
 export { Output } from './output';
 export { Mp4OutputFormat, MkvOutputFormat, WebMOutputFormat } from './output_format';
-export { EncodedVideoChunkSource, VideoFrameSource, CanvasSource, MediaStreamVideoTrackSource, EncodedAudioChunkSource, AudioDataSource, AudioBufferSource, MediaStreamAudioTrackSource } from './source';
+export { EncodedVideoChunkSource, VideoFrameSource, CanvasSource, MediaStreamVideoTrackSource, EncodedAudioChunkSource, AudioDataSource, AudioBufferSource, MediaStreamAudioTrackSource, TextSubtitleSource } from './source';
 export { Target, ArrayBufferTarget, StreamTarget, FileSystemWritableFileStreamTarget } from './target';

src/matroska/matroska_muxer.ts

@@ -1,13 +1,12 @@
 import { assert, readBits, toUint8Array, writeBits } from '../misc';
 import { Muxer } from '../muxer';
-import { Output, OutputAudioTrack, OutputTrack, OutputVideoTrack } from '../output';
+import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from '../output';
 import { MkvOutputFormat, WebMOutputFormat } from '../output_format';
-import { AudioCodec, VideoCodec } from '../source';
+import { AudioCodec, SubtitleCodec, VideoCodec } from '../source';
+import { EncodedSubtitleChunk, EncodedSubtitleChunkMetadata, SubtitleDecoderConfig } from '../subtitles';
 import { Writer } from '../writer';
 import { EBML, EBMLElement, EBMLFloat32, EBMLFloat64, EBMLId, EBMLSignedInt, measureEBMLVarInt, measureSignedInt, measureUnsignedInt } from './ebml';
 
-const VIDEO_TRACK_TYPE = 1;
-const AUDIO_TRACK_TYPE = 2;
 const MAX_CHUNK_LENGTH_MS = 2**15;
 const APP_NAME = 'https://github.com/Vanilagy/webm-muxer'; // TODO
 const SEGMENT_SIZE_BYTES = 6;
@@ -59,12 +58,19 @@ type MatroskaTrackData = {
 		sampleRate: number,
 		decoderConfig: AudioDecoderConfig
 	}
+} | {
+	track: OutputSubtitleTrack,
+	type: 'subtitle',
+	info: {
+		decoderConfig: SubtitleDecoderConfig
+	}
 });
 
 type MatroskaVideoTrackData = MatroskaTrackData & { type: 'video' };
 type MatroskaAudioTrackData = MatroskaTrackData & { type: 'audio' };
+type MatroskaSubtitleTrackData = MatroskaTrackData & { type: 'subtitle' };
 
-const CODEC_STRING_MAP: Record<VideoCodec | AudioCodec, string> = {
+const CODEC_STRING_MAP: Record<VideoCodec | AudioCodec | SubtitleCodec, string> = {
 	avc: 'V_MPEG4/ISO/AVC',
 	hevc: 'V_MPEGH/ISO/HEVC',
 	vp8: 'V_VP8',
@@ -73,6 +79,13 @@ const CODEC_STRING_MAP: Record<VideoCodec | AudioCodec, string> = {
 	aac: 'A_AAC',
 	opus: 'A_OPUS',
 	vorbis: 'A_VORBIS',
+	webvtt: 'S_TEXT/WEBVTT'
+};
+
+const TRACK_TYPE_MAP: Record<OutputTrack['type'], number> = {
+	video: 1,
+	audio: 2,
+	subtitle: 17
 };
 
 // TODO: Perhaps we can make this muxer always be streamable. We can do it similar to the MP4 muxer, where for each 
@@ -290,10 +303,16 @@ export class MatroskaMuxer extends Muxer {
 			if (!['vp8', 'vp9', 'av1'].includes(track.source.codec)) {
 				throw new Error(`WebM only supports VP8, VP9 and AV1 as video codecs. Switching to MKV removes this restriction.`);
 			}
-		} else {
+		} else if (track.type === 'audio') {
 			if (!['opus', 'vorbis'].includes(track.source.codec)) {
 				throw new Error(`WebM only supports Opus and Vorbis as audio codecs. Switching to MKV removes this restriction.`);
 			}
+		} else if (track.type === 'subtitle') {
+			if (track.source.codec !== 'webvtt') {
+				throw new Error(`WebM only supports WebVTT as subtitle codec. Switching to MKV removes this restriction.`);
+			}
+		} else {
+			throw new Error('WebM only supports video, audio and subtitle tracks. Switching to MKV removes this restriction.');
 		}
 	}
 
@@ -370,7 +389,7 @@ export class MatroskaMuxer extends Muxer {
 			tracksElement.data.push({ id: EBMLId.TrackEntry, data: [
 				{ id: EBMLId.TrackNumber, data: trackData.track.id },
 				{ id: EBMLId.TrackUID, data: trackData.track.id },
-				{ id: EBMLId.TrackType, data: trackData.type === 'video' ? VIDEO_TRACK_TYPE : AUDIO_TRACK_TYPE }, // TODO Subtitle case
+				{ id: EBMLId.TrackType, data: TRACK_TYPE_MAP[trackData.type] }, // TODO Subtitle case
 				{ id: EBMLId.CodecID, data: CODEC_STRING_MAP[trackData.track.source.codec] },
 				(trackData.info.decoderConfig.description ? { id: EBMLId.CodecPrivate, data: toUint8Array(trackData.info.decoderConfig.description) } : null),
 				...(trackData.type === 'video' ? [
@@ -419,18 +438,6 @@ export class MatroskaMuxer extends Muxer {
 				] : [])
 			] })
 		}
-
-		/*
-		if (this.#options.subtitles) {
-			tracksElement.data.push({ id: EBMLId.TrackEntry, data: [
-				{ id: EBMLId.TrackNumber, data: SUBTITLE_TRACK_NUMBER },
-				{ id: EBMLId.TrackUID, data: SUBTITLE_TRACK_NUMBER },
-				{ id: EBMLId.TrackType, data: SUBTITLE_TRACK_TYPE },
-				{ id: EBMLId.CodecID, data: this.#options.subtitles.codec },
-				this.#subtitleCodecPrivate
-			] });
-		}
-		*/
 	}
 
 	#createSegment() {
@@ -525,35 +532,75 @@ export class MatroskaMuxer extends Muxer {
 
 		return newTrackData;
 	}
+
+	#getSubtitleTrackData(track: OutputSubtitleTrack, meta?: EncodedSubtitleChunkMetadata) {
+		const existingTrackData = this.#trackDatas.find(x => x.track === track);
+		if (existingTrackData) {
+			return existingTrackData as MatroskaAudioTrackData;
+		}
+
+		// TODO Make proper errors for these
+		assert(meta);
+		assert(meta.decoderConfig);
+
+		const newTrackData: MatroskaSubtitleTrackData = {
+			track,
+			type: 'subtitle',
+			info: {
+				decoderConfig: meta.decoderConfig
+			},
+			chunkQueue: [],
+			firstTimestamp: null,
+			lastKeyFrameTimestamp: null,
+			lastWrittenMsTimestamp: null
+		};
+
+		this.#trackDatas.push(newTrackData);
+		this.#trackDatas.sort((a, b) => a.track.id - b.track.id);
+
+		return newTrackData;
+	}
 	
 	addEncodedVideoChunk(track: OutputVideoTrack, chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) {
 		const trackData = this.#getVideoTrackData(track, meta);
 
-		let videoChunk = this.#createInternalChunk(trackData, chunk);
+		let data = new Uint8Array(chunk.byteLength);
+		chunk.copyTo(data);
+
+		let videoChunk = this.#createInternalChunk(trackData, data, chunk.timestamp, chunk.duration ?? 0, chunk.type);
 		if (track.source.codec === 'vp9') this.#fixVP9ColorSpace(trackData, videoChunk);
 
 		trackData.chunkQueue.push(videoChunk);	
 		this.#interleaveChunks();
-		
-		//this.#writeSubtitleChunks();
-		this.#writer.flush();
 	}
 
 	addEncodedAudioChunk(track: OutputAudioTrack, chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) {
 		const trackData = this.#getAudioTrackData(track, meta);
 
-		let audioChunk = this.#createInternalChunk(trackData, chunk);
+		let data = new Uint8Array(chunk.byteLength);
+		chunk.copyTo(data);
+
+		let audioChunk = this.#createInternalChunk(trackData, data, chunk.timestamp, chunk.duration ?? 0, chunk.type);
 
 		trackData.chunkQueue.push(audioChunk);
 		this.#interleaveChunks();
-		
-		//this.#writeSubtitleChunks();
-		this.#writer.flush();
+	}
+	
+	addEncodedSubtitleChunk(track: OutputSubtitleTrack, chunk: EncodedSubtitleChunk, meta?: EncodedSubtitleChunkMetadata) {
+		const trackData = this.#getSubtitleTrackData(track, meta);
+
+		let subtitleChunk = this.#createInternalChunk(trackData, chunk.body, chunk.timestamp, chunk.duration, 'key', chunk.additions);
+
+		trackData.chunkQueue.push(subtitleChunk);
+		this.#interleaveChunks();
 	}
 
 	#interleaveChunks() {
-		if (this.#trackDatas.length < this.output.tracks.length) {
-			return; // We haven't seen a sample from each track yet
+		let openTrackCount = 0;
+		for (const trackData of this.#trackDatas) if (!trackData.track.source.closed) openTrackCount++;
+
+		if (this.#trackDatas.length < openTrackCount) {
+			return; // We haven't seen a sample from every open track yet
 		}
 
 		outer:
@@ -562,11 +609,11 @@ export class MatroskaMuxer extends Muxer {
 			let minTimestamp = Infinity;
 
 			for (let trackData of this.#trackDatas) {
-				if (trackData.chunkQueue.length === 0) {
+				if (trackData.chunkQueue.length === 0 && !trackData.track.source.closed) {
 					break outer;
 				}
 
-				if (trackData.chunkQueue[0]!.timestamp < minTimestamp) {
+				if (trackData.chunkQueue.length > 0 && trackData.chunkQueue[0]!.timestamp < minTimestamp) {
 					trackWithMinTimestamp = trackData;
 					minTimestamp = trackData.chunkQueue[0]!.timestamp;
 				}
@@ -579,6 +626,8 @@ export class MatroskaMuxer extends Muxer {
 			let chunk = trackWithMinTimestamp.chunkQueue.shift()!;
 			this.#writeBlock(trackWithMinTimestamp, chunk);
 		}
+
+		this.#writer.flush();
 	}
 
 	/** Due to [a bug in Chromium](https://bugs.chromium.org/p/chromium/issues/detail?id=1377842), VP9 streams often
@@ -617,99 +666,30 @@ export class MatroskaMuxer extends Muxer {
 		writeBits(chunk.data, i+0, i+3, colorSpaceID);
 	}
 
-	/*
-	addSubtitleChunk(chunk: EncodedSubtitleChunk, meta: EncodedSubtitleChunkMetadata, timestamp?: number) {
-		if (typeof chunk !== 'object' || !chunk) {
-			throw new TypeError("addSubtitleChunk's first argument (chunk) must be an object.");
-		} else {
-			// We can't simply do an instanceof check, so let's check the structure itself:
-			if (!(chunk.body instanceof Uint8Array)) {
-				throw new TypeError('body must be an instance of Uint8Array.');
-			}
-			if (!Number.isFinite(chunk.timestamp) || chunk.timestamp < 0) {
-				throw new TypeError('timestamp must be a non-negative real number.');
-			}
-			if (!Number.isFinite(chunk.duration) || chunk.duration < 0) {
-				throw new TypeError('duration must be a non-negative real number.');
-			}
-			if (chunk.additions && !(chunk.additions instanceof Uint8Array)) {
-				throw new TypeError('additions, when present, must be an instance of Uint8Array.');
-			}
-		}
-
-		if (typeof meta !== 'object') {
-			throw new TypeError("addSubtitleChunk's second argument (meta) must be an object.");
-		}
-
-		this.#ensureNotFinalized();
-		if (!this.#options.subtitles) throw new Error('No subtitle track declared.');
-
-		// Write possible subtitle decoder metadata to the file
-		if (meta?.decoderConfig) {
-			if (this.#options.streaming) {
-				this.#subtitleCodecPrivate = this.#createCodecPrivateElement(meta.decoderConfig.description);
-			} else {
-				this.#writeCodecPrivate(this.#subtitleCodecPrivate, meta.decoderConfig.description);
-			}
-		}
-
-		let subtitleChunk = this.#createInternalChunk(
-			chunk.body,
-			'key',
-			timestamp ?? chunk.timestamp,
-			SUBTITLE_TRACK_NUMBER,
-			chunk.duration,
-			chunk.additions
-		);
-
-		this.#lastSubtitleTimestamp = subtitleChunk.timestamp;
-		this.#subtitleChunkQueue.push(subtitleChunk);
-
-		this.#writeSubtitleChunks();
-		this.#maybeFlushStreamingTargetWriter();
-	}
-
-	#writeSubtitleChunks() {
-		// Writing subtitle chunks is different from video and audio: A subtitle chunk will be written if it's
-		// guaranteed that no more media chunks will be written before it, to ensure monotonicity. However, media chunks
-		// will NOT wait for subtitle chunks to arrive, as they may never arrive, so that's how non-monotonicity can
-		// arrive. But it should be fine, since it's all still in one cluster.
-
-		let lastWrittenMediaTimestamp = Math.min(
-			this.#options.video ? this.#lastVideoTimestamp : Infinity,
-			this.#options.audio ? this.#lastAudioTimestamp : Infinity
-		);
-
-		let queue = this.#subtitleChunkQueue;
-		while (queue.length > 0 && queue[0].timestamp <= lastWrittenMediaTimestamp) {
-			this.#writeBlock(queue.shift(), !this.#options.video && !this.#options.audio);
-		}
-	}
-	*/
-
 	/** Converts a read-only external chunk into an internal one for easier use. */
 	#createInternalChunk(
 		trackData: MatroskaTrackData,
-		chunk: EncodedVideoChunk | EncodedAudioChunk
+		data: Uint8Array,
+		timestamp: number,
+		duration: number,
+		type: 'key' | 'delta',
+		additions: Uint8Array | null = null
 	) {
-		let adjustedTimestamp = this.#validateTimestamp(trackData, chunk);
-
-		let data = new Uint8Array(chunk.byteLength);
-		chunk.copyTo(data);
+		let adjustedTimestamp = this.#validateTimestamp(trackData, timestamp, type === 'key');
 
 		let internalChunk: InternalMediaChunk = {
 			data,
-			type: chunk.type,
+			type,
 			timestamp: adjustedTimestamp,
-			duration: (chunk.duration ?? 0) / 1e6,
-			additions: null
+			duration: duration / 1e6,
+			additions
 		};
 
 		return internalChunk;
 	}
 
-	#validateTimestamp(trackData: MatroskaTrackData, chunk: EncodedVideoChunk | EncodedAudioChunk) {
-		let timestampInSeconds = chunk.timestamp / 1e6;
+	#validateTimestamp(trackData: MatroskaTrackData, timestamp: number, isKeyFrame: boolean) {
+		let timestampInSeconds = timestamp / 1e6;
 
 		if (timestampInSeconds < 0) {
 			throw new Error(`Timestamps must be non-negative (got ${timestampInSeconds}s).`);
@@ -725,7 +705,7 @@ export class MatroskaMuxer extends Muxer {
 			throw new Error(`Timestamp cannot be before last key frame's timestamp (got ${timestampInSeconds}s, last key frame at ${trackData.lastKeyFrameTimestamp}s).`);
 		}
 
-		if (chunk.type === 'key') {
+		if (isKeyFrame) {
 			trackData.lastKeyFrameTimestamp = timestampInSeconds;
 		}
 
@@ -860,7 +840,6 @@ export class MatroskaMuxer extends Muxer {
 		assert(this.#cues);
 
 		// Add a CuePoint to the Cues element for better seeking
-		// TODO: Should this include subtitle tracks?
 		(this.#cues.data as EBML[]).push({ id: EBMLId.CuePoint, data: [
 			{ id: EBMLId.CueTime, data: this.#currentClusterMsTimestamp! },
 			// We only write out cues for tracks that have at least one chunk in this cluster
@@ -873,6 +852,11 @@ export class MatroskaMuxer extends Muxer {
 		] });
 	}
 
+	override onTrackClose() {
+		// Since a track is now closed, we may be able to write out chunks that were previously waiting
+		this.#interleaveChunks();
+	}
+
 	/** Finalizes the file, making it ready for use. Must be called after all media chunks have been added. */
 	finalize() {
 		// Flush any remaining queued chunks to the file
@@ -886,14 +870,6 @@ export class MatroskaMuxer extends Muxer {
 			this.#finalizeCurrentCluster();
 		}
 
-		/*
-		while (this.#videoChunkQueue.length > 0) this.#writeBlock(this.#videoChunkQueue.shift(), true);
-		while (this.#audioChunkQueue.length > 0) this.#writeBlock(this.#audioChunkQueue.shift(), true);
-		while (this.#subtitleChunkQueue.length > 0 && this.#subtitleChunkQueue[0].timestamp <= this.#duration) {
-			this.#writeBlock(this.#subtitleChunkQueue.shift(), false);
-		}
-		*/
-
 		assert(this.#cues);
 		this.writeEBML(this.#cues);
 

src/muxer.ts

@@ -1,4 +1,5 @@
-import { Output, OutputAudioTrack, OutputTrack, OutputVideoTrack } from "./output";
+import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from "./output";
+import { EncodedSubtitleChunk, EncodedSubtitleChunkMetadata } from "./subtitles";
 
 export abstract class Muxer {
 	output: Output;
@@ -10,7 +11,9 @@ export abstract class Muxer {
 	abstract start(): void;
 	abstract addEncodedVideoChunk(track: OutputVideoTrack, chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata): void;
 	abstract addEncodedAudioChunk(track: OutputAudioTrack, chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata): void;
+	abstract addEncodedSubtitleChunk(track: OutputSubtitleTrack, chunk: EncodedSubtitleChunk, meta?: EncodedSubtitleChunkMetadata): void;
 	abstract finalize(): void;
 
 	beforeTrackAdd(track: OutputTrack) {}
+	onTrackClose(track: OutputTrack) {}
 }

src/output.ts

@@ -1,7 +1,7 @@
 import { TransformationMatrix } from "./misc";
 import { Muxer } from "./muxer";
 import { OutputFormat } from "./output_format";
-import { AudioSource, VideoSource } from "./source";
+import { AudioSource, MediaSource, SubtitleSource, VideoSource } from "./source";
 import { Target } from "./target";
 import { Writer } from "./writer";
 
@@ -21,16 +21,22 @@ export type OutputTrack = {
 	type: 'audio',
 	source: AudioSource,
 	metadata: AudioTrackMetadata
+} | {
+	type: 'subtitle',
+	source: SubtitleSource,
+	metadata: SubtitleTrackMetadata
 });
 
 export type OutputVideoTrack = OutputTrack & { type: 'video' };
 export type OutputAudioTrack = OutputTrack & { type: 'audio' };
+export type OutputSubtitleTrack = OutputTrack & { type: 'subtitle' };
 
 type VideoTrackMetadata = {
 	rotation?: 0 | 90 | 180 | 270 | TransformationMatrix,
 	frameRate?: number
 };
 type AudioTrackMetadata = {};
+type SubtitleTrackMetadata = {};
 
 export class Output {
 	muxer: Muxer;
@@ -40,11 +46,28 @@ export class Output {
 	finalizing = false;
 
 	constructor(options: OutputOptions) {
+		if (options.target.output) {
+			throw new Error('Target is already used for another output.');
+		}
+		options.target.output = this;
+
 		this.writer = options.target.createWriter();
 		this.muxer = options.format.createMuxer(this);
 	}
 
-	addTrack(source: VideoSource | AudioSource, metadata: VideoTrackMetadata | AudioTrackMetadata = {}) {
+	addVideoTrack(source: VideoSource, metadata: VideoTrackMetadata = {}) {
+		this.addTrack('video', source, metadata);
+	}
+
+	addAudioTrack(source: AudioSource, metadata: AudioTrackMetadata = {}) {
+		this.addTrack('audio', source, metadata);
+	}
+
+	addSubtitleTrack(source: SubtitleSource, metadata: SubtitleTrackMetadata = {}) {
+		this.addTrack('subtitle', source, metadata);
+	}
+
+	private addTrack(type: OutputTrack['type'], source: MediaSource, metadata: object) {
 		if (this.started) {
 			throw new Error('Cannot add track after output has started.');
 		}
@@ -55,8 +78,8 @@ export class Output {
 		const track = {
 			id: this.tracks.length + 1,
 			output: this,
-			type: source instanceof VideoSource ? 'video' : 'audio',
-			source,
+			type,
+			source: source as any,
 			metadata
 		} as OutputTrack;
 

src/source.ts

@@ -1,17 +1,15 @@
 import { buildAudioCodecString, buildVideoCodecString } from "./codec";
 import { assert, TransformationMatrix } from "./misc";
-import { OutputAudioTrack, OutputTrack, OutputVideoTrack } from "./output";
+import { OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from "./output";
+import { SubtitleEncoder } from "./subtitles";
 
 export type VideoCodec = 'avc' | 'hevc' | 'vp8' | 'vp9' | 'av1';
 export type AudioCodec = 'aac' | 'opus' | 'vorbis'; // TODO add the rest
+export type SubtitleCodec = 'webvtt';
 
-export abstract class VideoSource {
-	connectedTrack: OutputVideoTrack | null = null;
-	codec: VideoCodec;
-
-	constructor(codec: VideoCodec) {
-		this.codec = codec;
-	}
+export abstract class MediaSource {
+	connectedTrack: OutputTrack | null = null;
+	closed = false;
 
 	ensureValidDigest() {
 		if (!this.connectedTrack) {
@@ -25,39 +23,49 @@ export abstract class VideoSource {
 		if (this.connectedTrack.output.finalizing) {
 			throw new Error('Cannot call digest after output has started finalizing.');
 		}
-	}
 
-	start() {}
-	async flush() {}
-}
-
-export abstract class AudioSource {
-	connectedTrack: OutputAudioTrack | null = null;
-	codec: AudioCodec;
-
-	constructor(codec: AudioCodec) {
-		this.codec = codec;
-	}
-
-	ensureNotFinalizing() {
-		if (this.connectedTrack?.output.finalizing) {
-			throw new Error('Cannot call digest after output has started finalizing.');
+		if (this.closed) {
+			throw new Error('Cannot call digest after source has been closed.');
 		}
 	}
 
+	// TODO: These are should not be called from the outside lib
 	start() {}
 	async flush() {}
+
+	close() {
+		if (this.closed) {
+			throw new Error('Source already closed.');
+		}
+
+		if (!this.connectedTrack) {
+			throw new Error('Cannot call close without connecting the source to an output track.');
+		}
+
+		if (!this.connectedTrack.output.started) {
+			throw new Error('Cannot call close before output has been started.');
+		}
+
+		this.closed = true;
+
+		if (this.connectedTrack.output.finalizing) {
+			return;
+		}
+
+		this.connectedTrack.output.muxer.onTrackClose(this.connectedTrack);
+	}
 }
 
-export type VideoCodecConfig = {
-	codec: 'avc' | 'hevc' | 'vp8' | 'vp9' | 'av1',
-	bitrate: number 
-};
+export abstract class VideoSource extends MediaSource {
+	override connectedTrack: OutputVideoTrack | null = null;
+	codec: VideoCodec;
 
-export type AudioCodecConfig = {
-	codec: 'aac' | 'opus' | 'vorbis',
-	bitrate: number
-};
+	constructor(codec: VideoCodec) {
+		super();
+
+		this.codec = codec;
+	}
+}
 
 export class EncodedVideoChunkSource extends VideoSource {
 	constructor(codec: VideoCodec) {
@@ -74,6 +82,11 @@ export class EncodedVideoChunkSource extends VideoSource {
 
 const KEY_FRAME_INTERVAL = 5;
 
+type VideoCodecConfig = {
+	codec: 'avc' | 'hevc' | 'vp8' | 'vp9' | 'av1',
+	bitrate: number 
+};
+
 class VideoEncoderWrapper {
 	private encoder: VideoEncoder | null = null;
 	private lastMultipleOfKeyFrameInterval = -1;
@@ -200,17 +213,33 @@ export class MediaStreamVideoTrackSource extends VideoSource {
     }
 }
 
+export abstract class AudioSource extends MediaSource {
+	override connectedTrack: OutputAudioTrack | null = null;
+	codec: AudioCodec;
+
+	constructor(codec: AudioCodec) {
+		super();
+
+		this.codec = codec;
+	}
+}
+
 export class EncodedAudioChunkSource extends AudioSource {
 	constructor(codec: AudioCodec) {
 		super(codec);
 	}
 
 	digest(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) {
-		this.ensureNotFinalizing();
+		this.ensureValidDigest();
 		this.connectedTrack?.output.muxer.addEncodedAudioChunk(this.connectedTrack, chunk, meta);
 	}
 }
 
+type AudioCodecConfig = {
+	codec: 'aac' | 'opus' | 'vorbis',
+	bitrate: number
+};
+
 class AudioEncoderWrapper {
     private encoder: AudioEncoder | null = null;
 
@@ -218,7 +247,7 @@ class AudioEncoderWrapper {
 
 	// TODO: Ensure audio parameters remain constant
     digest(audioData: AudioData) {
-        this.source.ensureNotFinalizing();
+        this.source.ensureValidDigest();
 
         this.ensureEncoder(audioData);
         assert(this.encoder);
@@ -345,4 +374,35 @@ export class MediaStreamAudioTrackSource extends AudioSource {
 
         await this.encoder.flush();
     }
+}
+
+export abstract class SubtitleSource extends MediaSource {
+	override connectedTrack: OutputSubtitleTrack | null = null;
+	codec: SubtitleCodec;
+
+	constructor(codec: SubtitleCodec) {
+		super();
+
+		this.codec = codec;
+	}
+}
+
+export class TextSubtitleSource extends SubtitleSource {
+	private encoder: SubtitleEncoder;
+
+	constructor(codec: SubtitleCodec) {
+		super(codec);
+
+		this.encoder = new SubtitleEncoder({
+			output: (chunk, metadata) => this.connectedTrack?.output.muxer.addEncodedSubtitleChunk(this.connectedTrack, chunk, metadata),
+			error: (error) => console.error(error) // TODO
+		});
+
+		this.encoder.configure({ codec });
+	}
+
+	digest(text: string) {
+		this.ensureValidDigest();
+		this.encoder.encode(text);
+	}
 }

src/subtitles.ts

@@ -0,0 +1,151 @@
+export type EncodedSubtitleChunk = {
+	body: Uint8Array,
+	additions: Uint8Array | null,
+	timestamp: number,
+	duration: number
+};
+
+export type SubtitleDecoderConfig = {
+	description: Uint8Array
+};
+
+export type EncodedSubtitleChunkMetadata = {
+	decoderConfig?: SubtitleDecoderConfig
+};
+
+interface SubtitleEncoderOptions {
+	output: (chunk: EncodedSubtitleChunk, metadata: EncodedSubtitleChunkMetadata) => unknown,
+	error: (error: Error) => unknown
+}
+
+interface SubtitleEncoderConfig {
+	codec: 'webvtt'
+}
+
+const cueBlockHeaderRegex = /(?:(.+?)\n)?((?:\d{2}:)?\d{2}:\d{2}.\d{3})\s+-->\s+((?:\d{2}:)?\d{2}:\d{2}.\d{3})/g;
+const preambleStartRegex = /^WEBVTT.*?\n{2}/;
+const timestampRegex = /(?:(\d{2}):)?(\d{2}):(\d{2}).(\d{3})/;
+const inlineTimestampRegex = /<(?:(\d{2}):)?(\d{2}):(\d{2}).(\d{3})>/g;
+const textEncoder = new TextEncoder();
+
+export class SubtitleEncoder {
+	#options: SubtitleEncoderOptions;
+	#config: SubtitleEncoderConfig | null = null;
+	#preambleBytes: Uint8Array | null = null;
+	#preambleEmitted = false;
+
+	constructor(options: SubtitleEncoderOptions) {
+		this.#options = options;
+	}
+
+	configure(config: SubtitleEncoderConfig) {
+		if (config.codec !== 'webvtt') {
+			throw new Error("Codec must be 'webvtt'.");
+		}
+
+		this.#config = config;
+	}
+
+	encode(text: string) {
+		if (!this.#config) {
+			throw new Error('Encoder not configured.');
+		}
+
+		text = text.replace('\r\n', '\n').replace('\r', '\n');
+
+		cueBlockHeaderRegex.lastIndex = 0;
+		let match: RegExpMatchArray | null;
+
+		if (!this.#preambleBytes) {
+			if (!preambleStartRegex.test(text)) {
+				let error = new Error('WebVTT preamble incorrect.');
+				this.#options.error(error);
+				throw error;
+			}
+
+			match = cueBlockHeaderRegex.exec(text);
+			let preamble = text.slice(0, match?.index ?? text.length).trimEnd();
+
+			if (!preamble) {
+				let error = new Error('No WebVTT preamble provided.');
+				this.#options.error(error);
+				throw error;
+			}
+
+			this.#preambleBytes = textEncoder.encode(preamble);
+
+			if (match) {
+				text = text.slice(match.index);
+				cueBlockHeaderRegex.lastIndex = 0;
+			}
+		}
+
+		while (match = cueBlockHeaderRegex.exec(text)) {
+			let notes = text.slice(0, match.index);
+			let cueIdentifier = match[1] || '';
+			let matchEnd = match.index! + match[0].length;
+			let bodyStart = text.indexOf('\n', matchEnd) + 1;
+			let cueSettings = text.slice(matchEnd, bodyStart).trim();
+			let bodyEnd = text.indexOf('\n\n', matchEnd);
+			if (bodyEnd === -1) bodyEnd = text.length;
+
+			let startTime = this.#parseTimestamp(match[2]!);
+			let endTime = this.#parseTimestamp(match[3]!);
+			let duration = endTime - startTime;
+
+			let body = text.slice(bodyStart, bodyEnd);
+			let additions = `${cueSettings}\n${cueIdentifier}\n${notes}`;
+
+			// Replace in-body timestamps so that they're relative to the cue start time
+			inlineTimestampRegex.lastIndex = 0;
+			body = body.replace(inlineTimestampRegex, (match) => {
+				let time = this.#parseTimestamp(match.slice(1, -1));
+				let offsetTime = time - startTime;
+
+				return `<${this.#formatTimestamp(offsetTime)}>`;
+			});
+
+			text = text.slice(bodyEnd).trimStart();
+			cueBlockHeaderRegex.lastIndex = 0;
+
+			let chunk: EncodedSubtitleChunk = {
+				body: textEncoder.encode(body),
+				additions: additions.trim() === '' ? null : textEncoder.encode(additions),
+				timestamp: startTime * 1000,
+				duration: duration * 1000
+			};
+
+			let meta: EncodedSubtitleChunkMetadata = {};
+			if (!this.#preambleEmitted) {
+				meta.decoderConfig = {
+					description: this.#preambleBytes
+				};
+				this.#preambleEmitted = true;
+			}
+
+			this.#options.output(chunk, meta);
+		}
+	}
+
+	#parseTimestamp(string: string) {
+		let match = timestampRegex.exec(string);
+		if (!match) throw new Error('Expected match.');
+
+		return 60 * 60 * 1000 * Number(match[1] || '0') +
+			60 * 1000 * Number(match[2]) +
+			1000 * Number(match[3]) +
+			Number(match[4]);
+	}
+
+	#formatTimestamp(timestamp: number) {
+		let hours = Math.floor(timestamp / (60 * 60 * 1000));
+		let minutes = Math.floor((timestamp % (60 * 60 * 1000)) / (60 * 1000));
+		let seconds = Math.floor((timestamp % (60 * 1000)) / 1000);
+		let milliseconds = timestamp % 1000;
+
+		return hours.toString().padStart(2, '0') + ':' +
+			minutes.toString().padStart(2, '0') + ':' +
+			seconds.toString().padStart(2, '0') + '.' +
+			milliseconds.toString().padStart(3, '0');
+	}
+}

src/target.ts

@@ -1,9 +1,8 @@
+import { Output } from "./output";
 import { ArrayBufferTargetWriter, ChunkedStreamTargetWriter, FileSystemWritableFileStreamTargetWriter, StreamTargetWriter, Writer } from "./writer";
 
-const isTarget = Symbol('isTarget');
 export abstract class Target {
-	// If we didn't add this symbol, then {} would be assignable to Target
-	[isTarget]!: true;
+	output: Output | null = null;
 
 	abstract createWriter(): Writer;
 }