From 1145e0ebbe7fd5925e5e4b25f6f7a1d878db967c Mon Sep 17 00:00:00 2001 From: Vanilagy <1696106+Vanilagy@users.noreply.github.com> Date: Sat, 4 Jan 2025 00:28:54 +0100 Subject: [PATCH] Use new sample classes for muxers as well --- api-sketch.ts | 169 --------------------------------- dev/index.html | 2 +- src/index.ts | 4 +- src/isobmff/isobmff-muxer.ts | 42 ++++---- src/matroska/matroska-muxer.ts | 26 ++--- src/media-source.ts | 35 ++++--- src/muxer.ts | 13 ++- src/sample.ts | 32 +++++++ 8 files changed, 90 insertions(+), 233 deletions(-) delete mode 100644 api-sketch.ts diff --git a/api-sketch.ts b/api-sketch.ts deleted file mode 100644 index 1186ecb..0000000 --- a/api-sketch.ts +++ /dev/null @@ -1,169 +0,0 @@ -abstract class Target { - -} - -class ArrayBufferTarget extends Target { - -} - -class StreamTarget extends Target { - -} - -abstract class OutputFormat { - -} - -class Mp4OutputFormat extends OutputFormat { - constructor(options: { - fastStart: false | 'in-memory' | 'fragmented' | { - expectedVideoChunks?: number, - expectedAudioChunks?: number - }, - }) { - super(); - } -} - -class WebMOutputFormat extends OutputFormat { - -} - -class MkvOutputFormat extends OutputFormat { - -} - -type OutputOptions = { - format: OutputFormat, - target: Target -}; - -class Output { - constructor(options: OutputOptions) { - - } - - addVideoTrack(source: VideoSource) { - - } - - addAudioTrack(source: AudioSource) { - - } - - start() { - - } - - async finalize() { - - } -} - -abstract class InputSource { - -} - -class InputFormat { - constructor(format: string) {} -} - -const MP4_INPUT_FORMAT = new InputFormat('mp4'); -const WEB_M_INPUT_FORMAT = new InputFormat('webm'); -const MKV_INPUT_FORMAT = new InputFormat('mkv'); - -type InputOptions = { - formats: InputFormat[], - source: InputSource -}; - -class Input { - constructor(options: InputOptions) { - - } -} - -abstract class VideoSource { - -} - -abstract class AudioSource { - -} - -type VideoCodecConfig = { - codec: 'avc' | 'hevc' | 'vp8' | 'vp9' | 'av1', - bitrate: number -}; - -type AudioCodecConfig = { - codec: 'aac' | 'opus' | 'vorbis', - bitrate: number -}; - -class VideoFrameSource extends VideoSource { - constructor(codecConfig: VideoCodecConfig) { - super(); - } - - digest(videoFrame: VideoFrame) { - - } -} - -class CanvasSource extends VideoSource { - constructor(canvas: HTMLCanvasElement, codecConfig: VideoCodecConfig) { - super(); - } - - digest(timestamp: number) { - - } -} - -class MediaStreamTrackVideoSource extends VideoSource { - constructor(track: MediaStreamTrack, codecConfig: VideoCodecConfig) { - super(); - } -} - -class AudioDataSource extends AudioSource { - constructor(audioData: AudioData, codecConfig: AudioCodecConfig) { - super(); - } -} - -class AudioBufferSource extends AudioSource { - constructor(audioBuffer: AudioBuffer, codecConfig: AudioCodecConfig) { - super(); - } -} - -class MediaStreamTrackAudioSource extends AudioSource { - constructor(track: MediaStreamTrack, codecConfig: AudioCodecConfig) { - super(); - } -} - -let output = new Output({ - format: new Mp4OutputFormat({ - fastStart: false - }), - target: new ArrayBufferTarget() -}); - -let source = new CanvasSource(document.createElement('canvas'), { - codec: 'avc', - bitrate: 1e6 -}); - -output.addVideoTrack(source); - -output.start(); - -source.digest(0) -source.digest(1) -source.digest(2) -source.digest(3) - -output.finalize() \ No newline at end of file diff --git a/dev/index.html b/dev/index.html index 6994c3a..78c52d9 100644 --- a/dev/index.html +++ b/dev/index.html @@ -88,7 +88,7 @@ bitrate: 1e6 }); let audioSource = new Metamuxer.AudioBufferSource({ - codec: 'flac', + codec: 'opus', bitrate: 128e3, }); let subtitleSource = new Metamuxer.TextSubtitleSource('webvtt'); diff --git a/src/index.ts b/src/index.ts index a8e998d..c58cbed 100644 --- a/src/index.ts +++ b/src/index.ts @@ -13,12 +13,12 @@ export { AudioEncodingConfig, MediaSource, VideoSource, - EncodedVideoChunkSource, + EncodedVideoSampleSource, VideoFrameSource, CanvasSource, MediaStreamVideoTrackSource, AudioSource, - EncodedAudioChunkSource, + EncodedAudioSampleSource, AudioDataSource, AudioBufferSource, MediaStreamAudioTrackSource, diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index a144c1a..e280ec7 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -7,6 +7,7 @@ import { Mp4OutputFormat, Mp4OutputFormatOptions } from '../output-format'; import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles'; import { ArrayBufferTarget } from '../target'; import { validateAudioChunkMetadata, validateSubtitleMetadata, validateVideoChunkMetadata } from '../codec'; +import { EncodedAudioSample, EncodedVideoSample } from '../sample'; export const GLOBAL_TIMESCALE = 1000; const TIMESTAMP_OFFSET = 2_082_844_800; // Seconds between Jan 1 1904 and Jan 1 1970 @@ -269,58 +270,51 @@ export class IsobmffMuxer extends Muxer { return newTrackData; } - async addEncodedVideoChunk(track: OutputVideoTrack, chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) { + async addEncodedVideoSample(track: OutputVideoTrack, sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) { const release = await this.mutex.acquire(); try { const trackData = this.getVideoTrackData(track, meta); - const data = new Uint8Array(chunk.byteLength); - chunk.copyTo(data); - const timestamp = this.validateAndNormalizeTimestamp( trackData.track, - chunk.timestamp, - chunk.type === 'key', + sample.timestamp, + sample.type === 'key', ); - const sample = this.createSampleForTrack( + const internalSample = this.createSampleForTrack( trackData, - data, + sample.data, timestamp, - (chunk.duration ?? 0) / 1e6, - chunk.type, + sample.duration, + sample.type, ); - await this.registerSample(trackData, sample); + await this.registerSample(trackData, internalSample); } finally { release(); } } - async addEncodedAudioChunk(track: OutputAudioTrack, chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) { + async addEncodedAudioSample(track: OutputAudioTrack, sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) { const release = await this.mutex.acquire(); try { const trackData = this.getAudioTrackData(track, meta); - const data = new Uint8Array(chunk.byteLength); - chunk.copyTo(data); - - const chunkType = chunk.type as 'key' | 'delta'; // Types are weird for EncodedAudioChunk const timestamp = this.validateAndNormalizeTimestamp( trackData.track, - chunk.timestamp, - chunkType === 'key', + sample.timestamp, + sample.type === 'key', ); - const sample = this.createSampleForTrack( + const internalSample = this.createSampleForTrack( trackData, - data, + sample.data, timestamp, - (chunk.duration ?? 0) / 1e6, - chunkType, + sample.duration, + sample.type, ); - await this.registerSample(trackData, sample); + await this.registerSample(trackData, internalSample); } finally { release(); } @@ -332,7 +326,7 @@ export class IsobmffMuxer extends Muxer { try { const trackData = this.getSubtitleTrackData(track, meta); - this.validateAndNormalizeTimestamp(trackData.track, 1e6 * cue.timestamp, true); + this.validateAndNormalizeTimestamp(trackData.track, cue.timestamp, true); if (track.source._codec === 'webvtt') { trackData.cueQueue.push(cue); diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 0cd8740..f14bb50 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -40,6 +40,7 @@ import { } from '../codec'; import { Muxer } from '../muxer'; import { Writer } from '../writer'; +import { EncodedAudioSample, EncodedVideoSample } from '../sample'; const MAX_CHUNK_LENGTH_MS = 2 ** 15; const APP_NAME = 'https://github.com/Vanilagy/webm-muxer'; // TODO @@ -610,18 +611,15 @@ export class MatroskaMuxer extends Muxer { return newTrackData; } - async addEncodedVideoChunk(track: OutputVideoTrack, chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) { + async addEncodedVideoSample(track: OutputVideoTrack, sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) { const release = await this.mutex.acquire(); try { const trackData = this.getVideoTrackData(track, meta); - const data = new Uint8Array(chunk.byteLength); - chunk.copyTo(data); - - const isKeyFrame = chunk.type === 'key'; - const timestamp = this.validateAndNormalizeTimestamp(trackData.track, chunk.timestamp, isKeyFrame); - const videoChunk = this.createInternalChunk(data, timestamp, (chunk.duration ?? 0) / 1e6, chunk.type); + const isKeyFrame = sample.type === 'key'; + const timestamp = this.validateAndNormalizeTimestamp(trackData.track, sample.timestamp, isKeyFrame); + const videoChunk = this.createInternalChunk(sample.data, timestamp, sample.duration, sample.type); if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk); trackData.chunkQueue.push(videoChunk); @@ -631,19 +629,15 @@ export class MatroskaMuxer extends Muxer { } } - async addEncodedAudioChunk(track: OutputAudioTrack, chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) { + async addEncodedAudioSample(track: OutputAudioTrack, sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) { const release = await this.mutex.acquire(); try { const trackData = this.getAudioTrackData(track, meta); - const data = new Uint8Array(chunk.byteLength); - chunk.copyTo(data); - - const chunkType = chunk.type as 'key' | 'delta'; // Types are weird for EncodedAudioChunk - const isKeyFrame = chunkType === 'key'; - const timestamp = this.validateAndNormalizeTimestamp(trackData.track, chunk.timestamp, isKeyFrame); - const audioChunk = this.createInternalChunk(data, timestamp, (chunk.duration ?? 0) / 1e6, chunkType); + const isKeyFrame = sample.type === 'key'; + const timestamp = this.validateAndNormalizeTimestamp(trackData.track, sample.timestamp, isKeyFrame); + const audioChunk = this.createInternalChunk(sample.data, timestamp, sample.duration, sample.type); trackData.chunkQueue.push(audioChunk); await this.interleaveChunks(); @@ -658,7 +652,7 @@ export class MatroskaMuxer extends Muxer { try { const trackData = this.getSubtitleTrackData(track, meta); - const timestamp = this.validateAndNormalizeTimestamp(trackData.track, 1e6 * cue.timestamp, true); + const timestamp = this.validateAndNormalizeTimestamp(trackData.track, cue.timestamp, true); let bodyText = cue.text; const timestampMs = Math.floor(timestamp * 1000); diff --git a/src/media-source.ts b/src/media-source.ts index 995b622..33bc9f4 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -14,6 +14,7 @@ import { OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } import { assert } from './misc'; import { Muxer } from './muxer'; import { SubtitleParser } from './subtitles'; +import { EncodedAudioSample, EncodedVideoSample } from './sample'; /** @public */ export abstract class MediaSource { @@ -100,19 +101,18 @@ export abstract class VideoSource extends MediaSource { } /** @public */ -export class EncodedVideoChunkSource extends VideoSource { +export class EncodedVideoSampleSource extends VideoSource { constructor(codec: VideoCodec) { super(codec); } - digest(chunk: EncodedVideoChunk, meta?: EncodedVideoChunkMetadata) { - if (!(chunk instanceof EncodedVideoChunk)) { - // TODO add polyfill for browsers that don't have this - throw new TypeError('chunk must be an EncodedVideoChunk.'); + digest(sample: EncodedVideoSample, meta?: EncodedVideoChunkMetadata) { + if (!(sample instanceof EncodedVideoSample)) { + throw new TypeError('sample must be an EncodedVideoSample.'); } this._ensureValidDigest(); - return this._connectedTrack!.output._muxer.addEncodedVideoChunk(this._connectedTrack!, chunk, meta); + return this._connectedTrack!.output._muxer.addEncodedVideoSample(this._connectedTrack!, sample, meta); } } @@ -212,7 +212,11 @@ class VideoEncoderWrapper { this.encoder = new VideoEncoder({ output: (chunk, meta) => { this.encodingConfig.onEncodedChunk?.(chunk, meta); - void this.muxer!.addEncodedVideoChunk(this.source._connectedTrack!, chunk, meta); + void this.muxer!.addEncodedVideoSample( + this.source._connectedTrack!, + EncodedVideoSample.fromEncodedVideoChunk(chunk), + meta, + ); }, error: this.encodingConfig.onEncodingError ?? (error => console.error('VideoEncoder error:', error)), }); @@ -391,19 +395,18 @@ export abstract class AudioSource extends MediaSource { } /** @public */ -export class EncodedAudioChunkSource extends AudioSource { +export class EncodedAudioSampleSource extends AudioSource { constructor(codec: AudioCodec) { super(codec); } - digest(chunk: EncodedAudioChunk, meta?: EncodedAudioChunkMetadata) { - if (!(chunk instanceof EncodedAudioChunk)) { - // TODO add polyfill for browsers that don't have this - throw new TypeError('chunk must be an EncodedAudioChunk.'); + digest(sample: EncodedAudioSample, meta?: EncodedAudioChunkMetadata) { + if (!(sample instanceof EncodedAudioSample)) { + throw new TypeError('chunk must be an EncodedAudioSample.'); } this._ensureValidDigest(); - return this._connectedTrack!.output._muxer.addEncodedAudioChunk(this._connectedTrack!, chunk, meta); + return this._connectedTrack!.output._muxer.addEncodedAudioSample(this._connectedTrack!, sample, meta); } } /** @public */ @@ -479,7 +482,11 @@ class AudioEncoderWrapper { this.encoder = new AudioEncoder({ output: (chunk, meta) => { this.encodingConfig.onEncodedChunk?.(chunk, meta); - void this.muxer!.addEncodedAudioChunk(this.source._connectedTrack!, chunk, meta); + void this.muxer!.addEncodedAudioSample( + this.source._connectedTrack!, + EncodedAudioSample.fromEncodedAudioChunk(chunk), + meta, + ); }, error: this.encodingConfig.onEncodingError ?? (error => console.error('AudioEncoder error:', error)), }); diff --git a/src/muxer.ts b/src/muxer.ts index 4c940d3..f7c1802 100644 --- a/src/muxer.ts +++ b/src/muxer.ts @@ -1,5 +1,6 @@ import { AsyncMutex } from './misc'; import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from './output'; +import { EncodedAudioSample, EncodedVideoSample } from './sample'; import { SubtitleCue, SubtitleMetadata } from './subtitles'; export abstract class Muxer { @@ -11,14 +12,14 @@ export abstract class Muxer { } abstract start(): Promise; - abstract addEncodedVideoChunk( + abstract addEncodedVideoSample( track: OutputVideoTrack, - chunk: EncodedVideoChunk, + chunk: EncodedVideoSample, meta?: EncodedVideoChunkMetadata ): Promise; - abstract addEncodedAudioChunk( + abstract addEncodedAudioSample( track: OutputAudioTrack, - chunk: EncodedAudioChunk, + chunk: EncodedAudioSample, meta?: EncodedAudioChunkMetadata ): Promise; abstract addSubtitleCue(track: OutputSubtitleTrack, cue: SubtitleCue, meta?: SubtitleMetadata): Promise; @@ -34,9 +35,7 @@ export abstract class Muxer { }>(); abstract timestampsMustStartAtZero: boolean; - protected validateAndNormalizeTimestamp(track: OutputTrack, rawTimestampInUs: number, isKeyFrame: boolean) { - let timestampInSeconds = rawTimestampInUs / 1e6; - + protected validateAndNormalizeTimestamp(track: OutputTrack, timestampInSeconds: number, isKeyFrame: boolean) { let timestampInfo = this.trackTimestampInfo.get(track); if (!timestampInfo) { if (!isKeyFrame) { diff --git a/src/sample.ts b/src/sample.ts index 2e6aa3f..cba8d25 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -41,6 +41,22 @@ export class EncodedVideoSample { duration: this.microsecondDuration, }); } + + static fromEncodedVideoChunk(chunk: EncodedVideoChunk) { + if (typeof EncodedVideoChunk === 'undefined') { + throw new Error('Your browser does not support EncodedVideoChunk.'); + } + + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + + return new EncodedVideoSample( + data, + chunk.type, + chunk.timestamp / 1e6, + (chunk.duration ?? 0) / 1e6, + ); + } } /** @public */ @@ -81,4 +97,20 @@ export class EncodedAudioSample { duration: this.microsecondDuration, }); } + + static fromEncodedAudioChunk(chunk: EncodedAudioChunk) { + if (typeof EncodedAudioChunk === 'undefined') { + throw new Error('Your browser does not support EncodedAudioChunk.'); + } + + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + + return new EncodedAudioSample( + data, + chunk.type as SampleType, // Typing is weird + chunk.timestamp / 1e6, + (chunk.duration ?? 0) / 1e6, + ); + } }