import { expect, test } from 'vitest'; import { Output } from '../../src/output.js'; import { Mp4OutputFormat, WebMOutputFormat } from '../../src/output-format.js'; import { BufferTarget } from '../../src/target.js'; import { AudioSampleSource, VideoSampleSource } from '../../src/media-source.js'; import { AudioSample, VideoSample } from '../../src/sample.js'; import { QUALITY_MEDIUM } from '../../src/encode.js'; import { Input } from '../../src/input.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { BufferSource } from '../../src/source.js'; import { VideoSampleSink } from '../../src/media-sink.js'; import { assert, Rotation } from '../../src/misc.js'; import { InputAudioTrack, InputVideoTrack } from '../../src/input-track.js'; test('VideoSampleSource, normal usage', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM }, [{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(3); for (const sample of samples) { expect(sample.codedWidth).toBe(100); expect(sample.codedHeight).toBe(100); } }); test('VideoSampleSource, .close() should be idempotent after finalize()', async () => { const output = new Output({ format: new WebMOutputFormat(), target: new BufferTarget(), }); const videoSource = new VideoSampleSource({ codec: 'vp8', bitrate: QUALITY_MEDIUM, }); output.addVideoTrack(videoSource); await output.start(); const canvas = new OffscreenCanvas(100, 100); const ctx = canvas.getContext('2d')!; ctx.fillStyle = 'red'; ctx.fillRect(0, 0, 100, 100); const sample = new VideoSample(canvas, { timestamp: 0, duration: 1 / 30 }); await videoSource.add(sample); sample.close(); await output.finalize(); videoSource.close(); // This previously threw }); test('VideoSampleSource, changing input dimensions throws with deny (default)', async () => { const output = new Output({ format: new Mp4OutputFormat(), target: new BufferTarget(), }); const videoSource = new VideoSampleSource({ codec: 'vp8', bitrate: QUALITY_MEDIUM, }); output.addVideoTrack(videoSource); await output.start(); const sample1 = new VideoSample(makeCanvas(100, 100), { timestamp: 0, duration: 1 / 30 }); await videoSource.add(sample1); sample1.close(); const sample2 = new VideoSample(makeCanvas(200, 150), { timestamp: 1 / 30, duration: 1 / 30 }); await expect(videoSource.add(sample2)).rejects.toThrow(/Video sample size must remain constant/); sample2.close(); videoSource.close(); }); test('VideoSampleSource, changing input dimensions with passThrough preserves per-frame dimensions', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough' }, [{ width: 100, height: 100 }, { width: 200, height: 150 }], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(2); expect(samples[0]).toMatchObject({ codedWidth: 100, codedHeight: 100 }); expect( // Either are valid; WebCodecs encoders don't like changing dimensions sometimes (samples[1]!.codedWidth === 100 && samples[1]!.codedHeight === 100) || (samples[1]!.codedWidth === 200 && samples[1]!.codedHeight === 150), ).toBe(true); }); test( 'VideoSampleSource, changing input dimensions with fill/contain/cover locks output to first frame dimensions', async () => { for (const behavior of ['fill', 'contain', 'cover'] as const) { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: behavior }, [{ width: 100, height: 100 }, { width: 200, height: 150 }], ); const { input, track } = await readBackTrack(buffer); expect(await track.getCodedWidth()).toBe(100); expect(await track.getCodedHeight()).toBe(100); input.dispose(); } }, ); test('VideoSampleSource, same-sized frames with width and height set', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { width: 50, height: 80, fit: 'fill' } }, [{ width: 100, height: 100 }, { width: 100, height: 100 }], ); const { input, track } = await readBackTrack(buffer); expect(await track.getCodedWidth()).toBe(50); expect(await track.getCodedHeight()).toBe(80); input.dispose(); }); test('VideoSampleSource, same-sized frames with rotation set to 90', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90 } }, [{ width: 200, height: 100 }, { width: 200, height: 100 }], ); const { input, track } = await readBackTrack(buffer); expect(await track.getCodedWidth()).toBe(100); expect(await track.getCodedHeight()).toBe(200); input.dispose(); }); test('VideoSampleSource, same-sized frames with rotation, width and height', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { rotate: 90, width: 50, height: 80, fit: 'contain' } }, [{ width: 200, height: 100 }, { width: 200, height: 100 }], ); const { input, track } = await readBackTrack(buffer); expect(await track.getCodedWidth()).toBe(50); expect(await track.getCodedHeight()).toBe(80); input.dispose(); }); test('VideoSampleSource, changing dimensions with passThrough and rotation 90', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough', transform: { rotate: 90 } }, [{ width: 200, height: 100 }, { width: 300, height: 150 }], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(2); expect(samples[0]).toMatchObject({ codedWidth: 100, codedHeight: 200 }); expect( // Either are valid; WebCodecs encoders don't like changing dimensions sometimes (samples[1]!.codedWidth === 150 && samples[1]!.codedHeight === 300) || (samples[1]!.codedWidth === 100 && samples[1]!.codedHeight === 200), ).toBe(true); }); test('VideoSampleSource, changing dimensions with passThrough, width and height set', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, sizeChangeBehavior: 'passThrough', transform: { width: 50, height: 80, fit: 'fill', }, }, [{ width: 100, height: 100 }, { width: 200, height: 150 }], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(2); // Both frames should be resized to the fixed width/height expect(samples[0]).toMatchObject({ codedWidth: 50, codedHeight: 80 }); expect(samples[1]).toMatchObject({ codedWidth: 50, codedHeight: 80 }); }); test('VideoSampleSource, encoding rotated video frames', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM }, [{ width: 200, height: 100, rotation: 90 }, { width: 200, height: 100, rotation: 90 }], ); const samples = await readBackSamples(buffer); // They were encoded with no regard for the rotation metadata for (const sample of samples) { expect(sample.codedWidth).toBe(200); expect(sample.codedHeight).toBe(100); } }); test('VideoSampleSource, encoding rotated video frames with forced transform', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { force: true } }, [{ width: 200, height: 100, rotation: 90 }, { width: 200, height: 100, rotation: 90 }], ); const samples = await readBackSamples(buffer); // The rotation has been baked into the samples for (const sample of samples) { expect(sample.codedWidth).toBe(100); expect(sample.codedHeight).toBe(200); } }); test('VideoSampleSource, transform.process identity function', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { process: sample => sample } }, [{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(3); for (const sample of samples) { expect(sample.codedWidth).toBe(100); expect(sample.codedHeight).toBe(100); } }); test('VideoSampleSource, transform.process manual resize', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { process: (sample) => { const canvas = new OffscreenCanvas(60, 40); const ctx = canvas.getContext('2d')!; sample.draw(ctx, 0, 0, 60, 40); sample.close(); return canvas; }, }, }, [{ width: 100, height: 100 }, { width: 100, height: 100 }], ); const { input, track } = await readBackTrack(buffer); expect(await track.getCodedWidth()).toBe(60); expect(await track.getCodedHeight()).toBe(40); input.dispose(); }); test('VideoSampleSource, transform.process receives pre-transformed frames', async () => { const receivedDimensions: { width: number; height: number }[] = []; const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { width: 50, height: 80, fit: 'fill', process: (sample) => { receivedDimensions.push({ width: sample.codedWidth, height: sample.codedHeight, }); return sample; }, }, }, [{ width: 200, height: 200 }, { width: 200, height: 200 }], ); expect(receivedDimensions).toHaveLength(2); for (const dim of receivedDimensions) { expect(dim.width).toBe(50); expect(dim.height).toBe(80); } const { input, track } = await readBackTrack(buffer); expect(await track.getCodedWidth()).toBe(50); expect(await track.getCodedHeight()).toBe(80); input.dispose(); }); test('VideoSampleSource, transform.process drops all frames after the first', async () => { let frameIndex = 0; const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { process: (sample) => { if (frameIndex++ > 0) { return null; } return sample; }, }, }, [{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(1); }); test('VideoSampleSource, transform.process expands every frame into two', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { process: (sample) => { const t = sample.timestamp; const d = sample.duration; const clone = sample.clone(); clone.setTimestamp(2 * t); clone.setDuration(d); const clone2 = sample.clone(); clone2.setTimestamp(2 * t + d); clone2.setDuration(d); return [clone, clone2]; }, }, }, [{ width: 100, height: 100 }, { width: 100, height: 100 }, { width: 100, height: 100 }], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(6); const d = 1 / 30; for (let i = 0; i < 6; i++) { expect(samples[i]!.timestamp).toBe(i * d); expect(samples[i]!.duration).toBe(d); } }); test('VideoSampleSource, transform.frameRate normalizes variable-rate input to fixed rate', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ { width: 100, height: 100, timestamp: 0, duration: 0.15 }, { width: 100, height: 100, timestamp: 0.15, duration: 0.1 }, { width: 100, height: 100, timestamp: 0.25, duration: 0.05 }, ], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(3); for (let i = 0; i < 3; i++) { expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); expect(samples[i]!.duration).toBeCloseTo(0.1); } }); test('VideoSampleSource, transform.frameRate pads gaps by repeating last frame', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ { width: 100, height: 100, timestamp: 0, duration: 0.1 }, { width: 100, height: 100, timestamp: 0.3, duration: 0.1 }, ], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(4); for (let i = 0; i < 4; i++) { expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); expect(samples[i]!.duration).toBeCloseTo(0.1); } }); test('VideoSampleSource, transform.frameRate deduplicates frames in the same slot', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ { width: 100, height: 100, timestamp: 0, duration: 0.03 }, { width: 100, height: 100, timestamp: 0.03, duration: 0.03 }, { width: 100, height: 100, timestamp: 0.06, duration: 0.04 }, { width: 100, height: 100, timestamp: 0.1, duration: 0.1 }, ], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(2); expect(samples[0]!.timestamp).toBe(0); expect(samples[1]!.timestamp).toBeCloseTo(0.1); }); test('VideoSampleSource, transform.frameRate final padding fills remaining duration', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ { width: 100, height: 100, timestamp: 0, duration: 0.5 }, ], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(5); for (let i = 0; i < 5; i++) { expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); } }); test('VideoSampleSource, transform.frameRate skipping and padding combined', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10 } }, [ { width: 100, height: 100, timestamp: 0, duration: 0.02 }, { width: 100, height: 100, timestamp: 0.02, duration: 0.02 }, { width: 100, height: 100, timestamp: 0.04, duration: 0.02 }, { width: 100, height: 100, timestamp: 0.3, duration: 0.05 }, { width: 100, height: 100, timestamp: 0.35, duration: 0.05 }, ], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(4); for (let i = 0; i < 4; i++) { expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); expect(samples[i]!.duration).toBeCloseTo(0.1); } }); test('VideoSampleSource, transform.frameRate works with transform', async () => { const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { width: 50, height: 50, fit: 'fill', frameRate: 10 }, }, [ { width: 100, height: 100, timestamp: 0, duration: 0.3 }, ], ); const samples = await readBackSamples(buffer); expect(samples).toHaveLength(3); for (let i = 0; i < 3; i++) { expect(samples[i]!.timestamp).toBeCloseTo(i * 0.1); expect(samples[i]!.duration).toBe(0.1); expect(samples[i]!.codedWidth).toBe(50); expect(samples[i]!.codedHeight).toBe(50); } }); test('VideoSampleSource, transform.frameRate works with process', async () => { const processedTimestamps: number[] = []; const buffer = await encodeFrames( { codec: 'vp8', bitrate: QUALITY_MEDIUM, transform: { frameRate: 10, process: (sample) => { processedTimestamps.push(sample.timestamp); const canvas = new OffscreenCanvas(60, 40); const ctx = canvas.getContext('2d')!; sample.draw(ctx, 0, 0, 60, 40); return canvas; }, }, }, [ { width: 100, height: 100, timestamp: 0, duration: 0.3 }, ], ); expect(processedTimestamps).toHaveLength(3); for (let i = 0; i < 3; i++) { expect(processedTimestamps[i]).toBeCloseTo(i * 0.1); } const { input, track } = await readBackTrack(buffer); expect(await track.getCodedWidth()).toBe(60); expect(await track.getCodedHeight()).toBe(40); input.dispose(); }); const makeCanvas = (width: number, height: number) => { const canvas = new OffscreenCanvas(width, height); const ctx = canvas.getContext('2d')!; ctx.fillStyle = 'red'; ctx.fillRect(0, 0, width, height); return canvas; }; const encodeFrames = async ( encodingConfig: ConstructorParameters[0], frames: { width: number; height: number; timestamp?: number; duration?: number; rotation?: Rotation; }[], ) => { const output = new Output({ format: new Mp4OutputFormat(), target: new BufferTarget(), }); const videoSource = new VideoSampleSource(encodingConfig); output.addVideoTrack(videoSource); await output.start(); for (let i = 0; i < frames.length; i++) { const f = frames[i]!; const canvas = makeCanvas(f.width, f.height); const sample = new VideoSample(canvas, { timestamp: f.timestamp ?? i / 30, duration: f.duration ?? 1 / 30, rotation: f.rotation, }); await videoSource.add(sample); sample.close(); } await output.finalize(); return output.target.buffer!; }; const readBackTrack = async (buffer: ArrayBuffer) => { const input = new Input({ source: new BufferSource(buffer), formats: ALL_FORMATS, }); const track = await input.getPrimaryVideoTrack() as InputVideoTrack; assert(track); return { input, track }; }; const readBackSamples = async (buffer: ArrayBuffer) => { const { input, track } = await readBackTrack(buffer); const sink = new VideoSampleSink(track); const samples: { codedWidth: number; codedHeight: number; timestamp: number; duration: number }[] = []; for await (using sample of sink.samples()) { samples.push({ codedWidth: sample.codedWidth, codedHeight: sample.codedHeight, timestamp: sample.timestamp, duration: sample.duration, }); } input.dispose(); return samples; }; test('AudioSampleSource, normal usage', async () => { const sample = makeSineWave(48000, 2, 1); const buffer = await encodeAudio({ codec: 'pcm-s16' }, sample); const { input, track } = await readBackAudioTrack(buffer); expect(await track.getNumberOfChannels()).toBe(2); expect(await track.getSampleRate()).toBe(48000); expect(await track.computeDuration()).toBe(1); input.dispose(); }); test('AudioSampleSource, remixed to mono', async () => { const sample = makeSineWave(48000, 2, 1); const buffer = await encodeAudio( { codec: 'pcm-s16', transform: { numberOfChannels: 1 } }, sample, ); const { input, track } = await readBackAudioTrack(buffer); expect(await track.getNumberOfChannels()).toBe(1); expect(await track.getSampleRate()).toBe(48000); expect(await track.computeDuration()).toBe(1); input.dispose(); }); test('AudioSampleSource, resampled to 44100 Hz', async () => { const sample = makeSineWave(48000, 2, 1); const buffer = await encodeAudio( { codec: 'pcm-s16', transform: { sampleRate: 44100 } }, sample, ); const { input, track } = await readBackAudioTrack(buffer); expect(await track.getNumberOfChannels()).toBe(2); expect(await track.getSampleRate()).toBe(44100); expect(await track.computeDuration()).toBe(1); input.dispose(); }); test('AudioSampleSource, resampled stereo with non-zero start timestamp', async () => { const sample = makeSineWave(48000, 2, 1, 1); const buffer = await encodeAudio( { codec: 'pcm-s16', transform: { numberOfChannels: 2 } }, sample, ); const { input, track } = await readBackAudioTrack(buffer); expect(await track.getNumberOfChannels()).toBe(2); expect(await track.getSampleRate()).toBe(48000); expect(await track.getFirstTimestamp()).toBe(1); expect(await track.computeDuration()).toBe(2); input.dispose(); }); const makeSineWave = ( sampleRate: number, numberOfChannels: number, durationSeconds: number, timestamp = 0, ) => { const numberOfFrames = Math.round(sampleRate * durationSeconds); const data = new Float32Array(numberOfFrames * numberOfChannels); for (let frame = 0; frame < numberOfFrames; frame++) { const value = Math.sin(2 * Math.PI * 440 * frame / sampleRate); for (let ch = 0; ch < numberOfChannels; ch++) { data[frame * numberOfChannels + ch] = value; } } return new AudioSample({ data, format: 'f32-planar', sampleRate, numberOfChannels, numberOfFrames, timestamp, }); }; const encodeAudio = async ( encodingConfig: ConstructorParameters[0], sample: AudioSample, ) => { const output = new Output({ format: new Mp4OutputFormat({ fastStart: 'fragmented' }), // Fragmented to avoid the PCM transformation target: new BufferTarget(), }); const audioSource = new AudioSampleSource(encodingConfig); output.addAudioTrack(audioSource); await output.start(); await audioSource.add(sample); sample.close(); await output.finalize(); return output.target.buffer!; }; const readBackAudioTrack = async (buffer: ArrayBuffer) => { const input = new Input({ source: new BufferSource(buffer), formats: ALL_FORMATS, }); const track = await input.getPrimaryAudioTrack() as InputAudioTrack; assert(track); return { input, track }; };