import { afterEach, beforeEach, expect, test } from 'vitest'; import { Input } from '../../src/input.js'; import { BufferSource, UrlSource } from '../../src/source.js'; import { ALL_FORMATS } from '../../src/input-format.js'; import { AudioSampleCursor, canvasTransformer, VideoSampleCursor, WrappedCanvas, } from '../../src/cursors.js'; import { AudioSample, VideoSample } from '../../src/sample.js'; import { promiseAllEnsureOrder, promiseIterateAll } from '../../src/misc.js'; beforeEach(() => { VideoSample._openSampleCount = 0; AudioSample._openSampleCount = 0; }); afterEach(() => { expect(VideoSample._openSampleCount).toBe(0); expect(AudioSample._openSampleCount).toBe(0); }); test('Sample cursor seeking', { timeout: 30_000 }, async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; await using cursor = new VideoSampleCursor(videoTrack); cursor._debug.enabled = true; expect(cursor.current).toBe(null); expect(cursor._debug.pumpsStarted).toBe(0); // Seek to start const seekToResult1 = cursor.seekToFirst(); expect(seekToResult1).instanceOf(Promise); const sample1 = (await seekToResult1)!; expect(sample1).not.toBe(null); expect(sample1).toBe(cursor.current); expect(sample1.timestamp).toBeLessThanOrEqual(0); expect(sample1.closed).toBe(false); expect(cursor._debug.pumpsStarted).toBe(1); // Seek to a frame in the current GOP const seekToResult2 = cursor.seekTo(0.5); expect(seekToResult2).instanceOf(Promise); const sample2 = (await seekToResult2)!; expect(sample2).not.toBe(null); expect(sample2).toBe(cursor.current); expect(sample2.timestamp).toBeGreaterThan(0); expect(sample2.timestamp).toBeLessThanOrEqual(0.5); expect(sample2.closed).toBe(false); expect(sample1.closed).toBe(true); // Wait a little bit so the next sample is most definitely decoded and waiting in the queue await new Promise(resolve => setTimeout(resolve, 200)); const seekToResult3 = cursor.seekTo(0.55); expect(seekToResult3).instanceOf(VideoSample); // No promise this time const sample3 = (await seekToResult3)!; expect(sample3).toBe(cursor.current); expect(sample3.timestamp).toBeGreaterThan(0.5); expect(sample3.timestamp).toBeLessThanOrEqual(0.55); expect(sample3.closed).toBe(false); expect(sample2.closed).toBe(true); // Let's get the same sample again const seekToResult4 = cursor.seekTo(0.55); expect(seekToResult4).instanceOf(VideoSample); const sample4 = (await seekToResult4)!; expect(sample3).toBe(sample4); expect(cursor._debug.pumpsStarted).toBe(1); // Seek to a different GOP const seekToResult5 = cursor.seekTo(2); expect(seekToResult5).instanceOf(Promise); const sample5 = (await seekToResult5)!; expect(sample5).toBe(cursor.current); expect(sample5.timestamp).toBeGreaterThan(0.55); expect(sample5.timestamp).toBeLessThanOrEqual(2); expect(sample5.closed).toBe(false); expect(sample3.closed).toBe(true); expect(cursor._debug.pumpsStarted).toBe(2); // Seek to a frame in the current GOP const seekToResult6 = cursor.seekTo(2.5); expect(seekToResult6).instanceOf(Promise); const sample6 = (await seekToResult6)!; expect(sample6).toBe(cursor.current); expect(sample6.timestamp).toBeGreaterThan(2); expect(sample6.timestamp).toBeLessThanOrEqual(2.5); expect(sample6.closed).toBe(false); expect(sample5.closed).toBe(true); // Seek to a frame in the current GOP, but backwards, requiring decoding to start over const seekToResult7 = cursor.seekTo(2.4); expect(seekToResult7).instanceOf(Promise); const sample7 = (await seekToResult7)!; expect(sample7).toBe(cursor.current); expect(sample7.timestamp).toBeGreaterThan(2); expect(sample7.timestamp).toBeLessThanOrEqual(2.4); expect(sample7.closed).toBe(false); expect(sample6.closed).toBe(true); expect(cursor._debug.pumpsStarted).toBe(3); // Seek to a previous GOP const seekToResult8 = cursor.seekTo(1); expect(seekToResult8).instanceOf(Promise); const sample8 = (await seekToResult8)!; expect(sample8).toBe(cursor.current); expect(sample8.timestamp).toBeLessThanOrEqual(1); expect(sample8.closed).toBe(false); expect(sample7.closed).toBe(true); expect(cursor._debug.pumpsStarted).toBe(4); // Seek to past the end const seekToResult9 = cursor.seekTo(Infinity); expect(seekToResult9).instanceOf(Promise); const sample9 = (await seekToResult9)!; expect(sample9).toBe(cursor.current); expect(sample9.timestamp).toBe(5); // The length of the video expect(sample9.closed).toBe(false); expect(sample8.closed).toBe(true); expect(cursor._debug.pumpsStarted).toBe(5); await new Promise(resolve => setTimeout(resolve, 200)); expect(cursor.current).toBe(sample9); // Seek to the end again const seekToResult10 = cursor.seekTo(Infinity); expect(seekToResult10).not.toBeInstanceOf(Promise); const sample10 = await seekToResult10; expect(sample9).toBe(sample10); await new Promise(resolve => setTimeout(resolve, 200)); expect(cursor._debug.pumpsStarted).toBe(5); // Seek to before the start const seekToResult11 = cursor.seekTo(-Infinity); expect(seekToResult11).toBe(null); expect(sample9.closed).toBe(true); expect(cursor.current).toBe(null); const seekToResult12 = cursor.seekToKey(2.5); expect(seekToResult12).toBeInstanceOf(Promise); const sample12 = (await seekToResult12)!; expect(sample12).toBe(cursor.current); expect(sample12.timestamp).toBe(2); expect(sample12.closed).toBe(false); await cursor.close(); expect(sample12.closed).toBe(true); expect(cursor.current).toBe(null); expect(cursor._debug.pumpsStarted).toBe(6); await cursor.close(); }); test('Sample cursor advancing', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; const cursor = new VideoSampleCursor(videoTrack); cursor._debug.enabled = true; expect(cursor.current).toBe(null); expect(cursor._debug.pumpsStarted).toBe(0); const firstSample = (await cursor.seekToFirst())!; const secondSample = (await cursor.next())!; const thirdSample = (await cursor.next())!; expect(secondSample.timestamp).toBeGreaterThan(firstSample.timestamp); expect(thirdSample.timestamp).toBeGreaterThan(secondSample.timestamp); expect(firstSample.closed).toBe(true); expect(secondSample.closed).toBe(true); expect(thirdSample.closed).toBe(false); expect(cursor.current).toBe(thirdSample); await new Promise(resolve => setTimeout(resolve, 200)); const fourthSampleResult = cursor.next(); // It's available instantly expect(fourthSampleResult).not.toBeInstanceOf(Promise); const fourthSample = fourthSampleResult as VideoSample; expect(fourthSample.timestamp).toBeGreaterThan(thirdSample.timestamp); const lastSample = await cursor.seekTo(5); expect(lastSample).not.toBe(null); const nextSample = await cursor.next(); expect(nextSample).toBe(null); const nextNextSample = await cursor.next(); expect(nextNextSample).toBe(null); const middleSample = (await cursor.seekTo(3))!; const sampleAfterMiddle = (await cursor.next())!; expect(sampleAfterMiddle.timestamp).toBeGreaterThan(middleSample.timestamp); await cursor.seekTo(-Infinity); const firstSampleAgain = (await cursor.next())!; expect(firstSampleAgain.timestamp).toBe(firstSample.timestamp); let total = 0; let lastTimestamp = -Infinity; await cursor.iterate((sample) => { total++; expect(sample.timestamp).toBeGreaterThan(lastTimestamp); lastTimestamp = sample.timestamp; }); expect(total).toBe(121); expect(cursor.current).toBe(null); total = 0; await cursor.iterate(() => void total++); expect(total).toBe(0); // Since we're at the end await cursor.seekToFirst(); ; total = 0; await cursor.iterate((sample) => { if (sample.timestamp === 1) { return false; } total++; }); expect(total).toBe(24); expect(cursor.current!.timestamp).toBe(1); total = 0; for await (const sample of cursor) { if (total === 0) { expect(sample.timestamp).toBe(1); } total++; } expect(total).toBe(97); await cursor.close(); expect(cursor._debug.pumpsStarted).toBe(5); }); test('Sample cursor advancing, cold start', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; const cursor = new VideoSampleCursor(videoTrack); cursor._debug.enabled = true; let firstSample = (await cursor.next())!; expect(firstSample).not.toBe(null); expect(firstSample.timestamp).toBe(0); const secondSample = (await cursor.next())!; expect(secondSample.timestamp).toBeGreaterThan(firstSample.timestamp); await cursor.seekTo(-Infinity); firstSample = (await cursor.next())!; expect(firstSample).not.toBe(null); expect(firstSample.timestamp).toBe(0); await cursor.seekTo(-Infinity); firstSample = (await cursor.nextKey())!; expect(firstSample).not.toBe(null); expect(firstSample.timestamp).toBe(0); void cursor.next(); void cursor.seekTo(2); await cursor.close(); // Ensure the calls were serialized correctly expect(cursor._debug.seekPackets.map(x => x?.timestamp ?? null)).toEqual([0, null, 0, null, 0, 2]); const cursor2 = new VideoSampleCursor(videoTrack); for await (const sample of cursor2) { expect(sample.timestamp).toBe(0); break; } await cursor2.iterate((sample) => { expect(sample.timestamp).toBe(0); return false; }); await cursor2.close(); }); test('Sample cursor advancing, nextKey', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; await using cursor = new VideoSampleCursor(videoTrack); await cursor.seekToFirst(); let sample = await cursor.nextKey(); expect(sample!.timestamp).toBe(1); sample = await cursor.nextKey(); expect(sample!.timestamp).toBe(2); void cursor.nextKey(); void cursor.nextKey(); await cursor.nextKey(); expect(cursor.current!.timestamp).toBe(5); sample = await cursor.nextKey(); expect(sample).toBe(null); await cursor.seekTo(0.5); sample = await cursor.nextKey(); expect(sample!.timestamp).toBe(1); void cursor.seekTo(3.5); sample = await cursor.nextKey(); expect(sample!.timestamp).toBe(4); await cursor.seekTo(3.9); void cursor.next(); void cursor.next(); void cursor.next(); sample = await cursor.nextKey(); expect(sample!.timestamp).toBe(5); }); test('Sample cursor sample reuse', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; const cursor1 = new VideoSampleCursor(videoTrack); cursor1._debug.enabled = true; const sample1 = await cursor1.seekToFirst(); const sample2 = await cursor1.seekToFirst(); expect(cursor1._debug.pumpsStarted).toBe(1); expect(sample1!.timestamp).toBe(sample2!.timestamp); expect(sample1).toBe(sample2); await cursor1.next(); expect(sample1!.closed).toBe(true); expect(sample2!.closed).toBe(true); const cursor2 = new VideoSampleCursor(videoTrack, { closeSamples: false, }); const sample3 = await cursor2.seekToFirst(); const sample4 = await cursor2.seekToFirst(); expect(sample3!.timestamp).toBe(sample4!.timestamp); expect(sample3).not.toBe(sample4); sample3!.close(); expect(sample3!.closed).toBe(true); expect(sample4!.closed).toBe(false); sample4!.close(); let count = 0; const cursor3 = new VideoSampleCursor(videoTrack, { transform: () => count++, }); await cursor3.seekToFirst(); await cursor3.seekToFirst(); expect(count).toBe(1); count = 0; const cursor4 = new VideoSampleCursor(videoTrack, { closeSamples: false, transform: sample => (sample.close(), count++), }); await cursor4.seekToFirst(); await cursor4.seekToFirst(); expect(count).toBe(2); await cursor1.close(); await cursor2.close(); await cursor3.close(); await cursor4.close(); }); test('Sample cursor reset', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; await using cursor = new VideoSampleCursor(videoTrack); expect(cursor.closed).toBe(false); await cursor.close(); expect(cursor.closed).toBe(true); expect(() => cursor.seekToFirst()).toThrow('cursor has been closed'); await cursor.reset(); expect(cursor.closed).toBe(false); const firstSample = await cursor.seekToFirst(); expect(firstSample).not.toBe(null); await cursor.close(); await cursor.reset(); const nextSample = await cursor.next(); expect(nextSample).not.toBe(null); expect(nextSample!.timestamp).toBe(firstSample!.timestamp); // Absolutely deranged usage, but it's gotta work! const commands = [ cursor.next(), cursor.close(), cursor.reset(), cursor.next(), cursor.next(), cursor.reset(), cursor.next(), ]; const results = await promiseAllEnsureOrder(commands); expect(cursor.closed).toBe(false); expect(results[0]!.timestamp).toBeGreaterThan(firstSample!.timestamp); expect(results[3]!.timestamp).toBe(firstSample!.timestamp); expect(results[4]!.timestamp).toBe(results[0]!.timestamp); expect(results[6]!.timestamp).toBe(firstSample!.timestamp); await using cursor2 = new VideoSampleCursor(videoTrack); cursor2._debug.enabled = true; // Test if queueing a reset makes the decoder decode minimally many packets const commands2 = [ cursor2.seekToFirst(), cursor2.reset(), ]; await promiseAllEnsureOrder(commands2); expect(cursor2._debug.decodedPackets).toHaveLength(1); }); test('Decoder setup error & reset', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; await using cursor1 = new VideoSampleCursor(videoTrack); cursor1._debug.enabled = true; cursor1._debug.throwInDecoderInit = true; expect(cursor1.closed).toBe(false); await expect(cursor1.seekToFirst()).rejects.toThrow('Fake decoder init error'); expect(cursor1.closed).toBe(true); expect(cursor1.errored).toBe(true); expect(() => cursor1.seekToFirst()).toThrow('Fake decoder init error'); // Bricked await expect(cursor1.reset()).rejects.toThrow('Fake decoder init error'); expect(cursor1.closed).toBe(true); cursor1._debug.throwInDecoderInit = false; await cursor1.reset(); expect(cursor1.closed).toBe(false); expect(cursor1.errored).toBe(false); const firstSample = (await cursor1.seekToFirst())!; expect(firstSample).not.toBe(null); // Let's test directly closing after opening const cursor2 = new VideoSampleCursor(videoTrack); cursor2._debug.enabled = true; cursor2._debug.throwInDecoderInit = true; await cursor2.close(); }); test('Decoder pump error handling & reset', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; await using cursor = new VideoSampleCursor(videoTrack); cursor._debug.enabled = true; cursor._debug.throwInPump = true; await expect(async () => cursor.seekToFirst()).rejects.toThrow('Fake pump error'); expect(cursor.closed).toBe(true); expect(cursor.errored).toBe(true); expect(cursor.current).toBe(null); cursor._debug.throwInPump = false; await expect(async () => cursor.seekToFirst()).rejects.toThrow('Fake pump error'); // It's bricked await cursor.reset(); const firstSample = await cursor.seekToFirst(); expect(firstSample!.timestamp).toBe(0); }); test('Decoder errors & reset', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; const cursor1 = new VideoSampleCursor(videoTrack); cursor1._debug.enabled = true; cursor1._debug.throwDecoderError = true; await expect(cursor1.seekToFirst()).rejects.toThrow('Fake decoder error'); expect(cursor1.closed).toBe(true); expect(cursor1.errored).toBe(true); cursor1._debug.throwDecoderError = false; expect(() => cursor1.seekToFirst()).toThrow('Fake decoder error'); // Bricked await cursor1.reset(); const firstSample = await cursor1.seekToFirst(); expect(firstSample!.timestamp).toBe(0); const cursor2 = new VideoSampleCursor(videoTrack); cursor2._debug.enabled = true; await cursor2.seekToFirst(); cursor2._debug.throwDecoderError = true; await new Promise(resolve => setTimeout(resolve, 200)); expect(() => cursor2.next()).toThrow('Fake decoder error'); expect(cursor2.closed).toBe(true); }); test('Use after close', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; const cursor1 = new VideoSampleCursor(videoTrack); expect(cursor1.closed).toBe(false); await cursor1.close(); expect(cursor1.closed).toBe(true); await expect(async () => await cursor1.seekToFirst()).rejects.toThrow('cursor has been closed'); await expect(async () => await cursor1.seekTo(0)).rejects.toThrow('cursor has been closed'); await expect(async () => await cursor1.seekToKey(0)).rejects.toThrow('cursor has been closed'); await expect(async () => await cursor1.next()).rejects.toThrow('cursor has been closed'); const cursor2 = new VideoSampleCursor(videoTrack); const commands2 = [ cursor2.seekToFirst(), cursor2.next(), cursor2.close(), cursor2.seekTo(1), ]; await expect(commands2[0]).resolves.toBeInstanceOf(VideoSample); await expect(commands2[1]).resolves.toBeInstanceOf(VideoSample); await expect(commands2[2]).resolves.toBeUndefined(); await expect(commands2[3]).rejects.toThrow('cursor has been closed'); expect(cursor2._pumpRunning).toBe(false); expect(cursor2._decoder).toBe(null); const cursor3 = new VideoSampleCursor(videoTrack); const commands3 = [ cursor3.seekToFirst(), cursor3.next(), cursor3.close(), cursor3.next(), ]; await expect(commands3[0]).resolves.toBeInstanceOf(VideoSample); await expect(commands3[1]).resolves.toBeInstanceOf(VideoSample); await expect(commands3[2]).resolves.toBeUndefined(); await expect(commands3[3]).rejects.toThrow('cursor has been closed'); }); test('Wait until idle', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov', { maxCacheSize: 0, // HORRENDOUS for performance, but forces promises }), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; const cursor = new VideoSampleCursor(videoTrack); const promise1 = cursor.waitUntilIdle(); expect(promise1).toBeInstanceOf(Promise); // The decoder is setting up expect(cursor.isIdle()).toBe(false); await promise1; expect(cursor.waitUntilIdle()).toBe(null); expect(cursor.isIdle()).toBe(true); const promise2 = cursor.seekToFirst(); const promise3 = cursor.waitUntilIdle(); expect(cursor.isIdle()).toBe(false); expect(promise2).toBeInstanceOf(Promise); expect(promise3).toBeInstanceOf(Promise); expect(cursor.current).toBe(null); await promise3; expect(cursor.current!.timestamp).toBe(0); expect(cursor.isIdle()).toBe(true); expect(cursor.waitUntilIdle()).toBe(null); const promise4 = cursor.reset(); const promise5 = cursor.waitUntilIdle(); expect(promise4).toBeInstanceOf(Promise); expect(promise5).toBeInstanceOf(Promise); await promise5; expect(cursor.current).toBe(null); void cursor.seekToFirst(); const promise6 = cursor.close(); const promise7 = cursor.waitUntilIdle(); expect(promise6).toBeInstanceOf(Promise); expect(promise7).toBeInstanceOf(Promise); expect(cursor.closed).toBe(false); await promise7; expect(cursor.closed).toBe(true); expect(cursor.isIdle()).toBe(true); }); test('Command queuing', async () => { using input = new Input({ // Fetch the data into RAM to avoid packet lookups causing flaky timing source: new BufferSource(await fetch('/trim-buck-bunny.mov').then(x => x.arrayBuffer())), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; const cursor0 = new VideoSampleCursor(videoTrack); cursor0._debug.enabled = true; const commands0 = [ cursor0.seekToFirst(), cursor0.seekToFirst(), cursor0.close(), ]; expect(commands0.every(x => x instanceof Promise)).toBe(true); await promiseAllEnsureOrder(commands0); expect(cursor0._debug.pumpsStarted).toBe(1); const cursor1 = new VideoSampleCursor(videoTrack); cursor1._debug.enabled = true; const commands1 = [ cursor1.seekToFirst(), cursor1.close(), ]; expect(commands1.every(x => x instanceof Promise)).toBe(true); const results1 = await promiseAllEnsureOrder(commands1); expect(results1[0]!.timestamp).toBe(0); expect(cursor1._debug.decodedPackets.map(x => x.timestamp)).toEqual([0]); const cursor2 = new VideoSampleCursor(videoTrack); cursor2._debug.enabled = true; const commands2 = [ cursor2.seekTo(0), cursor2.seekTo(1), cursor2.seekTo(2), cursor2.waitUntilIdle()!.then(() => cursor2.current), cursor2.seekTo(3), cursor2.seekTo(4), cursor2.seekTo(5), cursor2.close(), ]; expect(commands2.every(x => x instanceof Promise)).toBe(true); const results2 = await promiseAllEnsureOrder(commands2); expect(results2[0]!.timestamp).toBe(0); expect(results2[1]!.timestamp).toBe(1); expect(results2[2]!.timestamp).toBe(2); expect(results2[3]!.timestamp).toBe(2); expect(results2[4]!.timestamp).toBe(3); expect(results2[5]!.timestamp).toBe(4); expect(results2[6]!.timestamp).toBe(5); expect(cursor2._debug.decodedPackets.map(x => x.timestamp)).toEqual([ 0, 1, 2, 3, 4, 5, ]); expect(cursor2._debug.pumpsStarted).toBe(6); const cursor3 = new VideoSampleCursor(videoTrack); cursor3._debug.enabled = true; const commands3 = [ cursor3.seekTo(0.5), cursor3.seekTo(0.4), cursor3.waitUntilIdle()!.then(() => cursor3.current), cursor3.seekTo(0.3), cursor3.seekTo(0.2), cursor3.seekTo(0.1), cursor3.seekTo(0), cursor3.close(), ]; expect(commands3.every(x => x instanceof Promise)).toBe(true); const results3 = await promiseAllEnsureOrder(commands3); expect(results3[0]!.timestamp).toBeLessThanOrEqual(0.5); expect(results3[1]!.timestamp).toBeLessThanOrEqual(0.4); expect(results3[2]!.timestamp).toBeLessThanOrEqual(0.4); expect(results3[3]!.timestamp).toBeLessThanOrEqual(0.3); expect(results3[4]!.timestamp).toBeLessThanOrEqual(0.2); expect(results3[5]!.timestamp).toBeLessThanOrEqual(0.1); expect(results3[6]!.timestamp).toBe(0); expect(cursor3._debug.decodedPackets.every(x => x.timestamp <= 0.5)).toBe(true); expect(cursor3._debug.pumpsStarted).toBe(6); const cursor4 = new VideoSampleCursor(videoTrack); cursor4._debug.enabled = true; const commands4 = [ cursor4.seekToFirst(), cursor4.next(), cursor4.next(), cursor4.nextKey(), cursor4.close(), ]; expect(commands4.every(x => x instanceof Promise)).toBe(true); const results4 = await promiseAllEnsureOrder(commands4); expect(results4[0]!.timestamp).toBe(0); expect(results4[1]!.timestamp).toBeGreaterThan(results4[0]!.timestamp); expect(results4[2]!.timestamp).toBeGreaterThan(results4[1]!.timestamp); expect(results4[3]!.timestamp).toBe(1); expect(cursor4._debug.decodedPackets.length).toBeGreaterThan(3); // Because .next() goes into "sequential mode" const cursor5 = new VideoSampleCursor(videoTrack); cursor5._debug.enabled = true; const commands5 = [ cursor5.seekTo(0), cursor5.next(), cursor5.next(), cursor5.seekTo(0.4), cursor5.next(), cursor5.next(), cursor5.seekTo(0.8), cursor5.next(), cursor5.next(), cursor5.seekTo(0), cursor5.next(), cursor5.close(), ]; const results5 = await promiseAllEnsureOrder(commands5); expect(results5[0]!.timestamp).toBe(0); expect(results5[1]!.timestamp).toBeGreaterThan(results5[0]!.timestamp); expect(results5[2]!.timestamp).toBeGreaterThan(results5[1]!.timestamp); expect(results5[3]!.timestamp).toBeLessThanOrEqual(0.4); expect(results5[3]!.timestamp).toBeGreaterThan(results5[2]!.timestamp); expect(results5[4]!.timestamp).toBeGreaterThan(results5[3]!.timestamp); expect(results5[5]!.timestamp).toBeGreaterThan(results5[4]!.timestamp); expect(results5[6]!.timestamp).toBeLessThanOrEqual(0.8); expect(results5[6]!.timestamp).toBeGreaterThan(results5[5]!.timestamp); expect(results5[7]!.timestamp).toBeGreaterThan(results5[6]!.timestamp); expect(results5[8]!.timestamp).toBeGreaterThan(results5[7]!.timestamp); expect(results5[9]!.timestamp).toBe(0); expect(results5[10]!.timestamp).toBeGreaterThan(results5[9]!.timestamp); expect(cursor5._debug.pumpsStarted).toBe(2); const cursor6 = new VideoSampleCursor(videoTrack); cursor6._debug.enabled = true; const commands6 = [ cursor6.seekTo(0), cursor6.seekTo(0.4), cursor6.seekTo(0.8), cursor6.seekTo(3.4), cursor6.waitUntilIdle()!.then(() => cursor6.current), cursor6.seekTo(3.8), cursor6.seekTo(5), cursor6.close(), ]; const results6 = await promiseAllEnsureOrder(commands6); expect(results6[0]!.timestamp).toBe(0); expect(results6[1]!.timestamp).toBeLessThanOrEqual(0.4); expect(results6[2]!.timestamp).toBeLessThanOrEqual(0.8); expect(results6[3]!.timestamp).toBeLessThanOrEqual(3.4); expect(results6[4]!.timestamp).toBeLessThanOrEqual(3.4); expect(results6[5]!.timestamp).toBeLessThanOrEqual(3.8); expect(results6[6]!.timestamp).toBe(5); expect(cursor6._debug.pumpsStarted).toBe(3); const cursor7 = new VideoSampleCursor(videoTrack); const commands7 = [ cursor7.close(), cursor7.close(), ]; expect(commands7[0]).toBe(commands7[1]); // Same Promise await promiseAllEnsureOrder(commands7); const cursor8 = new VideoSampleCursor(videoTrack, { closeSamples: false }); const firstSample = await cursor8.seekToFirst(); firstSample!.close(); await new Promise(resolve => setTimeout(resolve, 200)); const commands8 = [ cursor8.next(), cursor8.next(), cursor8.next(), ]; expect(commands8.every(x => !(x instanceof Promise))).toBe(true); for await (using sample of promiseIterateAll(commands8)) { expect(sample!.closed).toBe(false); } await cursor8.close(); const cursor9 = new VideoSampleCursor(videoTrack, { closeSamples: false }); const commands9 = [ cursor9.seekToFirst(), cursor9.seekToFirst(), cursor9.seekToFirst(), ]; expect(commands9.every(x => x instanceof Promise)).toBe(true); for await (using sample of promiseIterateAll(commands9)) { expect(sample!.timestamp).toBe(0); expect(sample!.closed).toBe(false); } await cursor9.close(); const cursor10 = new VideoSampleCursor(videoTrack); cursor10._debug.enabled = true; const commands10 = [ cursor10.seekToFirst(), cursor10.next(), cursor10.seekToFirst(), cursor10.next(), cursor10.close(), ]; const results10 = await promiseAllEnsureOrder(commands10); expect(results10[0]!.timestamp).toBe(0); expect(results10[1]!.timestamp).toBeGreaterThan(0); expect(results10[2]!.timestamp).toBe(0); expect(results10[3]!.timestamp).toBe(results10[1]!.timestamp); expect(cursor10._debug.pumpsStarted).toBe(2); }); test('Automatic cursor disposal', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; const cursor = new VideoSampleCursor(videoTrack); await cursor.seekToFirst(); // No cursor.close() here, but the disposed Input closes the cursor input.dispose(); expect(cursor.closed).toBe(true); // Make sure the close is actually complete using lock = cursor._mutex.lock(); if (lock.pending) await lock.ready; }); test('Video with stubborn first sample emit', async () => { using input = new Input({ // For some reason, this video is stubborn in the sense that it takes quite a lot of packets for the decoder to // emit its first samples. This used to cause issues in the past where the decoder got stuck, so good to have it // tested. source: new UrlSource('/sylvie-trimmed.mp4'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; await using cursor = new VideoSampleCursor(videoTrack); const firstSample = (await cursor.seekToFirst())!; expect(firstSample).not.toBe(null); expect(firstSample.timestamp).toBe(0); }); test('AudioSampleCursor', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const audioTrack = (await input.getPrimaryAudioTrack())!; const cursor = new AudioSampleCursor(audioTrack); cursor._debug.enabled = true; const firstSample = (await cursor.seekToFirst())!; expect(firstSample).not.toBe(null); expect(firstSample.timestamp).toBe(0); expect(firstSample).toBeInstanceOf(AudioSample); const secondSample = (await cursor.next())!; expect(secondSample.timestamp).toBeCloseTo(firstSample.timestamp + firstSample.duration); const thirdSample = (await cursor.seekTo(secondSample.timestamp + 0.05))!; expect(thirdSample.timestamp).toBeGreaterThan(secondSample.timestamp); expect(cursor._debug.pumpsStarted).toBe(1); await cursor.seekToFirst(); let lastTimestamp = -Infinity; let total = 0; for await (const sample of cursor) { if (total === 0) { expect(sample.timestamp).toBe(0); } expect(sample.timestamp).toBeGreaterThan(lastTimestamp); lastTimestamp = sample.timestamp; total++; } expect(total).toBe(235); expect(lastTimestamp).toBeCloseTo(5, 1); expect(cursor._debug.pumpsStarted).toBe(2); const middleSample = (await cursor.seekTo(2.5))!; expect(middleSample.timestamp).toBeLessThanOrEqual(2.5); expect(middleSample.timestamp).toBeGreaterThan(2.4); expect(cursor._debug.pumpsStarted).toBe(3); const commands = [ cursor.seekTo(0), cursor.seekTo(0.05), cursor.seekTo(0.1), cursor.seekTo(0.15), cursor.seekTo(0.2), ]; const result = await promiseAllEnsureOrder(commands); expect(result[0]!.timestamp).toBeLessThanOrEqual(0); expect(result[1]!.timestamp).toBeLessThanOrEqual(0.05); expect(result[2]!.timestamp).toBeLessThanOrEqual(0.1); expect(result[3]!.timestamp).toBeLessThanOrEqual(0.15); expect(result[4]!.timestamp).toBeLessThanOrEqual(0.2); // One pump was used for all the above commands, even tho they all seek to different key packets expect(cursor._debug.pumpsStarted).toBe(4); await cursor.seekToFirst(); const nextSample = (await cursor.nextKey())!; // nextKey acts like next for audio tracks const nextNextSample = (await cursor.nextKey())!; expect(nextSample.timestamp).toBeCloseTo(firstSample.timestamp + firstSample.duration); expect(nextNextSample.timestamp).toBeCloseTo(nextSample.timestamp + firstSample.duration); await cursor.seekToFirst(); const actualNextSample = (await cursor.next())!; expect(nextSample.timestamp).toBe(actualNextSample.timestamp); expect(cursor._debug.pumpsStarted).toBe(6); await cursor.close(); }); test('Sample mapping', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; let callCount = 0; await using cursor = new VideoSampleCursor(videoTrack, { transform: (sample) => { callCount++; return { original: sample, timestampMs: sample.timestamp * 1000, }; }, }); const firstSample = (await cursor.seekToFirst())!; expect(firstSample.original).toBeInstanceOf(VideoSample); expect(firstSample.timestampMs).toBe(0); expect(firstSample.original.closed).toBe(false); const secondSample = (await cursor.next())!; expect(secondSample.original).toBeInstanceOf(VideoSample); expect(secondSample.timestampMs).toBeCloseTo(firstSample.timestampMs + firstSample.original.duration * 1000); expect(firstSample.original.closed).toBe(true); expect(secondSample.original.closed).toBe(false); const thirdSample = (await cursor.seekTo(0.5))!; expect(thirdSample).not.toBe(null); expect(callCount).toBe(3); }); test('Canvas transformer', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; const cursor1 = new VideoSampleCursor(videoTrack, { transform: canvasTransformer(), }); const firstSample = (await cursor1.seekToFirst())!; expect(firstSample).toBeInstanceOf(WrappedCanvas); expect(firstSample.canvas).toBeInstanceOf(HTMLCanvasElement); expect(firstSample.canvas.width).toBe(await videoTrack.getDisplayWidth()); expect(firstSample.canvas.height).toBe(await videoTrack.getDisplayHeight()); expect(firstSample.timestamp).toBe(0); expect(firstSample.duration).toBeGreaterThan(0); const nextSample = (await cursor1.next())!; expect(nextSample.canvas).toBeInstanceOf(HTMLCanvasElement); expect(nextSample.timestamp).toBeGreaterThan(firstSample.timestamp); expect(nextSample.canvas).not.toBe(firstSample.canvas); await cursor1.close(); const cursor2 = new VideoSampleCursor(videoTrack, { transform: canvasTransformer({ width: 320, poolSize: 2, }), }); const sample1 = (await cursor2.seekToFirst())!; expect(sample1.canvas.width).toBe(320); expect(sample1.canvas.height).toBe(180); const sample2 = (await cursor2.next())!; expect(sample2.canvas.width).toBe(320); expect(sample2.canvas.height).toBe(180); expect(sample2.canvas).not.toBe(sample1.canvas); const sample3 = (await cursor2.next())!; const sample4 = (await cursor2.next())!; expect(sample3.canvas).toBe(sample1.canvas); expect(sample2.canvas).toBe(sample4.canvas); await cursor2.close(); }); test('Unthrottled decoder', async () => { // While the decoder is throttled in practice, these tests simulate what happens when the decoder naturally reaches // the end of the encodable packets without having been stopped; in this case, we still expect the cursor to keep // functioning normally, just that requests are now fully resolved through the sample queue only (when possible). using input = new Input({ source: new BufferSource(await fetch('/trim-buck-bunny.mov').then(x => x.arrayBuffer())), formats: ALL_FORMATS, }); const audioTrack = (await input.getPrimaryAudioTrack())!; const cursor1 = new AudioSampleCursor(audioTrack); cursor1._debug.enabled = true; cursor1._debug.unthrottledPump = true; const first = await cursor1.seekToFirst(); expect(first!.timestamp).toBe(0); await cursor1._debug.pumpEnded.wait(); expect(cursor1._pumpRunning).toBe(false); // All of these requests can be fully satisfied only by using the sample queue: const next = await cursor1.next(); expect(next).not.toBe(null); expect(next!.timestamp).toBeCloseTo(first!.timestamp + first!.duration); const nextNext = await cursor1.next(); expect(nextNext).not.toBe(null); expect(nextNext!.timestamp).toBeCloseTo(next!.timestamp + next!.duration); const seeked = await cursor1.seekTo(1); expect(seeked).not.toBe(null); expect(seeked!.timestamp).toBeGreaterThan(0.9); expect(seeked!.timestamp).toBeLessThanOrEqual(1); const nextKey = await cursor1.nextKey(); expect(nextKey).not.toBe(null); expect(nextKey!.timestamp).toBeCloseTo(seeked!.timestamp + seeked!.duration); const theLast = await cursor1.seekTo(Infinity); expect(theLast).not.toBe(null); const pastTheLast = await cursor1.next(); expect(pastTheLast).toBe(null); expect(cursor1._debug.pumpsStarted).toBe(1); const someRandomSample = await cursor1.seekTo(0.5); expect(someRandomSample).not.toBe(null); expect(someRandomSample!.timestamp).toBeGreaterThan(0.4); expect(someRandomSample!.timestamp).toBeLessThanOrEqual(0.5); expect(cursor1._debug.pumpsStarted).toBe(2); await cursor1.close(); const cursor2 = new AudioSampleCursor(audioTrack); cursor2._debug.enabled = true; cursor2._debug.unthrottledPump = true; await cursor2.seekToFirst(); await cursor2._debug.pumpEnded.wait(); expect(cursor2._pumpRunning).toBe(false); // Test that all samples in the queue get closed now await cursor2.close(); }); test('hasNext', async () => { using input = new Input({ source: new UrlSource('/trim-buck-bunny.mov'), formats: ALL_FORMATS, }); const videoTrack = (await input.getPrimaryVideoTrack())!; await using cursor = new VideoSampleCursor(videoTrack); expect(cursor.current).toBe(null); expect(await cursor.hasNext()).toBe(true); await cursor.seekToFirst(); expect(await cursor.hasNext()).toBe(true); await cursor.seekTo(Infinity); expect(cursor.current).not.toBe(null); expect(await cursor.hasNext()).toBe(false); await cursor.next(); expect(cursor.current).toBe(null); expect(await cursor.hasNext()).toBe(false); void cursor.seekTo(2.5); expect(await cursor.hasNext()).toBe(true); void cursor.seekTo(Infinity); expect(await cursor.hasNext()).toBe(false); expect(cursor.hasNext()).not.toBeInstanceOf(Promise); });