Ensure extensions work with huge timestamps, fix AC3 and AAC extension's behavior with huge timestamps

dev/convert.html

@@ -6,7 +6,7 @@
 
 <script type="module">
 	//MediabunnyMp3Encoder.registerMp3Encoder();
-	MediabunnyAc3.registerAc3Encoder();
+	MediabunnyAc3.registerAc3Decoder();
 
 	const fileInput = document.createElement('input');
 	fileInput.type = 'file';
@@ -59,7 +59,7 @@
 
 		if (true) {
 			input = new Mediabunny.Input({
-				source: new Mediabunny.UrlSource('https://stream.mux.com/v69RSHhFelSm4701snP22dYz2jICy4E4FUyk02rW4gxRM.m3u8'),
+				source: new Mediabunny.UrlSource('http://localhost:8000/index.m3u8'),
 				formats: Mediabunny.ALL_FORMATS,
 			});
 
@@ -87,6 +87,7 @@
 
 		const primaryTrack = await input.getPrimaryAudioTrack();
 		const startTime = await primaryTrack.getFirstTimestamp();
+		console.log(startTime)
 
 		let ctx = null;
 		let conversion = await Mediabunny.Conversion.init({

docs/guide/converting-media-files.md

@@ -500,3 +500,26 @@ const conversion = await Conversion.init({ input, output });
 conversion.utilizedTracks; // => InputTrack[]
 ```
 A track may appear multiple times in this list when [fan-out](#track-fan-out) produces multiple output tracks from it.
+
+## Converting live streams
+
+Live inputs, like HLS live streams, can also be used with the Conversion API. In this case, by default, the conversion will run until the live stream has ended.
+
+If you only want to convert a part of the live stream instead of waiting until it has ended, you can [trim](#trimming) the conversion. For example, here we're capturing the next 60 seconds of the live stream:
+```ts
+// Get the live edge
+const currentDuration = await input.getDurationFromMetadata(undefined, {
+	skipLiveWait: true,
+});
+
+const conversion = await Conversion.init({
+	input,
+	output,
+	trim: {
+		// Start at the current live edge
+		start: currentDuration,
+		// End at most 60 seconds later
+		end: currentDuration + 60,
+	},
+});
+```

packages/aac-encoder/src/bridge.c

@@ -7,6 +7,7 @@
  */
 
 #include <emscripten.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include "libavcodec/avcodec.h"
@@ -20,7 +21,7 @@ typedef struct {
 	AVFrame *frame;
 	float *input_buffer;
 	int input_buffer_size;
-	int encoded_pts;
+	int64_t encoded_pts;
 	int encoded_duration;
 } EncoderContext;
 
@@ -123,7 +124,7 @@ float *get_encode_input_ptr(EncoderContext *ctx, int size) {
 }
 
 EMSCRIPTEN_KEEPALIVE
-int send_frame(EncoderContext *ctx, int pts) {
+int send_frame(EncoderContext *ctx, int64_t pts) {
 	int channels = ctx->codec_ctx->ch_layout.nb_channels;
 	int frame_size = ctx->frame->nb_samples;
 
@@ -171,7 +172,7 @@ uint8_t *get_encoded_data(EncoderContext *ctx) {
 }
 
 EMSCRIPTEN_KEEPALIVE
-int get_encoded_pts(EncoderContext *ctx) {
+int64_t get_encoded_pts(EncoderContext *ctx) {
 	return ctx->encoded_pts;
 }
 

packages/aac-encoder/src/encode.worker.ts

@@ -21,12 +21,12 @@ let getEncoderFrameSize: (ctx: number) => number;
 let getEncoderExtradata: (ctx: number) => number;
 let getEncoderExtradataSize: (ctx: number) => number;
 let getEncodeInputPtr: (ctx: number, size: number) => number;
-let sendFrameFn: (ctx: number, pts: number) => number;
+let sendFrameFn: (ctx: number, pts: bigint) => number;
 let receivePacketFn: (ctx: number) => number;
 let flushEncoderStartFn: (ctx: number) => void;
 let resetEncoderFn: (ctx: number) => void;
 let getEncodedData: (ctx: number) => number;
-let getEncodedPts: (ctx: number) => number;
+let getEncodedPts: (ctx: number) => bigint;
 let getEncodedDuration: (ctx: number) => number;
 const ensureModule = async () => {
 	if (!module) {
@@ -43,12 +43,12 @@ const ensureModule = async () => {
 		getEncoderExtradata = module.cwrap('get_encoder_extradata', 'number', ['number']);
 		getEncoderExtradataSize = module.cwrap('get_encoder_extradata_size', 'number', ['number']);
 		getEncodeInputPtr = module.cwrap('get_encode_input_ptr', 'number', ['number', 'number']);
-		sendFrameFn = module.cwrap('send_frame', 'number', ['number', 'number']);
+		sendFrameFn = module.cwrap('send_frame', 'number', ['number', 'number']) as unknown as typeof sendFrameFn;
 		receivePacketFn = module.cwrap('receive_packet', 'number', ['number']);
 		flushEncoderStartFn = module.cwrap('flush_encoder_start', null, ['number']);
 		resetEncoderFn = module.cwrap('reset_encoder', null, ['number']);
 		getEncodedData = module.cwrap('get_encoded_data', 'number', ['number']);
-		getEncodedPts = module.cwrap('get_encoded_pts', 'number', ['number']);
+		getEncodedPts = module.cwrap('get_encoded_pts', 'number', ['number']) as unknown as typeof getEncodedPts;
 		getEncodedDuration = module.cwrap('get_encoded_duration', 'number', ['number']);
 	}
 };
@@ -81,7 +81,7 @@ const drainPackets = (ctx: number) => {
 	while ((size = receivePacketFn(ctx)) > 0) {
 		const ptr = getEncodedData(ctx);
 		const encodedData = module.HEAPU8.slice(ptr, ptr + size).buffer;
-		const pts = getEncodedPts(ctx);
+		const pts = Number(getEncodedPts(ctx));
 		const duration = getEncodedDuration(ctx);
 		packets.push({ encodedData, pts, duration });
 	}
@@ -98,7 +98,7 @@ const encode = (ctx: number, audioData: ArrayBuffer, timestamp: number) => {
 	}
 	module.HEAPU8.set(audioBytes, inputPtr);
 
-	const ret = sendFrameFn(ctx, timestamp);
+	const ret = sendFrameFn(ctx, BigInt(timestamp));
 	if (ret < 0) {
 		throw new Error(`Encode failed with error code ${ret}.`);
 	}

packages/ac3/src/bridge.c

@@ -7,6 +7,7 @@
  */
 
 #include <emscripten.h>
+#include <stdint.h>
 #include <stdlib.h>
 #include <string.h>
 #include "libavcodec/avcodec.h"
@@ -72,7 +73,7 @@ uint8_t *configure_decode_packet(DecoderContext *ctx, int size) {
 }
 
 EMSCRIPTEN_KEEPALIVE
-int decode_packet(DecoderContext *ctx, int pts) {
+int decode_packet(DecoderContext *ctx, int64_t pts) {
 	ctx->packet->pts = pts;
 	int ret = avcodec_send_packet(ctx->codec_ctx, ctx->packet);
 	av_packet_unref(ctx->packet);
@@ -110,8 +111,8 @@ int get_decoded_sample_count(DecoderContext *ctx) {
 }
 
 EMSCRIPTEN_KEEPALIVE
-int get_decoded_pts(DecoderContext *ctx) {
-	return (int)ctx->frame->pts;
+int64_t get_decoded_pts(DecoderContext *ctx) {
+	return ctx->frame->pts;
 }
 
 EMSCRIPTEN_KEEPALIVE
@@ -135,7 +136,7 @@ typedef struct {
 	AVFrame *frame;
 	float *input_buffer;
 	int input_buffer_size;
-	int encoded_pts;
+	int64_t encoded_pts;
 	int encoded_duration;
 } EncoderContext;
 
@@ -229,7 +230,7 @@ float *get_encode_input_ptr(EncoderContext *ctx, int size) {
 }
 
 EMSCRIPTEN_KEEPALIVE
-int encode_frame(EncoderContext *ctx, int pts) {
+int encode_frame(EncoderContext *ctx, int64_t pts) {
 	int channels = ctx->codec_ctx->ch_layout.nb_channels;
 	int frame_size = ctx->frame->nb_samples;
 
@@ -270,7 +271,7 @@ uint8_t *get_encoded_data(EncoderContext *ctx) {
 }
 
 EMSCRIPTEN_KEEPALIVE
-int get_encoded_pts(EncoderContext *ctx) {
+int64_t get_encoded_pts(EncoderContext *ctx) {
 	return ctx->encoded_pts;
 }
 

packages/ac3/src/codec.worker.ts

@@ -18,23 +18,23 @@ let modulePromise: Promise<ExtendedEmscriptenModule> | null = null;
 
 let initDecoderFn: (codecId: number) => number;
 let configureDecodePacket: (ctx: number, size: number) => number;
-let decodePacket: (ctx: number, pts: number) => number;
+let decodePacket: (ctx: number, pts: bigint) => number;
 let getDecodedFormat: (ctx: number) => number;
 let getDecodedPlanePtr: (ctx: number, plane: number) => number;
 let getDecodedChannels: (ctx: number) => number;
 let getDecodedSampleRate: (ctx: number) => number;
 let getDecodedSampleCount: (ctx: number) => number;
-let getDecodedPts: (ctx: number) => number;
+let getDecodedPts: (ctx: number) => bigint;
 let flushDecoderFn: (ctx: number) => void;
 let closeDecoderFn: (ctx: number) => void;
 
 let initEncoderFn: (codecId: number, channels: number, sampleRate: number, bitrate: number) => number;
 let getEncoderFrameSize: (ctx: number) => number;
 let getEncodeInputPtr: (ctx: number, size: number) => number;
-let encodeFrameFn: (ctx: number, pts: number) => number;
+let encodeFrameFn: (ctx: number, pts: bigint) => number;
 let flushEncoderFn: (ctx: number) => void;
 let getEncodedData: (ctx: number) => number;
-let getEncodedPts: (ctx: number) => number;
+let getEncodedPts: (ctx: number) => bigint;
 let getEncodedDuration: (ctx: number) => number;
 let closeEncoderFn: (ctx: number) => void;
 
@@ -53,23 +53,23 @@ const ensureModule = async () => {
 
 		initDecoderFn = module.cwrap('init_decoder', 'number', ['number']);
 		configureDecodePacket = module.cwrap('configure_decode_packet', 'number', ['number', 'number']);
-		decodePacket = module.cwrap('decode_packet', 'number', ['number', 'number']);
+		decodePacket = module.cwrap('decode_packet', 'number', ['number', 'number']) as unknown as typeof decodePacket;
 		getDecodedFormat = module.cwrap('get_decoded_format', 'number', ['number']);
 		getDecodedPlanePtr = module.cwrap('get_decoded_plane_ptr', 'number', ['number', 'number']);
 		getDecodedChannels = module.cwrap('get_decoded_channels', 'number', ['number']);
 		getDecodedSampleRate = module.cwrap('get_decoded_sample_rate', 'number', ['number']);
 		getDecodedSampleCount = module.cwrap('get_decoded_sample_count', 'number', ['number']);
-		getDecodedPts = module.cwrap('get_decoded_pts', 'number', ['number']);
+		getDecodedPts = module.cwrap('get_decoded_pts', 'number', ['number']) as unknown as typeof getDecodedPts;
 		flushDecoderFn = module.cwrap('flush_decoder', null, ['number']);
 		closeDecoderFn = module.cwrap('close_decoder', null, ['number']);
 
 		initEncoderFn = module.cwrap('init_encoder', 'number', ['number', 'number', 'number', 'number']);
 		getEncoderFrameSize = module.cwrap('get_encoder_frame_size', 'number', ['number']);
 		getEncodeInputPtr = module.cwrap('get_encode_input_ptr', 'number', ['number', 'number']);
-		encodeFrameFn = module.cwrap('encode_frame', 'number', ['number', 'number']);
+		encodeFrameFn = module.cwrap('encode_frame', 'number', ['number', 'number']) as unknown as typeof encodeFrameFn;
 		flushEncoderFn = module.cwrap('flush_encoder', null, ['number']);
 		getEncodedData = module.cwrap('get_encoded_data', 'number', ['number']);
-		getEncodedPts = module.cwrap('get_encoded_pts', 'number', ['number']);
+		getEncodedPts = module.cwrap('get_encoded_pts', 'number', ['number']) as unknown as typeof getEncodedPts;
 		getEncodedDuration = module.cwrap('get_encoded_duration', 'number', ['number']);
 		closeEncoderFn = module.cwrap('close_encoder', null, ['number']);
 	}
@@ -108,7 +108,7 @@ const decode = (ctx: number, encodedData: ArrayBuffer, timestamp: number) => {
 
 	module.HEAPU8.set(bytes, dataPtr);
 
-	const ret = decodePacket(ctx, timestamp);
+	const ret = decodePacket(ctx, BigInt(timestamp));
 	if (ret < 0) {
 		throw new Error(`Decode failed with error code ${ret}.`);
 	}
@@ -122,7 +122,7 @@ const decode = (ctx: number, encodedData: ArrayBuffer, timestamp: number) => {
 	const channels = getDecodedChannels(ctx);
 	const sampleRate = getDecodedSampleRate(ctx);
 	const sampleCount = getDecodedSampleCount(ctx);
-	const pts = getDecodedPts(ctx);
+	const pts = Number(getDecodedPts(ctx));
 
 	let pcmData: ArrayBuffer;
 	if (info.planar) {
@@ -169,14 +169,14 @@ const encode = (ctx: number, audioData: ArrayBuffer, timestamp: number) => {
 	}
 	module.HEAPU8.set(audioBytes, inputPtr);
 
-	const bytesWritten = encodeFrameFn(ctx, timestamp);
+	const bytesWritten = encodeFrameFn(ctx, BigInt(timestamp));
 	if (bytesWritten < 0) {
 		throw new Error(`Encode failed with error code ${bytesWritten}.`);
 	}
 
 	const ptr = getEncodedData(ctx);
 	const encodedData = module.HEAPU8.slice(ptr, ptr + bytesWritten).buffer;
-	const pts = getEncodedPts(ctx);
+	const pts = Number(getEncodedPts(ctx));
 	const duration = getEncodedDuration(ctx);
 
 	return { encodedData, pts, duration };

test/node/aac-encoder-extension.test.ts

@@ -10,6 +10,7 @@ import { EncodedPacketSink } from '../../src/media-sink.js';
 import { Mp4OutputFormat } from '../../src/output-format.js';
 import { AudioSample } from '../../src/sample.js';
 import { registerAacEncoder } from '@mediabunny/aac-encoder';
+import { assert } from '../../src/misc.js';
 
 const createSineWave = (sampleRate: number, channels: number, durationSeconds: number) => {
 	const totalFrames = sampleRate * durationSeconds;
@@ -80,3 +81,44 @@ test('AAC encoding', async () => {
 
 	expect(await track.computeDuration()).toBeCloseTo(2, 1);
 });
+
+test('AAC with huge timestamps', async () => {
+	registerAacEncoder();
+
+	const sampleRate = 48000;
+	const channels = 2;
+	const timestamp = 1e9;
+	const durationSeconds = 2;
+	const data = createSineWave(sampleRate, channels, durationSeconds);
+
+	const output = new Output({
+		format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
+		target: new BufferTarget(),
+	});
+
+	const audioSource = new AudioSampleSource({ codec: 'aac', bitrate: 128000 });
+	output.addAudioTrack(audioSource);
+
+	await output.start();
+	await audioSource.add(new AudioSample({
+		data,
+		format: 'f32',
+		numberOfChannels: channels,
+		sampleRate,
+		timestamp,
+	}));
+	audioSource.close();
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = (await input.getPrimaryAudioTrack())!;
+	const sink = new EncodedPacketSink(track);
+	const firstPacket = await sink.getFirstPacket();
+	assert(firstPacket);
+
+	expect(firstPacket.timestamp).toBe(timestamp);
+});

test/node/ac3.test.ts

@@ -14,6 +14,7 @@ import { AudioSampleSource } from '../../src/media-source.js';
 import { Mp4OutputFormat } from '../../src/output-format.js';
 import { AudioSample } from '../../src/sample.js';
 import { registerAc3Decoder, registerAc3Encoder } from '@mediabunny/ac3';
+import { assert } from '../../src/misc.js';
 
 const __dirname = new URL('.', import.meta.url).pathname;
 
@@ -329,3 +330,101 @@ test('E-AC-3 encoding', async () => {
 
 	expect(await track.computeDuration()).toBeCloseTo(2, 1);
 });
+
+test('AC-3 with huge timestamps', async () => {
+	registerAc3Decoder();
+	registerAc3Encoder();
+
+	const sampleRate = 48000;
+	const channels = 2;
+	const timestamp = 1e9;
+	const durationSeconds = 2;
+	const data = createSineWave(sampleRate, channels, durationSeconds);
+
+	const output = new Output({
+		format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
+		target: new BufferTarget(),
+	});
+
+	const audioSource = new AudioSampleSource({ codec: 'ac3', bitrate: 192000 });
+	output.addAudioTrack(audioSource);
+
+	await output.start();
+	await audioSource.add(new AudioSample({
+		data,
+		format: 'f32',
+		numberOfChannels: channels,
+		sampleRate,
+		timestamp,
+	}));
+	audioSource.close();
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = (await input.getPrimaryAudioTrack())!;
+	const packetSink = new EncodedPacketSink(track);
+	const firstPacket = await packetSink.getFirstPacket();
+	assert(firstPacket);
+
+	expect(firstPacket.timestamp).toBe(timestamp);
+
+	const sampleSink = new AudioSampleSink(track);
+	const iterator = sampleSink.samples(timestamp);
+	const firstSample = (await iterator.next()).value;
+	assert(firstSample);
+
+	expect(firstSample.timestamp).toBe(timestamp);
+});
+
+test('E-AC-3 with huge timestamps', async () => {
+	registerAc3Decoder();
+	registerAc3Encoder();
+
+	const sampleRate = 48000;
+	const channels = 2;
+	const timestamp = 1e9;
+	const durationSeconds = 2;
+	const data = createSineWave(sampleRate, channels, durationSeconds);
+
+	const output = new Output({
+		format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
+		target: new BufferTarget(),
+	});
+
+	const audioSource = new AudioSampleSource({ codec: 'eac3', bitrate: 192000 });
+	output.addAudioTrack(audioSource);
+
+	await output.start();
+	await audioSource.add(new AudioSample({
+		data,
+		format: 'f32',
+		numberOfChannels: channels,
+		sampleRate,
+		timestamp,
+	}));
+	audioSource.close();
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = (await input.getPrimaryAudioTrack())!;
+	const packetSink = new EncodedPacketSink(track);
+	const firstPacket = await packetSink.getFirstPacket();
+	assert(firstPacket);
+
+	expect(firstPacket.timestamp).toBe(timestamp);
+
+	const sampleSink = new AudioSampleSink(track);
+	const iterator = sampleSink.samples(timestamp);
+	const firstSample = (await iterator.next()).value;
+	assert(firstSample);
+
+	expect(firstSample.timestamp).toBe(timestamp);
+});

test/node/flac-encoder-extension.test.ts

@@ -7,9 +7,10 @@ import { BufferTarget } from '../../src/target.js';
 import { canEncode } from '../../src/encode.js';
 import { AudioSampleSource } from '../../src/media-source.js';
 import { EncodedPacketSink } from '../../src/media-sink.js';
-import { FlacOutputFormat } from '../../src/output-format.js';
+import { FlacOutputFormat, Mp4OutputFormat } from '../../src/output-format.js';
 import { AudioSample } from '../../src/sample.js';
 import { registerFlacEncoder } from '@mediabunny/flac-encoder';
+import { assert } from '../../src/misc.js';
 
 const createSineWave = (sampleRate: number, channels: number, durationSeconds: number) => {
 	const totalFrames = sampleRate * durationSeconds;
@@ -81,3 +82,44 @@ test('FLAC encoding', async () => {
 
 	expect(await track.computeDuration()).toBeCloseTo(2, 1);
 });
+
+test('FLAC with huge timestamps', async () => {
+	registerFlacEncoder();
+
+	const sampleRate = 48000;
+	const channels = 2;
+	const timestamp = 1e9;
+	const durationSeconds = 2;
+	const data = createSineWave(sampleRate, channels, durationSeconds);
+
+	const output = new Output({
+		format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
+		target: new BufferTarget(),
+	});
+
+	const audioSource = new AudioSampleSource({ codec: 'flac', bitrate: 1 });
+	output.addAudioTrack(audioSource);
+
+	await output.start();
+	await audioSource.add(new AudioSample({
+		data,
+		format: 'f32',
+		numberOfChannels: channels,
+		sampleRate,
+		timestamp,
+	}));
+	audioSource.close();
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = (await input.getPrimaryAudioTrack())!;
+	const sink = new EncodedPacketSink(track);
+	const firstPacket = await sink.getFirstPacket();
+	assert(firstPacket);
+
+	expect(firstPacket.timestamp).toBe(timestamp);
+});

test/node/mp3-encoder-extension.test.ts

@@ -7,9 +7,10 @@ import { BufferTarget } from '../../src/target.js';
 import { canEncode } from '../../src/encode.js';
 import { AudioSampleSource } from '../../src/media-source.js';
 import { EncodedPacketSink } from '../../src/media-sink.js';
-import { Mp3OutputFormat } from '../../src/output-format.js';
+import { Mp3OutputFormat, Mp4OutputFormat } from '../../src/output-format.js';
 import { AudioSample } from '../../src/sample.js';
 import { registerMp3Encoder } from '@mediabunny/mp3-encoder';
+import { assert } from '../../src/misc.js';
 
 const createSineWave = (sampleRate: number, channels: number, durationSeconds: number) => {
 	const totalFrames = sampleRate * durationSeconds;
@@ -81,3 +82,44 @@ test('MP3 encoding', async () => {
 
 	expect(await track.computeDuration()).toBeCloseTo(2, 0);
 });
+
+test('MP3 with huge timestamps', async () => {
+	registerMp3Encoder();
+
+	const sampleRate = 44100;
+	const channels = 2;
+	const timestamp = 1e9;
+	const durationSeconds = 2;
+	const data = createSineWave(sampleRate, channels, durationSeconds);
+
+	const output = new Output({
+		format: new Mp4OutputFormat({ fastStart: 'fragmented' }),
+		target: new BufferTarget(),
+	});
+
+	const audioSource = new AudioSampleSource({ codec: 'mp3', bitrate: 128_000 });
+	output.addAudioTrack(audioSource);
+
+	await output.start();
+	await audioSource.add(new AudioSample({
+		data,
+		format: 'f32',
+		numberOfChannels: channels,
+		sampleRate,
+		timestamp,
+	}));
+	audioSource.close();
+	await output.finalize();
+
+	using input = new Input({
+		source: new BufferSource(output.target.buffer!),
+		formats: ALL_FORMATS,
+	});
+
+	const track = (await input.getPrimaryAudioTrack())!;
+	const sink = new EncodedPacketSink(track);
+	const firstPacket = await sink.getFirstPacket();
+	assert(firstPacket);
+
+	expect(firstPacket.timestamp).toBe(timestamp);
+});