Add MPEG-TS muxer, fix a demuxer bug, remove Infinity from output format track counts

This commit is contained in:
Vanilagy
2026-01-16 16:33:28 +01:00
parent ae37973878
commit e8b27768a9
13 changed files with 1705 additions and 64 deletions
+681
View File
@@ -0,0 +1,681 @@
import { expect, test } from 'vitest';
import { Input } from '../../src/input.js';
import { BufferSource, UrlSource } from '../../src/source.js';
import { ALL_FORMATS, MPEG_TS } from '../../src/input-format.js';
import { Output } from '../../src/output.js';
import { MpegTsOutputFormat } from '../../src/output-format.js';
import { BufferTarget, StreamTarget, StreamTargetChunk } from '../../src/target.js';
import { AudioBufferSource, CanvasSource, EncodedAudioPacketSource } from '../../src/media-source.js';
import { QUALITY_HIGH } from '../../src/encode.js';
import { EncodedPacketSink } from '../../src/media-sink.js';
import { assert } from '../../src/misc.js';
import { Conversion } from '../../src/conversion.js';
test('MPEG-TS output format', async () => {
const format = new MpegTsOutputFormat();
expect(format.mimeType).toBe('video/MP2T');
expect(format.fileExtension).toBe('.ts');
expect(format.supportsVideoRotationMetadata).toBe(false);
expect(format.getSupportedCodecs()).toEqual(['avc', 'hevc', 'aac', 'mp3']);
expect(format.getSupportedTrackCounts()).toEqual({
video: { min: 0, max: 16 },
audio: { min: 0, max: 32 },
subtitle: { min: 0, max: 0 },
total: { min: 1, max: 48 },
});
});
test('MPEG-TS muxing with AVC and AAC', async () => {
let tsPacketCount = 0;
let started = false;
let finalized = false;
let mimeTypeResolved = false;
const output = new Output({
format: new MpegTsOutputFormat({
onPacket: (data) => {
expect(data[0]).toBe(0x47);
tsPacketCount++;
},
}),
target: new BufferTarget(),
});
// Test getMimeType - it resolves once all tracks are known (first packet arrives)
void output.getMimeType().then((mimeType) => {
expect(started).toBe(true);
expect(finalized).toBe(false);
expect(mimeType).toMatch(/^video\/MP2T; codecs="/);
expect(mimeType).toMatch(/avc1\./);
expect(mimeType).toMatch(/mp4a\.40\./);
mimeTypeResolved = true;
});
const canvas = new OffscreenCanvas(640, 480);
const context = canvas.getContext('2d')!;
context.fillStyle = '#000000';
context.fillRect(0, 0, canvas.width, canvas.height);
const videoSource = new CanvasSource(canvas, {
codec: 'avc',
bitrate: QUALITY_HIGH,
});
output.addVideoTrack(videoSource);
const audioSource = new AudioBufferSource({
codec: 'aac',
bitrate: QUALITY_HIGH,
});
output.addAudioTrack(audioSource);
await output.start();
started = true;
const fps = 30;
const duration = 5;
const frameCount = fps * duration;
const frameDuration = 1 / fps;
for (let i = 0; i < frameCount; i++) {
await videoSource.add(i * frameDuration, frameDuration);
}
const audioBuffer = new AudioBuffer({
length: 48000 * duration,
numberOfChannels: 2,
sampleRate: 48000,
});
await audioSource.add(audioBuffer);
await output.finalize();
finalized = true;
expect(mimeTypeResolved).toBe(true);
expect(tsPacketCount).toBeGreaterThan(100);
// Now let's read it back using the demuxer
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await input.getFormat()).toBe(MPEG_TS);
// Verify video track
const videoTrack = await input.getPrimaryVideoTrack();
assert(videoTrack);
expect(videoTrack.id).toBe(0x100);
expect(videoTrack.codec).toBe('avc');
expect(videoTrack.displayWidth).toBe(640);
expect(videoTrack.displayHeight).toBe(480);
const videoDecoderConfig = await videoTrack.getDecoderConfig();
assert(videoDecoderConfig);
expect(videoDecoderConfig.codec).toMatch(/^avc1\./);
expect(videoDecoderConfig.codedWidth).toBe(640);
expect(videoDecoderConfig.codedHeight).toBe(480);
expect(videoDecoderConfig.description).toBeUndefined(); // Annex B, no description
// Verify audio track
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
expect(audioTrack.id).toBe(0x101);
expect(audioTrack.codec).toBe('aac');
expect(audioTrack.numberOfChannels).toBe(2);
expect(audioTrack.sampleRate).toBe(48000);
const audioDecoderConfig = await audioTrack.getDecoderConfig();
assert(audioDecoderConfig);
expect(audioDecoderConfig.codec).toMatch(/^mp4a\.40\./);
expect(audioDecoderConfig.numberOfChannels).toBe(2);
expect(audioDecoderConfig.sampleRate).toBe(48000);
expect(audioDecoderConfig.description).toBeUndefined(); // ADTS, no description
// Verify video packets are Annex B
const videoSink = new EncodedPacketSink(videoTrack);
let videoPacketCount = 0;
const firstVideoPacket = await videoSink.getFirstPacket();
assert(firstVideoPacket);
expect(firstVideoPacket.type).toBe('key');
const secondVideoPacket = await videoSink.getNextPacket(firstVideoPacket);
assert(secondVideoPacket);
expect(secondVideoPacket.type).toBe('delta');
for await (const packet of videoSink.packets()) {
expect(packet.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1])); // Annex B start code
videoPacketCount++;
}
// Check that seeking works
const middlePacket = await videoSink.getPacket(2.5);
assert(middlePacket);
expect(middlePacket.timestamp).toBeCloseTo(2.5);
expect(videoPacketCount).toBe(frameCount);
// Verify audio packets are ADTS
const audioSink = new EncodedPacketSink(audioTrack);
let audioPacketCount = 0;
const firstAudioPacket = await audioSink.getFirstPacket();
assert(firstAudioPacket);
expect(firstAudioPacket.type).toBe('key');
const secondAudioPacket = await audioSink.getNextPacket(firstAudioPacket);
assert(secondAudioPacket);
expect(secondAudioPacket.type).toBe('key');
for await (const packet of audioSink.packets()) {
expect(packet.data[0]).toBe(0xff); // ADTS sync word
expect(packet.data[1]! & 0xf0).toBe(0xf0); // ADTS sync word continued
audioPacketCount++;
}
// Check that seeking works
const audioMiddlePacket = await audioSink.getPacket(2.5);
assert(audioMiddlePacket);
expect(audioMiddlePacket.timestamp).toBeCloseTo(2.5);
expect(audioPacketCount).toBeGreaterThan(0);
// Verify duration is approximately 5 seconds
const videoDuration = await videoTrack.computeDuration();
const audioDuration = await audioTrack.computeDuration();
expect(videoDuration).toBeCloseTo(5, 1);
expect(audioDuration).toBeCloseTo(5.077333333333334, 1);
});
test('MPEG-TS muxing with HEVC and MP3', async () => {
let tsPacketCount = 0;
const output = new Output({
format: new MpegTsOutputFormat({
onPacket: (data) => {
expect(data[0]).toBe(0x47);
tsPacketCount++;
},
}),
target: new BufferTarget(),
});
const canvas = new OffscreenCanvas(1280, 720);
const context = canvas.getContext('2d')!;
context.fillStyle = '#000000';
context.fillRect(0, 0, canvas.width, canvas.height);
const videoSource = new CanvasSource(canvas, {
codec: 'hevc',
bitrate: QUALITY_HIGH,
});
output.addVideoTrack(videoSource);
const audioSource = new EncodedAudioPacketSource('mp3');
output.addAudioTrack(audioSource);
await output.start();
const fps = 30;
const duration = 5;
const frameCount = fps * duration;
const frameDuration = 1 / fps;
for (let i = 0; i < frameCount; i++) {
await videoSource.add(i * frameDuration, frameDuration);
}
// Extract MP3 packets from existing file
using mp3Input = new Input({
source: new UrlSource('/Toothsome-Meme.VBRv2.mp3'),
formats: ALL_FORMATS,
});
const mp3Track = await mp3Input.getPrimaryAudioTrack();
assert(mp3Track);
const mp3Sink = new EncodedPacketSink(mp3Track);
let isFirst = true;
for await (const packet of mp3Sink.packets()) {
if (packet.timestamp >= duration) break;
await audioSource.add(packet, {
decoderConfig: isFirst
? (await mp3Track.getDecoderConfig())!
: undefined,
});
isFirst = false;
}
await output.finalize();
expect(tsPacketCount).toBeGreaterThan(100);
// Now let's read it back using the demuxer
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await input.getFormat()).toBe(MPEG_TS);
// Verify video track
const videoTrack = await input.getPrimaryVideoTrack();
assert(videoTrack);
expect(videoTrack.id).toBe(0x100);
expect(videoTrack.codec).toBe('hevc');
expect(videoTrack.displayWidth).toBe(1280);
expect(videoTrack.displayHeight).toBe(720);
const videoDecoderConfig = await videoTrack.getDecoderConfig();
assert(videoDecoderConfig);
expect(videoDecoderConfig.codec).toMatch(/^hev1\./);
expect(videoDecoderConfig.codedWidth).toBe(1280);
expect(videoDecoderConfig.codedHeight).toBe(720);
expect(videoDecoderConfig.description).toBeUndefined(); // Annex B, no description
// Verify audio track
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
expect(audioTrack.id).toBe(0x101);
expect(audioTrack.codec).toBe('mp3');
expect(audioTrack.numberOfChannels).toBe(2);
expect(audioTrack.sampleRate).toBe(24000);
const audioDecoderConfig = await audioTrack.getDecoderConfig();
assert(audioDecoderConfig);
expect(audioDecoderConfig.codec).toBe('mp3');
expect(audioDecoderConfig.numberOfChannels).toBe(2);
expect(audioDecoderConfig.sampleRate).toBe(24000);
expect(audioDecoderConfig.description).toBeUndefined(); // MP3 has no description
// Verify video packets are Annex B
const videoSink = new EncodedPacketSink(videoTrack);
let videoPacketCount = 0;
for await (const packet of videoSink.packets()) {
expect(packet.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1])); // Annex B start code
videoPacketCount++;
}
expect(videoPacketCount).toBe(frameCount);
// Verify audio packets are MP3 frames
const audioSink = new EncodedPacketSink(audioTrack);
let audioPacketCount = 0;
for await (const packet of audioSink.packets()) {
expect(packet.data[0]).toBe(0xff); // MP3 sync word
audioPacketCount++;
}
expect(audioPacketCount).toBeGreaterThan(0);
// Verify duration is approximately 5 seconds
const videoDuration = await videoTrack.computeDuration();
const audioDuration = await audioTrack.computeDuration();
expect(videoDuration).toBeCloseTo(5, 1);
expect(audioDuration).toBeCloseTo(5, 1);
});
test('MPEG-TS muxing with no data', async () => {
const output = new Output({
format: new MpegTsOutputFormat(),
target: new BufferTarget(),
});
const canvas = new OffscreenCanvas(640, 480);
const videoSource = new CanvasSource(canvas, {
codec: 'avc',
bitrate: QUALITY_HIGH,
});
output.addVideoTrack(videoSource);
await output.start();
await output.finalize();
// Read it back - should have zero tracks since no packets were written
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
const tracks = await input.getTracks();
expect(tracks.length).toBe(0);
});
test('MPEG-TS muxing with video only', async () => {
const output = new Output({
format: new MpegTsOutputFormat(),
target: new BufferTarget(),
});
const canvas = new OffscreenCanvas(640, 480);
const context = canvas.getContext('2d')!;
context.fillStyle = '#ff0000';
context.fillRect(0, 0, canvas.width, canvas.height);
const videoSource = new CanvasSource(canvas, {
codec: 'avc',
bitrate: QUALITY_HIGH,
});
output.addVideoTrack(videoSource);
await output.start();
const fps = 30;
const duration = 1;
const frameCount = fps * duration;
const frameDuration = 1 / fps;
for (let i = 0; i < frameCount; i++) {
await videoSource.add(i * frameDuration, frameDuration);
}
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await input.getFormat()).toBe(MPEG_TS);
const videoTrack = await input.getPrimaryVideoTrack();
assert(videoTrack);
expect(videoTrack.codec).toBe('avc');
const audioTrack = await input.getPrimaryAudioTrack();
expect(audioTrack).toBeNull();
const videoSink = new EncodedPacketSink(videoTrack);
let videoPacketCount = 0;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for await (const packet of videoSink.packets()) {
videoPacketCount++;
}
expect(videoPacketCount).toBe(frameCount);
});
test('MPEG-TS muxing with audio only', async () => {
const output = new Output({
format: new MpegTsOutputFormat(),
target: new BufferTarget(),
});
const audioSource = new AudioBufferSource({
codec: 'aac',
bitrate: QUALITY_HIGH,
});
output.addAudioTrack(audioSource);
await output.start();
const duration = 1;
const audioBuffer = new AudioBuffer({
length: 48000 * duration,
numberOfChannels: 2,
sampleRate: 48000,
});
await audioSource.add(audioBuffer);
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await input.getFormat()).toBe(MPEG_TS);
const videoTrack = await input.getPrimaryVideoTrack();
expect(videoTrack).toBeNull();
const audioTrack = await input.getPrimaryAudioTrack();
assert(audioTrack);
expect(audioTrack.codec).toBe('aac');
const audioSink = new EncodedPacketSink(audioTrack);
let audioPacketCount = 0;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for await (const packet of audioSink.packets()) {
audioPacketCount++;
}
expect(audioPacketCount).toBeGreaterThan(0);
});
test('MPEG-TS muxing with two video tracks', async () => {
const output = new Output({
format: new MpegTsOutputFormat(),
target: new BufferTarget(),
});
const canvas1 = new OffscreenCanvas(640, 480);
const ctx1 = canvas1.getContext('2d')!;
ctx1.fillStyle = '#ff0000';
ctx1.fillRect(0, 0, canvas1.width, canvas1.height);
const canvas2 = new OffscreenCanvas(320, 240);
const ctx2 = canvas2.getContext('2d')!;
ctx2.fillStyle = '#00ff00';
ctx2.fillRect(0, 0, canvas2.width, canvas2.height);
const videoSource1 = new CanvasSource(canvas1, { codec: 'hevc', bitrate: QUALITY_HIGH });
const videoSource2 = new CanvasSource(canvas2, { codec: 'hevc', bitrate: QUALITY_HIGH });
output.addVideoTrack(videoSource1);
output.addVideoTrack(videoSource2);
await output.start();
const fps = 30;
const duration = 1;
const frameCount = fps * duration;
const frameDuration = 1 / fps;
for (let i = 0; i < frameCount; i++) {
await videoSource1.add(i * frameDuration, frameDuration);
await videoSource2.add(i * frameDuration, frameDuration);
}
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await input.getFormat()).toBe(MPEG_TS);
const tracks = await input.getTracks();
const videoTracks = tracks.filter(t => t.type === 'video');
expect(videoTracks.length).toBe(2);
expect(videoTracks[0]!.codec).toBe('hevc');
expect(videoTracks[1]!.codec).toBe('hevc');
expect(videoTracks[0]!.id).toBe(0x100);
expect(videoTracks[1]!.id).toBe(0x101);
});
test('MPEG-TS muxing with two audio tracks', async () => {
const output = new Output({
format: new MpegTsOutputFormat(),
target: new BufferTarget(),
});
const audioSource1 = new AudioBufferSource({ codec: 'aac', bitrate: QUALITY_HIGH });
const audioSource2 = new AudioBufferSource({ codec: 'aac', bitrate: QUALITY_HIGH });
output.addAudioTrack(audioSource1);
output.addAudioTrack(audioSource2);
await output.start();
const duration = 1;
const audioBuffer1 = new AudioBuffer({
length: 48000 * duration,
numberOfChannels: 2,
sampleRate: 48000,
});
const audioBuffer2 = new AudioBuffer({
length: 44100 * duration,
numberOfChannels: 1,
sampleRate: 44100,
});
await audioSource1.add(audioBuffer1);
await audioSource2.add(audioBuffer2);
await output.finalize();
using input = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await input.getFormat()).toBe(MPEG_TS);
const tracks = await input.getTracks();
const audioTracks = tracks.filter(t => t.type === 'audio');
expect(audioTracks.length).toBe(2);
expect(audioTracks[0]!.codec).toBe('aac');
expect(audioTracks[1]!.codec).toBe('aac');
expect(audioTracks[0]!.id).toBe(0x100);
expect(audioTracks[1]!.id).toBe(0x101);
});
test('MPEG-TS transmux (Annex B and ADTS passthrough)', async () => {
using input = new Input({
source: new UrlSource('/0.ts'),
formats: ALL_FORMATS,
});
expect(await input.getFormat()).toBe(MPEG_TS);
const output = new Output({
format: new MpegTsOutputFormat(),
target: new BufferTarget(),
});
const conversion = await Conversion.init({ input, output });
expect(conversion.isValid).toBe(true);
await conversion.execute();
// Read the output back
using outputInput = new Input({
source: new BufferSource(output.target.buffer!),
formats: ALL_FORMATS,
});
expect(await outputInput.getFormat()).toBe(MPEG_TS);
const inputVideoTrack = await input.getPrimaryVideoTrack();
const inputAudioTrack = await input.getPrimaryAudioTrack();
const outputVideoTrack = await outputInput.getPrimaryVideoTrack();
const outputAudioTrack = await outputInput.getPrimaryAudioTrack();
assert(inputVideoTrack);
assert(inputAudioTrack);
assert(outputVideoTrack);
assert(outputAudioTrack);
// Codecs should match
expect(outputVideoTrack.codec).toBe(inputVideoTrack.codec);
expect(outputAudioTrack.codec).toBe(inputAudioTrack.codec);
// Verify video packets are Annex B
const videoSink = new EncodedPacketSink(outputVideoTrack);
const firstVideoPacket = await videoSink.getFirstPacket();
assert(firstVideoPacket);
expect(firstVideoPacket.data.slice(0, 4)).toEqual(new Uint8Array([0, 0, 0, 1]));
// Verify audio packets are ADTS
const audioSink = new EncodedPacketSink(outputAudioTrack);
const firstAudioPacket = await audioSink.getFirstPacket();
assert(firstAudioPacket);
expect(firstAudioPacket.data[0]).toBe(0xff);
expect(firstAudioPacket.data[1]! & 0xf0).toBe(0xf0);
expect(await outputInput.getFirstTimestamp()).toBe(10.012);
expect(await outputInput.computeDuration()).toBe(15.004);
});
test('MPEG-TS muxing with StreamTarget', async () => {
let nextPos = 0;
const chunks: Uint8Array[] = [];
const writable = new WritableStream<StreamTargetChunk>({
write(chunk) {
chunks.push(chunk.data);
expect(chunk.position).toBe(nextPos);
nextPos += chunk.data.byteLength;
},
});
const output = new Output({
format: new MpegTsOutputFormat(),
target: new StreamTarget(writable),
});
const canvas = new OffscreenCanvas(640, 480);
const context = canvas.getContext('2d')!;
context.fillStyle = '#0000ff';
context.fillRect(0, 0, canvas.width, canvas.height);
const videoSource = new CanvasSource(canvas, {
codec: 'avc',
bitrate: QUALITY_HIGH,
});
output.addVideoTrack(videoSource);
await output.start();
const fps = 30;
const duration = 1;
const frameCount = fps * duration;
const frameDuration = 1 / fps;
for (let i = 0; i < frameCount; i++) {
await videoSource.add(i * frameDuration, frameDuration);
}
await output.finalize();
expect(chunks.length).toBe(frameCount);
const buffer = new Uint8Array(nextPos);
nextPos = 0;
for (const chunk of chunks) {
buffer.set(chunk, nextPos);
nextPos += chunk.byteLength;
}
// Verify the concatenated output
using input = new Input({
source: new BufferSource(buffer),
formats: ALL_FORMATS,
});
expect(await input.getFormat()).toBe(MPEG_TS);
const videoTrack = await input.getPrimaryVideoTrack();
assert(videoTrack);
expect(videoTrack.codec).toBe('avc');
const videoSink = new EncodedPacketSink(videoTrack);
let videoPacketCount = 0;
// eslint-disable-next-line @typescript-eslint/no-unused-vars
for await (const packet of videoSink.packets()) {
videoPacketCount++;
}
expect(videoPacketCount).toBe(frameCount);
});
+18
View File
@@ -12,6 +12,11 @@ import { MpegTsTrackBacking } from '../../src/mpeg-ts/mpeg-ts-demuxer.js';
const __dirname = new URL('.', import.meta.url).pathname;
test('MPEG-TS input format', async () => {
expect(MPEG_TS.mimeType).toBe('video/MP2T');
expect(MPEG_TS.name).toBe('MPEG Transport Stream');
});
test('MPEG-TS metadata reading', async () => {
using input = new Input({
source: new FilePathSource(path.join(__dirname, '../public/0.ts')),
@@ -356,6 +361,11 @@ test('MPEG-TS video key packets', async () => {
expect(firstKeyPacket.type).toBe('key');
expect(firstKeyPacket.sequenceNumber).toBe(firstPacket.sequenceNumber);
const afterKeyPacket = await sink.getNextPacket(firstKeyPacket);
expect(afterKeyPacket).not.toBe(null);
expect(afterKeyPacket!.type).toBe('delta');
expect(afterKeyPacket!.sequenceNumber).toBe(secondPacket.sequenceNumber);
const secondKeyPacket = await sink.getKeyPacket(15);
assert(secondKeyPacket);
expect(secondKeyPacket.type).toBe('key');
@@ -407,6 +417,14 @@ test('MPEG-TS audio key packets', async () => {
expect(nextKeyPacket.type).toBe('key');
expect(nextKeyPacket.sequenceNumber).toBe(secondPacket.sequenceNumber); // All audio packets are key packets
const middleKeyPacket = await sink.getKeyPacket(12.5);
assert(middleKeyPacket);
expect(middleKeyPacket.type).toBe('key');
const afterKeyPacket = await sink.getNextPacket(middleKeyPacket);
expect(afterKeyPacket).not.toBe(null);
expect(afterKeyPacket!.type).toBe('key');
const lastPacket = await sink.getPacket(Infinity);
assert(lastPacket);
Binary file not shown.