Files
mediabunny/src/codec.ts
T
2024-12-08 15:19:56 +01:00

492 lines
20 KiB
TypeScript

import {
COLOR_PRIMARIES_MAP,
MATRIX_COEFFICIENTS_MAP,
TRANSFER_CHARACTERISTICS_MAP,
bytesToHexString,
isAllowSharedBufferSource,
last,
reverseBitsU32,
} from './misc';
import { SubtitleMetadata } from './subtitles';
/** @public */
export const VIDEO_CODECS = ['avc', 'hevc', 'vp8', 'vp9', 'av1'] as const;
/** @public */
export const AUDIO_CODECS = ['aac', 'opus'] as const; // TODO add the rest
/** @public */
export const SUBTITLE_CODECS = ['webvtt'] as const; // TODO add the rest
/** @public */
export type VideoCodec = typeof VIDEO_CODECS[number];
/** @public */
export type AudioCodec = typeof AUDIO_CODECS[number];
/** @public */
export type SubtitleCodec = typeof SUBTITLE_CODECS[number];
/** @public */
export type MediaCodec = VideoCodec | AudioCodec | SubtitleCodec;
// https://en.wikipedia.org/wiki/Advanced_Video_Coding
const AVC_LEVEL_TABLE = [
{ maxMacroblocks: 99, maxBitrate: 64000, level: 0x0A }, // Level 1
{ maxMacroblocks: 396, maxBitrate: 192000, level: 0x0B }, // Level 1.1
{ maxMacroblocks: 396, maxBitrate: 384000, level: 0x0C }, // Level 1.2
{ maxMacroblocks: 396, maxBitrate: 768000, level: 0x0D }, // Level 1.3
{ maxMacroblocks: 396, maxBitrate: 2000000, level: 0x14 }, // Level 2
{ maxMacroblocks: 792, maxBitrate: 4000000, level: 0x15 }, // Level 2.1
{ maxMacroblocks: 1620, maxBitrate: 4000000, level: 0x16 }, // Level 2.2
{ maxMacroblocks: 1620, maxBitrate: 10000000, level: 0x1E }, // Level 3
{ maxMacroblocks: 3600, maxBitrate: 14000000, level: 0x1F }, // Level 3.1
{ maxMacroblocks: 5120, maxBitrate: 20000000, level: 0x20 }, // Level 3.2
{ maxMacroblocks: 8192, maxBitrate: 20000000, level: 0x28 }, // Level 4
{ maxMacroblocks: 8192, maxBitrate: 50000000, level: 0x29 }, // Level 4.1
{ maxMacroblocks: 8704, maxBitrate: 50000000, level: 0x2A }, // Level 4.2
{ maxMacroblocks: 22080, maxBitrate: 135000000, level: 0x32 }, // Level 5
{ maxMacroblocks: 36864, maxBitrate: 240000000, level: 0x33 }, // Level 5.1
{ maxMacroblocks: 36864, maxBitrate: 240000000, level: 0x34 }, // Level 5.2
{ maxMacroblocks: 139264, maxBitrate: 240000000, level: 0x3C }, // Level 6
{ maxMacroblocks: 139264, maxBitrate: 480000000, level: 0x3D }, // Level 6.1
{ maxMacroblocks: 139264, maxBitrate: 800000000, level: 0x3E }, // Level 6.2
];
// https://en.wikipedia.org/wiki/High_Efficiency_Video_Coding
const HEVC_LEVEL_TABLE = [
{ maxPictureSize: 36864, maxBitrate: 128000, tier: 'L', level: 30 }, // Level 1 (Low Tier)
{ maxPictureSize: 122880, maxBitrate: 1500000, tier: 'L', level: 60 }, // Level 2 (Low Tier)
{ maxPictureSize: 245760, maxBitrate: 3000000, tier: 'L', level: 63 }, // Level 2.1 (Low Tier)
{ maxPictureSize: 552960, maxBitrate: 6000000, tier: 'L', level: 90 }, // Level 3 (Low Tier)
{ maxPictureSize: 983040, maxBitrate: 10000000, tier: 'L', level: 93 }, // Level 3.1 (Low Tier)
{ maxPictureSize: 2228224, maxBitrate: 12000000, tier: 'L', level: 120 }, // Level 4 (Low Tier)
{ maxPictureSize: 2228224, maxBitrate: 30000000, tier: 'H', level: 120 }, // Level 4 (High Tier)
{ maxPictureSize: 2228224, maxBitrate: 20000000, tier: 'L', level: 123 }, // Level 4.1 (Low Tier)
{ maxPictureSize: 2228224, maxBitrate: 50000000, tier: 'H', level: 123 }, // Level 4.1 (High Tier)
{ maxPictureSize: 8912896, maxBitrate: 25000000, tier: 'L', level: 150 }, // Level 5 (Low Tier)
{ maxPictureSize: 8912896, maxBitrate: 100000000, tier: 'H', level: 150 }, // Level 5 (High Tier)
{ maxPictureSize: 8912896, maxBitrate: 40000000, tier: 'L', level: 153 }, // Level 5.1 (Low Tier)
{ maxPictureSize: 8912896, maxBitrate: 160000000, tier: 'H', level: 153 }, // Level 5.1 (High Tier)
{ maxPictureSize: 8912896, maxBitrate: 60000000, tier: 'L', level: 156 }, // Level 5.2 (Low Tier)
{ maxPictureSize: 8912896, maxBitrate: 240000000, tier: 'H', level: 156 }, // Level 5.2 (High Tier)
{ maxPictureSize: 35651584, maxBitrate: 60000000, tier: 'L', level: 180 }, // Level 6 (Low Tier)
{ maxPictureSize: 35651584, maxBitrate: 240000000, tier: 'H', level: 180 }, // Level 6 (High Tier)
{ maxPictureSize: 35651584, maxBitrate: 120000000, tier: 'L', level: 183 }, // Level 6.1 (Low Tier)
{ maxPictureSize: 35651584, maxBitrate: 480000000, tier: 'H', level: 183 }, // Level 6.1 (High Tier)
{ maxPictureSize: 35651584, maxBitrate: 240000000, tier: 'L', level: 186 }, // Level 6.2 (Low Tier)
{ maxPictureSize: 35651584, maxBitrate: 800000000, tier: 'H', level: 186 }, // Level 6.2 (High Tier)
];
// https://en.wikipedia.org/wiki/VP9
const VP9_LEVEL_TABLE = [
{ maxPictureSize: 36864, maxBitrate: 200000, level: 10 }, // Level 1
{ maxPictureSize: 73728, maxBitrate: 800000, level: 11 }, // Level 1.1
{ maxPictureSize: 122880, maxBitrate: 1800000, level: 20 }, // Level 2
{ maxPictureSize: 245760, maxBitrate: 3600000, level: 21 }, // Level 2.1
{ maxPictureSize: 552960, maxBitrate: 7200000, level: 30 }, // Level 3
{ maxPictureSize: 983040, maxBitrate: 12000000, level: 31 }, // Level 3.1
{ maxPictureSize: 2228224, maxBitrate: 18000000, level: 40 }, // Level 4
{ maxPictureSize: 2228224, maxBitrate: 30000000, level: 41 }, // Level 4.1
{ maxPictureSize: 8912896, maxBitrate: 60000000, level: 50 }, // Level 5
{ maxPictureSize: 8912896, maxBitrate: 120000000, level: 51 }, // Level 5.1
{ maxPictureSize: 8912896, maxBitrate: 180000000, level: 52 }, // Level 5.2
{ maxPictureSize: 35651584, maxBitrate: 180000000, level: 60 }, // Level 6
{ maxPictureSize: 35651584, maxBitrate: 240000000, level: 61 }, // Level 6.1
{ maxPictureSize: 35651584, maxBitrate: 480000000, level: 62 }, // Level 6.2
];
// https://en.wikipedia.org/wiki/AV1
const AV1_LEVEL_TABLE = [
{ maxPictureSize: 147456, maxBitrate: 1500000, tier: 'M', level: 0 }, // Level 2.0 (Main Tier)
{ maxPictureSize: 278784, maxBitrate: 3000000, tier: 'M', level: 1 }, // Level 2.1 (Main Tier)
{ maxPictureSize: 665856, maxBitrate: 6000000, tier: 'M', level: 4 }, // Level 3.0 (Main Tier)
{ maxPictureSize: 1065024, maxBitrate: 10000000, tier: 'M', level: 5 }, // Level 3.1 (Main Tier)
{ maxPictureSize: 2359296, maxBitrate: 12000000, tier: 'M', level: 8 }, // Level 4.0 (Main Tier)
{ maxPictureSize: 2359296, maxBitrate: 30000000, tier: 'H', level: 8 }, // Level 4.0 (High Tier)
{ maxPictureSize: 2359296, maxBitrate: 20000000, tier: 'M', level: 9 }, // Level 4.1 (Main Tier)
{ maxPictureSize: 2359296, maxBitrate: 50000000, tier: 'H', level: 9 }, // Level 4.1 (High Tier)
{ maxPictureSize: 8912896, maxBitrate: 30000000, tier: 'M', level: 12 }, // Level 5.0 (Main Tier)
{ maxPictureSize: 8912896, maxBitrate: 100000000, tier: 'H', level: 12 }, // Level 5.0 (High Tier)
{ maxPictureSize: 8912896, maxBitrate: 40000000, tier: 'M', level: 13 }, // Level 5.1 (Main Tier)
{ maxPictureSize: 8912896, maxBitrate: 160000000, tier: 'H', level: 13 }, // Level 5.1 (High Tier)
{ maxPictureSize: 8912896, maxBitrate: 60000000, tier: 'M', level: 14 }, // Level 5.2 (Main Tier)
{ maxPictureSize: 8912896, maxBitrate: 240000000, tier: 'H', level: 14 }, // Level 5.2 (High Tier)
{ maxPictureSize: 35651584, maxBitrate: 60000000, tier: 'M', level: 15 }, // Level 5.3 (Main Tier)
{ maxPictureSize: 35651584, maxBitrate: 240000000, tier: 'H', level: 15 }, // Level 5.3 (High Tier)
{ maxPictureSize: 35651584, maxBitrate: 60000000, tier: 'M', level: 16 }, // Level 6.0 (Main Tier)
{ maxPictureSize: 35651584, maxBitrate: 240000000, tier: 'H', level: 16 }, // Level 6.0 (High Tier)
{ maxPictureSize: 35651584, maxBitrate: 100000000, tier: 'M', level: 17 }, // Level 6.1 (Main Tier)
{ maxPictureSize: 35651584, maxBitrate: 480000000, tier: 'H', level: 17 }, // Level 6.1 (High Tier)
{ maxPictureSize: 35651584, maxBitrate: 160000000, tier: 'M', level: 18 }, // Level 6.2 (Main Tier)
{ maxPictureSize: 35651584, maxBitrate: 800000000, tier: 'H', level: 18 }, // Level 6.2 (High Tier)
{ maxPictureSize: 35651584, maxBitrate: 160000000, tier: 'M', level: 19 }, // Level 6.3 (Main Tier)
{ maxPictureSize: 35651584, maxBitrate: 800000000, tier: 'H', level: 19 }, // Level 6.3 (High Tier)
];
export const buildVideoCodecString = (codec: VideoCodec, width: number, height: number, bitrate: number) => {
if (codec === 'avc') {
const profileIndication = 0x64; // High Profile
const totalMacroblocks = Math.ceil(width / 16) * Math.ceil(height / 16);
// Determine the level based on the table
const levelInfo = AVC_LEVEL_TABLE.find(
level => totalMacroblocks <= level.maxMacroblocks && bitrate <= level.maxBitrate,
) ?? last(AVC_LEVEL_TABLE)!;
const levelIndication = levelInfo ? levelInfo.level : 0;
const hexProfileIndication = profileIndication.toString(16).padStart(2, '0');
const hexProfileCompatibility = '00';
const hexLevelIndication = levelIndication.toString(16).padStart(2, '0');
return `avc1.${hexProfileIndication}${hexProfileCompatibility}${hexLevelIndication}`;
} else if (codec === 'hevc') {
const profilePrefix = ''; // Profile space 0
const profileIdc = 1; // Main Profile
const compatibilityFlags = '6'; // Taken from the example in ISO 14496-15
const pictureSize = width * height;
const levelInfo = HEVC_LEVEL_TABLE.find(
level => pictureSize <= level.maxPictureSize && bitrate <= level.maxBitrate,
) ?? last(HEVC_LEVEL_TABLE)!;
const constraintFlags = 'B0'; // Progressive source flag
return 'hev1.'
+ `${profilePrefix}${profileIdc}.`
+ `${compatibilityFlags}.`
+ `${levelInfo.tier}${levelInfo.level}.`
+ `${constraintFlags}`;
} else if (codec === 'vp8') {
return 'vp8'; // Easy, this one
} else if (codec === 'vp9') {
const profile = '00'; // Profile 0
const pictureSize = width * height;
const levelInfo = VP9_LEVEL_TABLE.find(
level => pictureSize <= level.maxPictureSize && bitrate <= level.maxBitrate,
) ?? last(VP9_LEVEL_TABLE)!;
const bitDepth = '08'; // 8-bit
return `vp09.${profile}.${levelInfo.level}.${bitDepth}`;
} else if (codec === 'av1') {
const profile = 0; // Main Profile
const pictureSize = width * height;
const levelInfo = AV1_LEVEL_TABLE.find(
level => pictureSize <= level.maxPictureSize && bitrate <= level.maxBitrate,
) ?? last(AV1_LEVEL_TABLE)!;
const bitDepth = '08'; // 8-bit
return `av01.${profile}.${levelInfo.level.toString().padStart(2, '0')}${levelInfo.tier}.${bitDepth}`;
}
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
throw new TypeError(`Unhandled codec '${codec}'.`);
};
export const extractVideoCodecString = (codec: VideoCodec, description: Uint8Array | null) => {
if (codec === 'avc') {
if (!description || description.byteLength < 4) {
throw new TypeError('AVC description must be at least 4 bytes long.');
}
// TODO: The "temp hack". Something is amiss with the second byte in the hex string, check the specification
return `avc1.${bytesToHexString(description.subarray(1, 4))}`;
} else if (codec === 'hevc') {
if (!description) {
throw new TypeError('HEVC description must be provided.');
}
const view = new DataView(description.buffer, description.byteOffset, description.byteLength);
let codecString = 'hev1.';
// general_profile_space and general_profile_idc
const generalProfileSpace = (description[1]! >> 6) & 0x03;
const generalProfileIdc = description[1]! & 0x1F;
codecString += ['', 'A', 'B', 'C'][generalProfileSpace]! + generalProfileIdc;
codecString += '.';
// general_profile_compatibility_flags (in reverse bit order)
const compatibilityFlags = reverseBitsU32(view.getUint32(2));
codecString += compatibilityFlags.toString(16);
codecString += '.';
// general_tier_flag and general_level_idc
const generalTierFlag = (description[1]! >> 5) & 0x01;
const generalLevelIdc = description[12]!;
codecString += generalTierFlag === 0 ? 'L' : 'H';
codecString += generalLevelIdc;
codecString += '.';
// constraint_flags (6 bytes)
const constraintFlags: number[] = [];
for (let i = 0; i < 6; i++) {
const byte = description[i + 13]!;
constraintFlags.push(byte);
}
while (constraintFlags[constraintFlags.length - 1] === 0) {
constraintFlags.pop();
}
codecString += constraintFlags.map(x => x.toString(16)).join('.');
return codecString;
}
// TODO
throw new TypeError(`Unhandled codec '${codec}'.`);
};
export const buildAudioCodecString = (codec: AudioCodec, numberOfChannels: number, sampleRate: number) => {
if (codec === 'aac') {
// If stereo or higher channels and lower sample rate, likely using HE-AAC v2 with PS
if (numberOfChannels >= 2 && sampleRate <= 24000) {
return 'mp4a.40.29'; // HE-AAC v2 (AAC LC + SBR + PS)
}
// If sample rate is low, likely using HE-AAC v1 with SBR
if (sampleRate <= 24000) {
return 'mp4a.40.5'; // HE-AAC v1 (AAC LC + SBR)
}
// Default to standard AAC-LC for higher sample rates
return 'mp4a.40.2'; // AAC-LC
} else if (codec === 'opus') {
return 'opus'; // Easy, this one
} else if (codec === 'vorbis') {
return 'vorbis'; // Also easy, this one
}
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
throw new TypeError(`Unhandled codec '${codec}'.`);
};
export const extractAudioCodecString = (codec: AudioCodec, description: Uint8Array | null) => {
if (codec === 'aac') {
if (!description || description.byteLength < 2) {
throw new TypeError('AAC description must be at least 2 bytes long.');
}
// TODO: Is this correct? Give a source/reason
const mpeg4AudioObjectType = description[0]! >> 3;
return `mp4a.40.${mpeg4AudioObjectType}`;
} else if (codec === 'opus') {
return 'opus';
} else if (codec === 'vorbis') {
return 'vorbis';
}
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
throw new TypeError(`Unhandled codec '${codec}'.`);
};
export const getVideoEncoderConfigExtension = (codec: VideoCodec) => {
if (codec === 'avc') {
return {
avc: {
format: 'avc' as const, // Ensure the format is not Annex B
},
};
} else if (codec === 'hevc') {
return {
hevc: {
format: 'hevc' as const, // Ensure the format is not Annex B
},
};
}
return {};
};
export const getAudioEncoderConfigExtension = (codec: AudioCodec) => {
if (codec === 'aac') {
return {
aac: {
format: 'aac' as const, // Ensure the format is not ADTS
},
};
} else if (codec === 'opus') {
return {
opus: {
format: 'opus' as const,
},
};
}
return {};
};
export const validateVideoChunkMetadata = (metadata: EncodedVideoChunkMetadata | undefined) => {
if (!metadata) {
throw new TypeError('Video chunk metadata must be provided.');
}
if (typeof metadata !== 'object') {
throw new TypeError('Video chunk metadata must be an object.');
}
if (!metadata.decoderConfig) {
throw new TypeError('Video chunk metadata must include a decoder configuration.');
}
if (typeof metadata.decoderConfig !== 'object') {
throw new TypeError('Video chunk metadata decoder configuration must be an object.');
}
if (typeof metadata.decoderConfig.codec !== 'string') {
throw new TypeError('Video chunk metadata decoder configuration must specify a codec string.');
}
if (!Number.isInteger(metadata.decoderConfig.codedWidth) || metadata.decoderConfig.codedWidth! <= 0) {
throw new TypeError(
'Video chunk metadata decoder configuration must specify a valid codedWidth (positive integer).',
);
}
if (!Number.isInteger(metadata.decoderConfig.codedHeight) || metadata.decoderConfig.codedHeight! <= 0) {
throw new TypeError(
'Video chunk metadata decoder configuration must specify a valid codedHeight (positive integer).',
);
}
if (metadata.decoderConfig.description !== undefined) {
if (!isAllowSharedBufferSource(metadata.decoderConfig.description)) {
throw new TypeError(
'Video chunk metadata decoder configuration description, when defined, must be an ArrayBuffer or an'
+ ' ArrayBuffer view.',
);
}
}
if (metadata.decoderConfig.colorSpace !== undefined) {
const { colorSpace } = metadata.decoderConfig;
if (typeof colorSpace !== 'object') {
throw new TypeError(
'Video chunk metadata decoder configuration colorSpace, when provided, must be an object.',
);
}
const primariesValues = Object.keys(COLOR_PRIMARIES_MAP);
if (colorSpace.primaries != null && !primariesValues.includes(colorSpace.primaries)) {
throw new TypeError(
`Video chunk metadata decoder configuration colorSpace primaries, when defined, must be one of`
+ ` ${primariesValues.join(', ')}.`,
);
}
const transferValues = Object.keys(TRANSFER_CHARACTERISTICS_MAP);
if (colorSpace.transfer != null && !transferValues.includes(colorSpace.transfer)) {
throw new TypeError(
`Video chunk metadata decoder configuration colorSpace transfer, when defined, must be one of`
+ ` ${transferValues.join(', ')}.`,
);
}
const matrixValues = Object.keys(MATRIX_COEFFICIENTS_MAP);
if (colorSpace.matrix != null && !matrixValues.includes(colorSpace.matrix)) {
throw new TypeError(
`Video chunk metadata decoder configuration colorSpace matrix, when defined, must be one of`
+ ` ${matrixValues.join(', ')}.`,
);
}
if (colorSpace.fullRange != null && typeof colorSpace.fullRange !== 'boolean') {
throw new TypeError(
'Video chunk metadata decoder configuration colorSpace fullRange, when defined, must be a boolean.',
);
}
}
if (
(metadata.decoderConfig.codec.startsWith('avc1') || metadata.decoderConfig.codec.startsWith('avc3'))
&& !metadata.decoderConfig.description
) {
throw new TypeError(
'Video chunk metadata decoder configuration for AVC must include a description, which is expected to be an'
+ ' AVCDecoderConfigurationRecord as specified in ISO 14496-15.',
);
}
if (
(metadata.decoderConfig.codec.startsWith('hev1') || metadata.decoderConfig.codec.startsWith('hvc1'))
&& !metadata.decoderConfig.description
) {
throw new TypeError(
'Video chunk metadata decoder configuration for HEVC must include a description, which is expected to be an'
+ ' HEVCDecoderConfigurationRecord as specified in ISO 14496-15.',
);
}
if (
(metadata.decoderConfig.codec === 'vp8' || metadata.decoderConfig.codec.startsWith('vp09'))
&& metadata.decoderConfig.colorSpace === undefined
) {
throw new TypeError('Video chunk metadata decoder configuration for VP8/VP9 must include a colorSpace.');
}
// No added requirements for AV1 (based)
};
export const validateAudioChunkMetadata = (metadata: EncodedAudioChunkMetadata | undefined) => {
if (!metadata) {
throw new TypeError('Audio chunk metadata must be provided.');
}
if (typeof metadata !== 'object') {
throw new TypeError('Audio chunk metadata must be an object.');
}
if (!metadata.decoderConfig) {
throw new TypeError('Audio chunk metadata must include a decoder configuration.');
}
if (typeof metadata.decoderConfig !== 'object') {
throw new TypeError('Audio chunk metadata decoder configuration must be an object.');
}
if (typeof metadata.decoderConfig.codec !== 'string') {
throw new TypeError('Audio chunk metadata decoder configuration must specify a codec string.');
}
if (!Number.isInteger(metadata.decoderConfig.sampleRate) || metadata.decoderConfig.sampleRate <= 0) {
throw new TypeError(
'Audio chunk metadata decoder configuration must specify a valid sampleRate (positive integer).',
);
}
if (!Number.isInteger(metadata.decoderConfig.numberOfChannels) || metadata.decoderConfig.numberOfChannels <= 0) {
throw new TypeError(
'Audio chunk metadata decoder configuration must specify a valid numberOfChannels (positive integer).',
);
}
if (metadata.decoderConfig.description !== undefined) {
if (!isAllowSharedBufferSource(metadata.decoderConfig.description)) {
throw new TypeError(
'Audio chunk metadata decoder configuration description, when defined, must be an ArrayBuffer or an'
+ ' ArrayBuffer view.',
);
}
}
if (metadata.decoderConfig.codec.startsWith('mp4a') && !metadata.decoderConfig.description) {
throw new TypeError(
'Audio chunk metadata decoder configuration for AAC must include a description, which is expected to be an'
+ ' AudioSpecificConfig as specified in ISO 14496-3.',
);
}
if (
metadata.decoderConfig.codec === 'opus'
&& metadata.decoderConfig.description
&& metadata.decoderConfig.description.byteLength < 18
) {
throw new TypeError('Invalid decoder description provided for Opus; must be at least 18 bytes long.');
}
};
export const validateSubtitleMetadata = (metadata: SubtitleMetadata | undefined) => {
if (!metadata) {
throw new TypeError('Subtitle metadata must be provided.');
}
if (typeof metadata !== 'object') {
throw new TypeError('Subtitle metadata must be an object.');
}
if (!metadata.config) {
throw new TypeError('Subtitle metadata must include a config object.');
}
if (typeof metadata.config !== 'object') {
throw new TypeError('Subtitle metadata config must be an object.');
}
if (typeof metadata.config.description !== 'string') {
throw new TypeError('Subtitle metadata config description must be a string.');
}
};