diff --git a/src/codec-data.ts b/src/codec-data.ts index 2fd8ca7..2fad0ce 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -1856,6 +1856,10 @@ export const determineVideoPacketType = ( return null; }; + case 'prores': { + return 'key'; + }; + default: { assertNever(codec); assert(false); diff --git a/src/codec.ts b/src/codec.ts index b9414fd..2ff9788 100644 --- a/src/codec.ts +++ b/src/codec.ts @@ -18,6 +18,7 @@ import { MATRIX_COEFFICIENTS_MAP, TRANSFER_CHARACTERISTICS_MAP, assert, + assertNever, bytesToHexString, isAllowSharedBufferSource, last, @@ -37,6 +38,7 @@ export const VIDEO_CODECS = [ 'vp9', 'av1', 'vp8', + 'prores', ] as const; /** * List of known PCM (uncompressed) audio codecs, ordered by encoding preference. @@ -212,6 +214,17 @@ const AV1_LEVEL_TABLE = [ const VP9_DEFAULT_SUFFIX = '.01.01.01.01.00'; const AV1_DEFAULT_SUFFIX = '.0.110.01.01.01.0'; +export const PRORES_FOURCCS = [ + 'ap4x', + 'ap4h', + 'apch', + 'apcn', + 'apcs', + 'apco', + 'aprh', + 'aprn', +]; + export const buildVideoCodecString = (codec: VideoCodec, width: number, height: number, bitrate: number) => { if (codec === 'avc') { const profileIndication = 0x64; // High Profile @@ -271,10 +284,13 @@ export const buildVideoCodecString = (codec: VideoCodec, width: number, height: const bitDepth = '08'; // 8-bit return `av01.${profile}.${level}${levelInfo.tier}.${bitDepth}`; + } else if (codec === 'prores') { + return 'apr1.apch'; + } else { + assertNever(codec); } - // eslint-disable-next-line @typescript-eslint/restrict-template-expressions - throw new TypeError(`Unhandled codec '${codec}'.`); + throw new TypeError(`Unhandled codec '${String(codec)}'.`); }; export const generateVp9CodecConfigurationFromCodecString = (codecString: string) => { @@ -342,8 +358,18 @@ export const extractVideoCodecString = (trackInfo: { hevcCodecInfo: HevcDecoderConfigurationRecord | null; vp9CodecInfo: Vp9CodecInfo | null; av1CodecInfo: Av1CodecInfo | null; + proresFormat: string | null; }) => { - const { codec, codecDescription, colorSpace, avcCodecInfo, hevcCodecInfo, vp9CodecInfo, av1CodecInfo } = trackInfo; + const { + codec, + codecDescription, + colorSpace, + avcCodecInfo, + hevcCodecInfo, + vp9CodecInfo, + av1CodecInfo, + proresFormat, + } = trackInfo; if (codec === 'avc') { assert(trackInfo.avcType !== null); @@ -501,6 +527,10 @@ export const extractVideoCodecString = (trackInfo: { } return string; + } else if (codec === 'prores') { + return `apr1.${proresFormat ?? 'apch'}`; + } else if (codec !== null) { + assertNever(codec); } throw new TypeError(`Unhandled codec '${codec}'.`); @@ -786,7 +816,7 @@ export const getAudioEncoderConfigExtension = (codec: AudioCodec) => { return {}; }; -const VALID_VIDEO_CODEC_STRING_PREFIXES = ['avc1', 'avc3', 'hev1', 'hvc1', 'vp8', 'vp09', 'av01']; +const VALID_VIDEO_CODEC_STRING_PREFIXES = ['avc1', 'avc3', 'hev1', 'hvc1', 'vp8', 'vp09', 'av01', 'apr1']; const AVC_CODEC_STRING_REGEX = /^(avc1|avc3)\.[0-9a-fA-F]{6}$/; const HEVC_CODEC_STRING_REGEX = /^(hev1|hvc1)\.(?:[ABC]?\d+)\.[0-9a-fA-F]{1,8}\.[LH]\d+(?:\.[0-9a-fA-F]{1,2}){0,6}$/; const VP9_CODEC_STRING_REGEX = /^vp09(?:\.\d{2}){3}(?:(?:\.\d{2}){5})?$/; @@ -811,7 +841,7 @@ export const validateVideoChunkMetadata = (metadata: EncodedVideoChunkMetadata | if (!VALID_VIDEO_CODEC_STRING_PREFIXES.some(prefix => metadata.decoderConfig!.codec.startsWith(prefix))) { throw new TypeError( 'Video chunk metadata decoder configuration codec string must be a valid video codec string as specified in' - + ' the WebCodecs Codec Registry.', + + ' the WebCodecs Codec Registry.', // todo? ); } if (!Number.isInteger(metadata.decoderConfig.codedWidth) || metadata.decoderConfig.codedWidth! <= 0) { @@ -922,6 +952,16 @@ export const validateVideoChunkMetadata = (metadata: EncodedVideoChunkMetadata | + ' specified in Section "Codecs Parameter String" of https://aomediacodec.github.io/av1-isobmff/.', ); } + } else if (metadata.decoderConfig.codec.startsWith('apr1')) { + // ProRes-specific validation + + const parts = metadata.decoderConfig.codec.split('.'); + if (parts.length !== 2 || parts[0] !== 'apr1' || !PRORES_FOURCCS.includes(parts[1]!)) { + throw new TypeError( + 'Video chunk metadata decoder configuration codec string for ProRes must be a valid ProRes codec' + + ' string as specified in the Mediabunny Codec Registry.', + ); + } } }; diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 0079451..4c2a871 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -629,7 +629,7 @@ export const videoSampleDescription = ( u16(0x0018), // Depth i16(0xffff), // Pre-defined ], [ - VIDEO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec](trackData), + VIDEO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec]?.(trackData) ?? null, colorSpaceIsComplete(trackData.info.decoderConfig.colorSpace) ? colr(trackData) : null, ]); @@ -1584,15 +1584,20 @@ const videoCodecToBoxName = (codec: VideoCodec, fullCodecString: string) => { case 'vp8': return 'vp08'; case 'vp9': return 'vp09'; case 'av1': return 'av01'; + case 'prores': return fullCodecString.split('.')[1]!; } }; -const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record Box | null> = { +const VIDEO_CODEC_TO_CONFIGURATION_BOX: Record< + VideoCodec, + ((trackData: IsobmffVideoTrackData) => Box | null) | null +> = { avc: avcC, hevc: hvcC, vp8: vpcC, vp9: vpcC, av1: av1C, + prores: null, }; const audioCodecToBoxName = (codec: AudioCodec, isQuickTime: boolean): string => { diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index b524bb2..245c856 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -17,6 +17,7 @@ import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, + PRORES_FOURCCS, VideoCodec, } from '../codec'; import { @@ -132,6 +133,7 @@ type InternalTrack = { hevcCodecInfo: HevcDecoderConfigurationRecord | null; vp9CodecInfo: Vp9CodecInfo | null; av1CodecInfo: Av1CodecInfo | null; + proresFormat: string | null; }; } | { info: { @@ -849,6 +851,7 @@ export class IsobmffDemuxer extends Demuxer { hevcCodecInfo: null, vp9CodecInfo: null, av1CodecInfo: null, + proresFormat: null, }; } else if (handlerType === 'soun') { track.info = { @@ -910,6 +913,9 @@ export class IsobmffDemuxer extends Demuxer { track.info.codec = 'vp9'; } else if (lowercaseBoxName === 'av01') { track.info.codec = 'av1'; + } else if (PRORES_FOURCCS.includes(lowercaseBoxName)) { + track.info.codec = 'prores'; + track.info.proresFormat = lowercaseBoxName; } else { console.warn(`Unsupported video codec (sample entry type '${sampleBoxInfo.name}').`); } diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index eb27c40..982dddf 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -701,6 +701,7 @@ export const CODEC_STRING_MAP: Partial> = { 'vp8': 'V_VP8', 'vp9': 'V_VP9', 'av1': 'V_AV1', + 'prores': 'V_PRORES', 'aac': 'A_AAC', 'mp3': 'A_MPEG/L3', diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index db421b6..510f990 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -44,6 +44,7 @@ import { normalizeRotation, Rotation, roundIfAlmostInteger, + textDecoder, TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE, } from '../misc'; @@ -1042,6 +1043,8 @@ export class MatroskaDemuxer extends Demuxer { this.currentTrack.info.codec = 'vp9'; } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.av1) { this.currentTrack.info.codec = 'av1'; + } else if (codecIdWithoutSuffix === CODEC_STRING_MAP.prores) { + this.currentTrack.info.codec = 'prores'; } const videoTrack = this.currentTrack as InternalVideoTrack; @@ -2356,6 +2359,9 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid av1CodecInfo: this.internalTrack.info.codec === 'av1' && firstPacket ? extractAv1CodecInfoFromPacket(firstPacket.data) : null, + proresFormat: this.internalTrack.info.codec === 'prores' && this.internalTrack.codecPrivate + ? textDecoder.decode(this.internalTrack.codecPrivate) + : null, }), codedWidth: this.internalTrack.info.width, codedHeight: this.internalTrack.info.height, diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index 73af407..d0327ba 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -82,6 +82,7 @@ type InternalMediaChunk = { type MatroskaTrackData = { chunkQueue: InternalMediaChunk[]; lastWrittenMsTimestamp: number | null; + codecPrivate: AllowSharedBufferSource | null; } & ({ track: OutputVideoTrack; type: 'video'; @@ -325,6 +326,9 @@ export class MatroskaMuxer extends Muxer { { id: EBMLId.FlagLacing, data: 0 }, { id: EBMLId.Language, data: trackData.track.metadata.languageCode ?? UNDETERMINED_LANGUAGE }, { id: EBMLId.CodecID, data: codecId }, + trackData.codecPrivate + ? { id: EBMLId.CodecPrivate, data: toUint8Array(trackData.codecPrivate) } + : null, { id: EBMLId.CodecDelay, data: 0 }, { id: EBMLId.SeekPreRoll, data: seekPreRollNs }, trackData.track.metadata.name !== undefined @@ -341,12 +345,6 @@ export class MatroskaMuxer extends Muxer { const { frameRate, rotation } = trackData.track.metadata; const elements: EBMLElement['data'] = [ - (trackData.info.decoderConfig.description - ? { - id: EBMLId.CodecPrivate, - data: toUint8Array(trackData.info.decoderConfig.description), - } - : null), (frameRate ? { id: EBMLId.DefaultDuration, @@ -414,12 +412,6 @@ export class MatroskaMuxer extends Muxer { : null; return [ - (trackData.info.decoderConfig.description - ? { - id: EBMLId.CodecPrivate, - data: toUint8Array(trackData.info.decoderConfig.description), - } - : null), { id: EBMLId.Audio, data: [ { id: EBMLId.SamplingFrequency, data: new EBMLFloat32(trackData.info.sampleRate) }, { id: EBMLId.Channels, data: trackData.info.numberOfChannels }, @@ -428,10 +420,9 @@ export class MatroskaMuxer extends Muxer { ]; } + // eslint-disable-next-line @typescript-eslint/no-unused-vars private subtitleSpecificTrackInfo(trackData: MatroskaSubtitleTrackData) { - return [ - { id: EBMLId.CodecPrivate, data: textEncoder.encode(trackData.info.config.description) }, - ]; + return []; } private maybeCreateTags() { @@ -739,27 +730,28 @@ export class MatroskaMuxer extends Muxer { }, chunkQueue: [], lastWrittenMsTimestamp: null, + codecPrivate: meta.decoderConfig.description ?? null, }; if (track.source._codec === 'vp9') { // https://www.webmproject.org/docs/container specifies that VP9 "SHOULD" make use of the CodecPrivate // field. Since WebCodecs makes no use of the description field for VP9, we need to derive it ourselves: - newTrackData.info.decoderConfig = { - ...newTrackData.info.decoderConfig, - description: new Uint8Array( - generateVp9CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec), - ), - }; + newTrackData.codecPrivate = new Uint8Array( + generateVp9CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec), + ); } else if (track.source._codec === 'av1') { // Per https://github.com/ietf-wg-cellar/matroska-specification/blob/master/codec/av1.md, AV1 requires // CodecPrivate to be set, but WebCodecs makes no use of the description field for AV1. Thus, let's derive // it ourselves: - newTrackData.info.decoderConfig = { - ...newTrackData.info.decoderConfig, - description: new Uint8Array( - generateAv1CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec), - ), - }; + newTrackData.codecPrivate = new Uint8Array( + generateAv1CodecConfigurationFromCodecString(newTrackData.info.decoderConfig.codec), + ); + } else if (track.source._codec === 'prores') { + const format = meta.decoderConfig.codec.split('.')[1]; + assert(format); + + // "The Private Data contains the FourCC as found in MP4 movies" + newTrackData.codecPrivate = textEncoder.encode(format); } this.trackDatas.push(newTrackData); @@ -793,6 +785,7 @@ export class MatroskaMuxer extends Muxer { }, chunkQueue: [], lastWrittenMsTimestamp: null, + codecPrivate: meta.decoderConfig.description ?? null, }; this.trackDatas.push(newTrackData); @@ -824,6 +817,7 @@ export class MatroskaMuxer extends Muxer { }, chunkQueue: [], lastWrittenMsTimestamp: null, + codecPrivate: textEncoder.encode(meta.config.description), }; this.trackDatas.push(newTrackData); diff --git a/test/node/prores.test.ts b/test/node/prores.test.ts new file mode 100644 index 0000000..76d6eda --- /dev/null +++ b/test/node/prores.test.ts @@ -0,0 +1,101 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { BufferSource, UrlSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { Output } from '../../src/output.js'; +import { MkvOutputFormat, MovOutputFormat } from '../../src/output-format.js'; +import { BufferTarget } from '../../src/target.js'; +import { Conversion } from '../../src/conversion.js'; + +test.concurrent('ProRes MOV file reading', async () => { + using input = new Input({ + source: new UrlSource('https://pub-cf9fcfcb5c0a44e9b1bb5ff890e041ae.r2.dev/IMG_0158-prores-raw.MOV'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + expect(videoTrack.codec).toBe('prores'); + expect(videoTrack.codedWidth).toBe(1920); + expect(videoTrack.codedHeight).toBe(1080); + + const decoderConfig = (await videoTrack.getDecoderConfig())!; + expect(decoderConfig.codec).toBe('apr1.apch'); + expect(decoderConfig.description).toBeUndefined(); +}); + +test.concurrent('ProRes transcoding into MOV', { timeout: 60_000 }, async () => { + using input = new Input({ + source: new UrlSource('https://pub-cf9fcfcb5c0a44e9b1bb5ff890e041ae.r2.dev/IMG_0158-prores-raw.MOV'), + formats: ALL_FORMATS, + }); + + const output = new Output({ + format: new MovOutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ + input, + output, + audio: { + discard: true, + }, + trim: { + end: 0.5, + }, + }); + await conversion.execute(); + + using newInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const videoTrack = (await newInput.getPrimaryVideoTrack())!; + expect(videoTrack.codec).toBe('prores'); + expect((await videoTrack.computePacketStats()).packetCount).toBe(15); + + const decoderConfig = (await videoTrack.getDecoderConfig())!; + expect(decoderConfig.codec).toBe('apr1.apch'); + expect(decoderConfig.description).toBeUndefined(); +}); + +test.concurrent('ProRes transcoding into MKV', { timeout: 60_000 }, async () => { + using input = new Input({ + source: new UrlSource('https://pub-cf9fcfcb5c0a44e9b1bb5ff890e041ae.r2.dev/IMG_0158-prores-raw.MOV'), + formats: ALL_FORMATS, + }); + + const output = new Output({ + format: new MkvOutputFormat(), + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ + input, + output, + video: { + rotate: 270, // To undo the source rotation metadata so the copy path is taken + }, + audio: { + discard: true, + }, + trim: { + end: 0.5, + }, + }); + await conversion.execute(); + + using newInput = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const videoTrack = (await newInput.getPrimaryVideoTrack())!; + expect(videoTrack.codec).toBe('prores'); + expect((await videoTrack.computePacketStats()).packetCount).toBe(15); + + const decoderConfig = (await videoTrack.getDecoderConfig())!; + expect(decoderConfig.codec).toBe('apr1.apch'); + expect(decoderConfig.description).toBeUndefined(); +}); diff --git a/todo.txt b/todo.txt new file mode 100644 index 0000000..65834c2 --- /dev/null +++ b/todo.txt @@ -0,0 +1,2 @@ +- different codec strings +- separate out prores and prores raw (?) ffmpeg does this, so probably good reasons. Matroska does not. \ No newline at end of file