diff --git a/README.md b/README.md index 064d3ad..e97c50f 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,10 @@ Mediabunny is a JavaScript library for reading, writing, and converting media fi Screen Studio +      + + Tella + ### Silver sponsors diff --git a/dev/convert.html b/dev/convert.html index 0609f8d..96c013a 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -26,7 +26,7 @@ chunked: true, chunkSize: 2**20 }); - const outputFormat = new Mediabunny.MpegTsOutputFormat({}); + const outputFormat = new Mediabunny.Mp4OutputFormat({}); const button = document.createElement('button'); button.textContent = 'Cancel'; @@ -104,8 +104,9 @@ }, */ video: () => ({ - forceTranscode: true, - width: 1280, + //forceTranscode: true, + //forceTranscode: true, + //width: 1280, //forceTranscode: true, //allowRotationMetadata: false, //width: 720, diff --git a/docs/guide/media-sinks.md b/docs/guide/media-sinks.md index ac76161..84747cb 100644 --- a/docs/guide/media-sinks.md +++ b/docs/guide/media-sinks.md @@ -350,7 +350,7 @@ type CanvasSinkOptions = { - `rotation`\ The clockwise rotation by which to rotate the raw video frame. Defaults to the rotation set in the file metadata. Rotation is applied before cropping and resizing. - `crop`\ - Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to the dimensions of the input video track. Cropping is performed after rotation but before resizing. + Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to the dimensions of the input video track. Cropping is performed after rotation but before resizing. The crop region is in the _display pixel space_ of the underlying video data. - `poolSize`\ See [Canvas pool](#canvas-pool). @@ -374,7 +374,7 @@ new CanvasSink(videoTrack, { fit: 'cover', }); -// This sink yields canvases with the unaltered coded dimensions of the track, +// This sink yields canvases with the unrotated dimensions of the track, // and without applying any rotation. new CanvasSink(videoTrack, { rotation: 0, diff --git a/docs/guide/packets-and-samples.md b/docs/guide/packets-and-samples.md index e16d1c2..85247ee 100644 --- a/docs/guide/packets-and-samples.md +++ b/docs/guide/packets-and-samples.md @@ -281,6 +281,10 @@ videoSample.format; // => VideoPixelFormat | null videoSample.codedWidth; // => number videoSample.codedHeight; // => number +// Pixel aspect ratio-corrected dimensions of the sample +videoSample.squarePixelWidth; +videoSample.squarePixelHeight; + // Transformed display dimensions of the sample (after rotation) videoSample.displayWidth; // => number videoSample.displayHeight; // => number @@ -289,6 +293,9 @@ videoSample.displayHeight; // => number // rotated by this amount when it is presented. videoSample.rotation; // => 0 | 90 | 180 | 270 +// The sample's pixel aspect ratio +videoSample.pixelAspectRatio; // => { num: number, den: number } + // Timing information videoSample.timestamp; // => Presentation timestamp in seconds videoSample.duration; // => Duration in seconds @@ -297,6 +304,8 @@ videoSample.microsecondDuration; // => Duration in microseconds // Color space of the sample videoSample.colorSpace; // => VideoColorSpace + +videoSample.visibleRect; // Rectangle ``` While all of these properties are read-only, you can use the `setTimestamp`, `setDuration` and `setRotation` methods to modify some of the metadata of the video sample. diff --git a/docs/guide/reading-media-files.md b/docs/guide/reading-media-files.md index a3e1a9e..8193df6 100644 --- a/docs/guide/reading-media-files.md +++ b/docs/guide/reading-media-files.md @@ -216,17 +216,26 @@ This will only look at the first ~50 packets and then return the result. This is In addition to the [common track metadata](#common-track-metadata), video tracks have additional metadata you can query: ```ts -// Get the raw pixel dimensions of the track's coded samples, before rotation: +// Get the raw pixel dimensions of the track's coded samples: videoTrack.codedWidth; // => number videoTrack.codedHeight; // => number -// Get the displayed pixel dimensions of the track's samples, after rotation: +// Get the pixel dimensions of the track after aspect ratio adjustments, +// but before rotation: +videoTrack.squarePixelWidth; // => number +videoTrack.squarePixelHeight; // => number + +// Get the displayed pixel dimensions of the track's samples, after +// aspect ratio adjustments and rotation: videoTrack.displayWidth; // => number videoTrack.displayHeight; // => number // Get the clockwise rotation in degrees by which the // track's frames should be rotated: videoTrack.rotation; // => 0 | 90 | 180 | 270 + +// Get the aspect ratio of the track's pixels (usually 1:1): +videoTrack.pixelAspectRatio; // => { num: number, den: number } ``` To compute a video track's average frame rate (FPS), use [`computePacketStats`](#packet-statistics): diff --git a/docs/index.md b/docs/index.md index 7922f13..a98654a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -96,6 +96,7 @@ const sponsors = { { image: '/sponsors/diffusionstudio.png', name: 'Diffusion Studio', url: 'https://diffusion.studio/' }, { image: '/sponsors/kino.jpg', name: 'Kino', url: 'https://kino.ai/' }, { image: '/sponsors/screen-studio.webp', name: 'Screen Studio', url: 'https://screen.studio/' }, + { image: '/sponsors/tella.svg', name: 'Tella', url: 'https://www.tella.com/' }, ], silver: [ { image: '/sponsors/ponder.png', name: 'Ponder', url: 'https://ponder.ai/' }, diff --git a/docs/public/sponsors/tella.svg b/docs/public/sponsors/tella.svg new file mode 100644 index 0000000..dd9e447 --- /dev/null +++ b/docs/public/sponsors/tella.svg @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/examples/metadata-extraction/metadata-extraction.ts b/examples/metadata-extraction/metadata-extraction.ts index 61cf332..ace5e9e 100644 --- a/examples/metadata-extraction/metadata-extraction.ts +++ b/examples/metadata-extraction/metadata-extraction.ts @@ -57,6 +57,9 @@ const extractMetadata = (resource: File | string) => { 'Coded width': `${track.codedWidth} pixels`, 'Coded height': `${track.codedHeight} pixels`, 'Rotation': `${track.rotation}° clockwise`, + 'Pixel aspect ratio': `${track.pixelAspectRatio.num}:${track.pixelAspectRatio.den}`, + 'Display width': `${track.displayWidth} pixels`, + 'Display height': `${track.displayHeight} pixels`, 'Transparency': track.canBeTransparent(), } : track.isAudioTrack() diff --git a/package-lock.json b/package-lock.json index e146134..57c2cdd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.34.4", + "version": "1.35.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.34.4", + "version": "1.35.0", "license": "MPL-2.0", "workspaces": [ "packages/*" @@ -7743,9 +7743,9 @@ } }, "node_modules/mediabunny": { - "version": "1.34.3", - "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.34.3.tgz", - "integrity": "sha512-Jb0x3jyvadGVpo8aZSn53NZduPbPdbx9foQsAGOmKBz7Hai9B38Kf+7eXbxXFcXoXxcTyNaZeDhoA0agoBQldA==", + "version": "1.34.5", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.34.5.tgz", + "integrity": "sha512-Hs31rd+ane6GQJClM77KfNnf49DenK7yRU9MC4JeHwpNIFCSww/XUoGyKleAVrylw6Snrep2Sm+BhD2CS5ARkg==", "license": "MPL-2.0", "peer": true, "workspaces": [ @@ -12069,7 +12069,7 @@ }, "packages/ac3": { "name": "@mediabunny/ac3", - "version": "1.34.4", + "version": "1.35.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" @@ -12084,7 +12084,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.34.4", + "version": "1.35.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" diff --git a/package.json b/package.json index 6700376..f4cdf84 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.34.4", + "version": "1.35.0", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ diff --git a/packages/ac3/package.json b/packages/ac3/package.json index 225fc05..35a3339 100644 --- a/packages/ac3/package.json +++ b/packages/ac3/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/ac3", "author": "Vanilagy", - "version": "1.34.4", + "version": "1.35.0", "description": "AC-3 and E-AC-3 (Dolby Digital) decoder and encoder extension for Mediabunny, based on FFmpeg.", "main": "./dist/bundles/mediabunny-ac3.mjs", "module": "./dist/bundles/mediabunny-ac3.mjs", @@ -18,6 +18,9 @@ "dist", "src" ], + "browser": { + "worker_threads": false + }, "sideEffects": false, "license": "MPL-2.0", "repository": { diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index 63e6769..cbe3af7 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.34.4", + "version": "1.35.0", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", @@ -18,6 +18,9 @@ "dist", "src" ], + "browser": { + "worker_threads": false + }, "sideEffects": false, "license": "MPL-2.0", "repository": { diff --git a/scripts/esbuild/inlined-workers.ts b/scripts/esbuild/inlined-workers.ts index f93098d..79a5bd8 100644 --- a/scripts/esbuild/inlined-workers.ts +++ b/scripts/esbuild/inlined-workers.ts @@ -39,11 +39,10 @@ export default async function inlineWorker(scriptText) { // Node, Bun (Bun's Worker is flaky, worker_threads works much better) let Worker; - const workerModule = 'node:worker_threads'; try { - Worker = (await import(workerModule)).Worker; + Worker = (await import('worker_threads')).Worker; } catch { - Worker = require(workerModule).Worker; + Worker = require('worker_threads').Worker; } const worker = new Worker(scriptText, { eval: true }); @@ -56,6 +55,9 @@ export default async function inlineWorker(scriptText) { build.onResolve({ filter: /^__inline-worker$/ }, ({ path }) => { return { path, namespace: 'inline-worker' }; }); + build.onResolve({ filter: /^worker_threads$/ }, ({ path }) => { + return { path, external: true }; // Keep it in the bundle + }); build.onLoad({ filter: /.*/, namespace: 'inline-worker' }, () => { return { contents: inlineWorkerFunctionCode, loader: 'js' }; }); diff --git a/src/adts/adts-muxer.ts b/src/adts/adts-muxer.ts index 0cadd8a..f004ae3 100644 --- a/src/adts/adts-muxer.ts +++ b/src/adts/adts-muxer.ts @@ -111,5 +111,8 @@ export class AdtsMuxer extends Muxer { throw new Error('ADTS does not support subtitles.'); } - async finalize() {} + async finalize() { + const release = await this.mutex.acquire(); // Required so that finalize() can't resolve before other calls + release(); + } } diff --git a/src/codec-data.ts b/src/codec-data.ts index 5f2ffb5..0cb5135 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -18,6 +18,7 @@ import { last, readExpGolomb, readSignedExpGolomb, + Rational, textDecoder, textEncoder, toDataView, @@ -494,6 +495,7 @@ export type AvcSpsInfo = { codedHeight: number; displayWidth: number; displayHeight: number; + pixelAspectRatio: Rational; colourPrimaries: number; transferCharacteristics: number; matrixCoefficients: number; @@ -502,6 +504,25 @@ export type AvcSpsInfo = { maxDecFrameBuffering: number; }; +const AVC_HEVC_ASPECT_RATIO_IDC_TABLE: Partial> = { + 1: { num: 1, den: 1 }, + 2: { num: 12, den: 11 }, + 3: { num: 10, den: 11 }, + 4: { num: 16, den: 11 }, + 5: { num: 40, den: 33 }, + 6: { num: 24, den: 11 }, + 7: { num: 20, den: 11 }, + 8: { num: 32, den: 11 }, + 9: { num: 80, den: 33 }, + 10: { num: 18, den: 11 }, + 11: { num: 15, den: 11 }, + 12: { num: 64, den: 33 }, + 13: { num: 160, den: 99 }, + 14: { num: 4, den: 3 }, + 15: { num: 3, den: 2 }, + 16: { num: 2, den: 1 }, +}; + /** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { try { @@ -634,6 +655,7 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { let transferCharacteristics = 2; let matrixCoefficients = 2; let fullRangeFlag = 0; + let pixelAspectRatio: Rational = { num: 1, den: 1 }; let numReorderFrames: number | null = null; let maxDecFrameBuffering: number | null = null; @@ -643,9 +665,17 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { const aspectRatioInfoPresentFlag = bitstream.readBits(1); if (aspectRatioInfoPresentFlag) { const aspectRatioIdc = bitstream.readBits(8); + if (aspectRatioIdc === 255) { // Extended_SAR - bitstream.skipBits(16); // sar_width - bitstream.skipBits(16); // sar_height + pixelAspectRatio = { + num: bitstream.readBits(16), + den: bitstream.readBits(16), + }; + } else { + const aspectRatio = AVC_HEVC_ASPECT_RATIO_IDC_TABLE[aspectRatioIdc]; + if (aspectRatio) { + pixelAspectRatio = aspectRatio; + } } } @@ -756,6 +786,7 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { codedHeight, displayWidth, displayHeight, + pixelAspectRatio, colourPrimaries, matrixCoefficients, transferCharacteristics, @@ -815,6 +846,7 @@ export type HevcDecoderConfigurationRecord = { export type HevcSpsInfo = { displayWidth: number; displayHeight: number; + pixelAspectRatio: Rational; colourPrimaries: number; transferCharacteristics: number; matrixCoefficients: number; @@ -962,9 +994,11 @@ export const parseHevcSps = (sps: Uint8Array): HevcSpsInfo | null => { let matrixCoefficients = 2; let fullRangeFlag = 0; let minSpatialSegmentationIdc = 0; + let pixelAspectRatio: Rational = { num: 1, den: 1 }; if (bitstream.readBits(1)) { // vui_parameters_present_flag const vui = parseHevcVui(bitstream, spsMaxSubLayersMinus1); + pixelAspectRatio = vui.pixelAspectRatio; colourPrimaries = vui.colourPrimaries; transferCharacteristics = vui.transferCharacteristics; matrixCoefficients = vui.matrixCoefficients; @@ -975,6 +1009,7 @@ export const parseHevcSps = (sps: Uint8Array): HevcSpsInfo | null => { return { displayWidth, displayHeight, + pixelAspectRatio, colourPrimaries, transferCharacteristics, matrixCoefficients, @@ -1259,12 +1294,20 @@ const parseHevcVui = (bitstream: Bitstream, sps_max_sub_layers_minus1: number) = let matrixCoefficients = 2; let fullRangeFlag = 0; let minSpatialSegmentationIdc = 0; + let pixelAspectRatio: Rational = { num: 1, den: 1 }; if (bitstream.readBits(1)) { // aspect_ratio_info_present_flag const aspect_ratio_idc = bitstream.readBits(8); if (aspect_ratio_idc === 255) { - bitstream.readBits(16); // sar_width - bitstream.readBits(16); // sar_height + pixelAspectRatio = { + num: bitstream.readBits(16), + den: bitstream.readBits(16), + }; + } else { + const aspectRatio = AVC_HEVC_ASPECT_RATIO_IDC_TABLE[aspect_ratio_idc]; + if (aspectRatio) { + pixelAspectRatio = aspectRatio; + } } } if (bitstream.readBits(1)) { // overscan_info_present_flag @@ -1314,6 +1357,7 @@ const parseHevcVui = (bitstream: Bitstream, sps_max_sub_layers_minus1: number) = } return { + pixelAspectRatio, colourPrimaries, transferCharacteristics, matrixCoefficients, diff --git a/src/conversion.ts b/src/conversion.ts index 0a4ac13..14bc516 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -882,12 +882,13 @@ export class Conversion { let videoSource: VideoSource; const totalRotation = normalizeRotation(track.rotation + (trackOptions.rotate ?? 0)); + let outputTrackRotation = totalRotation; const canUseRotationMetadata = this.output.format.supportsVideoRotationMetadata && (trackOptions.allowRotationMetadata ?? true); const [rotatedWidth, rotatedHeight] = totalRotation % 180 === 0 - ? [track.codedWidth, track.codedHeight] - : [track.codedHeight, track.codedWidth]; + ? [track.squarePixelWidth, track.squarePixelHeight] + : [track.squarePixelHeight, track.squarePixelWidth]; const crop = trackOptions.crop; if (crop) { @@ -917,29 +918,27 @@ export class Conversion { } const firstTimestamp = await track.getFirstTimestamp(); + let videoCodecs = this.output.format.getSupportedVideoCodecs(); + const needsTranscode = !!trackOptions.forceTranscode || firstTimestamp < this._startTimestamp || !!trackOptions.frameRate || trackOptions.keyFrameInterval !== undefined - || trackOptions.process !== undefined; - let needsRerender = width !== originalWidth + || trackOptions.process !== undefined + || trackOptions.bitrate !== undefined + || !videoCodecs.includes(sourceCodec) + || (trackOptions.codec && trackOptions.codec !== sourceCodec) + || width !== originalWidth || height !== originalHeight // TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not // performance-optimal, but right now there's no other way because we can't change the track rotation // metadata after the output has already started. Should be possible with API changes in v2, though! - || (totalRotation !== 0 && (!canUseRotationMetadata || trackOptions.process !== undefined)) + || (totalRotation !== 0 && !canUseRotationMetadata) || !!crop; const alpha = trackOptions.alpha ?? 'discard'; - let videoCodecs = this.output.format.getSupportedVideoCodecs(); - if ( - !needsTranscode - && !trackOptions.bitrate - && !needsRerender - && videoCodecs.includes(sourceCodec) - && (!trackOptions.codec || trackOptions.codec === sourceCodec) - ) { + if (!needsTranscode) { // Fast path, we can simply copy over the encoded packets const source = new EncodedVideoPacketSource(sourceCodec); @@ -1026,6 +1025,14 @@ export class Conversion { const source = new VideoSampleSource(encodingConfig); videoSource = source; + let needsRerender = width !== originalWidth + || height !== originalHeight + || (totalRotation !== 0 && (!canUseRotationMetadata || trackOptions.process !== undefined)) + || !!crop + // Don't expect encoders to reliably handle non-square pixels: + || track.squarePixelWidth !== track.codedWidth + || track.squarePixelHeight !== track.codedHeight; + if (!needsRerender) { // If we're directly passing decoded samples back to the encoder, sometimes the encoder may error due // to lack of support of certain video frame formats, like when HDR is at play. To check for this, we @@ -1079,6 +1086,8 @@ export class Conversion { const iterator = sink.canvases(this._startTimestamp, this._endTimestamp); const frameRate = trackOptions.frameRate; + outputTrackRotation = 0; // Since the rotation is baked into the output + let lastCanvas: HTMLCanvasElement | OffscreenCanvas | null = null; let lastCanvasTimestamp: number | null = null; let lastCanvasEndTimestamp: number | null = null; @@ -1242,7 +1251,7 @@ export class Conversion { languageCode: isIso639Dash2LanguageCode(track.languageCode) ? track.languageCode : undefined, name: track.name ?? undefined, disposition: track.disposition, - rotation: needsRerender ? 0 : totalRotation, // Rerendering will bake the rotation into the output + rotation: outputTrackRotation, }); this._addedCounts.video++; this._totalTrackCount++; diff --git a/src/encode.ts b/src/encode.ts index 165dc3b..06ac5b2 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -192,6 +192,8 @@ export const buildVideoEncoderConfig = (options: { height: number; bitrate: number | Quality; framerate: number | undefined; + squarePixelWidth?: number; + squarePixelHeight?: number; } & VideoEncodingAdditionalOptions): VideoEncoderConfig => { const resolvedBitrate = options.bitrate instanceof Quality ? options.bitrate._toVideoBitrate(options.codec, options.width, options.height) @@ -206,6 +208,8 @@ export const buildVideoEncoderConfig = (options: { ), width: options.width, height: options.height, + displayWidth: options.squarePixelWidth, + displayHeight: options.squarePixelHeight, bitrate: resolvedBitrate, bitrateMode: options.bitrateMode, alpha: options.alpha ?? 'discard', diff --git a/src/index.ts b/src/index.ts index d353898..4f92765 100644 --- a/src/index.ts +++ b/src/index.ts @@ -115,6 +115,8 @@ export { export { AnyIterable, MaybePromise, + Rational, + Rectangle, Rotation, SetRequired, } from './misc'; diff --git a/src/input-track.ts b/src/input-track.ts index ee10a77..4c31acd 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -11,7 +11,7 @@ import { determineVideoPacketType } from './codec-data'; import { customAudioDecoders, customVideoDecoders } from './custom-coder'; import { Input } from './input'; import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink'; -import { assert, Rotation } from './misc'; +import { assert, Rational, Rotation, simplifyRational } from './misc'; import { TrackType } from './output'; import { EncodedPacket, PacketType } from './packet'; import { TrackDisposition } from './metadata'; @@ -214,6 +214,8 @@ export interface InputVideoTrackBacking extends InputTrackBacking { getCodec(): VideoCodec | null; getCodedWidth(): number; getCodedHeight(): number; + getSquarePixelWidth(): number; + getSquarePixelHeight(): number; getRotation(): Rotation; getColorSpace(): Promise; canBeTransparent(): Promise; @@ -229,11 +231,21 @@ export class InputVideoTrack extends InputTrack { /** @internal */ override _backing: InputVideoTrackBacking; + /** + * The pixel aspect ratio of the track's frames, as a rational number in its reduced form. Most videos use + * square pixels (1:1). + */ + readonly pixelAspectRatio: Rational; + /** @internal */ constructor(input: Input, backing: InputVideoTrackBacking) { super(input, backing); this._backing = backing; + this.pixelAspectRatio = simplifyRational({ + num: this._backing.getSquarePixelWidth() * this._backing.getCodedHeight(), + den: this._backing.getSquarePixelHeight() * this._backing.getCodedWidth(), + }); } get type(): TrackType { @@ -259,16 +271,26 @@ export class InputVideoTrack extends InputTrack { return this._backing.getRotation(); } - /** The width in pixels of the track's frames after rotation. */ - get displayWidth() { - const rotation = this._backing.getRotation(); - return rotation % 180 === 0 ? this._backing.getCodedWidth() : this._backing.getCodedHeight(); + /** The width of the track's frames in square pixels, adjusted for pixel aspect ratio but before rotation. */ + get squarePixelWidth() { + return this._backing.getSquarePixelWidth(); } - /** The height in pixels of the track's frames after rotation. */ + /** The height of the track's frames in square pixels, adjusted for pixel aspect ratio but before rotation. */ + get squarePixelHeight() { + return this._backing.getSquarePixelHeight(); + } + + /** The display width of the track's frames in pixels, after aspect ratio adjustment and rotation. */ + get displayWidth() { + const rotation = this._backing.getRotation(); + return rotation % 180 === 0 ? this.squarePixelWidth : this.squarePixelHeight; + } + + /** The display height of the track's frames in pixels, after aspect ratio adjustment and rotation. */ get displayHeight() { const rotation = this._backing.getRotation(); - return rotation % 180 === 0 ? this._backing.getCodedHeight() : this._backing.getCodedWidth(); + return rotation % 180 === 0 ? this.squarePixelHeight : this.squarePixelWidth; } /** Returns the color space of the track's samples. */ diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 63330ba..c2952dd 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -631,9 +631,22 @@ export const videoSampleDescription = ( i16(0xffff), // Pre-defined ], [ VIDEO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec](trackData), + pasp(trackData), colorSpaceIsComplete(trackData.info.decoderConfig.colorSpace) ? colr(trackData) : null, ]); +/** Pixel Aspect Ratio Box: Specifies pixel width:height spacing for non-square pixels. */ +export const pasp = (trackData: IsobmffVideoTrackData) => { + if (trackData.info.pixelAspectRatio.num === trackData.info.pixelAspectRatio.den) { + return null; + } + + return box('pasp', [ + u32(trackData.info.pixelAspectRatio.num), + u32(trackData.info.pixelAspectRatio.den), + ]); +}; + /** Colour Information Box: Specifies the color space of the video. */ export const colr = (trackData: IsobmffVideoTrackData) => box('colr', [ ascii('nclx'), // Colour type diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index cd07457..638205f 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -129,6 +129,8 @@ type InternalTrack = { type: 'video'; width: number; height: number; + squarePixelWidth: number; + squarePixelHeight: number; codec: VideoCodec | null; codecDescription: Uint8Array | null; colorSpace: VideoColorSpaceInit | null; @@ -927,6 +929,8 @@ export class IsobmffDemuxer extends Demuxer { type: 'video', width: -1, height: -1, + squarePixelWidth: -1, + squarePixelHeight: -1, codec: null, codecDescription: null, colorSpace: null, @@ -1004,6 +1008,8 @@ export class IsobmffDemuxer extends Demuxer { track.info.width = readU16Be(slice); track.info.height = readU16Be(slice); + track.info.squarePixelWidth = track.info.width; + track.info.squarePixelHeight = track.info.height; slice.skip(4 + 4 + 4 + 2 + 32 + 2 + 2); @@ -1279,6 +1285,23 @@ export class IsobmffDemuxer extends Demuxer { } as VideoColorSpaceInit; }; break; + case 'pasp': { + const track = this.currentTrack; + if (!track) { + break; + } + assert(track.info?.type === 'video'); + + const num = readU32Be(slice); + const den = readU32Be(slice); + + if (num > den) { + track.info.squarePixelWidth = Math.round(track.info.width * num / den); + } else { + track.info.squarePixelHeight = Math.round(track.info.height * den / num); + } + }; break; + case 'wave': { this.readContiguousBoxes(slice.slice(contentStartPos, boxInfo.contentSize)); }; break; @@ -2967,6 +2990,14 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo return this.internalTrack.info.height; } + getSquarePixelWidth() { + return this.internalTrack.info.squarePixelWidth; + } + + getSquarePixelHeight() { + return this.internalTrack.info.squarePixelHeight; + } + getRotation() { return this.internalTrack.rotation; } @@ -3002,6 +3033,8 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo codec: extractVideoCodecString(this.internalTrack.info), codedWidth: this.internalTrack.info.width, codedHeight: this.internalTrack.info.height, + displayAspectWidth: this.internalTrack.info.squarePixelWidth, + displayAspectHeight: this.internalTrack.info.squarePixelHeight, description: this.internalTrack.info.codecDescription ?? undefined, colorSpace: this.internalTrack.info.colorSpace ?? undefined, }; diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 5074a1d..7c56b7c 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -10,7 +10,7 @@ import { Box, free, ftyp, IsobmffBoxWriter, mdat, mfra, moof, moov, vtta, vttc, import { Muxer } from '../muxer'; import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from '../output'; import { BufferTargetWriter, Writer } from '../writer'; -import { assert, computeRationalApproximation, last, promiseWithResolvers } from '../misc'; +import { assert, computeRationalApproximation, last, promiseWithResolvers, Rational, simplifyRational } from '../misc'; import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from '../output-format'; import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles'; import { @@ -86,6 +86,7 @@ export type IsobmffTrackData = { info: { width: number; height: number; + pixelAspectRatio: Rational; decoderConfig: VideoDecoderConfig; /** * The "Annex B transformation" involves converting the raw packet data from Annex B to @@ -343,6 +344,15 @@ export class IsobmffMuxer extends Muxer { // as the timescale. const timescale = computeRationalApproximation(1 / (track.metadata.frameRate ?? 57600), 1e6).denominator; + const displayAspectWidth = decoderConfig.displayAspectWidth; + const displayAspectHeight = decoderConfig.displayAspectHeight; + const pixelAspectRatio = displayAspectWidth === undefined || displayAspectHeight === undefined + ? { num: 1, den: 1 } + : simplifyRational({ + num: displayAspectWidth * decoderConfig.codedHeight, + den: displayAspectHeight * decoderConfig.codedWidth, + }); + const newTrackData: IsobmffVideoTrackData = { muxer: this, track, @@ -350,6 +360,7 @@ export class IsobmffMuxer extends Muxer { info: { width: decoderConfig.codedWidth, height: decoderConfig.codedHeight, + pixelAspectRatio, decoderConfig: decoderConfig, requiresAnnexBTransformation, }, diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index ba1b736..1edb0d7 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -104,6 +104,9 @@ export enum EBMLId { Video = 0xe0, PixelWidth = 0xb0, PixelHeight = 0xba, + DisplayWidth = 0x54b0, + DisplayHeight = 0x54ba, + DisplayUnit = 0x54b2, AlphaMode = 0x53c0, Audio = 0xe1, SamplingFrequency = 0xb5, diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index c7c8f26..5d5902d 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -204,6 +204,11 @@ type InternalTrack = { type: 'video'; width: number; height: number; + displayWidth: number | null; + displayHeight: number | null; + displayUnit: number | null; + squarePixelWidth: number; + squarePixelHeight: number; rotation: Rotation; codec: VideoCodec | null; codecDescription: Uint8Array | null; @@ -1036,6 +1041,27 @@ export class MatroskaDemuxer extends Demuxer { && this.currentTrack.info.width !== -1 && this.currentTrack.info.height !== -1 ) { + this.currentTrack.info.squarePixelWidth = this.currentTrack.info.width; + this.currentTrack.info.squarePixelHeight = this.currentTrack.info.height; + + if ( + this.currentTrack.info.displayWidth !== null + && this.currentTrack.info.displayHeight !== null + ) { + const num = this.currentTrack.info.displayWidth * this.currentTrack.info.height; + const den = this.currentTrack.info.displayHeight * this.currentTrack.info.width; + + if (num > den) { + this.currentTrack.info.squarePixelWidth = Math.round( + this.currentTrack.info.width * num / den, + ); + } else { + this.currentTrack.info.squarePixelHeight = Math.round( + this.currentTrack.info.height * den / num, + ); + } + } + if (this.currentTrack.codecId === CODEC_STRING_MAP.avc) { this.currentTrack.info.codec = 'avc'; this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; @@ -1137,6 +1163,11 @@ export class MatroskaDemuxer extends Demuxer { type: 'video', width: -1, height: -1, + displayWidth: null, + displayHeight: null, + displayUnit: null, + squarePixelWidth: -1, + squarePixelHeight: -1, rotation: 0, codec: null, codecDescription: null, @@ -1273,6 +1304,24 @@ export class MatroskaDemuxer extends Demuxer { this.currentTrack.info.height = readUnsignedInt(slice, size); }; break; + case EBMLId.DisplayWidth: { + if (this.currentTrack?.info?.type !== 'video') break; + + this.currentTrack.info.displayWidth = readUnsignedInt(slice, size); + }; break; + + case EBMLId.DisplayHeight: { + if (this.currentTrack?.info?.type !== 'video') break; + + this.currentTrack.info.displayHeight = readUnsignedInt(slice, size); + }; break; + + case EBMLId.DisplayUnit: { + if (this.currentTrack?.info?.type !== 'video') break; + + this.currentTrack.info.displayUnit = readUnsignedInt(slice, size); + }; break; + case EBMLId.AlphaMode: { if (this.currentTrack?.info?.type !== 'video') break; @@ -2325,6 +2374,14 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid return this.internalTrack.info.height; } + getSquarePixelWidth() { + return this.internalTrack.info.squarePixelWidth; + } + + getSquarePixelHeight() { + return this.internalTrack.info.squarePixelHeight; + } + getRotation() { return this.internalTrack.info.rotation; } @@ -2384,6 +2441,8 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid }), codedWidth: this.internalTrack.info.width, codedHeight: this.internalTrack.info.height, + displayAspectWidth: this.internalTrack.info.squarePixelWidth, + displayAspectHeight: this.internalTrack.info.squarePixelHeight, description: this.internalTrack.info.codecDescription ?? undefined, colorSpace: this.internalTrack.info.colorSpace ?? undefined, }; diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index d4a73af..6554f20 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -19,7 +19,9 @@ import { keyValueIterator, normalizeRotation, promiseWithResolvers, + Rational, roundToMultiple, + simplifyRational, textEncoder, toUint8Array, uint8ArraysAreEqual, @@ -93,6 +95,7 @@ type MatroskaTrackData = { info: { width: number; height: number; + aspectRatio: Rational | null; decoderConfig: VideoDecoderConfig; alphaMode: boolean; }; @@ -368,10 +371,19 @@ export class MatroskaMuxer extends Muxer { // Convert from clockwise to counter-clockwise const flippedRotation = rotation ? normalizeRotation(-rotation) : 0; + const hasNonSquarePixelAspectRatio + = !!trackData.info.aspectRatio && ( + trackData.info.aspectRatio.num * trackData.info.height + !== trackData.info.aspectRatio.den * trackData.info.width + ); + const colorSpace = trackData.info.decoderConfig.colorSpace; const videoElement: EBMLElement = { id: EBMLId.Video, data: [ { id: EBMLId.PixelWidth, data: trackData.info.width }, { id: EBMLId.PixelHeight, data: trackData.info.height }, + (hasNonSquarePixelAspectRatio ? { id: EBMLId.DisplayWidth, data: trackData.info.aspectRatio!.num } : null), + (hasNonSquarePixelAspectRatio ? { id: EBMLId.DisplayHeight, data: trackData.info.aspectRatio!.den } : null), + (hasNonSquarePixelAspectRatio ? { id: EBMLId.DisplayUnit, data: 3 } : null), // 3 = display aspect ratio trackData.info.alphaMode ? { id: EBMLId.AlphaMode, data: 1 } : null, (colorSpaceIsComplete(colorSpace) ? { @@ -738,12 +750,23 @@ export class MatroskaMuxer extends Muxer { assert(meta.decoderConfig.codedWidth !== undefined); assert(meta.decoderConfig.codedHeight !== undefined); + const displayAspectWidth = meta.decoderConfig.displayAspectWidth; + const displayAspectHeight = meta.decoderConfig.displayAspectHeight; + + const aspectRatio = displayAspectWidth === undefined || displayAspectHeight === undefined + ? null + : simplifyRational({ + num: displayAspectWidth, + den: displayAspectHeight, + }); + const newTrackData: MatroskaVideoTrackData = { track, type: 'video', info: { width: meta.decoderConfig.codedWidth, height: meta.decoderConfig.codedHeight, + aspectRatio, decoderConfig: meta.decoderConfig, alphaMode: !!packet.sideData.alpha, // The first packet determines if this track has alpha or not }, diff --git a/src/media-sink.ts b/src/media-sink.ts index 5afc541..140ee10 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1507,7 +1507,8 @@ export type CanvasSinkOptions = { rotation?: Rotation; /** * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to - * the dimensions of the input video track. Cropping is performed after rotation but before resizing. + * the dimensions of the input video track. Cropping is performed after rotation but before resizing. The crop + * region is in the _display pixel space_ of the underlying video data. */ crop?: CropRectangle; /** @@ -1596,8 +1597,8 @@ export class CanvasSink { const rotation = options.rotation ?? videoTrack.rotation; const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 - ? [videoTrack.codedWidth, videoTrack.codedHeight] - : [videoTrack.codedHeight, videoTrack.codedWidth]; + ? [videoTrack.squarePixelWidth, videoTrack.squarePixelHeight] + : [videoTrack.squarePixelHeight, videoTrack.squarePixelWidth]; const crop = options.crop; if (crop) { diff --git a/src/media-source.ts b/src/media-source.ts index a65a9da..faf6243 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -416,6 +416,8 @@ class VideoEncoderWrapper { const encoderConfig = buildVideoEncoderConfig({ width: videoSample.codedWidth, height: videoSample.codedHeight, + squarePixelWidth: videoSample.squarePixelWidth, + squarePixelHeight: videoSample.squarePixelHeight, ...this.encodingConfig, framerate: this.source._connectedTrack?.metadata.frameRate, }); diff --git a/src/misc.ts b/src/misc.ts index 4dc150a..301e978 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -931,3 +931,69 @@ export const arrayArgmin = (array: T[], getValue: (item: T) => number): numbe return minIndex; }; + +/** + * A rational number; a ratio of two integers. + * @group Miscellaneous + * @public + */ +export type Rational = { + /** The numerator of the rational number. */ + num: number; + /** The denominator of the rational number. */ + den: number; +}; + +export const simplifyRational = (rational: Rational): Rational => { + assert(rational.den !== 0); + + let a = Math.abs(rational.num); + let b = Math.abs(rational.den); + + // Euclidean algorithm + while (b !== 0) { + const t = a % b; + a = b; + b = t; + } + + const gcd = a || 1; + return { + num: rational.num / gcd, + den: rational.den / gcd, + }; +}; + +/** + * Specifies a rectangular region where all quantities must be non-negative integers. + * @group Miscellaneous + * @public + */ +export type Rectangle = { + /** The distance in pixels to the left edge of the rectangle . */ + left: number; + /** The distance in pixels to the top edge of the rectangle. */ + top: number; + /** The width in pixels of the rectangle. */ + width: number; + /** The height in pixels of the rectangle. */ + height: number; +}; + +export const validateRectangle = (rect: Rectangle, propertyPath: string) => { + if (typeof rect !== 'object' || !rect) { + throw new TypeError(`${propertyPath} must be an object.`); + } + if (!Number.isInteger(rect.left) || rect.left < 0) { + throw new TypeError(`${propertyPath}.left must be a non-negative integer.`); + } + if (!Number.isInteger(rect.top) || rect.top < 0) { + throw new TypeError(`${propertyPath}.top must be a non-negative integer.`); + } + if (!Number.isInteger(rect.width) || rect.width < 0) { + throw new TypeError(`${propertyPath}.width must be a non-negative integer.`); + } + if (!Number.isInteger(rect.height) || rect.height < 0) { + throw new TypeError(`${propertyPath}.height must be a non-negative integer.`); + } +}; diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index 771fbe0..3e9a2da 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -99,6 +99,8 @@ type ElementaryStream = { colorSpace: VideoColorSpaceInit; width: number; height: number; + squarePixelWidth: number; + squarePixelHeight: number; reorderSize: number; } | { type: 'audio'; @@ -328,6 +330,8 @@ export class MpegTsDemuxer extends Demuxer { }, width: -1, height: -1, + squarePixelWidth: -1, + squarePixelHeight: -1, reorderSize: -1, }; }; break; @@ -447,8 +451,8 @@ export class MpegTsDemuxer extends Demuxer { if (elementaryStream.info.type === 'video') { // We loop because in some files, the video parameters are not in the first packet while (true) { - const contextAlias = context; - contextAlias.suppliedPacket = null; // TyyyyypeScript 😩 + const contextAlias = context; // TyyyyypeScript 😩 + contextAlias.suppliedPacket = null; await context.markNextPacket(); if (elementaryStream.info.codec === 'avc') { @@ -472,6 +476,20 @@ export class MpegTsDemuxer extends Demuxer { elementaryStream.info.width = spsInfo.displayWidth; elementaryStream.info.height = spsInfo.displayHeight; + if (spsInfo.pixelAspectRatio.num > spsInfo.pixelAspectRatio.den) { + elementaryStream.info.squarePixelWidth = Math.round( + elementaryStream.info.width + * spsInfo.pixelAspectRatio.num / spsInfo.pixelAspectRatio.den, + ); + elementaryStream.info.squarePixelHeight = elementaryStream.info.height; + } else { + elementaryStream.info.squarePixelWidth = elementaryStream.info.width; + elementaryStream.info.squarePixelHeight = Math.round( + elementaryStream.info.height + * spsInfo.pixelAspectRatio.den / spsInfo.pixelAspectRatio.num, + ); + } + elementaryStream.info.colorSpace = { primaries: COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as @@ -511,6 +529,20 @@ export class MpegTsDemuxer extends Demuxer { elementaryStream.info.width = spsInfo.displayWidth; elementaryStream.info.height = spsInfo.displayHeight; + if (spsInfo.pixelAspectRatio.num > spsInfo.pixelAspectRatio.den) { + elementaryStream.info.squarePixelWidth = Math.round( + elementaryStream.info.width + * spsInfo.pixelAspectRatio.num / spsInfo.pixelAspectRatio.den, + ); + elementaryStream.info.squarePixelHeight = elementaryStream.info.height; + } else { + elementaryStream.info.squarePixelWidth = elementaryStream.info.width; + elementaryStream.info.squarePixelHeight = Math.round( + elementaryStream.info.height + * spsInfo.pixelAspectRatio.den / spsInfo.pixelAspectRatio.num, + ); + } + elementaryStream.info.colorSpace = { primaries: COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as @@ -546,6 +578,8 @@ export class MpegTsDemuxer extends Demuxer { }), codedWidth: elementaryStream.info.width, codedHeight: elementaryStream.info.height, + displayAspectWidth: elementaryStream.info.squarePixelWidth, + displayAspectHeight: elementaryStream.info.squarePixelHeight, colorSpace: elementaryStream.info.colorSpace, }; elementaryStream.initialized = true; @@ -1598,6 +1632,14 @@ class MpegTsVideoTrackBacking extends MpegTsTrackBacking implements InputVideoTr return this.elementaryStream.info.height; } + getSquarePixelWidth() { + return this.elementaryStream.info.squarePixelWidth; + } + + getSquarePixelHeight() { + return this.elementaryStream.info.squarePixelHeight; + } + getRotation(): Rotation { return 0; } diff --git a/src/sample.ts b/src/sample.ts index f88a3c8..5a086b8 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -9,16 +9,23 @@ import { assert, clamp, + COLOR_PRIMARIES_MAP, isAllowSharedBufferSource, + MATRIX_COEFFICIENTS_MAP, Rotation, SECOND_TO_MICROSECOND_FACTOR, toDataView, toUint8Array, SetRequired, + TRANSFER_CHARACTERISTICS_MAP, isFirefox, polyfillSymbolDispose, assertNever, isWebKit, + Rational, + simplifyRational, + Rectangle, + validateRectangle, } from './misc'; polyfillSymbolDispose(); @@ -147,6 +154,12 @@ export type VideoSampleInit = { colorSpace?: VideoColorSpaceInit; /** The byte layout of the planes of the frame. */ layout?: PlaneLayout[]; + /** Visible region in the coded frame. When omitted, the rect defaults to `(0, 0, codedWidth, codedHeight)`. */ + visibleRect?: Rectangle | undefined; + /** Width of the frame in pixels after applying aspect ratio adjustments and rotation. */ + displayWidth?: number | undefined; + /** Height of the frame in pixels after applying aspect ratio adjustments and rotation. */ + displayHeight?: number | undefined; }; /** @@ -172,12 +185,19 @@ export class VideoSample implements Disposable { * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format) */ readonly format!: VideoSamplePixelFormat | null; - /** The width of the frame in pixels. */ - readonly codedWidth!: number; - /** The height of the frame in pixels. */ - readonly codedHeight!: number; + /** The visible region of the frame in the coded pixel grid. */ + readonly visibleRect!: Rectangle; + /** The width of the frame in square pixels, before rotation is applied. */ + readonly squarePixelWidth!: number; + /** The height of the frame in square pixels, before rotation is applied. */ + readonly squarePixelHeight!: number; /** The rotation of the frame in degrees, clockwise. */ readonly rotation!: Rotation; + /** + * The pixel aspect ratio of the frame, as a rational number in its reduced form. Most videos use + * square pixels (1:1). + */ + readonly pixelAspectRatio!: Rational; /** * The presentation timestamp of the frame in seconds. May be negative. Frames with negative end timestamps should * not be presented. @@ -188,14 +208,26 @@ export class VideoSample implements Disposable { /** The color space of the frame. */ readonly colorSpace!: VideoSampleColorSpace; - /** The width of the frame in pixels after rotation. */ - get displayWidth() { - return this.rotation % 180 === 0 ? this.codedWidth : this.codedHeight; + /** The width of the frame in pixels. */ + get codedWidth() { + // This is wrong, but the fix is a v2 thing + return this.visibleRect.width; } - /** The height of the frame in pixels after rotation. */ + /** The height of the frame in pixels. */ + get codedHeight() { + // Same here + return this.visibleRect.height; + } + + /** The display width of the frame in pixels, after aspect ratio adjustment and rotation. */ + get displayWidth() { + return this.rotation % 180 === 0 ? this.squarePixelWidth : this.squarePixelHeight; + } + + /** The display height of the frame in pixels, after aspect ratio adjustment and rotation. */ get displayHeight() { - return this.rotation % 180 === 0 ? this.codedHeight : this.codedWidth; + return this.rotation % 180 === 0 ? this.squarePixelHeight : this.squarePixelWidth; } /** The presentation timestamp of the frame in microseconds. */ @@ -268,17 +300,66 @@ export class VideoSample implements Disposable { if (init.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) { throw new TypeError('init.duration, when provided, must be a non-negative number.'); } + if (init.layout !== undefined) { + if (!Array.isArray(init.layout)) { + throw new TypeError('init.layout, when provided, must be an array.'); + } + + for (const plane of init.layout) { + if (!plane || typeof plane !== 'object' || Array.isArray(plane)) { + throw new TypeError('Each entry in init.layout must be an object.'); + } + if (!Number.isInteger(plane.offset) || plane.offset < 0) { + throw new TypeError('plane.offset must be a non-negative integer.'); + } + if (!Number.isInteger(plane.stride) || plane.stride < 0) { + throw new TypeError('plane.stride must be a non-negative integer.'); + } + } + } + if (init.visibleRect !== undefined) { + validateRectangle(init.visibleRect, 'init.visibleRect'); + } + if ( + init.displayWidth !== undefined + && (!Number.isInteger(init.displayWidth) || init.displayWidth <= 0) + ) { + throw new TypeError('init.displayWidth, when provided, must be a positive integer.'); + } + if ( + init.displayHeight !== undefined + && (!Number.isInteger(init.displayHeight) || init.displayHeight <= 0) + ) { + throw new TypeError('init.displayHeight, when provided, must be a positive integer.'); + } + if ((init.displayWidth !== undefined) !== (init.displayHeight !== undefined)) { + throw new TypeError( + 'init.displayWidth and init.displayHeight must be either both provided or both omitted.', + ); + } this._data = toUint8Array(data).slice(); // Copy it this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!); this.format = init.format; - this.codedWidth = init.codedWidth!; - this.codedHeight = init.codedHeight!; this.rotation = init.rotation ?? 0; this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; this.colorSpace = new VideoSampleColorSpace(init.colorSpace); + this.visibleRect = { + left: init.visibleRect?.left ?? 0, + top: init.visibleRect?.top ?? 0, + width: init.visibleRect?.width ?? init.codedWidth!, + height: init.visibleRect?.height ?? init.codedHeight!, + }; + + if (init.displayWidth !== undefined) { + this.squarePixelWidth = this.rotation % 180 === 0 ? init.displayWidth : init.displayHeight!; + this.squarePixelHeight = this.rotation % 180 === 0 ? init.displayHeight! : init.displayWidth; + } else { + this.squarePixelWidth = this.codedWidth; + this.squarePixelHeight = this.codedHeight; + } } else if (typeof VideoFrame !== 'undefined' && data instanceof VideoFrame) { if (init?.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) { throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.'); @@ -289,17 +370,28 @@ export class VideoSample implements Disposable { if (init?.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) { throw new TypeError('init.duration, when provided, must be a non-negative number.'); } + if (init?.visibleRect !== undefined) { + validateRectangle(init.visibleRect, 'init.visibleRect'); + } this._data = data; this._layout = null; this.format = data.format; - // Copying the display dimensions here, assuming no innate VideoFrame rotation - this.codedWidth = data.displayWidth; - this.codedHeight = data.displayHeight; + this.visibleRect = { + left: data.visibleRect?.x ?? 0, + top: data.visibleRect?.y ?? 0, + width: data.visibleRect?.width ?? data.codedWidth, + height: data.visibleRect?.height ?? data.codedHeight, + }; // The VideoFrame's rotation is ignored here. It's still a new field, and I'm not sure of any application // where the browser makes use of it. If a case gets found, I'll add it. this.rotation = init?.rotation ?? 0; + + // Assuming no innate VideoFrame rotation here + this.squarePixelWidth = data.displayWidth; + this.squarePixelHeight = data.displayHeight; + this.timestamp = init?.timestamp ?? data.timestamp / 1e6; this.duration = init?.duration ?? (data.duration ?? 0) / 1e6; this.colorSpace = new VideoSampleColorSpace(data.colorSpace); @@ -367,8 +459,9 @@ export class VideoSample implements Disposable { this._layout = null; this.format = 'RGBX'; - this.codedWidth = width; - this.codedHeight = height; + this.visibleRect = { left: 0, top: 0, width, height }; + this.squarePixelWidth = width; + this.squarePixelHeight = height; this.rotation = init.rotation ?? 0; this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; @@ -382,6 +475,10 @@ export class VideoSample implements Disposable { throw new TypeError('Invalid data type: Must be a BufferSource or CanvasImageSource.'); } + this.pixelAspectRatio = simplifyRational({ + num: this.squarePixelWidth * this.codedHeight, + den: this.squarePixelHeight * this.codedWidth, + }); finalizationRegistry?.register(this, { type: 'video', data: this._data }, this); } @@ -411,6 +508,9 @@ export class VideoSample implements Disposable { duration: this.duration, colorSpace: this.colorSpace, rotation: this.rotation, + visibleRect: this.visibleRect, + displayWidth: this.displayWidth, + displayHeight: this.displayHeight, }); } else { return new VideoSample(this._data, { @@ -421,6 +521,9 @@ export class VideoSample implements Disposable { duration: this.duration, colorSpace: this.colorSpace, rotation: this.rotation, + visibleRect: this.visibleRect, + displayWidth: this.displayWidth, + displayHeight: this.displayHeight, }); } } @@ -755,6 +858,7 @@ export class VideoSample implements Disposable { /** * Specifies the rectangular region of the video sample to crop to. The crop region will automatically be * clamped to the dimensions of the video sample. Cropping is performed after rotation but before resizing. + * The crop region is in the _display pixel space_ of the underlying video data. */ crop?: CropRectangle; }) { @@ -785,8 +889,8 @@ export class VideoSample implements Disposable { const rotation = options.rotation ?? this.rotation; const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 - ? [this.codedWidth, this.codedHeight] - : [this.codedHeight, this.codedWidth]; + ? [this.squarePixelWidth, this.squarePixelHeight] + : [this.squarePixelHeight, this.squarePixelWidth]; if (options.crop) { clampCropRectangle(options.crop, rotatedWidth, rotatedHeight); @@ -849,18 +953,18 @@ export class VideoSample implements Disposable { if (rotation === 90) { [sx, sy, sWidth, sHeight] = [ sy, - this.codedHeight - sx - sWidth, + this.squarePixelHeight - sx - sWidth, sHeight, sWidth, ]; } else if (rotation === 180) { [sx, sy] = [ - this.codedWidth - sx - sWidth, - this.codedHeight - sy - sHeight, + this.squarePixelWidth - sx - sWidth, + this.squarePixelHeight - sy - sHeight, ]; } else if (rotation === 270) { [sx, sy, sWidth, sHeight] = [ - this.codedWidth - sy - sHeight, + this.squarePixelWidth - sy - sHeight, sx, sHeight, sWidth, @@ -948,6 +1052,37 @@ export class VideoSampleColorSpace { /** Creates a new VideoSampleColorSpace. */ constructor(init?: VideoColorSpaceInit) { + if (init !== undefined) { + if (!init || typeof init !== 'object') { + throw new TypeError('init.colorSpace, when provided, must be an object.'); + } + + const primariesValues = Object.keys(COLOR_PRIMARIES_MAP); + if (init.primaries != null && !primariesValues.includes(init.primaries)) { + throw new TypeError( + `init.colorSpace.primaries, when provided, must be one of ${primariesValues.join(', ')}.`, + ); + } + + const transferValues = Object.keys(TRANSFER_CHARACTERISTICS_MAP); + if (init.transfer != null && !transferValues.includes(init.transfer)) { + throw new TypeError( + `init.colorSpace.transfer, when provided, must be one of ${transferValues.join(', ')}.`, + ); + } + + const matrixValues = Object.keys(MATRIX_COEFFICIENTS_MAP); + if (init.matrix != null && !matrixValues.includes(init.matrix)) { + throw new TypeError( + `init.colorSpace.matrix, when provided, must be one of ${matrixValues.join(', ')}.`, + ); + } + + if (init.fullRange != null && typeof init.fullRange !== 'boolean') { + throw new TypeError('init.colorSpace.fullRange, when provided, must be a boolean.'); + } + } + this.primaries = init?.primaries ?? null; this.transfer = init?.transfer ?? null; this.matrix = init?.matrix ?? null; diff --git a/src/writer.ts b/src/writer.ts index da40e1b..73a62db 100644 --- a/src/writer.ts +++ b/src/writer.ts @@ -215,6 +215,7 @@ export class StreamTargetWriter extends Writer { private lastWriteEnd = 0; private lastFlushEnd = 0; private writer: WritableStreamDefaultWriter | null = null; + private writeError: unknown = null; // These variables regard chunked mode: private chunked: boolean; @@ -267,6 +268,11 @@ export class StreamTargetWriter extends Writer { } async flush() { + if (this.writeError !== null) { + // eslint-disable-next-line @typescript-eslint/only-throw-error + throw this.writeError; + } + if (this.pos > this.lastWriteEnd) { // There's a "void" between the last written byte and the next byte we're about to write. Let's pad that // void with zeroes explicitly. @@ -329,11 +335,12 @@ export class StreamTargetWriter extends Writer { throw new Error('Internal error: Monotonicity violation.'); } - // Write out the data immediately void this.writer.write({ type: 'write', data: chunk.data, position: chunk.start, + }).catch((error) => { + this.writeError ??= error; }); this.lastFlushEnd = chunk.start + chunk.data.byteLength; @@ -440,6 +447,8 @@ export class StreamTargetWriter extends Writer { type: 'write', data: chunk.data.subarray(section.start, section.end), position, + }).catch((error) => { + this.writeError ??= error; }); this.lastFlushEnd = chunk.start + section.end; @@ -449,12 +458,18 @@ export class StreamTargetWriter extends Writer { } } - finalize() { + async finalize() { if (this.chunked) { this.tryToFlushChunks(true); } + if (this.writeError !== null) { + // eslint-disable-next-line @typescript-eslint/only-throw-error + throw this.writeError; + } + assert(this.writer); + await this.writer.ready; return this.writer.close(); } diff --git a/test/browser/adts-muxing.test.ts b/test/browser/adts-muxing.test.ts new file mode 100644 index 0000000..51f98b1 --- /dev/null +++ b/test/browser/adts-muxing.test.ts @@ -0,0 +1,130 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { BufferSource, UrlSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { EncodedPacketSink } from '../../src/media-sink.js'; +import { EncodedAudioPacketSource } from '../../src/media-source.js'; +import { Output } from '../../src/output.js'; +import { StreamTarget, type StreamTargetChunk } from '../../src/target.js'; +import { AdtsOutputFormat } from '../../src/output-format.js'; +import { assert } from '../../src/misc.js'; + +const createBufferingStreamTarget = () => { + const written = new Map(); + + const stream = new WritableStream({ + async write(chunk: StreamTargetChunk) { + written.set(chunk.position, chunk.data.slice()); + }, + }); + + const toBuffer = () => { + let maxEnd = 0; + for (const [offset, data] of written) { + maxEnd = Math.max(maxEnd, offset + data.byteLength); + } + const buffer = new Uint8Array(maxEnd); + for (const [offset, data] of written) { + buffer.set(data, offset); + } + return buffer; + }; + + return { stream, toBuffer }; +}; + +test('ADTS with metadata over StreamTarget', async () => { + const target = createBufferingStreamTarget(); + + const output = new Output({ + format: new AdtsOutputFormat(), + target: new StreamTarget(target.stream), + }); + + output.setMetadataTags({ comment: 'Remotion' }); + + const audioSource = new EncodedAudioPacketSource('aac'); + output.addAudioTrack(audioSource); + + await output.start(); + + using input = new Input({ + source: new UrlSource('/sample3.aac'), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const sink = new EncodedPacketSink(audioTrack); + + let isFirst = true; + for await (const packet of sink.packets()) { + await audioSource.add(packet, { + decoderConfig: isFirst ? (await audioTrack.getDecoderConfig())! : undefined, + }); + isFirst = false; + } + + await output.finalize(); + + const buffer = target.toBuffer(); + using outputAsInput = new Input({ + source: new BufferSource(buffer.buffer), + formats: ALL_FORMATS, + }); + + const readTags = await outputAsInput.getMetadataTags(); + expect(readTags.comment).toBe('Remotion'); + + const outputAudioTrack = await outputAsInput.getPrimaryAudioTrack(); + assert(outputAudioTrack); + expect(outputAudioTrack.codec).toBe('aac'); +}); + +// Previously, write handler rejections were silently swallowed and surfaced as +// "Cannot write to a closing writable stream" instead of the actual error. +test('StreamTarget write errors surface directly', async () => { + let writeCount = 0; + const stream = new WritableStream({ + async write() { + writeCount++; + if (writeCount === 2) { + throw new Error('OPFS write failed'); + } + }, + }); + + const output = new Output({ + format: new AdtsOutputFormat(), + target: new StreamTarget(stream), + }); + + const audioSource = new EncodedAudioPacketSource('aac'); + output.addAudioTrack(audioSource); + + await output.start(); + + using input = new Input({ + source: new UrlSource('/sample3.aac'), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const sink = new EncodedPacketSink(audioTrack); + + const run = async () => { + let isFirst = true; + for await (const packet of sink.packets()) { + await audioSource.add(packet, { + decoderConfig: isFirst ? (await audioTrack.getDecoderConfig())! : undefined, + }); + isFirst = false; + } + await output.finalize(); + }; + + await expect(run()).rejects.toThrow('OPFS write failed'); +}); diff --git a/test/browser/par.test.ts b/test/browser/par.test.ts new file mode 100644 index 0000000..744999d --- /dev/null +++ b/test/browser/par.test.ts @@ -0,0 +1,185 @@ +import { expect, test } from 'vitest'; +import { Conversion } from '../../src/conversion.js'; +import { ALL_FORMATS, MATROSKA, MP4, MPEG_TS } from '../../src/input-format.js'; +import { Input } from '../../src/input.js'; +import { VideoSampleSink } from '../../src/media-sink.js'; +import { assert } from '../../src/misc.js'; +import { Output } from '../../src/output.js'; +import { MkvOutputFormat, Mp4OutputFormat, MpegTsOutputFormat } from '../../src/output-format.js'; +import { BufferSource, UrlSource } from '../../src/source.js'; +import { BufferTarget } from '../../src/target.js'; + +const SOURCE_PATH = '/sar_2x1.mp4'; + +test('Pixel aspect ratio reading', async () => { + using input = new Input({ + source: new UrlSource(SOURCE_PATH), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MP4); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + expect(videoTrack.rotation).toBe(0); + expectPar2x1Geometry(videoTrack); + + const decoderConfig = await videoTrack.getDecoderConfig(); + assert(decoderConfig); + expect(decoderConfig.displayAspectWidth).toBe(videoTrack.squarePixelWidth); + expect(decoderConfig.displayAspectHeight).toBe(videoTrack.squarePixelHeight); + + const sink = new VideoSampleSink(videoTrack); + using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + + expect(sample.rotation).toBe(0); + expect(sample.visibleRect.width).toBe(sample.codedWidth); + expect(sample.visibleRect.height).toBe(sample.codedHeight); + expect(sample.codedWidth).toBe(videoTrack.codedWidth); + expect(sample.codedHeight).toBe(videoTrack.codedHeight); + expectPar2x1Geometry(sample); + expect(sample.squarePixelWidth).toBe(videoTrack.squarePixelWidth); + expect(sample.squarePixelHeight).toBe(videoTrack.squarePixelHeight); +}); + +test('Pixel aspect ratio copy conversion', async () => { + using input = new Input({ + source: new UrlSource(SOURCE_PATH), + formats: ALL_FORMATS, + }); + + const sourceTrack = await input.getPrimaryVideoTrack(); + assert(sourceTrack); + const expected = await snapshotTrack(sourceTrack); + expectPar2x1Geometry(expected); + + const mkv = await convertAndReadVideoTrack(new MkvOutputFormat()); + expect(mkv.format).toBe(MATROSKA); + expectPar2x1Geometry(mkv.snapshot); + expect(mkv.snapshot).toEqual(expected); + + const ts = await convertAndReadVideoTrack(new MpegTsOutputFormat()); + expect(ts.format).toBe(MPEG_TS); + expectPar2x1Geometry(ts.snapshot); + expect(ts.snapshot).toEqual(expected); +}); + +test('Pixel aspect ratio transcode conversion', async () => { + using input = new Input({ + source: new UrlSource(SOURCE_PATH), + formats: ALL_FORMATS, + }); + + const sourceTrack = await input.getPrimaryVideoTrack(); + assert(sourceTrack); + const source = await snapshotTrack(sourceTrack); + expectPar2x1Geometry(source); + + const mp4 = await convertAndReadVideoTrack( + new Mp4OutputFormat(), + true, + ); + + expect(mp4.format).toBe(MP4); + expect(mp4.snapshot.pixelAspectRatio).toEqual({ num: 1, den: 1 }); + expect(mp4.snapshot.squarePixelWidth).toBe(mp4.snapshot.codedWidth); + expect(mp4.snapshot.squarePixelHeight).toBe(mp4.snapshot.codedHeight); + expect(mp4.snapshot.displayWidth).toBe(mp4.snapshot.codedWidth); + expect(mp4.snapshot.displayHeight).toBe(mp4.snapshot.codedHeight); + expect(mp4.snapshot.decoderDisplayAspectWidth).toBe(mp4.snapshot.codedWidth); + expect(mp4.snapshot.decoderDisplayAspectHeight).toBe(mp4.snapshot.codedHeight); + + expect(mp4.snapshot.codedWidth).toBe(source.squarePixelWidth); + expect(mp4.snapshot.codedHeight).toBe(source.squarePixelHeight); + expect(mp4.snapshot.displayWidth).toBe(source.displayWidth); + expect(mp4.snapshot.displayHeight).toBe(source.displayHeight); +}); + +const expectPar2x1Geometry = (value: { + codedWidth: number; + codedHeight: number; + squarePixelWidth: number; + squarePixelHeight: number; + displayWidth: number; + displayHeight: number; + pixelAspectRatio: { num: number; den: number }; +}) => { + expect(value.pixelAspectRatio).toEqual({ num: 2, den: 1 }); + expect(value.squarePixelWidth).toBe(value.codedWidth * 2); + expect(value.squarePixelHeight).toBe(value.codedHeight); + expect(value.displayWidth).toBe(value.squarePixelWidth); + expect(value.displayHeight).toBe(value.squarePixelHeight); +}; + +const snapshotTrack = async (track: { + codedWidth: number; + codedHeight: number; + squarePixelWidth: number; + squarePixelHeight: number; + displayWidth: number; + displayHeight: number; + rotation: number; + pixelAspectRatio: { num: number; den: number }; + getDecoderConfig(): Promise; +}) => { + const decoderConfig = await track.getDecoderConfig(); + assert(decoderConfig); + + return { + codedWidth: track.codedWidth, + codedHeight: track.codedHeight, + squarePixelWidth: track.squarePixelWidth, + squarePixelHeight: track.squarePixelHeight, + displayWidth: track.displayWidth, + displayHeight: track.displayHeight, + rotation: track.rotation, + pixelAspectRatio: track.pixelAspectRatio, + decoderDisplayAspectWidth: decoderConfig.displayAspectWidth, + decoderDisplayAspectHeight: decoderConfig.displayAspectHeight, + }; +}; + +const convertAndReadVideoTrack = async ( + format: MkvOutputFormat | MpegTsOutputFormat | Mp4OutputFormat, + forceTranscode = false, +) => { + using input = new Input({ + source: new UrlSource(SOURCE_PATH), + formats: ALL_FORMATS, + }); + + const output = new Output({ + format, + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ + input, + output, + video: { + forceTranscode, + }, + trim: { + end: 1, + }, + }); + expect(conversion.isValid).toBe(true); + await conversion.execute(); + + const buffer = output.target.buffer; + assert(buffer); + + using outputInput = new Input({ + source: new BufferSource(buffer), + formats: ALL_FORMATS, + }); + + const track = await outputInput.getPrimaryVideoTrack(); + assert(track); + + return { + format: await outputInput.getFormat(), + snapshot: await snapshotTrack(track), + }; +}; diff --git a/test/node/mpeg-ts-demuxing.test.ts b/test/node/mpeg-ts-demuxing.test.ts index dbff147..b23dc19 100644 --- a/test/node/mpeg-ts-demuxing.test.ts +++ b/test/node/mpeg-ts-demuxing.test.ts @@ -46,6 +46,8 @@ test('MPEG-TS metadata reading', async () => { codec: 'avc1.640020', codedWidth: 720, codedHeight: 720, + displayAspectWidth: 720, + displayAspectHeight: 720, colorSpace: { primaries: 'bt2020', transfer: 'hlg', @@ -559,6 +561,8 @@ test('MPEG-TS with HEVC video', async () => { codec: 'hev1.1.6.L120.90', codedWidth: 1920, codedHeight: 1080, + displayAspectWidth: 1920, + displayAspectHeight: 1080, colorSpace: { primaries: 'bt709', transfer: 'bt709', diff --git a/test/public/sar_2x1.mp4 b/test/public/sar_2x1.mp4 new file mode 100644 index 0000000..3c4a74a Binary files /dev/null and b/test/public/sar_2x1.mp4 differ