diff --git a/dev/convert.html b/dev/convert.html index 0609f8d..96c013a 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -26,7 +26,7 @@ chunked: true, chunkSize: 2**20 }); - const outputFormat = new Mediabunny.MpegTsOutputFormat({}); + const outputFormat = new Mediabunny.Mp4OutputFormat({}); const button = document.createElement('button'); button.textContent = 'Cancel'; @@ -104,8 +104,9 @@ }, */ video: () => ({ - forceTranscode: true, - width: 1280, + //forceTranscode: true, + //forceTranscode: true, + //width: 1280, //forceTranscode: true, //allowRotationMetadata: false, //width: 720, diff --git a/dev/demux.html b/dev/demux.html index ab3c3ac..233aae3 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -17,18 +17,11 @@ source: new Mediabunny.BlobSource(file), }); - for (let i = 0; i < 1000; i++) { - const track = await input.getPrimaryAudioTrack(); - const sink = new Mediabunny.AudioSampleSink(track); - - for await (const sample of sink.samples()) { - sample.close(); - } + const videoTrack = await input.getPrimaryVideoTrack(); + const sink = new Mediabunny.VideoSampleSink(videoTrack); - console.log(i); - } - - console.log("Done") + const sample = await sink.getSample(await videoTrack.getFirstTimestamp()); + console.log(sample); /* diff --git a/docs/guide/media-sinks.md b/docs/guide/media-sinks.md index ac76161..84747cb 100644 --- a/docs/guide/media-sinks.md +++ b/docs/guide/media-sinks.md @@ -350,7 +350,7 @@ type CanvasSinkOptions = { - `rotation`\ The clockwise rotation by which to rotate the raw video frame. Defaults to the rotation set in the file metadata. Rotation is applied before cropping and resizing. - `crop`\ - Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to the dimensions of the input video track. Cropping is performed after rotation but before resizing. + Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to the dimensions of the input video track. Cropping is performed after rotation but before resizing. The crop region is in the _display pixel space_ of the underlying video data. - `poolSize`\ See [Canvas pool](#canvas-pool). @@ -374,7 +374,7 @@ new CanvasSink(videoTrack, { fit: 'cover', }); -// This sink yields canvases with the unaltered coded dimensions of the track, +// This sink yields canvases with the unrotated dimensions of the track, // and without applying any rotation. new CanvasSink(videoTrack, { rotation: 0, diff --git a/docs/guide/packets-and-samples.md b/docs/guide/packets-and-samples.md index e16d1c2..85247ee 100644 --- a/docs/guide/packets-and-samples.md +++ b/docs/guide/packets-and-samples.md @@ -281,6 +281,10 @@ videoSample.format; // => VideoPixelFormat | null videoSample.codedWidth; // => number videoSample.codedHeight; // => number +// Pixel aspect ratio-corrected dimensions of the sample +videoSample.squarePixelWidth; +videoSample.squarePixelHeight; + // Transformed display dimensions of the sample (after rotation) videoSample.displayWidth; // => number videoSample.displayHeight; // => number @@ -289,6 +293,9 @@ videoSample.displayHeight; // => number // rotated by this amount when it is presented. videoSample.rotation; // => 0 | 90 | 180 | 270 +// The sample's pixel aspect ratio +videoSample.pixelAspectRatio; // => { num: number, den: number } + // Timing information videoSample.timestamp; // => Presentation timestamp in seconds videoSample.duration; // => Duration in seconds @@ -297,6 +304,8 @@ videoSample.microsecondDuration; // => Duration in microseconds // Color space of the sample videoSample.colorSpace; // => VideoColorSpace + +videoSample.visibleRect; // Rectangle ``` While all of these properties are read-only, you can use the `setTimestamp`, `setDuration` and `setRotation` methods to modify some of the metadata of the video sample. diff --git a/docs/guide/reading-media-files.md b/docs/guide/reading-media-files.md index a3e1a9e..8193df6 100644 --- a/docs/guide/reading-media-files.md +++ b/docs/guide/reading-media-files.md @@ -216,17 +216,26 @@ This will only look at the first ~50 packets and then return the result. This is In addition to the [common track metadata](#common-track-metadata), video tracks have additional metadata you can query: ```ts -// Get the raw pixel dimensions of the track's coded samples, before rotation: +// Get the raw pixel dimensions of the track's coded samples: videoTrack.codedWidth; // => number videoTrack.codedHeight; // => number -// Get the displayed pixel dimensions of the track's samples, after rotation: +// Get the pixel dimensions of the track after aspect ratio adjustments, +// but before rotation: +videoTrack.squarePixelWidth; // => number +videoTrack.squarePixelHeight; // => number + +// Get the displayed pixel dimensions of the track's samples, after +// aspect ratio adjustments and rotation: videoTrack.displayWidth; // => number videoTrack.displayHeight; // => number // Get the clockwise rotation in degrees by which the // track's frames should be rotated: videoTrack.rotation; // => 0 | 90 | 180 | 270 + +// Get the aspect ratio of the track's pixels (usually 1:1): +videoTrack.pixelAspectRatio; // => { num: number, den: number } ``` To compute a video track's average frame rate (FPS), use [`computePacketStats`](#packet-statistics): diff --git a/examples/metadata-extraction/metadata-extraction.ts b/examples/metadata-extraction/metadata-extraction.ts index 61cf332..ace5e9e 100644 --- a/examples/metadata-extraction/metadata-extraction.ts +++ b/examples/metadata-extraction/metadata-extraction.ts @@ -57,6 +57,9 @@ const extractMetadata = (resource: File | string) => { 'Coded width': `${track.codedWidth} pixels`, 'Coded height': `${track.codedHeight} pixels`, 'Rotation': `${track.rotation}° clockwise`, + 'Pixel aspect ratio': `${track.pixelAspectRatio.num}:${track.pixelAspectRatio.den}`, + 'Display width': `${track.displayWidth} pixels`, + 'Display height': `${track.displayHeight} pixels`, 'Transparency': track.canBeTransparent(), } : track.isAudioTrack() diff --git a/src/codec-data.ts b/src/codec-data.ts index 5f2ffb5..0cb5135 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -18,6 +18,7 @@ import { last, readExpGolomb, readSignedExpGolomb, + Rational, textDecoder, textEncoder, toDataView, @@ -494,6 +495,7 @@ export type AvcSpsInfo = { codedHeight: number; displayWidth: number; displayHeight: number; + pixelAspectRatio: Rational; colourPrimaries: number; transferCharacteristics: number; matrixCoefficients: number; @@ -502,6 +504,25 @@ export type AvcSpsInfo = { maxDecFrameBuffering: number; }; +const AVC_HEVC_ASPECT_RATIO_IDC_TABLE: Partial> = { + 1: { num: 1, den: 1 }, + 2: { num: 12, den: 11 }, + 3: { num: 10, den: 11 }, + 4: { num: 16, den: 11 }, + 5: { num: 40, den: 33 }, + 6: { num: 24, den: 11 }, + 7: { num: 20, den: 11 }, + 8: { num: 32, den: 11 }, + 9: { num: 80, den: 33 }, + 10: { num: 18, den: 11 }, + 11: { num: 15, den: 11 }, + 12: { num: 64, den: 33 }, + 13: { num: 160, den: 99 }, + 14: { num: 4, den: 3 }, + 15: { num: 3, den: 2 }, + 16: { num: 2, den: 1 }, +}; + /** Parses an AVC SPS (Sequence Parameter Set) to extract basic information. */ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { try { @@ -634,6 +655,7 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { let transferCharacteristics = 2; let matrixCoefficients = 2; let fullRangeFlag = 0; + let pixelAspectRatio: Rational = { num: 1, den: 1 }; let numReorderFrames: number | null = null; let maxDecFrameBuffering: number | null = null; @@ -643,9 +665,17 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { const aspectRatioInfoPresentFlag = bitstream.readBits(1); if (aspectRatioInfoPresentFlag) { const aspectRatioIdc = bitstream.readBits(8); + if (aspectRatioIdc === 255) { // Extended_SAR - bitstream.skipBits(16); // sar_width - bitstream.skipBits(16); // sar_height + pixelAspectRatio = { + num: bitstream.readBits(16), + den: bitstream.readBits(16), + }; + } else { + const aspectRatio = AVC_HEVC_ASPECT_RATIO_IDC_TABLE[aspectRatioIdc]; + if (aspectRatio) { + pixelAspectRatio = aspectRatio; + } } } @@ -756,6 +786,7 @@ export const parseAvcSps = (sps: Uint8Array): AvcSpsInfo | null => { codedHeight, displayWidth, displayHeight, + pixelAspectRatio, colourPrimaries, matrixCoefficients, transferCharacteristics, @@ -815,6 +846,7 @@ export type HevcDecoderConfigurationRecord = { export type HevcSpsInfo = { displayWidth: number; displayHeight: number; + pixelAspectRatio: Rational; colourPrimaries: number; transferCharacteristics: number; matrixCoefficients: number; @@ -962,9 +994,11 @@ export const parseHevcSps = (sps: Uint8Array): HevcSpsInfo | null => { let matrixCoefficients = 2; let fullRangeFlag = 0; let minSpatialSegmentationIdc = 0; + let pixelAspectRatio: Rational = { num: 1, den: 1 }; if (bitstream.readBits(1)) { // vui_parameters_present_flag const vui = parseHevcVui(bitstream, spsMaxSubLayersMinus1); + pixelAspectRatio = vui.pixelAspectRatio; colourPrimaries = vui.colourPrimaries; transferCharacteristics = vui.transferCharacteristics; matrixCoefficients = vui.matrixCoefficients; @@ -975,6 +1009,7 @@ export const parseHevcSps = (sps: Uint8Array): HevcSpsInfo | null => { return { displayWidth, displayHeight, + pixelAspectRatio, colourPrimaries, transferCharacteristics, matrixCoefficients, @@ -1259,12 +1294,20 @@ const parseHevcVui = (bitstream: Bitstream, sps_max_sub_layers_minus1: number) = let matrixCoefficients = 2; let fullRangeFlag = 0; let minSpatialSegmentationIdc = 0; + let pixelAspectRatio: Rational = { num: 1, den: 1 }; if (bitstream.readBits(1)) { // aspect_ratio_info_present_flag const aspect_ratio_idc = bitstream.readBits(8); if (aspect_ratio_idc === 255) { - bitstream.readBits(16); // sar_width - bitstream.readBits(16); // sar_height + pixelAspectRatio = { + num: bitstream.readBits(16), + den: bitstream.readBits(16), + }; + } else { + const aspectRatio = AVC_HEVC_ASPECT_RATIO_IDC_TABLE[aspect_ratio_idc]; + if (aspectRatio) { + pixelAspectRatio = aspectRatio; + } } } if (bitstream.readBits(1)) { // overscan_info_present_flag @@ -1314,6 +1357,7 @@ const parseHevcVui = (bitstream: Bitstream, sps_max_sub_layers_minus1: number) = } return { + pixelAspectRatio, colourPrimaries, transferCharacteristics, matrixCoefficients, diff --git a/src/conversion.ts b/src/conversion.ts index 0a4ac13..14bc516 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -882,12 +882,13 @@ export class Conversion { let videoSource: VideoSource; const totalRotation = normalizeRotation(track.rotation + (trackOptions.rotate ?? 0)); + let outputTrackRotation = totalRotation; const canUseRotationMetadata = this.output.format.supportsVideoRotationMetadata && (trackOptions.allowRotationMetadata ?? true); const [rotatedWidth, rotatedHeight] = totalRotation % 180 === 0 - ? [track.codedWidth, track.codedHeight] - : [track.codedHeight, track.codedWidth]; + ? [track.squarePixelWidth, track.squarePixelHeight] + : [track.squarePixelHeight, track.squarePixelWidth]; const crop = trackOptions.crop; if (crop) { @@ -917,29 +918,27 @@ export class Conversion { } const firstTimestamp = await track.getFirstTimestamp(); + let videoCodecs = this.output.format.getSupportedVideoCodecs(); + const needsTranscode = !!trackOptions.forceTranscode || firstTimestamp < this._startTimestamp || !!trackOptions.frameRate || trackOptions.keyFrameInterval !== undefined - || trackOptions.process !== undefined; - let needsRerender = width !== originalWidth + || trackOptions.process !== undefined + || trackOptions.bitrate !== undefined + || !videoCodecs.includes(sourceCodec) + || (trackOptions.codec && trackOptions.codec !== sourceCodec) + || width !== originalWidth || height !== originalHeight // TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not // performance-optimal, but right now there's no other way because we can't change the track rotation // metadata after the output has already started. Should be possible with API changes in v2, though! - || (totalRotation !== 0 && (!canUseRotationMetadata || trackOptions.process !== undefined)) + || (totalRotation !== 0 && !canUseRotationMetadata) || !!crop; const alpha = trackOptions.alpha ?? 'discard'; - let videoCodecs = this.output.format.getSupportedVideoCodecs(); - if ( - !needsTranscode - && !trackOptions.bitrate - && !needsRerender - && videoCodecs.includes(sourceCodec) - && (!trackOptions.codec || trackOptions.codec === sourceCodec) - ) { + if (!needsTranscode) { // Fast path, we can simply copy over the encoded packets const source = new EncodedVideoPacketSource(sourceCodec); @@ -1026,6 +1025,14 @@ export class Conversion { const source = new VideoSampleSource(encodingConfig); videoSource = source; + let needsRerender = width !== originalWidth + || height !== originalHeight + || (totalRotation !== 0 && (!canUseRotationMetadata || trackOptions.process !== undefined)) + || !!crop + // Don't expect encoders to reliably handle non-square pixels: + || track.squarePixelWidth !== track.codedWidth + || track.squarePixelHeight !== track.codedHeight; + if (!needsRerender) { // If we're directly passing decoded samples back to the encoder, sometimes the encoder may error due // to lack of support of certain video frame formats, like when HDR is at play. To check for this, we @@ -1079,6 +1086,8 @@ export class Conversion { const iterator = sink.canvases(this._startTimestamp, this._endTimestamp); const frameRate = trackOptions.frameRate; + outputTrackRotation = 0; // Since the rotation is baked into the output + let lastCanvas: HTMLCanvasElement | OffscreenCanvas | null = null; let lastCanvasTimestamp: number | null = null; let lastCanvasEndTimestamp: number | null = null; @@ -1242,7 +1251,7 @@ export class Conversion { languageCode: isIso639Dash2LanguageCode(track.languageCode) ? track.languageCode : undefined, name: track.name ?? undefined, disposition: track.disposition, - rotation: needsRerender ? 0 : totalRotation, // Rerendering will bake the rotation into the output + rotation: outputTrackRotation, }); this._addedCounts.video++; this._totalTrackCount++; diff --git a/src/encode.ts b/src/encode.ts index 165dc3b..06ac5b2 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -192,6 +192,8 @@ export const buildVideoEncoderConfig = (options: { height: number; bitrate: number | Quality; framerate: number | undefined; + squarePixelWidth?: number; + squarePixelHeight?: number; } & VideoEncodingAdditionalOptions): VideoEncoderConfig => { const resolvedBitrate = options.bitrate instanceof Quality ? options.bitrate._toVideoBitrate(options.codec, options.width, options.height) @@ -206,6 +208,8 @@ export const buildVideoEncoderConfig = (options: { ), width: options.width, height: options.height, + displayWidth: options.squarePixelWidth, + displayHeight: options.squarePixelHeight, bitrate: resolvedBitrate, bitrateMode: options.bitrateMode, alpha: options.alpha ?? 'discard', diff --git a/src/index.ts b/src/index.ts index 1ee6ded..b83f7bb 100644 --- a/src/index.ts +++ b/src/index.ts @@ -115,6 +115,8 @@ export { export { AnyIterable, MaybePromise, + Rational, + Rectangle, Rotation, SetRequired, } from './misc'; diff --git a/src/input-track.ts b/src/input-track.ts index 607e56a..d990449 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -11,7 +11,7 @@ import { determineVideoPacketType } from './codec-data'; import { customAudioDecoders, customVideoDecoders } from './custom-coder'; import { Input } from './input'; import { EncodedPacketSink, PacketRetrievalOptions } from './media-sink'; -import { assert, Rotation } from './misc'; +import { assert, Rational, Rotation, simplifyRational } from './misc'; import { TrackType } from './output'; import { EncodedPacket, PacketType } from './packet'; import { TrackDisposition } from './metadata'; @@ -208,6 +208,8 @@ export interface InputVideoTrackBacking extends InputTrackBacking { getCodec(): VideoCodec | null; getCodedWidth(): number; getCodedHeight(): number; + getSquarePixelWidth(): number; + getSquarePixelHeight(): number; getRotation(): Rotation; getColorSpace(): Promise; canBeTransparent(): Promise; @@ -223,11 +225,21 @@ export class InputVideoTrack extends InputTrack { /** @internal */ override _backing: InputVideoTrackBacking; + /** + * The pixel aspect ratio of the track's frames, as a rational number in its reduced form. Most videos use + * square pixels (1:1). + */ + readonly pixelAspectRatio: Rational; + /** @internal */ constructor(input: Input, backing: InputVideoTrackBacking) { super(input, backing); this._backing = backing; + this.pixelAspectRatio = simplifyRational({ + num: this._backing.getSquarePixelWidth() * this._backing.getCodedHeight(), + den: this._backing.getSquarePixelHeight() * this._backing.getCodedWidth(), + }); } get type(): TrackType { @@ -253,16 +265,26 @@ export class InputVideoTrack extends InputTrack { return this._backing.getRotation(); } - /** The width in pixels of the track's frames after rotation. */ - get displayWidth() { - const rotation = this._backing.getRotation(); - return rotation % 180 === 0 ? this._backing.getCodedWidth() : this._backing.getCodedHeight(); + /** The width of the track's frames in square pixels, adjusted for pixel aspect ratio but before rotation. */ + get squarePixelWidth() { + return this._backing.getSquarePixelWidth(); } - /** The height in pixels of the track's frames after rotation. */ + /** The height of the track's frames in square pixels, adjusted for pixel aspect ratio but before rotation. */ + get squarePixelHeight() { + return this._backing.getSquarePixelHeight(); + } + + /** The display width of the track's frames in pixels, after aspect ratio adjustment and rotation. */ + get displayWidth() { + const rotation = this._backing.getRotation(); + return rotation % 180 === 0 ? this.squarePixelWidth : this.squarePixelHeight; + } + + /** The display height of the track's frames in pixels, after aspect ratio adjustment and rotation. */ get displayHeight() { const rotation = this._backing.getRotation(); - return rotation % 180 === 0 ? this._backing.getCodedHeight() : this._backing.getCodedWidth(); + return rotation % 180 === 0 ? this.squarePixelHeight : this.squarePixelWidth; } /** Returns the color space of the track's samples. */ diff --git a/src/isobmff/isobmff-boxes.ts b/src/isobmff/isobmff-boxes.ts index 63330ba..c2952dd 100644 --- a/src/isobmff/isobmff-boxes.ts +++ b/src/isobmff/isobmff-boxes.ts @@ -631,9 +631,22 @@ export const videoSampleDescription = ( i16(0xffff), // Pre-defined ], [ VIDEO_CODEC_TO_CONFIGURATION_BOX[trackData.track.source._codec](trackData), + pasp(trackData), colorSpaceIsComplete(trackData.info.decoderConfig.colorSpace) ? colr(trackData) : null, ]); +/** Pixel Aspect Ratio Box: Specifies pixel width:height spacing for non-square pixels. */ +export const pasp = (trackData: IsobmffVideoTrackData) => { + if (trackData.info.pixelAspectRatio.num === trackData.info.pixelAspectRatio.den) { + return null; + } + + return box('pasp', [ + u32(trackData.info.pixelAspectRatio.num), + u32(trackData.info.pixelAspectRatio.den), + ]); +}; + /** Colour Information Box: Specifies the color space of the video. */ export const colr = (trackData: IsobmffVideoTrackData) => box('colr', [ ascii('nclx'), // Colour type diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index c70efcc..df68ff9 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -129,6 +129,8 @@ type InternalTrack = { type: 'video'; width: number; height: number; + squarePixelWidth: number; + squarePixelHeight: number; codec: VideoCodec | null; codecDescription: Uint8Array | null; colorSpace: VideoColorSpaceInit | null; @@ -846,6 +848,8 @@ export class IsobmffDemuxer extends Demuxer { type: 'video', width: -1, height: -1, + squarePixelWidth: -1, + squarePixelHeight: -1, codec: null, codecDescription: null, colorSpace: null, @@ -923,6 +927,8 @@ export class IsobmffDemuxer extends Demuxer { track.info.width = readU16Be(slice); track.info.height = readU16Be(slice); + track.info.squarePixelWidth = track.info.width; + track.info.squarePixelHeight = track.info.height; slice.skip(4 + 4 + 4 + 2 + 32 + 2 + 2); @@ -1198,6 +1204,23 @@ export class IsobmffDemuxer extends Demuxer { } as VideoColorSpaceInit; }; break; + case 'pasp': { + const track = this.currentTrack; + if (!track) { + break; + } + assert(track.info?.type === 'video'); + + const num = readU32Be(slice); + const den = readU32Be(slice); + + if (num > den) { + track.info.squarePixelWidth = Math.round(track.info.width * num / den); + } else { + track.info.squarePixelHeight = Math.round(track.info.height * den / num); + } + }; break; + case 'wave': { this.readContiguousBoxes(slice.slice(contentStartPos, boxInfo.contentSize)); }; break; @@ -2885,6 +2908,14 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo return this.internalTrack.info.height; } + getSquarePixelWidth() { + return this.internalTrack.info.squarePixelWidth; + } + + getSquarePixelHeight() { + return this.internalTrack.info.squarePixelHeight; + } + getRotation() { return this.internalTrack.rotation; } @@ -2920,6 +2951,8 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo codec: extractVideoCodecString(this.internalTrack.info), codedWidth: this.internalTrack.info.width, codedHeight: this.internalTrack.info.height, + displayAspectWidth: this.internalTrack.info.squarePixelWidth, + displayAspectHeight: this.internalTrack.info.squarePixelHeight, description: this.internalTrack.info.codecDescription ?? undefined, colorSpace: this.internalTrack.info.colorSpace ?? undefined, }; diff --git a/src/isobmff/isobmff-muxer.ts b/src/isobmff/isobmff-muxer.ts index 5074a1d..7c56b7c 100644 --- a/src/isobmff/isobmff-muxer.ts +++ b/src/isobmff/isobmff-muxer.ts @@ -10,7 +10,7 @@ import { Box, free, ftyp, IsobmffBoxWriter, mdat, mfra, moof, moov, vtta, vttc, import { Muxer } from '../muxer'; import { Output, OutputAudioTrack, OutputSubtitleTrack, OutputTrack, OutputVideoTrack } from '../output'; import { BufferTargetWriter, Writer } from '../writer'; -import { assert, computeRationalApproximation, last, promiseWithResolvers } from '../misc'; +import { assert, computeRationalApproximation, last, promiseWithResolvers, Rational, simplifyRational } from '../misc'; import { IsobmffOutputFormatOptions, IsobmffOutputFormat, MovOutputFormat } from '../output-format'; import { inlineTimestampRegex, SubtitleConfig, SubtitleCue, SubtitleMetadata } from '../subtitles'; import { @@ -86,6 +86,7 @@ export type IsobmffTrackData = { info: { width: number; height: number; + pixelAspectRatio: Rational; decoderConfig: VideoDecoderConfig; /** * The "Annex B transformation" involves converting the raw packet data from Annex B to @@ -343,6 +344,15 @@ export class IsobmffMuxer extends Muxer { // as the timescale. const timescale = computeRationalApproximation(1 / (track.metadata.frameRate ?? 57600), 1e6).denominator; + const displayAspectWidth = decoderConfig.displayAspectWidth; + const displayAspectHeight = decoderConfig.displayAspectHeight; + const pixelAspectRatio = displayAspectWidth === undefined || displayAspectHeight === undefined + ? { num: 1, den: 1 } + : simplifyRational({ + num: displayAspectWidth * decoderConfig.codedHeight, + den: displayAspectHeight * decoderConfig.codedWidth, + }); + const newTrackData: IsobmffVideoTrackData = { muxer: this, track, @@ -350,6 +360,7 @@ export class IsobmffMuxer extends Muxer { info: { width: decoderConfig.codedWidth, height: decoderConfig.codedHeight, + pixelAspectRatio, decoderConfig: decoderConfig, requiresAnnexBTransformation, }, diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index ba1b736..1edb0d7 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -104,6 +104,9 @@ export enum EBMLId { Video = 0xe0, PixelWidth = 0xb0, PixelHeight = 0xba, + DisplayWidth = 0x54b0, + DisplayHeight = 0x54ba, + DisplayUnit = 0x54b2, AlphaMode = 0x53c0, Audio = 0xe1, SamplingFrequency = 0xb5, diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index ad1295a..28106c9 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -204,6 +204,11 @@ type InternalTrack = { type: 'video'; width: number; height: number; + displayWidth: number | null; + displayHeight: number | null; + displayUnit: number | null; + squarePixelWidth: number; + squarePixelHeight: number; rotation: Rotation; codec: VideoCodec | null; codecDescription: Uint8Array | null; @@ -1042,6 +1047,27 @@ export class MatroskaDemuxer extends Demuxer { && this.currentTrack.info.width !== -1 && this.currentTrack.info.height !== -1 ) { + this.currentTrack.info.squarePixelWidth = this.currentTrack.info.width; + this.currentTrack.info.squarePixelHeight = this.currentTrack.info.height; + + if ( + this.currentTrack.info.displayWidth !== null + && this.currentTrack.info.displayHeight !== null + ) { + const num = this.currentTrack.info.displayWidth * this.currentTrack.info.height; + const den = this.currentTrack.info.displayHeight * this.currentTrack.info.width; + + if (num > den) { + this.currentTrack.info.squarePixelWidth = Math.round( + this.currentTrack.info.width * num / den, + ); + } else { + this.currentTrack.info.squarePixelHeight = Math.round( + this.currentTrack.info.height * den / num, + ); + } + } + if (this.currentTrack.codecId === CODEC_STRING_MAP.avc) { this.currentTrack.info.codec = 'avc'; this.currentTrack.info.codecDescription = this.currentTrack.codecPrivate; @@ -1143,6 +1169,11 @@ export class MatroskaDemuxer extends Demuxer { type: 'video', width: -1, height: -1, + displayWidth: null, + displayHeight: null, + displayUnit: null, + squarePixelWidth: -1, + squarePixelHeight: -1, rotation: 0, codec: null, codecDescription: null, @@ -1279,6 +1310,24 @@ export class MatroskaDemuxer extends Demuxer { this.currentTrack.info.height = readUnsignedInt(slice, size); }; break; + case EBMLId.DisplayWidth: { + if (this.currentTrack?.info?.type !== 'video') break; + + this.currentTrack.info.displayWidth = readUnsignedInt(slice, size); + }; break; + + case EBMLId.DisplayHeight: { + if (this.currentTrack?.info?.type !== 'video') break; + + this.currentTrack.info.displayHeight = readUnsignedInt(slice, size); + }; break; + + case EBMLId.DisplayUnit: { + if (this.currentTrack?.info?.type !== 'video') break; + + this.currentTrack.info.displayUnit = readUnsignedInt(slice, size); + }; break; + case EBMLId.AlphaMode: { if (this.currentTrack?.info?.type !== 'video') break; @@ -2337,6 +2386,14 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid return this.internalTrack.info.height; } + getSquarePixelWidth() { + return this.internalTrack.info.squarePixelWidth; + } + + getSquarePixelHeight() { + return this.internalTrack.info.squarePixelHeight; + } + getRotation() { return this.internalTrack.info.rotation; } @@ -2396,6 +2453,8 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid }), codedWidth: this.internalTrack.info.width, codedHeight: this.internalTrack.info.height, + displayAspectWidth: this.internalTrack.info.squarePixelWidth, + displayAspectHeight: this.internalTrack.info.squarePixelHeight, description: this.internalTrack.info.codecDescription ?? undefined, colorSpace: this.internalTrack.info.colorSpace ?? undefined, }; diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index d4a73af..6554f20 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -19,7 +19,9 @@ import { keyValueIterator, normalizeRotation, promiseWithResolvers, + Rational, roundToMultiple, + simplifyRational, textEncoder, toUint8Array, uint8ArraysAreEqual, @@ -93,6 +95,7 @@ type MatroskaTrackData = { info: { width: number; height: number; + aspectRatio: Rational | null; decoderConfig: VideoDecoderConfig; alphaMode: boolean; }; @@ -368,10 +371,19 @@ export class MatroskaMuxer extends Muxer { // Convert from clockwise to counter-clockwise const flippedRotation = rotation ? normalizeRotation(-rotation) : 0; + const hasNonSquarePixelAspectRatio + = !!trackData.info.aspectRatio && ( + trackData.info.aspectRatio.num * trackData.info.height + !== trackData.info.aspectRatio.den * trackData.info.width + ); + const colorSpace = trackData.info.decoderConfig.colorSpace; const videoElement: EBMLElement = { id: EBMLId.Video, data: [ { id: EBMLId.PixelWidth, data: trackData.info.width }, { id: EBMLId.PixelHeight, data: trackData.info.height }, + (hasNonSquarePixelAspectRatio ? { id: EBMLId.DisplayWidth, data: trackData.info.aspectRatio!.num } : null), + (hasNonSquarePixelAspectRatio ? { id: EBMLId.DisplayHeight, data: trackData.info.aspectRatio!.den } : null), + (hasNonSquarePixelAspectRatio ? { id: EBMLId.DisplayUnit, data: 3 } : null), // 3 = display aspect ratio trackData.info.alphaMode ? { id: EBMLId.AlphaMode, data: 1 } : null, (colorSpaceIsComplete(colorSpace) ? { @@ -738,12 +750,23 @@ export class MatroskaMuxer extends Muxer { assert(meta.decoderConfig.codedWidth !== undefined); assert(meta.decoderConfig.codedHeight !== undefined); + const displayAspectWidth = meta.decoderConfig.displayAspectWidth; + const displayAspectHeight = meta.decoderConfig.displayAspectHeight; + + const aspectRatio = displayAspectWidth === undefined || displayAspectHeight === undefined + ? null + : simplifyRational({ + num: displayAspectWidth, + den: displayAspectHeight, + }); + const newTrackData: MatroskaVideoTrackData = { track, type: 'video', info: { width: meta.decoderConfig.codedWidth, height: meta.decoderConfig.codedHeight, + aspectRatio, decoderConfig: meta.decoderConfig, alphaMode: !!packet.sideData.alpha, // The first packet determines if this track has alpha or not }, diff --git a/src/media-sink.ts b/src/media-sink.ts index 0f2243d..7054f97 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -1490,7 +1490,8 @@ export type CanvasSinkOptions = { rotation?: Rotation; /** * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to - * the dimensions of the input video track. Cropping is performed after rotation but before resizing. + * the dimensions of the input video track. Cropping is performed after rotation but before resizing. The crop + * region is in the _display pixel space_ of the underlying video data. */ crop?: CropRectangle; /** @@ -1579,8 +1580,8 @@ export class CanvasSink { const rotation = options.rotation ?? videoTrack.rotation; const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 - ? [videoTrack.codedWidth, videoTrack.codedHeight] - : [videoTrack.codedHeight, videoTrack.codedWidth]; + ? [videoTrack.squarePixelWidth, videoTrack.squarePixelHeight] + : [videoTrack.squarePixelHeight, videoTrack.squarePixelWidth]; const crop = options.crop; if (crop) { diff --git a/src/media-source.ts b/src/media-source.ts index a65a9da..faf6243 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -416,6 +416,8 @@ class VideoEncoderWrapper { const encoderConfig = buildVideoEncoderConfig({ width: videoSample.codedWidth, height: videoSample.codedHeight, + squarePixelWidth: videoSample.squarePixelWidth, + squarePixelHeight: videoSample.squarePixelHeight, ...this.encodingConfig, framerate: this.source._connectedTrack?.metadata.frameRate, }); diff --git a/src/misc.ts b/src/misc.ts index eaefa9d..72b33df 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -850,3 +850,69 @@ export const polyfillSymbolDispose = () => { export const isNumber = (x: unknown) => { return typeof x === 'number' && !Number.isNaN(x); }; + +/** + * A rational number; a ratio of two integers. + * @group Miscellaneous + * @public + */ +export type Rational = { + /** The numerator of the rational number. */ + num: number; + /** The denominator of the rational number. */ + den: number; +}; + +export const simplifyRational = (rational: Rational): Rational => { + assert(rational.den !== 0); + + let a = Math.abs(rational.num); + let b = Math.abs(rational.den); + + // Euclidean algorithm + while (b !== 0) { + const t = a % b; + a = b; + b = t; + } + + const gcd = a || 1; + return { + num: rational.num / gcd, + den: rational.den / gcd, + }; +}; + +/** + * Specifies a rectangular region where all quantities must be non-negative integers. + * @group Miscellaneous + * @public + */ +export type Rectangle = { + /** The distance in pixels to the left edge of the rectangle . */ + left: number; + /** The distance in pixels to the top edge of the rectangle. */ + top: number; + /** The width in pixels of the rectangle. */ + width: number; + /** The height in pixels of the rectangle. */ + height: number; +}; + +export const validateRectangle = (rect: Rectangle, propertyPath: string) => { + if (typeof rect !== 'object' || !rect) { + throw new TypeError(`${propertyPath} must be an object.`); + } + if (!Number.isInteger(rect.left) || rect.left < 0) { + throw new TypeError(`${propertyPath}.left must be a non-negative integer.`); + } + if (!Number.isInteger(rect.top) || rect.top < 0) { + throw new TypeError(`${propertyPath}.top must be a non-negative integer.`); + } + if (!Number.isInteger(rect.width) || rect.width < 0) { + throw new TypeError(`${propertyPath}.width must be a non-negative integer.`); + } + if (!Number.isInteger(rect.height) || rect.height < 0) { + throw new TypeError(`${propertyPath}.height must be a non-negative integer.`); + } +}; diff --git a/src/mpeg-ts/mpeg-ts-demuxer.ts b/src/mpeg-ts/mpeg-ts-demuxer.ts index fa8d14a..562ee45 100644 --- a/src/mpeg-ts/mpeg-ts-demuxer.ts +++ b/src/mpeg-ts/mpeg-ts-demuxer.ts @@ -89,6 +89,8 @@ type ElementaryStream = { colorSpace: VideoColorSpaceInit; width: number; height: number; + squarePixelWidth: number; + squarePixelHeight: number; reorderSize: number; } | { type: 'audio'; @@ -327,6 +329,8 @@ export class MpegTsDemuxer extends Demuxer { }, width: -1, height: -1, + squarePixelWidth: -1, + squarePixelHeight: -1, reorderSize: -1, }; }; break; @@ -422,6 +426,21 @@ export class MpegTsDemuxer extends Demuxer { elementaryStream.info.width = spsInfo.displayWidth; elementaryStream.info.height = spsInfo.displayHeight; + + if (spsInfo.pixelAspectRatio.num > spsInfo.pixelAspectRatio.den) { + elementaryStream.info.squarePixelWidth = Math.round( + elementaryStream.info.width + * spsInfo.pixelAspectRatio.num / spsInfo.pixelAspectRatio.den, + ); + elementaryStream.info.squarePixelHeight = elementaryStream.info.height; + } else { + elementaryStream.info.squarePixelWidth = elementaryStream.info.width; + elementaryStream.info.squarePixelHeight = Math.round( + elementaryStream.info.height + * spsInfo.pixelAspectRatio.den / spsInfo.pixelAspectRatio.num, + ); + } + elementaryStream.info.colorSpace = { primaries: COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as VideoColorPrimaries | undefined, @@ -454,6 +473,21 @@ export class MpegTsDemuxer extends Demuxer { elementaryStream.info.width = spsInfo.displayWidth; elementaryStream.info.height = spsInfo.displayHeight; + + if (spsInfo.pixelAspectRatio.num > spsInfo.pixelAspectRatio.den) { + elementaryStream.info.squarePixelWidth = Math.round( + elementaryStream.info.width + * spsInfo.pixelAspectRatio.num / spsInfo.pixelAspectRatio.den, + ); + elementaryStream.info.squarePixelHeight = elementaryStream.info.height; + } else { + elementaryStream.info.squarePixelWidth = elementaryStream.info.width; + elementaryStream.info.squarePixelHeight = Math.round( + elementaryStream.info.height + * spsInfo.pixelAspectRatio.den / spsInfo.pixelAspectRatio.num, + ); + } + elementaryStream.info.colorSpace = { primaries: COLOR_PRIMARIES_MAP_INVERSE[spsInfo.colourPrimaries] as VideoColorPrimaries | undefined, @@ -1481,6 +1515,8 @@ class MpegTsVideoTrackBacking extends MpegTsTrackBacking implements InputVideoTr }), codedWidth: this.elementaryStream.info.width, codedHeight: this.elementaryStream.info.height, + displayAspectWidth: this.elementaryStream.info.squarePixelWidth, + displayAspectHeight: this.elementaryStream.info.squarePixelHeight, colorSpace: this.elementaryStream.info.colorSpace, }; } @@ -1497,6 +1533,14 @@ class MpegTsVideoTrackBacking extends MpegTsTrackBacking implements InputVideoTr return this.elementaryStream.info.height; } + getSquarePixelWidth() { + return this.elementaryStream.info.squarePixelWidth; + } + + getSquarePixelHeight() { + return this.elementaryStream.info.squarePixelHeight; + } + getRotation(): Rotation { return 0; } diff --git a/src/sample.ts b/src/sample.ts index f88a3c8..5a086b8 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -9,16 +9,23 @@ import { assert, clamp, + COLOR_PRIMARIES_MAP, isAllowSharedBufferSource, + MATRIX_COEFFICIENTS_MAP, Rotation, SECOND_TO_MICROSECOND_FACTOR, toDataView, toUint8Array, SetRequired, + TRANSFER_CHARACTERISTICS_MAP, isFirefox, polyfillSymbolDispose, assertNever, isWebKit, + Rational, + simplifyRational, + Rectangle, + validateRectangle, } from './misc'; polyfillSymbolDispose(); @@ -147,6 +154,12 @@ export type VideoSampleInit = { colorSpace?: VideoColorSpaceInit; /** The byte layout of the planes of the frame. */ layout?: PlaneLayout[]; + /** Visible region in the coded frame. When omitted, the rect defaults to `(0, 0, codedWidth, codedHeight)`. */ + visibleRect?: Rectangle | undefined; + /** Width of the frame in pixels after applying aspect ratio adjustments and rotation. */ + displayWidth?: number | undefined; + /** Height of the frame in pixels after applying aspect ratio adjustments and rotation. */ + displayHeight?: number | undefined; }; /** @@ -172,12 +185,19 @@ export class VideoSample implements Disposable { * [See pixel formats](https://www.w3.org/TR/webcodecs/#pixel-format) */ readonly format!: VideoSamplePixelFormat | null; - /** The width of the frame in pixels. */ - readonly codedWidth!: number; - /** The height of the frame in pixels. */ - readonly codedHeight!: number; + /** The visible region of the frame in the coded pixel grid. */ + readonly visibleRect!: Rectangle; + /** The width of the frame in square pixels, before rotation is applied. */ + readonly squarePixelWidth!: number; + /** The height of the frame in square pixels, before rotation is applied. */ + readonly squarePixelHeight!: number; /** The rotation of the frame in degrees, clockwise. */ readonly rotation!: Rotation; + /** + * The pixel aspect ratio of the frame, as a rational number in its reduced form. Most videos use + * square pixels (1:1). + */ + readonly pixelAspectRatio!: Rational; /** * The presentation timestamp of the frame in seconds. May be negative. Frames with negative end timestamps should * not be presented. @@ -188,14 +208,26 @@ export class VideoSample implements Disposable { /** The color space of the frame. */ readonly colorSpace!: VideoSampleColorSpace; - /** The width of the frame in pixels after rotation. */ - get displayWidth() { - return this.rotation % 180 === 0 ? this.codedWidth : this.codedHeight; + /** The width of the frame in pixels. */ + get codedWidth() { + // This is wrong, but the fix is a v2 thing + return this.visibleRect.width; } - /** The height of the frame in pixels after rotation. */ + /** The height of the frame in pixels. */ + get codedHeight() { + // Same here + return this.visibleRect.height; + } + + /** The display width of the frame in pixels, after aspect ratio adjustment and rotation. */ + get displayWidth() { + return this.rotation % 180 === 0 ? this.squarePixelWidth : this.squarePixelHeight; + } + + /** The display height of the frame in pixels, after aspect ratio adjustment and rotation. */ get displayHeight() { - return this.rotation % 180 === 0 ? this.codedHeight : this.codedWidth; + return this.rotation % 180 === 0 ? this.squarePixelHeight : this.squarePixelWidth; } /** The presentation timestamp of the frame in microseconds. */ @@ -268,17 +300,66 @@ export class VideoSample implements Disposable { if (init.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) { throw new TypeError('init.duration, when provided, must be a non-negative number.'); } + if (init.layout !== undefined) { + if (!Array.isArray(init.layout)) { + throw new TypeError('init.layout, when provided, must be an array.'); + } + + for (const plane of init.layout) { + if (!plane || typeof plane !== 'object' || Array.isArray(plane)) { + throw new TypeError('Each entry in init.layout must be an object.'); + } + if (!Number.isInteger(plane.offset) || plane.offset < 0) { + throw new TypeError('plane.offset must be a non-negative integer.'); + } + if (!Number.isInteger(plane.stride) || plane.stride < 0) { + throw new TypeError('plane.stride must be a non-negative integer.'); + } + } + } + if (init.visibleRect !== undefined) { + validateRectangle(init.visibleRect, 'init.visibleRect'); + } + if ( + init.displayWidth !== undefined + && (!Number.isInteger(init.displayWidth) || init.displayWidth <= 0) + ) { + throw new TypeError('init.displayWidth, when provided, must be a positive integer.'); + } + if ( + init.displayHeight !== undefined + && (!Number.isInteger(init.displayHeight) || init.displayHeight <= 0) + ) { + throw new TypeError('init.displayHeight, when provided, must be a positive integer.'); + } + if ((init.displayWidth !== undefined) !== (init.displayHeight !== undefined)) { + throw new TypeError( + 'init.displayWidth and init.displayHeight must be either both provided or both omitted.', + ); + } this._data = toUint8Array(data).slice(); // Copy it this._layout = init.layout ?? createDefaultPlaneLayout(init.format, init.codedWidth!, init.codedHeight!); this.format = init.format; - this.codedWidth = init.codedWidth!; - this.codedHeight = init.codedHeight!; this.rotation = init.rotation ?? 0; this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; this.colorSpace = new VideoSampleColorSpace(init.colorSpace); + this.visibleRect = { + left: init.visibleRect?.left ?? 0, + top: init.visibleRect?.top ?? 0, + width: init.visibleRect?.width ?? init.codedWidth!, + height: init.visibleRect?.height ?? init.codedHeight!, + }; + + if (init.displayWidth !== undefined) { + this.squarePixelWidth = this.rotation % 180 === 0 ? init.displayWidth : init.displayHeight!; + this.squarePixelHeight = this.rotation % 180 === 0 ? init.displayHeight! : init.displayWidth; + } else { + this.squarePixelWidth = this.codedWidth; + this.squarePixelHeight = this.codedHeight; + } } else if (typeof VideoFrame !== 'undefined' && data instanceof VideoFrame) { if (init?.rotation !== undefined && ![0, 90, 180, 270].includes(init.rotation)) { throw new TypeError('init.rotation, when provided, must be 0, 90, 180, or 270.'); @@ -289,17 +370,28 @@ export class VideoSample implements Disposable { if (init?.duration !== undefined && (!Number.isFinite(init.duration) || init.duration < 0)) { throw new TypeError('init.duration, when provided, must be a non-negative number.'); } + if (init?.visibleRect !== undefined) { + validateRectangle(init.visibleRect, 'init.visibleRect'); + } this._data = data; this._layout = null; this.format = data.format; - // Copying the display dimensions here, assuming no innate VideoFrame rotation - this.codedWidth = data.displayWidth; - this.codedHeight = data.displayHeight; + this.visibleRect = { + left: data.visibleRect?.x ?? 0, + top: data.visibleRect?.y ?? 0, + width: data.visibleRect?.width ?? data.codedWidth, + height: data.visibleRect?.height ?? data.codedHeight, + }; // The VideoFrame's rotation is ignored here. It's still a new field, and I'm not sure of any application // where the browser makes use of it. If a case gets found, I'll add it. this.rotation = init?.rotation ?? 0; + + // Assuming no innate VideoFrame rotation here + this.squarePixelWidth = data.displayWidth; + this.squarePixelHeight = data.displayHeight; + this.timestamp = init?.timestamp ?? data.timestamp / 1e6; this.duration = init?.duration ?? (data.duration ?? 0) / 1e6; this.colorSpace = new VideoSampleColorSpace(data.colorSpace); @@ -367,8 +459,9 @@ export class VideoSample implements Disposable { this._layout = null; this.format = 'RGBX'; - this.codedWidth = width; - this.codedHeight = height; + this.visibleRect = { left: 0, top: 0, width, height }; + this.squarePixelWidth = width; + this.squarePixelHeight = height; this.rotation = init.rotation ?? 0; this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; @@ -382,6 +475,10 @@ export class VideoSample implements Disposable { throw new TypeError('Invalid data type: Must be a BufferSource or CanvasImageSource.'); } + this.pixelAspectRatio = simplifyRational({ + num: this.squarePixelWidth * this.codedHeight, + den: this.squarePixelHeight * this.codedWidth, + }); finalizationRegistry?.register(this, { type: 'video', data: this._data }, this); } @@ -411,6 +508,9 @@ export class VideoSample implements Disposable { duration: this.duration, colorSpace: this.colorSpace, rotation: this.rotation, + visibleRect: this.visibleRect, + displayWidth: this.displayWidth, + displayHeight: this.displayHeight, }); } else { return new VideoSample(this._data, { @@ -421,6 +521,9 @@ export class VideoSample implements Disposable { duration: this.duration, colorSpace: this.colorSpace, rotation: this.rotation, + visibleRect: this.visibleRect, + displayWidth: this.displayWidth, + displayHeight: this.displayHeight, }); } } @@ -755,6 +858,7 @@ export class VideoSample implements Disposable { /** * Specifies the rectangular region of the video sample to crop to. The crop region will automatically be * clamped to the dimensions of the video sample. Cropping is performed after rotation but before resizing. + * The crop region is in the _display pixel space_ of the underlying video data. */ crop?: CropRectangle; }) { @@ -785,8 +889,8 @@ export class VideoSample implements Disposable { const rotation = options.rotation ?? this.rotation; const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 - ? [this.codedWidth, this.codedHeight] - : [this.codedHeight, this.codedWidth]; + ? [this.squarePixelWidth, this.squarePixelHeight] + : [this.squarePixelHeight, this.squarePixelWidth]; if (options.crop) { clampCropRectangle(options.crop, rotatedWidth, rotatedHeight); @@ -849,18 +953,18 @@ export class VideoSample implements Disposable { if (rotation === 90) { [sx, sy, sWidth, sHeight] = [ sy, - this.codedHeight - sx - sWidth, + this.squarePixelHeight - sx - sWidth, sHeight, sWidth, ]; } else if (rotation === 180) { [sx, sy] = [ - this.codedWidth - sx - sWidth, - this.codedHeight - sy - sHeight, + this.squarePixelWidth - sx - sWidth, + this.squarePixelHeight - sy - sHeight, ]; } else if (rotation === 270) { [sx, sy, sWidth, sHeight] = [ - this.codedWidth - sy - sHeight, + this.squarePixelWidth - sy - sHeight, sx, sHeight, sWidth, @@ -948,6 +1052,37 @@ export class VideoSampleColorSpace { /** Creates a new VideoSampleColorSpace. */ constructor(init?: VideoColorSpaceInit) { + if (init !== undefined) { + if (!init || typeof init !== 'object') { + throw new TypeError('init.colorSpace, when provided, must be an object.'); + } + + const primariesValues = Object.keys(COLOR_PRIMARIES_MAP); + if (init.primaries != null && !primariesValues.includes(init.primaries)) { + throw new TypeError( + `init.colorSpace.primaries, when provided, must be one of ${primariesValues.join(', ')}.`, + ); + } + + const transferValues = Object.keys(TRANSFER_CHARACTERISTICS_MAP); + if (init.transfer != null && !transferValues.includes(init.transfer)) { + throw new TypeError( + `init.colorSpace.transfer, when provided, must be one of ${transferValues.join(', ')}.`, + ); + } + + const matrixValues = Object.keys(MATRIX_COEFFICIENTS_MAP); + if (init.matrix != null && !matrixValues.includes(init.matrix)) { + throw new TypeError( + `init.colorSpace.matrix, when provided, must be one of ${matrixValues.join(', ')}.`, + ); + } + + if (init.fullRange != null && typeof init.fullRange !== 'boolean') { + throw new TypeError('init.colorSpace.fullRange, when provided, must be a boolean.'); + } + } + this.primaries = init?.primaries ?? null; this.transfer = init?.transfer ?? null; this.matrix = init?.matrix ?? null; diff --git a/test/browser/par.test.ts b/test/browser/par.test.ts new file mode 100644 index 0000000..744999d --- /dev/null +++ b/test/browser/par.test.ts @@ -0,0 +1,185 @@ +import { expect, test } from 'vitest'; +import { Conversion } from '../../src/conversion.js'; +import { ALL_FORMATS, MATROSKA, MP4, MPEG_TS } from '../../src/input-format.js'; +import { Input } from '../../src/input.js'; +import { VideoSampleSink } from '../../src/media-sink.js'; +import { assert } from '../../src/misc.js'; +import { Output } from '../../src/output.js'; +import { MkvOutputFormat, Mp4OutputFormat, MpegTsOutputFormat } from '../../src/output-format.js'; +import { BufferSource, UrlSource } from '../../src/source.js'; +import { BufferTarget } from '../../src/target.js'; + +const SOURCE_PATH = '/sar_2x1.mp4'; + +test('Pixel aspect ratio reading', async () => { + using input = new Input({ + source: new UrlSource(SOURCE_PATH), + formats: ALL_FORMATS, + }); + + expect(await input.getFormat()).toBe(MP4); + + const videoTrack = await input.getPrimaryVideoTrack(); + assert(videoTrack); + + expect(videoTrack.rotation).toBe(0); + expectPar2x1Geometry(videoTrack); + + const decoderConfig = await videoTrack.getDecoderConfig(); + assert(decoderConfig); + expect(decoderConfig.displayAspectWidth).toBe(videoTrack.squarePixelWidth); + expect(decoderConfig.displayAspectHeight).toBe(videoTrack.squarePixelHeight); + + const sink = new VideoSampleSink(videoTrack); + using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + + expect(sample.rotation).toBe(0); + expect(sample.visibleRect.width).toBe(sample.codedWidth); + expect(sample.visibleRect.height).toBe(sample.codedHeight); + expect(sample.codedWidth).toBe(videoTrack.codedWidth); + expect(sample.codedHeight).toBe(videoTrack.codedHeight); + expectPar2x1Geometry(sample); + expect(sample.squarePixelWidth).toBe(videoTrack.squarePixelWidth); + expect(sample.squarePixelHeight).toBe(videoTrack.squarePixelHeight); +}); + +test('Pixel aspect ratio copy conversion', async () => { + using input = new Input({ + source: new UrlSource(SOURCE_PATH), + formats: ALL_FORMATS, + }); + + const sourceTrack = await input.getPrimaryVideoTrack(); + assert(sourceTrack); + const expected = await snapshotTrack(sourceTrack); + expectPar2x1Geometry(expected); + + const mkv = await convertAndReadVideoTrack(new MkvOutputFormat()); + expect(mkv.format).toBe(MATROSKA); + expectPar2x1Geometry(mkv.snapshot); + expect(mkv.snapshot).toEqual(expected); + + const ts = await convertAndReadVideoTrack(new MpegTsOutputFormat()); + expect(ts.format).toBe(MPEG_TS); + expectPar2x1Geometry(ts.snapshot); + expect(ts.snapshot).toEqual(expected); +}); + +test('Pixel aspect ratio transcode conversion', async () => { + using input = new Input({ + source: new UrlSource(SOURCE_PATH), + formats: ALL_FORMATS, + }); + + const sourceTrack = await input.getPrimaryVideoTrack(); + assert(sourceTrack); + const source = await snapshotTrack(sourceTrack); + expectPar2x1Geometry(source); + + const mp4 = await convertAndReadVideoTrack( + new Mp4OutputFormat(), + true, + ); + + expect(mp4.format).toBe(MP4); + expect(mp4.snapshot.pixelAspectRatio).toEqual({ num: 1, den: 1 }); + expect(mp4.snapshot.squarePixelWidth).toBe(mp4.snapshot.codedWidth); + expect(mp4.snapshot.squarePixelHeight).toBe(mp4.snapshot.codedHeight); + expect(mp4.snapshot.displayWidth).toBe(mp4.snapshot.codedWidth); + expect(mp4.snapshot.displayHeight).toBe(mp4.snapshot.codedHeight); + expect(mp4.snapshot.decoderDisplayAspectWidth).toBe(mp4.snapshot.codedWidth); + expect(mp4.snapshot.decoderDisplayAspectHeight).toBe(mp4.snapshot.codedHeight); + + expect(mp4.snapshot.codedWidth).toBe(source.squarePixelWidth); + expect(mp4.snapshot.codedHeight).toBe(source.squarePixelHeight); + expect(mp4.snapshot.displayWidth).toBe(source.displayWidth); + expect(mp4.snapshot.displayHeight).toBe(source.displayHeight); +}); + +const expectPar2x1Geometry = (value: { + codedWidth: number; + codedHeight: number; + squarePixelWidth: number; + squarePixelHeight: number; + displayWidth: number; + displayHeight: number; + pixelAspectRatio: { num: number; den: number }; +}) => { + expect(value.pixelAspectRatio).toEqual({ num: 2, den: 1 }); + expect(value.squarePixelWidth).toBe(value.codedWidth * 2); + expect(value.squarePixelHeight).toBe(value.codedHeight); + expect(value.displayWidth).toBe(value.squarePixelWidth); + expect(value.displayHeight).toBe(value.squarePixelHeight); +}; + +const snapshotTrack = async (track: { + codedWidth: number; + codedHeight: number; + squarePixelWidth: number; + squarePixelHeight: number; + displayWidth: number; + displayHeight: number; + rotation: number; + pixelAspectRatio: { num: number; den: number }; + getDecoderConfig(): Promise; +}) => { + const decoderConfig = await track.getDecoderConfig(); + assert(decoderConfig); + + return { + codedWidth: track.codedWidth, + codedHeight: track.codedHeight, + squarePixelWidth: track.squarePixelWidth, + squarePixelHeight: track.squarePixelHeight, + displayWidth: track.displayWidth, + displayHeight: track.displayHeight, + rotation: track.rotation, + pixelAspectRatio: track.pixelAspectRatio, + decoderDisplayAspectWidth: decoderConfig.displayAspectWidth, + decoderDisplayAspectHeight: decoderConfig.displayAspectHeight, + }; +}; + +const convertAndReadVideoTrack = async ( + format: MkvOutputFormat | MpegTsOutputFormat | Mp4OutputFormat, + forceTranscode = false, +) => { + using input = new Input({ + source: new UrlSource(SOURCE_PATH), + formats: ALL_FORMATS, + }); + + const output = new Output({ + format, + target: new BufferTarget(), + }); + + const conversion = await Conversion.init({ + input, + output, + video: { + forceTranscode, + }, + trim: { + end: 1, + }, + }); + expect(conversion.isValid).toBe(true); + await conversion.execute(); + + const buffer = output.target.buffer; + assert(buffer); + + using outputInput = new Input({ + source: new BufferSource(buffer), + formats: ALL_FORMATS, + }); + + const track = await outputInput.getPrimaryVideoTrack(); + assert(track); + + return { + format: await outputInput.getFormat(), + snapshot: await snapshotTrack(track), + }; +}; diff --git a/test/node/mpeg-ts-demuxing.test.ts b/test/node/mpeg-ts-demuxing.test.ts index 62349b9..e2a6a99 100644 --- a/test/node/mpeg-ts-demuxing.test.ts +++ b/test/node/mpeg-ts-demuxing.test.ts @@ -46,6 +46,8 @@ test('MPEG-TS metadata reading', async () => { codec: 'avc1.640020', codedWidth: 720, codedHeight: 720, + displayAspectWidth: 720, + displayAspectHeight: 720, colorSpace: { primaries: 'bt2020', transfer: 'hlg', @@ -559,6 +561,8 @@ test('MPEG-TS with HEVC video', async () => { codec: 'hev1.1.6.L120.90', codedWidth: 1920, codedHeight: 1080, + displayAspectWidth: 1920, + displayAspectHeight: 1080, colorSpace: { primaries: 'bt709', transfer: 'bt709', diff --git a/test/public/sar_2x1.mp4 b/test/public/sar_2x1.mp4 new file mode 100644 index 0000000..3c4a74a Binary files /dev/null and b/test/public/sar_2x1.mp4 differ