diff --git a/packages/server/src/misc.ts b/packages/server/src/misc.ts index ab6ab63..e3bb3b1 100644 --- a/packages/server/src/misc.ts +++ b/packages/server/src/misc.ts @@ -1,12 +1,14 @@ -import { VideoSamplePixelFormat, VideoCodec } from 'mediabunny'; +import { VideoSamplePixelFormat, MediaCodec } from 'mediabunny'; import * as NodeAv from 'node-av'; -export const CODEC_TO_CODEC_ID: Record = { +export const CODEC_TO_CODEC_ID: Partial> = { avc: NodeAv.AV_CODEC_ID_H264, hevc: NodeAv.AV_CODEC_ID_HEVC, vp8: NodeAv.AV_CODEC_ID_VP8, vp9: NodeAv.AV_CODEC_ID_VP9, av1: NodeAv.AV_CODEC_ID_AV1, + + aac: NodeAv.AV_CODEC_ID_AAC, }; let cachedHardwareContext: NodeAv.HardwareContext | null | undefined = undefined; @@ -180,3 +182,44 @@ export const fromPixelFormat = (pixelFormat: VideoSamplePixelFormat) => { default: return NodeAv.AV_PIX_FMT_NONE; } }; + +export const toAudioSampleFormat = (ffmpegSampleFormat: NodeAv.AVSampleFormat): AudioSampleFormat | null => { + switch (ffmpegSampleFormat) { + case NodeAv.AV_SAMPLE_FMT_U8: return 'u8'; + case NodeAv.AV_SAMPLE_FMT_S16: return 's16'; + case NodeAv.AV_SAMPLE_FMT_S32: return 's32'; + case NodeAv.AV_SAMPLE_FMT_FLT: return 'f32'; + case NodeAv.AV_SAMPLE_FMT_U8P: return 'u8-planar'; + case NodeAv.AV_SAMPLE_FMT_S16P: return 's16-planar'; + case NodeAv.AV_SAMPLE_FMT_S32P: return 's32-planar'; + case NodeAv.AV_SAMPLE_FMT_FLTP: return 'f32-planar'; + + default: return null; + } +}; + +export const fromAudioSampleFormat = (sampleFormat: AudioSampleFormat): NodeAv.AVSampleFormat => { + switch (sampleFormat) { + case 'u8': return NodeAv.AV_SAMPLE_FMT_U8; + case 's16': return NodeAv.AV_SAMPLE_FMT_S16; + case 's32': return NodeAv.AV_SAMPLE_FMT_S32; + case 'f32': return NodeAv.AV_SAMPLE_FMT_FLT; + case 'u8-planar': return NodeAv.AV_SAMPLE_FMT_U8P; + case 's16-planar': return NodeAv.AV_SAMPLE_FMT_S16P; + case 's32-planar': return NodeAv.AV_SAMPLE_FMT_S32P; + case 'f32-planar': return NodeAv.AV_SAMPLE_FMT_FLTP; + + default: return NodeAv.AV_SAMPLE_FMT_NONE; + } +}; + +export const getChannelLayout = (numChannels: number): NodeAv.ChannelLayout => { + switch (numChannels) { + case 1: return NodeAv.AV_CHANNEL_LAYOUT_MONO; + case 2: return NodeAv.AV_CHANNEL_LAYOUT_STEREO; + case 4: return NodeAv.AV_CHANNEL_LAYOUT_QUAD; + case 6: return NodeAv.AV_CHANNEL_LAYOUT_5POINT1; + case 8: return NodeAv.AV_CHANNEL_LAYOUT_7POINT1; + default: return { nbChannels: numChannels, order: NodeAv.AV_CHANNEL_ORDER_UNSPEC, mask: 0n }; + } +}; diff --git a/packages/server/src/video-decoder.ts b/packages/server/src/video-decoder.ts index 6a08f06..b77e6ce 100644 --- a/packages/server/src/video-decoder.ts +++ b/packages/server/src/video-decoder.ts @@ -1,13 +1,7 @@ import { CustomVideoDecoder, VideoCodec, EncodedPacket, VideoSample, MaybePromise, Rational } from 'mediabunny'; import * as NodeAv from 'node-av'; -import { - CODEC_TO_CODEC_ID, - getHardwareDecoderCodec, - mapColorPrimaries, - mapMatrixCoefficients, - mapTransferCharacteristics, -} from './misc'; -import { binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; +import { CODEC_TO_CODEC_ID, getHardwareDecoderCodec } from './misc'; +import { assert, binarySearchLessOrEqual, simplifyRational, toUint8Array } from '../../../src/misc'; import { NodeAvFrameVideoSampleResource } from './video-sample'; export class NodeAvVideoDecoder extends CustomVideoDecoder { @@ -37,6 +31,7 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { this.packet.alloc(); const codecId = CODEC_TO_CODEC_ID[this.codec]; + assert(codecId !== undefined); let codec: NodeAv.Codec | null; if (this.config.hardwareAcceleration !== 'prefer-hardware' || this.codec === 'av1') { @@ -67,30 +62,6 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { : null; codecContext.sampleAspectRatio = new NodeAv.Rational(this.pixelAspectRatio.num, this.pixelAspectRatio.den); - if (this.config.colorSpace?.primaries) { - const mapped = mapColorPrimaries(this.config.colorSpace.primaries); - if (mapped !== null) { - codecContext.colorPrimaries = mapped; - } - } - if (this.config.colorSpace?.transfer) { - const mapped = mapTransferCharacteristics(this.config.colorSpace.transfer); - if (mapped !== null) { - codecContext.colorTrc = mapped; - } - } - if (this.config.colorSpace?.matrix) { - const mapped = mapMatrixCoefficients(this.config.colorSpace.matrix); - if (mapped !== null) { - codecContext.colorSpace = mapped; - } - } - if (this.config.colorSpace?.fullRange != null) { - codecContext.colorRange = this.config.colorSpace.fullRange - ? NodeAv.AVCOL_RANGE_JPEG - : NodeAv.AVCOL_RANGE_MPEG; - } - const ret = await codecContext.open2(); NodeAv.FFmpegError.throwIfError(ret, 'Open codec context'); @@ -178,7 +149,12 @@ export class NodeAvVideoDecoder extends CustomVideoDecoder { } } - this.onSample(new VideoSample(new NodeAvFrameVideoSampleResource(this.frame), { + const clone = this.frame.clone(); + if (!clone) { + throw new Error('Frame clone allocation failed.'); + } + + this.onSample(new VideoSample(new NodeAvFrameVideoSampleResource(clone), { timestamp, duration, })); diff --git a/packages/server/src/video-encoder.ts b/packages/server/src/video-encoder.ts index 4dbea2d..752fbc2 100644 --- a/packages/server/src/video-encoder.ts +++ b/packages/server/src/video-encoder.ts @@ -35,7 +35,6 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { avCodec!: NodeAv.Codec; codecContext: NodeAv.CodecContext | null = null; lastBuffer: Buffer | null = null; - colorSpaceSet = false; packetEmitted = false; scaler: NodeAv.SoftwareScaleContext | null = null; @@ -65,6 +64,7 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { this.packet.alloc(); const codecId = CODEC_TO_CODEC_ID[this.codec]; + assert(codecId !== undefined); let codec: NodeAv.Codec | null = null; if (this.config.hardwareAcceleration === 'prefer-software') { @@ -203,15 +203,6 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { this.frame.fromBuffer(this.lastBuffer); } - if (!this.colorSpaceSet) { - this.codecContext.colorPrimaries = this.frame.colorPrimaries; - this.codecContext.colorTrc = this.frame.colorTrc; - this.codecContext.colorSpace = this.frame.colorSpace; - this.codecContext.colorRange = this.frame.colorRange; - - this.colorSpaceSet = true; - } - let frameToEncode = this.frame; const requiresScaler @@ -550,7 +541,6 @@ export class NodeAvVideoEncoder extends CustomVideoEncoder { } this.packetEmitted = false; - this.colorSpaceSet = false; } close(): MaybePromise { diff --git a/packages/server/src/video-sample.ts b/packages/server/src/video-sample.ts index d142708..d1d798f 100644 --- a/packages/server/src/video-sample.ts +++ b/packages/server/src/video-sample.ts @@ -1,20 +1,27 @@ import assert from 'assert'; -import { Rectangle, VideoSamplePixelFormat } from 'mediabunny'; +import { VideoSamplePixelFormat } from 'mediabunny'; import { VideoSampleColorSpace } from 'mediabunny'; import { VideoSampleResource } from 'mediabunny'; import * as NodeAv from 'node-av'; -import { toUint8Array } from '../../../src/misc'; -import { getPlaneConfigs } from '../../../src/sample'; +import { MaybePromise, toUint8Array } from '../../../src/misc'; +import { VideoDataPlane, VideoSample } from '../../../src/sample'; import { toPixelFormat, unmapColorPrimaries, unmapTransferCharacteristics, unmapMatrixCoefficients, fromPixelFormat, - mapColorPrimaries, - mapTransferCharacteristics, - mapMatrixCoefficients, } from './misc'; +import { SetRequired } from 'mediabunny'; +import { VideoSampleInit } from 'mediabunny'; + +const JPEG_RANGE_PIX_FORMATS = new Set([ + NodeAv.AV_PIX_FMT_YUVJ411P, + NodeAv.AV_PIX_FMT_YUVJ420P, + NodeAv.AV_PIX_FMT_YUVJ422P, + NodeAv.AV_PIX_FMT_YUVJ440P, + NodeAv.AV_PIX_FMT_YUVJ444P, +]); export class NodeAvFrameVideoSampleResource extends VideoSampleResource { frame: NodeAv.Frame; @@ -22,12 +29,7 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource { constructor(frame: NodeAv.Frame) { super(); - const clone = frame.clone(); - if (!clone) { - throw new Error('Allocation failure during frame clone.'); - } - - this.frame = clone; + this.frame = frame; } getFormat(): VideoSamplePixelFormat | null { @@ -63,7 +65,12 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource { primaries: unmapColorPrimaries(this.frame.colorPrimaries) as VideoColorPrimaries | null, transfer: unmapTransferCharacteristics(this.frame.colorTrc) as VideoTransferCharacteristics | null, matrix: unmapMatrixCoefficients(this.frame.colorSpace) as VideoMatrixCoefficients | null, - fullRange: this.frame.colorRange === NodeAv.AVCOL_RANGE_JPEG, + fullRange: this.frame.colorRange === NodeAv.AVCOL_RANGE_JPEG + || JPEG_RANGE_PIX_FORMATS.has(this.frame.format as NodeAv.AVPixelFormat) + ? true + : this.frame.colorRange === NodeAv.AVCOL_RANGE_MPEG + ? false + : null, }); } @@ -71,450 +78,53 @@ export class NodeAvFrameVideoSampleResource extends VideoSampleResource { this.frame.free(); } - allocationSize(options: VideoFrameCopyToOptions): number { - // 3. Let combinedLayout be the result of running the Parse VideoFrameCopyToOptions algorithm with options. - // 4. If combinedLayout is an exception, throw combinedLayout. - const combinedLayout = ParseVideoFrameCopyToOptions(this, options); + getDataPlanes(): MaybePromise { + assert(this.frame.data); - // 5. Return combinedLayout's allocationSize. - return combinedLayout.allocationSize; + return this.frame.data.map((data, i) => ({ + data: toUint8Array(data), + stride: this.frame.linesize[i]!, + })); } - async copyTo( - destination: AllowSharedBufferSource, - options: VideoFrameCopyToOptions, - ): Promise { - const format = this.getFormat(); - assert(format !== null); + async toRgbSample( + init: SetRequired, + format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX', + // Will respect it when somebody complains + // eslint-disable-next-line @typescript-eslint/no-unused-vars + colorSpace: PredefinedColorSpace, + ): Promise { + const width = this.frame.width; + const height = this.frame.height; - // 3. Let combinedLayout be the result of running the Parse VideoFrameCopyToOptions algorithm with options. - const combinedLayout = ParseVideoFrameCopyToOptions(this, options); + const scaler = new NodeAv.SoftwareScaleContext(); + const srcFmt = this.frame.format as NodeAv.AVPixelFormat; + const dstFmt = fromPixelFormat(format); - // 4. If destination.byteLength is less than combinedLayout’s allocationSize, return a promise rejected with - // // eslint-disable-next-line @stylistic/max-len - const destBytes = toUint8Array(destination); - if (destBytes.byteLength < combinedLayout.allocationSize) { - throw new TypeError( - `Destination buffer too small. Required: ${combinedLayout.allocationSize},` - + ` Available: ${destBytes.byteLength}`, - ); + scaler.getContext( + width, height, srcFmt, + width, height, dstFmt, + NodeAv.SWS_BILINEAR, + ); + + const dstFrame = new NodeAv.Frame(); + dstFrame.width = width; + dstFrame.height = height; + dstFrame.format = dstFmt; + dstFrame.alloc(); + dstFrame.allocBuffer(); + + const srcFrame = this.frame; + + try { + await scaler.scaleFrame(dstFrame, srcFrame); + } finally { + scaler.freeContext(); } - // 5. If options.format is equal to one of RGBA, RGBX, BGRA, BGRX then: - if (options.format && ['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format) && options.format !== format) { - // Let newOptions be the result of running the Clone Configuration algorithm with options. - // Assign undefined to newOptions.format. - const newOptions = { ...options }; - delete newOptions.format; + dstFrame.format = dstFmt; // FFmpeg messes up RGBA and RGBX + dstFrame.sampleAspectRatio = srcFrame.sampleAspectRatio; - // Let rgbFrame be the result of running the Convert to RGB frame algorithm with this, options.format, - // and options.colorSpace. - const rgbFrame = await ConvertToRGBFrame(this, options.format, options.colorSpace); - - // Return the result of calling copyTo() on rgbFrame with destination and newOptions. - try { - const result = await rgbFrame.copyTo(destination, newOptions); - rgbFrame.close(); - return result; - } catch (e) { - rgbFrame.close(); - throw e; - } - } - - // 6. Let p be a new Promise. (Implicit) - // 7. Let copyStepsQueue be the result of starting a new parallel queue. (Implicit) - // 8. Let planeLayouts be a new list. - const planeLayouts: PlaneLayout[] = []; - - // Enqueue the following steps to copyStepsQueue: (fuck the queuing part) - - // Let resource be the media resource referenced by [[resource reference]]. - // (this.data) - - // Let numPlanes be the number of planes as defined by [[format]]. - const planes = getPlaneConfigs(format); - const numPlanes = planes.length; - - // Let planeIndex be 0. - // While planeIndex is less than combinedLayout’s numPlanes: - for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { - const computedLayout = combinedLayout.computedLayouts[planeIndex]!; - - // Let sourceStride be the stride of the plane in resource as identified by planeIndex. - let sourceStride = 0; - let sourceStartOffset = 0; - - sourceStride = this.frame.linesize[planeIndex]!; - const planeBuffer = this.frame.data![planeIndex]!; - if (!planeBuffer) { - throw new Error(`Missing data for plane ${planeIndex}`); - } - const sourceData = new Uint8Array(planeBuffer.buffer, planeBuffer.byteOffset, planeBuffer.byteLength); - sourceStartOffset = 0; - - // Let sourceOffset be the product of multiplying computedLayout’s sourceTop by sourceStride - let sourceOffset = computedLayout.sourceTop * sourceStride; - - // Add computedLayout’s sourceLeftBytes to sourceOffset. - sourceOffset += computedLayout.sourceLeftBytes; - - // Adjust for the base offset of the plane in the source buffer - sourceOffset += sourceStartOffset; - - // Let destinationOffset be computedLayout’s destinationOffset. - let destinationOffset = computedLayout.destinationOffset; - - // Let rowBytes be computedLayout’s sourceWidthBytes. - const rowBytes = computedLayout.sourceWidthBytes; - - // Let layout be a new PlaneLayout, with offset set to destinationOffset and stride set to rowBytes. - // This is a spec error actually (https://github.com/w3c/webcodecs/issues/918) - const layout: PlaneLayout = { - offset: destinationOffset, - stride: computedLayout.destinationStride, - }; - - // Let row be 0. - // While row is less than computedLayout’s sourceHeight: - for (let row = 0; row < computedLayout.sourceHeight; row++) { - // Copy rowBytes bytes from resource starting at sourceOffset to destination starting - // at destinationOffset. - if (sourceOffset + rowBytes > sourceData.byteLength) { - throw new Error(`Source buffer OOB read`); - } - if (destinationOffset + rowBytes > destBytes.byteLength) { - throw new Error(`Destination buffer OOB write`); - } - - const srcSub = sourceData.subarray(sourceOffset, sourceOffset + rowBytes); - destBytes.set(srcSub, destinationOffset); - - // Increment sourceOffset by sourceStride. - sourceOffset += sourceStride; - - // Increment destinationOffset by computedLayout’s destinationStride. - destinationOffset += computedLayout.destinationStride; - } - - // Append layout to planeLayouts. - planeLayouts.push(layout); - } - - // Queue a task to resolve p with planeLayouts. - return planeLayouts; + return new VideoSample(new NodeAvFrameVideoSampleResource(dstFrame), init); } } - -type CombinedBufferLayout = { - allocationSize: number; - computedLayouts: ComputedPlaneLayout[]; -}; - -type ComputedPlaneLayout = { - destinationOffset: number; - destinationStride: number; - sourceTop: number; - sourceHeight: number; - sourceLeftBytes: number; - sourceWidthBytes: number; -}; - -// Taken from the WebCodecs spec -const ParseVideoFrameCopyToOptions = ( - frame: VideoSampleResource, - options: VideoFrameCopyToOptions, -): CombinedBufferLayout => { - // 1. Let defaultRect be the result of performing the getter steps for visibleRect. - const defaultRect: Rectangle = { - left: 0, - top: 0, - width: frame.getCodedWidth(), - height: frame.getCodedHeight(), - }; - - // 2. Let overrideRect be undefined. - // 3. If options.rect exists, assign the value of options.rect to overrideRect. - const overrideRect = options.rect; - - // 4. Let parsedRect be the result of running the Parse Visible Rect algorithm... - const parsedRect = ParseVisibleRect( - defaultRect, - overrideRect, - frame.getCodedWidth(), - frame.getCodedHeight(), - frame.getFormat(), - ); - - // 5. If parsedRect is an exception, return parsedRect. (Handled by throw) - - // 6. Let optLayout be undefined. - // 7. If options.layout exists, assign its value to optLayout. - const optLayout = options.layout; - - // 8. Let format be undefined. - let format: VideoSamplePixelFormat | undefined; - - // 9. If options.format does not exist, assign [[format]] to format. - if (!options.format || options.format === frame.getFormat()) { - format = frame.getFormat()!; - } else if (['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format)) { - // 10. Otherwise, if options.format is equal to one of RGBA, RGBX, BGRA, BGRX, then assign options.format - // to format... - format = options.format; - } else { - throw new Error('NotSupportedError: Invalid destination format'); - } - - // 11. Let combinedLayout be the result of running the Compute Layout and Allocation Size algorithm... - return ComputeLayoutAndAllocationSize(parsedRect, format, optLayout); -}; - -// Taken from the WebCodecs spec -const ParseVisibleRect = ( - defaultRect: DOMRectInit, - overrideRect: DOMRectInit | undefined, - codedWidth: number, - codedHeight: number, - format: VideoSamplePixelFormat | null, -): DOMRectInit => { - // 1. Let sourceRect be defaultRect - const sourceRect = { ...defaultRect }; - - // 2. If overrideRect is not undefined: - if (overrideRect !== undefined) { - // If either of overrideRect.width or height is 0, return a TypeError. - if (overrideRect.width === 0 || overrideRect.height === 0) { - throw new TypeError('visibleRect dimensions cannot be zero'); - } - // If the sum of overrideRect.x and overrideRect.width is greater than codedWidth, return a TypeError. - if ((overrideRect.x || 0) + (overrideRect.width || 0) > codedWidth) { - throw new TypeError('visibleRect exceeds codedWidth'); - } - // If the sum of overrideRect.y and overrideRect.height is greater than codedHeight, return a TypeError. - if ((overrideRect.y || 0) + (overrideRect.height || 0) > codedHeight) { - throw new TypeError('visibleRect exceeds codedHeight'); - } - // Assign overrideRect to sourceRect. - sourceRect.x = overrideRect.x || 0; - sourceRect.y = overrideRect.y || 0; - sourceRect.width = overrideRect.width || 0; - sourceRect.height = overrideRect.height || 0; - } - - // 3. Let validAlignment be the result of running the Verify Rect Offset Alignment algorithm. - const validAlignment = VerifyRectOffsetAlignment(format, sourceRect); - - // 4. If validAlignment is false, throw a TypeError. - if (!validAlignment) { - throw new TypeError('visibleRect alignment is invalid for the format'); - } - - // 5. Return sourceRect. - return sourceRect; -}; - -// Taken from the WebCodecs spec -const VerifyRectOffsetAlignment = (format: VideoSamplePixelFormat | null, rect: DOMRectInit): boolean => { - // 1. If format is null, return true. - if (format === null) return true; - - const planes = getPlaneConfigs(format); - - // 2. Let planeIndex be 0. - // 3. Let numPlanes be the number of planes as defined by format. - // 4. While planeIndex is less than numPlanes: - for (let planeIndex = 0; planeIndex < planes.length; planeIndex++) { - const plane = planes[planeIndex]!; - const sampleWidth = plane.widthDivisor; - const sampleHeight = plane.heightDivisor; - - // If rect.x is not a multiple of sampleWidth, return false. - if ((rect.x || 0) % sampleWidth !== 0) return false; - // If rect.y is not a multiple of sampleHeight, return false. - if ((rect.y || 0) % sampleHeight !== 0) return false; - } - - return true; -}; - -// Taken from the WebCodecs spec -const ComputeLayoutAndAllocationSize = ( - parsedRect: DOMRectInit, - format: VideoSamplePixelFormat, - layout?: PlaneLayout[], -): CombinedBufferLayout => { - const planes = getPlaneConfigs(format); - - // 1. Let numPlanes be the number of planes as defined by format. - const numPlanes = planes.length; - - // 2. If layout is not undefined and its length does not equal numPlanes, throw a TypeError. - if (layout !== undefined && layout.length !== numPlanes) { - throw new TypeError(`Layout must have ${numPlanes} planes`); - } - - // 3. Let minAllocationSize be 0. - let minAllocationSize = 0; - - // 4. Let computedLayouts be a new list. - const computedLayouts: ComputedPlaneLayout[] = []; - - // 5. Let endOffsets be a new list. - const endOffsets: number[] = []; - - // 6. Let planeIndex be 0. - // 7. While planeIndex < numPlanes: - for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { - const plane = planes[planeIndex]!; - const sampleBytes = plane.sampleBytes; - const sampleWidth = plane.widthDivisor; - const sampleHeight = plane.heightDivisor; - - // Let computedLayout be a new computed plane layout. - const computedLayout: ComputedPlaneLayout = { - destinationOffset: 0, - destinationStride: 0, - sourceTop: 0, - sourceHeight: 0, - sourceLeftBytes: 0, - sourceWidthBytes: 0, - }; - - // Set computedLayout’s sourceTop... - computedLayout.sourceTop = Math.ceil(Math.trunc(parsedRect.y || 0) / sampleHeight); - // Set computedLayout’s sourceHeight... - computedLayout.sourceHeight = Math.ceil(Math.trunc(parsedRect.height || 0) / sampleHeight); - // Set computedLayout’s sourceLeftBytes... - computedLayout.sourceLeftBytes = Math.floor(Math.trunc(parsedRect.x || 0) / sampleWidth) * sampleBytes; - // Set computedLayout’s sourceWidthBytes... - computedLayout.sourceWidthBytes = Math.floor(Math.trunc(parsedRect.width || 0) / sampleWidth) * sampleBytes; - - // If layout is not undefined: - if (layout !== undefined) { - const planeLayout = layout[planeIndex]!; - // If planeLayout.stride is less than computedLayout’s sourceWidthBytes, return a TypeError. - if (planeLayout.stride < computedLayout.sourceWidthBytes) { - throw new TypeError(`Stride for plane ${planeIndex} is too small`); - } - // Assign planeLayout.offset to computedLayout’s destinationOffset. - computedLayout.destinationOffset = planeLayout.offset; - // Assign planeLayout.stride to computedLayout’s destinationStride. - computedLayout.destinationStride = planeLayout.stride; - } else { - // Otherwise: - // Assign minAllocationSize to computedLayout’s destinationOffset. - computedLayout.destinationOffset = minAllocationSize; - // Assign computedLayout’s sourceWidthBytes to computedLayout’s destinationStride. - computedLayout.destinationStride = computedLayout.sourceWidthBytes; - } - - // Let planeSize be the product of multiplying computedLayout’s destinationStride and sourceHeight. - const planeSize = computedLayout.destinationStride * computedLayout.sourceHeight; - - // Let planeEnd be the sum of planeSize and computedLayout’s destinationOffset. - const planeEnd = planeSize + computedLayout.destinationOffset; - - // If planeSize or planeEnd is greater than maximum range of unsigned long, return a TypeError. - if (planeEnd > 4294967295) throw new TypeError('Allocation size exceeds limit'); - - // Append planeEnd to endOffsets. - endOffsets.push(planeEnd); - - // Assign the maximum of minAllocationSize and planeEnd to minAllocationSize. - minAllocationSize = Math.max(minAllocationSize, planeEnd); - - // Check for overlap - for (let earlierPlaneIndex = 0; earlierPlaneIndex < planeIndex; earlierPlaneIndex++) { - const earlierLayout = computedLayouts[earlierPlaneIndex]!; - // If plane A ends before plane B starts, they do not overlap. - if ( - endOffsets[planeIndex]! <= earlierLayout.destinationOffset - || endOffsets[earlierPlaneIndex]! <= computedLayout.destinationOffset - ) { - continue; - } - throw new TypeError('Planes overlap'); - } - - computedLayouts.push(computedLayout); - } - - // 12. Return combinedLayout. - return { - allocationSize: minAllocationSize, - computedLayouts: computedLayouts, - }; -}; - -// Taken from the WebCodecs spec -const ConvertToRGBFrame = async ( - frame: NodeAvFrameVideoSampleResource, - format: VideoPixelFormat, - colorSpace?: PredefinedColorSpace, -): Promise => { - // 1. Let convertedFrame be a new VideoFrame... - // (We construct it at the end, but we prepare the resource here) - - const width = frame.getCodedWidth(); - const height = frame.getCodedHeight(); - - const scaler = new NodeAv.SoftwareScaleContext(); - const srcFmt = fromPixelFormat(frame.getFormat()!); - const dstFmt = fromPixelFormat(format); - - // Configure scaling: Source -> Destination (Visible Size) - scaler.getContext( - width, height, srcFmt, - width, height, dstFmt, - NodeAv.SWS_BILINEAR, - ); - - // Allocate destination frame - const dstFrame = new NodeAv.Frame(); - dstFrame.width = width; - dstFrame.height = height; - dstFrame.format = dstFmt; - dstFrame.alloc(); - dstFrame.allocBuffer(); - - // Apply destination color space settings to dstFrame - // If colorSpace is not provided, srgb is used - const targetColorSpace = colorSpace || 'srgb'; - if (targetColorSpace === 'srgb') { - dstFrame.colorPrimaries = NodeAv.AVCOL_PRI_BT709; - dstFrame.colorTrc = NodeAv.AVCOL_TRC_IEC61966_2_1; - dstFrame.colorSpace = NodeAv.AVCOL_SPC_RGB; - dstFrame.colorRange = NodeAv.AVCOL_RANGE_JPEG; - } else if (targetColorSpace === 'display-p3') { - dstFrame.colorPrimaries = NodeAv.AVCOL_PRI_SMPTE432; - dstFrame.colorTrc = NodeAv.AVCOL_TRC_IEC61966_2_1; - dstFrame.colorSpace = NodeAv.AVCOL_SPC_RGB; - dstFrame.colorRange = NodeAv.AVCOL_RANGE_JPEG; - } - - const srcFrame = frame.frame; - - // Apply source color space settings to srcFrame (if known) - // This ensures the scaler knows the input color space for correct conversion - const frameColorSpace = frame.getColorSpace(); - if (colorSpace) { - srcFrame.colorPrimaries = mapColorPrimaries(frameColorSpace.primaries ?? 'unknown') - ?? NodeAv.AVCOL_PRI_UNSPECIFIED; - srcFrame.colorTrc = mapTransferCharacteristics(frameColorSpace.transfer ?? 'unknown') - ?? NodeAv.AVCOL_TRC_UNSPECIFIED; - srcFrame.colorSpace = mapMatrixCoefficients(frameColorSpace.matrix ?? 'unknown') - ?? NodeAv.AVCOL_SPC_UNSPECIFIED; - srcFrame.colorRange = frameColorSpace.fullRange - ? NodeAv.AVCOL_RANGE_JPEG - : NodeAv.AVCOL_RANGE_MPEG; - } - - try { - await scaler.scaleFrame(dstFrame, srcFrame); - } finally { - scaler.freeContext(); - } - - return new NodeAvFrameVideoSampleResource(dstFrame); -}; diff --git a/src/sample.ts b/src/sample.ts index 15f16d2..f341553 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -116,16 +116,20 @@ export abstract class VideoSampleResource { */ abstract close(): void; - /** Returns the number of bytes required to hold this video sample's pixel data. */ - abstract allocationSize(options: VideoFrameCopyToOptions): number; + abstract getDataPlanes(): MaybePromise; - /** Copies this video sample's pixel data to an ArrayBuffer or ArrayBufferView. */ - abstract copyTo( - destination: AllowSharedBufferSource, - options: VideoFrameCopyToOptions - ): MaybePromise; + abstract toRgbSample( + init: SetRequired, + format: 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX', + colorSpace: PredefinedColorSpace, + ): MaybePromise; } +export type VideoDataPlane = { + data: Uint8Array; + stride: number; +}; + /** * The list of {@link VideoSample} pixel formats. * @group Samples @@ -567,9 +571,33 @@ export class VideoSample implements Disposable { data._referenceCount++; this.format = data.getFormat(); - this.visibleRect = { left: 0, top: 0, width: data.getCodedWidth(), height: data.getCodedHeight() }; + if (this.format !== null && !VIDEO_SAMPLE_PIXEL_FORMATS.includes(this.format)) { + throw new TypeError('getFormat() must return a VideoSamplePixelFormat or null.'); + } + + this.visibleRect = { + left: 0, + top: 0, + width: data.getCodedWidth(), + height: data.getCodedHeight(), + }; + if (!Number.isInteger(this.visibleRect.width) || this.visibleRect.width <= 0) { + throw new TypeError('getCodedWidth() must return a positive integer.'); + } + if (!Number.isInteger(this.visibleRect.height) || this.visibleRect.height <= 0) { + throw new TypeError('getCodedHeight() must return a positive integer.'); + } + this.squarePixelWidth = data.getSquarePixelWidth(); + if (!Number.isInteger(this.squarePixelWidth) || this.squarePixelWidth <= 0) { + throw new TypeError('getSquarePixelWidth() must return a positive integer.'); + } + this.squarePixelHeight = data.getSquarePixelHeight(); + if (!Number.isInteger(this.squarePixelHeight) || this.squarePixelHeight <= 0) { + throw new TypeError('getSquarePixelHeight() must return a positive integer.'); + } + this.rotation = init.rotation ?? 0; this.timestamp = init.timestamp!; this.duration = init.duration ?? 0; @@ -682,42 +710,20 @@ export class VideoSample implements Disposable { if (this._closed) { throw new Error('VideoSample is closed.'); } - if (this.format === null) { + + if ((options.format ?? this.format) == null) { // https://github.com/Vanilagy/mediabunny/issues/267 // https://github.com/w3c/webcodecs/issues/920 - throw new Error('Cannot get allocation size when format is null. Sorry!'); - } - - assert(this._data !== null); - - if (this._data instanceof VideoSampleResource) { - return this._data.allocationSize(options); - } - - if (!isVideoFrame(this._data)) { - if ( - options.colorSpace - || (options.format && options.format !== this.format) - || options.layout - || options.rect - ) { - // Temporarily convert to VideoFrame to get it done - // TODO: Compute this directly without needing to go through VideoFrame - const videoFrame = this.toVideoFrame(); - const size = videoFrame.allocationSize(options); - videoFrame.close(); - - return size; - } + throw new Error('Cannot get allocation size when format is null.'); } if (isVideoFrame(this._data)) { + // Call the native method purely for performance return this._data.allocationSize(options); - } else if (this._data instanceof Uint8Array) { - return this._data.byteLength; - } else { - return this.codedWidth * this.codedHeight * 4; // RGBX } + + const combinedLayout = ParseVideoFrameCopyToOptions(this, options); + return combinedLayout.allocationSize; } /** @@ -733,56 +739,200 @@ export class VideoSample implements Disposable { if (this._closed) { throw new Error('VideoSample is closed.'); } - if (this.format === null) { + if ((options.format ?? this.format) == null) { throw new Error('Cannot copy video sample data when format is null. Sorry!'); } assert(this._data !== null); - if (this._data instanceof VideoSampleResource) { + if (isVideoFrame(this._data)) { return this._data.copyTo(destination, options); } - if (!isVideoFrame(this._data)) { - if ( - options.colorSpace - || (options.format && options.format !== this.format) - || options.layout - || options.rect - ) { - // Temporarily convert to VideoFrame to get it done - // TODO: Do this directly without needing to go through VideoFrame - const videoFrame = this.toVideoFrame(); - const layout = await videoFrame.copyTo(destination, options); - videoFrame.close(); + if ( + options.format + && this.format !== options.format + && ['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format) + ) { + // RGB conversion for custom VideoSampleResource + if (this._data instanceof VideoSampleResource) { + using rgbSample = await this._data.toRgbSample( + { + timestamp: this.timestamp, + duration: this.duration, + rotation: this.rotation, + }, + options.format as 'RGBA' | 'RGBX' | 'BGRA' | 'BGRX', + options.colorSpace ?? 'srgb', + ); + if (!(rgbSample instanceof VideoSample)) { + throw new TypeError('toRgbSample() must return a VideoSample.'); + } + if (rgbSample.format !== options.format) { + throw new Error( + `Sample returned by toRgbSample was expected to have format '${options.format}', got` + + ` '${rgbSample.format}' instead.`, + ); + } - return layout; + return await rgbSample.copyTo(destination, options); // 'await' is intentional here cuz of using + } else if (!['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(this.format!)) { + if (typeof VideoFrame === 'undefined') { + throw new Error( + 'For this sample, converting from a non-RGB to an RGB format requires VideoFrame to' + + ' be defined.', + ); + } + + const tempFrame = this.toVideoFrame(); + const result = await tempFrame.copyTo(destination, options); + tempFrame.close(); + + return result; } } - if (isVideoFrame(this._data)) { - return this._data.copyTo(destination, options); + const combinedLayout = ParseVideoFrameCopyToOptions(this, options); + assert(this.format); + + // 4. If destination.byteLength is less than combinedLayout’s allocationSize, return a promise rejected with + const destBytes = toUint8Array(destination); + if (destBytes.byteLength < combinedLayout.allocationSize) { + throw new TypeError( + `Destination buffer too small. Required: ${combinedLayout.allocationSize},` + + ` Available: ${destBytes.byteLength}`, + ); + } + + const planeConfigs = getPlaneConfigs(this.format); + let dataPlanes: VideoDataPlane[]; + + if (this._data instanceof VideoSampleResource) { + let result = this._data.getDataPlanes(); + if (result instanceof Promise) result = await result; + + if ( + !Array.isArray(result) + || result.some(x => !(x.data instanceof Uint8Array) || !Number.isInteger(x.stride) || x.stride < 0) + ) { + throw new TypeError( + 'getDataPlanes() must return an array of objects with a Uint8Array "data" property and a' + + ' non-negative integer "stride" property.', + ); + } + + dataPlanes = result; } else if (this._data instanceof Uint8Array) { assert(this._layout); + assert(this._layout.length === planeConfigs.length); - const dest = toUint8Array(destination); - dest.set(this._data); + dataPlanes = this._layout.map((planeLayout, i) => { + const height = Math.ceil(this.codedHeight / planeConfigs[i]!.heightDivisor); - return this._layout; + return { + data: (this._data as Uint8Array).subarray( + planeLayout.offset, + planeLayout.offset + planeLayout.stride * height, + ), + stride: planeLayout.stride, + }; + }); } else { const canvas = this._data; const context = canvas.getContext('2d'); assert(context); const imageData = context.getImageData(0, 0, this.codedWidth, this.codedHeight); - const dest = toUint8Array(destination); - dest.set(imageData.data); - return [{ - offset: 0, + dataPlanes = [{ + data: toUint8Array(imageData.data), stride: 4 * this.codedWidth, }]; } + + // 6. Let p be a new Promise. (Implicit) + // 7. Let copyStepsQueue be the result of starting a new parallel queue. (Implicit) + // 8. Let planeLayouts be a new list. + const planeLayouts: PlaneLayout[] = []; + + // Enqueue the following steps to copyStepsQueue: (fuck the queuing part) + + // Let resource be the media resource referenced by [[resource reference]]. + // (this.data) + + // Let numPlanes be the number of planes as defined by [[format]]. + const numPlanes = planeConfigs.length; + + // Let planeIndex be 0. + // While planeIndex is less than combinedLayout’s numPlanes: + for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { + const computedLayout = combinedLayout.computedLayouts[planeIndex]!; + + // Let sourceStride be the stride of the plane in resource as identified by planeIndex. + const sourceStride = dataPlanes[planeIndex]!.stride; + const sourceData = dataPlanes[planeIndex]!.data; + + // Let sourceOffset be the product of multiplying computedLayout’s sourceTop by sourceStride + let sourceOffset = computedLayout.sourceTop * sourceStride; + + // Add computedLayout’s sourceLeftBytes to sourceOffset. + sourceOffset += computedLayout.sourceLeftBytes; + + // Let destinationOffset be computedLayout’s destinationOffset. + let destinationOffset = computedLayout.destinationOffset; + + // Let rowBytes be computedLayout’s sourceWidthBytes. + const rowBytes = computedLayout.sourceWidthBytes; + + // Let layout be a new PlaneLayout, with offset set to destinationOffset and stride set to rowBytes. + // This is a spec error actually (https://github.com/w3c/webcodecs/issues/918) + const layout: PlaneLayout = { + offset: destinationOffset, + stride: computedLayout.destinationStride, + }; + + // Let row be 0. + // While row is less than computedLayout’s sourceHeight: + for (let row = 0; row < computedLayout.sourceHeight; row++) { + // Copy rowBytes bytes from resource starting at sourceOffset to destination starting + // at destinationOffset. + if (sourceOffset + rowBytes > sourceData.byteLength) { + throw new Error(`Source buffer OOB read.`); + } + if (destinationOffset + rowBytes > destBytes.byteLength) { + throw new Error(`Destination buffer OOB write.`); + } + + const srcSub = sourceData.subarray(sourceOffset, sourceOffset + rowBytes); + destBytes.set(srcSub, destinationOffset); + + // Increment sourceOffset by sourceStride. + sourceOffset += sourceStride; + + // Increment destinationOffset by computedLayout’s destinationStride. + destinationOffset += computedLayout.destinationStride; + } + + // Append layout to planeLayouts. + planeLayouts.push(layout); + } + + // RGB conversion for ArrayBuffer and canvas-backed samples + if (options.format !== undefined && !(this._data instanceof VideoSampleResource)) { + const needsRgbConversion = this.format.startsWith('RGB') !== options.format.startsWith('RGB'); + if (needsRgbConversion) { + // Loop over the destination bytes, swapping R and B + for (let i = 0; i < combinedLayout.allocationSize; i += 4) { + const r = destBytes[i]!; + const b = destBytes[i + 2]!; + destBytes[i] = b; + destBytes[i + 2] = r; + } + } + } + + // Queue a task to resolve p with planeLayouts. + return planeLayouts; } /** @@ -797,35 +947,10 @@ export class VideoSample implements Disposable { assert(this._data !== null); if (this._data instanceof VideoSampleResource) { - const format = this._data.getFormat() ?? 'RGBA'; - const allocationSize = this._data.allocationSize({ format: format as VideoPixelFormat }); - - let data: ArrayBuffer; - if (this._data._lastAllocationBuffer?.byteLength === allocationSize) { - data = this._data._lastAllocationBuffer; - } else { - data = this._data._lastAllocationBuffer = new ArrayBuffer(allocationSize); - } - const layoutResult = this._data.copyTo(data, { format: format as VideoPixelFormat }); - - if (layoutResult instanceof Promise) { - throw new Error( - 'Cannot create a VideoFrame from a VideoSampleResource if copyTo returns a Promise. To work' - + ' around this, copy the data into a buffer and then create a VideoFrame from it.', - ); - } - - const layout = layoutResult; - - return new VideoFrame(data, { - format: format as VideoPixelFormat, - codedWidth: this._data.getCodedWidth(), - codedHeight: this._data.getCodedHeight(), - colorSpace: this._data.getColorSpace(), - layout, - timestamp: this.microsecondTimestamp, - duration: this.microsecondDuration, - }); + throw new Error( + 'Creating a VideoFrame from a VideoSampleResource is currently not supported. To work around this,' + + ' copy the data into a buffer and then create a VideoFrame from it.', + ); } else if (isVideoFrame(this._data)) { return new VideoFrame(this._data, { timestamp: this.microsecondTimestamp, @@ -1474,6 +1599,250 @@ export const getPlaneConfigs = (format: VideoSamplePixelFormat): PlaneConfig[] = } }; +type CombinedBufferLayout = { + allocationSize: number; + computedLayouts: ComputedPlaneLayout[]; +}; + +type ComputedPlaneLayout = { + destinationOffset: number; + destinationStride: number; + sourceTop: number; + sourceHeight: number; + sourceLeftBytes: number; + sourceWidthBytes: number; +}; + +/** Taken from the WebCodecs spec. */ +const ParseVideoFrameCopyToOptions = ( + sample: VideoSample, + options: VideoFrameCopyToOptions, +): CombinedBufferLayout => { + // 1. Let defaultRect be the result of performing the getter steps for visibleRect. + const defaultRect: Rectangle = { + left: 0, + top: 0, + width: sample.codedWidth, + height: sample.codedHeight, + }; + + // 2. Let overrideRect be undefined. + // 3. If options.rect exists, assign the value of options.rect to overrideRect. + const overrideRect = options.rect; + + // 4. Let parsedRect be the result of running the Parse Visible Rect algorithm... + const parsedRect = ParseVisibleRect( + defaultRect, + overrideRect, + sample.codedWidth, + sample.codedHeight, + sample.format, + ); + + // 5. If parsedRect is an exception, return parsedRect. (Handled by throw) + + // 6. Let optLayout be undefined. + // 7. If options.layout exists, assign its value to optLayout. + const optLayout = options.layout; + + // 8. Let format be undefined. + let format: VideoSamplePixelFormat | undefined; + + // 9. If options.format does not exist, assign [[format]] to format. + if (!options.format || options.format === sample.format) { + format = sample.format!; + } else if (['RGBA', 'RGBX', 'BGRA', 'BGRX'].includes(options.format)) { + // 10. Otherwise, if options.format is equal to one of RGBA, RGBX, BGRA, BGRX, then assign options.format + // to format... + format = options.format; + } else { + throw new Error('NotSupportedError: Invalid destination format.'); + } + + // 11. Let combinedLayout be the result of running the Compute Layout and Allocation Size algorithm... + return ComputeLayoutAndAllocationSize(parsedRect, format, optLayout); +}; + +/** Taken from the WebCodecs spec. */ +const ParseVisibleRect = ( + defaultRect: DOMRectInit, + overrideRect: DOMRectInit | undefined, + codedWidth: number, + codedHeight: number, + format: VideoSamplePixelFormat | null, +): DOMRectInit => { + // 1. Let sourceRect be defaultRect + const sourceRect = { ...defaultRect }; + + // 2. If overrideRect is not undefined: + if (overrideRect !== undefined) { + // If either of overrideRect.width or height is 0, return a TypeError. + if (overrideRect.width === 0 || overrideRect.height === 0) { + throw new TypeError('visibleRect dimensions cannot be zero.'); + } + // If the sum of overrideRect.x and overrideRect.width is greater than codedWidth, return a TypeError. + if ((overrideRect.x || 0) + (overrideRect.width || 0) > codedWidth) { + throw new TypeError('visibleRect exceeds codedWidth.'); + } + // If the sum of overrideRect.y and overrideRect.height is greater than codedHeight, return a TypeError. + if ((overrideRect.y || 0) + (overrideRect.height || 0) > codedHeight) { + throw new TypeError('visibleRect exceeds codedHeight.'); + } + // Assign overrideRect to sourceRect. + sourceRect.x = overrideRect.x || 0; + sourceRect.y = overrideRect.y || 0; + sourceRect.width = overrideRect.width || 0; + sourceRect.height = overrideRect.height || 0; + } + + // 3. Let validAlignment be the result of running the Verify Rect Offset Alignment algorithm. + const validAlignment = VerifyRectOffsetAlignment(format, sourceRect); + + // 4. If validAlignment is false, throw a TypeError. + if (!validAlignment) { + throw new TypeError('visibleRect alignment is invalid for the format.'); + } + + // 5. Return sourceRect. + return sourceRect; +}; + +/** Taken from the WebCodecs spec. */ +const VerifyRectOffsetAlignment = (format: VideoSamplePixelFormat | null, rect: DOMRectInit): boolean => { + // 1. If format is null, return true. + if (format === null) return true; + + const planes = getPlaneConfigs(format); + + // 2. Let planeIndex be 0. + // 3. Let numPlanes be the number of planes as defined by format. + // 4. While planeIndex is less than numPlanes: + for (let planeIndex = 0; planeIndex < planes.length; planeIndex++) { + const plane = planes[planeIndex]!; + const sampleWidth = plane.widthDivisor; + const sampleHeight = plane.heightDivisor; + + // If rect.x is not a multiple of sampleWidth, return false. + if ((rect.x || 0) % sampleWidth !== 0) return false; + // If rect.y is not a multiple of sampleHeight, return false. + if ((rect.y || 0) % sampleHeight !== 0) return false; + } + + return true; +}; + +/** Taken from the WebCodecs spec. */ +const ComputeLayoutAndAllocationSize = ( + parsedRect: DOMRectInit, + format: VideoSamplePixelFormat, + layout?: PlaneLayout[], +): CombinedBufferLayout => { + const planes = getPlaneConfigs(format); + + // 1. Let numPlanes be the number of planes as defined by format. + const numPlanes = planes.length; + + // 2. If layout is not undefined and its length does not equal numPlanes, throw a TypeError. + if (layout !== undefined && layout.length !== numPlanes) { + throw new TypeError(`Layout must have ${numPlanes} planes.`); + } + + // 3. Let minAllocationSize be 0. + let minAllocationSize = 0; + + // 4. Let computedLayouts be a new list. + const computedLayouts: ComputedPlaneLayout[] = []; + + // 5. Let endOffsets be a new list. + const endOffsets: number[] = []; + + // 6. Let planeIndex be 0. + // 7. While planeIndex < numPlanes: + for (let planeIndex = 0; planeIndex < numPlanes; planeIndex++) { + const plane = planes[planeIndex]!; + const sampleBytes = plane.sampleBytes; + const sampleWidth = plane.widthDivisor; + const sampleHeight = plane.heightDivisor; + + // Let computedLayout be a new computed plane layout. + const computedLayout: ComputedPlaneLayout = { + destinationOffset: 0, + destinationStride: 0, + sourceTop: 0, + sourceHeight: 0, + sourceLeftBytes: 0, + sourceWidthBytes: 0, + }; + + // Set computedLayout’s sourceTop... + computedLayout.sourceTop = Math.ceil(Math.trunc(parsedRect.y || 0) / sampleHeight); + // Set computedLayout’s sourceHeight... + computedLayout.sourceHeight = Math.ceil(Math.trunc(parsedRect.height || 0) / sampleHeight); + // Set computedLayout’s sourceLeftBytes... + computedLayout.sourceLeftBytes = Math.floor(Math.trunc(parsedRect.x || 0) / sampleWidth) * sampleBytes; + // Set computedLayout’s sourceWidthBytes... + computedLayout.sourceWidthBytes = Math.floor(Math.trunc(parsedRect.width || 0) / sampleWidth) * sampleBytes; + + // If layout is not undefined: + if (layout !== undefined) { + const planeLayout = layout[planeIndex]!; + // If planeLayout.stride is less than computedLayout’s sourceWidthBytes, return a TypeError. + if (planeLayout.stride < computedLayout.sourceWidthBytes) { + throw new TypeError(`Stride for plane ${planeIndex} is too small.`); + } + // Assign planeLayout.offset to computedLayout’s destinationOffset. + computedLayout.destinationOffset = planeLayout.offset; + // Assign planeLayout.stride to computedLayout’s destinationStride. + computedLayout.destinationStride = planeLayout.stride; + } else { + // Otherwise: + // Assign minAllocationSize to computedLayout’s destinationOffset. + computedLayout.destinationOffset = minAllocationSize; + // Assign computedLayout’s sourceWidthBytes to computedLayout’s destinationStride. + computedLayout.destinationStride = computedLayout.sourceWidthBytes; + } + + // Let planeSize be the product of multiplying computedLayout’s destinationStride and sourceHeight. + const planeSize = computedLayout.destinationStride * computedLayout.sourceHeight; + + // Let planeEnd be the sum of planeSize and computedLayout’s destinationOffset. + const planeEnd = planeSize + computedLayout.destinationOffset; + + // If planeSize or planeEnd is greater than maximum range of unsigned long, return a TypeError. + if (planeEnd > 4294967295) { + throw new TypeError('Allocation size exceeds limit.'); + } + + // Append planeEnd to endOffsets. + endOffsets.push(planeEnd); + + // Assign the maximum of minAllocationSize and planeEnd to minAllocationSize. + minAllocationSize = Math.max(minAllocationSize, planeEnd); + + // Check for overlap + for (let earlierPlaneIndex = 0; earlierPlaneIndex < planeIndex; earlierPlaneIndex++) { + const earlierLayout = computedLayouts[earlierPlaneIndex]!; + // If plane A ends before plane B starts, they do not overlap. + if ( + endOffsets[planeIndex]! <= earlierLayout.destinationOffset + || endOffsets[earlierPlaneIndex]! <= computedLayout.destinationOffset + ) { + continue; + } + + throw new TypeError('Planes overlap.'); + } + + computedLayouts.push(computedLayout); + } + + // 12. Return combinedLayout. + return { + allocationSize: minAllocationSize, + computedLayouts: computedLayouts, + }; +}; + const AUDIO_SAMPLE_FORMATS = new Set( ['f32', 'f32-planar', 's16', 's16-planar', 's32', 's32-planar', 'u8', 'u8-planar'], ); diff --git a/test/browser/video-samples.test.ts b/test/browser/video-samples.test.ts index 8f080dc..e1cf0ff 100644 --- a/test/browser/video-samples.test.ts +++ b/test/browser/video-samples.test.ts @@ -154,7 +154,7 @@ test('copyTo and plane layouts', async () => { expect(layout).toEqual([{ offset: 0, - stride: 1300 * 4, + stride: 1280 * 4, }]); using clone = sample.clone(); @@ -162,7 +162,7 @@ test('copyTo and plane layouts', async () => { expect(clonedLayout).toEqual([{ offset: 0, - stride: 1300 * 4, + stride: 1280 * 4, }]); } @@ -186,6 +186,184 @@ test('copyTo and plane layouts', async () => { offset: 1280 * 720 + (1280 / 2) * (720 / 2), stride: 1280 / 2, }]); + + const rgbLayout = await sample.copyTo(buffer, { format: 'RGBA' }); + + expect(rgbLayout).toEqual([{ + offset: 0, + stride: 4 * 1280, + }]); + } +}); + +test('RGB conversion for ArrayBuffer-backed data', async () => { + // 2x2 RGBA image with distinct, easily-verifiable pixels + const src = new Uint8Array([ + 10, 20, 30, 255, 40, 50, 60, 255, + 70, 80, 90, 255, 100, 110, 120, 255, + ]); + + { + // RGBA -> BGRA: R and B should swap + using sample = new VideoSample(src.slice(), { + timestamp: 0, + codedWidth: 2, + codedHeight: 2, + format: 'RGBA', + }); + + const dest = new Uint8Array(2 * 2 * 4); + const layout = await sample.copyTo(dest, { format: 'BGRA' }); + + expect(layout).toEqual([{ offset: 0, stride: 2 * 4 }]); + expect(Array.from(dest)).toEqual([ + 30, 20, 10, 255, 60, 50, 40, 255, + 90, 80, 70, 255, 120, 110, 100, 255, + ]); + } + + { + // RGBA -> RGBA: no swap, bytes copied through unchanged + using sample = new VideoSample(src.slice(), { + timestamp: 0, + codedWidth: 2, + codedHeight: 2, + format: 'RGBA', + }); + + const dest = new Uint8Array(2 * 2 * 4); + await sample.copyTo(dest, { format: 'RGBA' }); + + expect(Array.from(dest)).toEqual(Array.from(src)); + } + + { + // RGBA -> BGRX: R and B should swap + using sample = new VideoSample(src.slice(), { + timestamp: 0, + codedWidth: 2, + codedHeight: 2, + format: 'RGBA', + }); + + const dest = new Uint8Array(2 * 2 * 4); + await sample.copyTo(dest, { format: 'BGRX' }); + + expect(Array.from(dest)).toEqual([ + 30, 20, 10, 255, 60, 50, 40, 255, + 90, 80, 70, 255, 120, 110, 100, 255, + ]); + } + + { + // BGRA -> RGBA: R and B should swap + using sample = new VideoSample(src.slice(), { + timestamp: 0, + codedWidth: 2, + codedHeight: 2, + format: 'BGRA', + }); + + const dest = new Uint8Array(2 * 2 * 4); + await sample.copyTo(dest, { format: 'RGBA' }); + + expect(Array.from(dest)).toEqual([ + 30, 20, 10, 255, 60, 50, 40, 255, + 90, 80, 70, 255, 120, 110, 100, 255, + ]); + } + + { + // BGRA -> BGRA: no swap + using sample = new VideoSample(src.slice(), { + timestamp: 0, + codedWidth: 2, + codedHeight: 2, + format: 'BGRA', + }); + + const dest = new Uint8Array(2 * 2 * 4); + await sample.copyTo(dest, { format: 'BGRA' }); + + expect(Array.from(dest)).toEqual(Array.from(src)); + } +}); + +test('crop (rect option) for ArrayBuffer-backed data', async () => { + // 4x4 RGBA image where each pixel's R/G channels encode (x, y) + const width = 4; + const height = 4; + const src = new Uint8Array(width * height * 4); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const i = (y * width + x) * 4; + src[i] = x; + src[i + 1] = y; + src[i + 2] = 0; + src[i + 3] = 255; + } + } + + { + // Crop a 2x2 region at (1, 1) + using sample = new VideoSample(src.slice(), { + timestamp: 0, + codedWidth: width, + codedHeight: height, + format: 'RGBA', + }); + + const size = sample.allocationSize({ rect: { x: 1, y: 1, width: 2, height: 2 } }); + expect(size).toBe(2 * 2 * 4); + + const dest = new Uint8Array(size); + const layout = await sample.copyTo(dest, { rect: { x: 1, y: 1, width: 2, height: 2 } }); + + expect(layout).toEqual([{ offset: 0, stride: 2 * 4 }]); + expect(Array.from(dest)).toEqual([ + 1, 1, 0, 255, 2, 1, 0, 255, + 1, 2, 0, 255, 2, 2, 0, 255, + ]); + } + + { + // Crop an offset rect at the top-right corner + using sample = new VideoSample(src.slice(), { + timestamp: 0, + codedWidth: width, + codedHeight: height, + format: 'RGBA', + }); + + const dest = new Uint8Array(2 * 1 * 4); + const layout = await sample.copyTo(dest, { rect: { x: 2, y: 0, width: 2, height: 1 } }); + + expect(layout).toEqual([{ offset: 0, stride: 2 * 4 }]); + expect(Array.from(dest)).toEqual([ + 2, 0, 0, 255, 3, 0, 0, 255, + ]); + } + + { + // Crop combined with a custom destination stride (padding between rows) + using sample = new VideoSample(src.slice(), { + timestamp: 0, + codedWidth: width, + codedHeight: height, + format: 'RGBA', + }); + + const stride = 3 * 4; // wider than the crop, leaving trailing padding + const dest = new Uint8Array(stride * 2); + const layout = await sample.copyTo(dest, { + rect: { x: 0, y: 2, width: 2, height: 2 }, + layout: [{ offset: 0, stride }], + }); + + expect(layout).toEqual([{ offset: 0, stride }]); + // Row 0 of crop (y=2): pixels (0,2) and (1,2), then 4 bytes of untouched padding + expect(Array.from(dest.subarray(0, 8))).toEqual([0, 2, 0, 255, 1, 2, 0, 255]); + expect(Array.from(dest.subarray(stride, stride + 8))).toEqual([0, 3, 0, 255, 1, 3, 0, 255]); } }); @@ -201,12 +379,7 @@ test('null format', async () => { expect(() => sample.allocationSize()).toThrow('when format is null'); await expect(async () => sample.copyTo(new ArrayBuffer())).rejects.toThrow('when format is null'); - // Even this throws :( - // See https://github.com/Vanilagy/mediabunny/issues/267 - expect(() => sample.allocationSize({ format: 'RGBA' })).toThrow('when format is null'); - - // Uncomment this if the RGBA conversion works again: - /* + // BUT: See https://github.com/Vanilagy/mediabunny/issues/267 const size = sample.allocationSize({ format: 'RGBA' }); expect(size).toBe(1280 * 720 * 4); const buffer = new ArrayBuffer(size); @@ -223,5 +396,4 @@ test('null format', async () => { await sample.copyTo(buffer, { format: 'RGBX' }); await sample.copyTo(buffer, { format: 'BGRA' }); await sample.copyTo(buffer, { format: 'BGRX' }); - */ }); diff --git a/test/node/server-extension.test.ts b/test/node/server-extension.test.ts index 3fd10ce..145d94c 100644 --- a/test/node/server-extension.test.ts +++ b/test/node/server-extension.test.ts @@ -7,7 +7,8 @@ import { assert, toUint8Array } from '../../src/misc.js'; import { EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; import { NodeAvVideoDecoder } from '../../packages/server/src/video-decoder.js'; import { NodeAvVideoEncoder } from '../../packages/server/src/video-encoder.js'; -import { VideoSample } from '../../src/sample.js'; +import { NodeAvAudioDecoder } from '../../packages/server/src/audio-decoder.js'; +import { AudioSample, VideoSample } from '../../src/sample.js'; import { buildVideoCodecString, VideoCodec } from '../../src/codec.js'; import { EncodedPacket } from '../../src/packet.js'; import { @@ -122,12 +123,6 @@ describe('Video', async () => { expect(meta.decoderConfig!.codec.startsWith('avc1.')).toBe(true); expect(meta.decoderConfig!.codedWidth).toBe(1280); expect(meta.decoderConfig!.codedHeight).toBe(720); - expect(meta.decoderConfig!.colorSpace).toEqual({ - primaries: 'bt709', - transfer: 'iec61966-2-1', - matrix: 'rgb', - fullRange: true, - }); expect(meta.decoderConfig!.description).toBeDefined(); } @@ -208,12 +203,6 @@ describe('Video', async () => { expect(sample.codedHeight).toBe(720); expect(sample.timestamp).toBe(i / 30); expect(sample.duration).toBe(1 / 30); - expect(sample.colorSpace).toEqual({ - primaries: 'bt709', - transfer: 'iec61966-2-1', - matrix: 'rgb', - fullRange: true, - }); const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' })); await sample.copyTo(buf, { format: 'RGBX' }); @@ -248,12 +237,6 @@ describe('Video', async () => { expect(sample.codedHeight).toBe(720); expect(sample.timestamp).toBe(i / 30); expect(sample.duration).toBe(1 / 30); - expect(sample.colorSpace).toEqual({ - primaries: 'bt709', - transfer: 'iec61966-2-1', - matrix: 'rgb', - fullRange: true, - }); const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' })); await sample.copyTo(buf, { format: 'RGBX' }); @@ -292,14 +275,6 @@ describe('Video', async () => { expect(sample.timestamp).toBe(i / 30); expect(sample.duration).toBe(1 / 30); - // Undefined, for some reason: - expect(sample.colorSpace).toEqual({ - primaries: null, - transfer: null, - matrix: null, - fullRange: false, - }); - const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' })); await sample.copyTo(buf, { format: 'RGBX' }); @@ -336,14 +311,6 @@ describe('Video', async () => { expect(sample.timestamp).toBe(i / 30); expect(sample.duration).toBe(1 / 30); - // Undefined, for some reason: - expect(sample.colorSpace).toEqual({ - primaries: null, - transfer: null, - matrix: null, - fullRange: false, - }); - const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' })); await sample.copyTo(buf, { format: 'RGBX' }); @@ -368,12 +335,6 @@ describe('Video', async () => { expect(sample.codedHeight).toBe(720); expect(sample.timestamp).toBe(i / 30); expect(sample.duration).toBe(1 / 30); - expect(sample.colorSpace).toEqual({ - primaries: 'bt709', - transfer: 'iec61966-2-1', - matrix: 'bt470bg', - fullRange: false, - }); const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' })); await sample.copyTo(buf, { format: 'RGBX' }); @@ -399,12 +360,6 @@ describe('Video', async () => { expect(sample.codedHeight).toBe(720); expect(sample.timestamp).toBe(i / 30); expect(sample.duration).toBe(1 / 30); - expect(sample.colorSpace).toEqual({ - primaries: 'bt709', - transfer: 'iec61966-2-1', - matrix: null, - fullRange: false, - }); const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' })); await sample.copyTo(buf, { format: 'RGBX' }); @@ -430,12 +385,6 @@ describe('Video', async () => { expect(sample.codedHeight).toBe(720); expect(sample.timestamp).toBe(i / 30); expect(sample.duration).toBe(1 / 30); - expect(sample.colorSpace).toEqual({ - primaries: 'bt709', - transfer: 'iec61966-2-1', - matrix: 'rgb', - fullRange: false, - }); const buf = new Uint8Array(sample.allocationSize({ format: 'RGBX' })); await sample.copyTo(buf, { format: 'RGBX' }); @@ -668,12 +617,6 @@ describe('Video', async () => { expect(sample.rotation).toBe(0); expect(sample.timestamp).toBe(0); expect(sample.duration).toBe(1 / 25); - expect(sample.colorSpace).toEqual({ - primaries: null, - transfer: null, - matrix: 'bt470bg', - fullRange: true, - }); // Default expected YUV size expect(sample.allocationSize()).toBe(1920 * 1080 * 1.5); @@ -792,3 +735,77 @@ describe('Video', async () => { ]); }); }); + +/* +describe('Audio', async () => { + test('Decoder lifecycle', async () => { + using input = new Input({ + source: new FilePathSource('./test/public/trim-buck-bunny-ffmpeg.ts'), + formats: ALL_FORMATS, + }); + + const audioTrack = await input.getPrimaryAudioTrack(); + assert(audioTrack); + + const decoder = new NodeAvAudioDecoder(); + // @ts-expect-error Readonly + decoder.codec = await audioTrack.getCodec(); + // @ts-expect-error Readonly + decoder.config = await audioTrack.getDecoderConfig(); + + let sampleCount = 0; + const packetTimestamps: number[] = []; + + // @ts-expect-error Readonly + decoder.onSample = (sample: AudioSample) => { + expect(sample.timestamp).toBe(packetTimestamps[sampleCount]); + + if (sampleCount > 0) { + expect(sample.duration).toBeCloseTo( + packetTimestamps[sampleCount]! - packetTimestamps[sampleCount - 1]!, + ); + } + + sampleCount++; + sample.close(); + }; + + await decoder.init(); + + const sink = new EncodedPacketSink(audioTrack); + let packetCount = 0; + for await (const packet of sink.packets()) { + packetTimestamps.push(packet.timestamp); + await decoder.decode(packet); + + if (++packetCount === 10) { + break; + } + } + + await decoder.flush(); + + expect(sampleCount).toBe(10); + + // And, go again + sampleCount = 0; + packetTimestamps.length = 0; + + packetCount = 0; + for await (const packet of sink.packets((await sink.getKeyPacket(5))!)) { + packetTimestamps.push(packet.timestamp); + await decoder.decode(packet); + + if (++packetCount === 10) { + break; + } + } + + await decoder.flush(); + + expect(sampleCount).toBe(10); + + await decoder.close(); + }); +}); +*/