diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 31a0ce8..31d25a7 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -3,6 +3,7 @@ name: Lint on: push: pull_request: + types: [opened, reopened] jobs: lint: @@ -21,11 +22,11 @@ jobs: - name: Install dependencies run: npm ci - - name: Run build - run: npm run build - - name: Run TypeScript run: npm run check - name: Run ESLint run: npm run lint + + - name: Run build + run: npm run build diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 28d920c..2538950 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -3,6 +3,7 @@ name: Test on: push: pull_request: + types: [opened, reopened] jobs: test: @@ -22,4 +23,4 @@ jobs: run: npm ci - name: Run tests - run: npm run test + run: xvfb-run npm test diff --git a/.gitignore b/.gitignore index 355e7e1..a359208 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,6 @@ node_modules .DS_Store /docs/.vitepress/cache /docs/api +*.tsbuildinfo packages/mp3-encoder/dist \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 7a1398e..3b881ed 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -2,5 +2,6 @@ "editor.defaultFormatter": "dbaeumer.vscode-eslint", "editor.codeActionsOnSave": { "source.fixAll.eslint": "explicit" - } + }, + "typescript.tsdk": "node_modules/typescript/lib" } diff --git a/dev/demux.html b/dev/demux.html index 799b92f..bfbdb68 100644 --- a/dev/demux.html +++ b/dev/demux.html @@ -14,16 +14,20 @@ source: new Mediabunny.BlobSource(file), }); - window.doDispose = () => input.dispose(); - const videoTrack = await input.getPrimaryVideoTrack(); + const sink = new Mediabunny.VideoSampleSink(videoTrack); + + const sample = await sink.getSample(0); + + console.log(sample); + + /* const sink = new Mediabunny.EncodedPacketSink(videoTrack); - for await (const sample of sink.packets()) { - console.log(sample.timestamp) - //sample.close(); - await new Promise(resolve => setTimeout(resolve, 0)) + for await (const packet of sink.packets()) { + console.log(packet.timestamp, packet.type, await videoTrack.determinePacketType(packet)); } + */ /* console.log(await input.computeDuration()); diff --git a/dev/mux.html b/dev/mux.html index 21509c3..7afcc7b 100644 --- a/dev/mux.html +++ b/dev/mux.html @@ -34,8 +34,8 @@ } const canvas = document.createElement('canvas'); - canvas.width = 1280; - canvas.height = 720; + canvas.width = 640; + canvas.height = 480; const context = canvas.getContext('2d'); let format = new Mediabunny.MkvOutputFormat({ streamable: false }); @@ -47,6 +47,7 @@ format = new Mediabunny.MkvOutputFormat(); format = new Mediabunny.MovOutputFormat(); format = new Mediabunny.Mp4OutputFormat({ fastStart: 'reserve' }); + format = new Mediabunny.MkvOutputFormat(); let target = new Mediabunny.BufferTarget(); /* @@ -120,6 +121,7 @@ codec: 'vp9', //fullCodecString: 'avc1.42001f', bitrate: 1e6, + alpha: 'keep', onEncoderConfig: console.log, }); let audioSource = new Mediabunny.AudioBufferSource({ @@ -209,5 +211,5 @@ Testing... <00:17.350>One... <00:18.125>Two... await output.finalize(); console.log(target); - //download(new Blob([target.buffer]), 'test' + format.fileExtension); + download(new Blob([target.buffer]), 'test' + format.fileExtension); \ No newline at end of file diff --git a/docs/guide/media-sources.md b/docs/guide/media-sources.md index 96dd142..a013bf4 100644 --- a/docs/guide/media-sources.md +++ b/docs/guide/media-sources.md @@ -47,6 +47,7 @@ All video sources that handle encoding internally require you to specify a `Vide type VideoEncodingConfig = { codec: VideoCodec; bitrate: number | Quality; + alpha?: 'discard' | 'keep'; bitrateMode?: 'constant' | 'variable'; latencyMode?: 'quality' | 'realtime'; keyFrameInterval?: number; @@ -67,6 +68,9 @@ type VideoEncodingConfig = { ``` - `codec`: The [video codec](./supported-formats-and-codecs#video-codecs) used for encoding. - `bitrate`: The target number of bits per second. Alternatively, this can be a [subjective quality](#subjective-qualities). +- `alpha`: What to do with alpha data contained in the video samples. + - `'discard'` (default): Only the samples' color data is kept; the video is opaque. + - `'keep'`: The samples' alpha data is also encoded as side data. Make sure to pair this mode with a container format that supports transparency (such as WebM or Matroska). - `bitrateMode`: Can be used to control constant vs. variable bitrate. - `latencyMode`: The latency mode as specified by the WebCodecs API. Browsers default to `quality`. Media stream-driven video sources will automatically use the `realtime` setting. - `keyFrameInterval`: The maximum interval in seconds between two adjacent key frames. Defaults to 5 seconds. More frequent key frames improve seeking behavior but increase file size. When using multiple video tracks, this value should be set to the same value for all tracks. diff --git a/docs/guide/quick-start.md b/docs/guide/quick-start.md index 94094f1..8e9b8ac 100644 --- a/docs/guide/quick-start.md +++ b/docs/guide/quick-start.md @@ -397,6 +397,42 @@ await output.finalize(); - This is basically [`MediaRecorder`](https://developer.mozilla.org/en-US/docs/Web/API/MediaRecorder), but less sucky. ::: +## Creating transparent video + +```ts +import { + Output, + WebMOutputFormat, + BufferTarget, + CanvasSource, + QUALITY_MEDIUM, +} from 'mediabunny'; + +const output = new Output({ + // Use a format that supports transparency: + format: new WebMOutputFormat(), + target: new BufferTarget(), +}); + +const canvas = new OffscreenCanvas(1280, 720); +const context = canvas.getContext('2d', { alpha: true })!; + +const source = new CanvasSource(canvas, { + codec: 'vp9', + quality: QUALITY_MEDIUM, + alpha: 'keep', // => Also encode alpha data +}); +output.addVideoTrack(source); + +await output.start(); + +// Add data... +await source.add(0, 1 / 30); +// ... + +await output.finalize(); +``` + ## Check encoding support ```ts diff --git a/examples/media-player/media-player.ts b/examples/media-player/media-player.ts index 7827cff..868183c 100644 --- a/examples/media-player/media-player.ts +++ b/examples/media-player/media-player.ts @@ -35,7 +35,7 @@ const fullscreenButton = document.querySelector('#fullscreen-button') as HTMLBut const errorElement = document.querySelector('#error-element') as HTMLDivElement; const warningElement = document.querySelector('#warning-element') as HTMLDivElement; -const context = canvas.getContext('2d', { alpha: false, desynchronized: true })!; +const context = canvas.getContext('2d')!; let audioContext: AudioContext | null = null; let gainNode: GainNode | null = null; @@ -149,11 +149,18 @@ const initMediaPlayer = async (resource: File | string) => { gainNode.connect(audioContext.destination); updateVolume(); + const videoCanBeTransparent = videoTrack + ? await videoTrack.canBeTransparent() + : false; + + playerContainer.style.background = videoCanBeTransparent ? 'transparent' : ''; + // For video, let's use a CanvasSink as it handles rotation and closing video samples for us. // Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases. videoSink = videoTrack && new CanvasSink(videoTrack, { poolSize: 2, fit: 'contain', // In case the video changes dimensions over time + alpha: videoCanBeTransparent, }); // For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API audioSink = audioTrack && new AudioBufferSink(audioTrack); @@ -226,6 +233,7 @@ const startVideoIterator = async () => { if (firstFrame) { // Draw the first frame + context.clearRect(0, 0, canvas.width, canvas.height); context.drawImage(firstFrame.canvas, 0, 0); } }; @@ -242,6 +250,7 @@ const render = (requestFrame = true) => { // Check if the current playback time has caught up to the next frame if (nextFrame && nextFrame.timestamp <= playbackTime) { + context.clearRect(0, 0, canvas.width, canvas.height); context.drawImage(nextFrame.canvas, 0, 0); nextFrame = null; @@ -281,6 +290,7 @@ const updateNextFrame = async () => { const playbackTime = getPlaybackTime(); if (newNextFrame.timestamp <= playbackTime) { // Draw it immediately + context.clearRect(0, 0, canvas.width, canvas.height); context.drawImage(newNextFrame.canvas, 0, 0); } else { // Save it for later @@ -594,9 +604,7 @@ fullscreenButton.addEventListener('click', () => { // I'm sorry for this const isTouchDevice = () => { - return (('ontouchstart' in window) - || (navigator.maxTouchPoints > 0) - || ('msMaxTouchPoints' in navigator && (navigator.msMaxTouchPoints as number) > 0)); + return 'ontouchstart' in window; }; playerContainer.addEventListener('click', () => { diff --git a/examples/metadata-extraction/metadata-extraction.ts b/examples/metadata-extraction/metadata-extraction.ts index a071e95..e5cf198 100644 --- a/examples/metadata-extraction/metadata-extraction.ts +++ b/examples/metadata-extraction/metadata-extraction.ts @@ -55,6 +55,7 @@ const extractMetadata = (resource: File | string) => { 'Coded width': `${track.codedWidth} pixels`, 'Coded height': `${track.codedHeight} pixels`, 'Rotation': `${track.rotation}° clockwise`, + 'Transparency': track.canBeTransparent(), } : track.isAudioTrack() ? { diff --git a/package.json b/package.json index 066f00c..9876f85 100644 --- a/package.json +++ b/package.json @@ -32,10 +32,10 @@ "build": "./build.sh", "watch": "tsx scripts/bundle.ts --watch", "lint": "eslint .", - "test-node": "cd test && vitest node --run", - "test-browser": "cd test && vitest browser --run --browser", - "test": "npm run test-node && npm run test-browser", - "check": "tsc -p src --noEmit && tsc -p packages/mp3-encoder/src --noEmit && tsc -p scripts --noEmit && tsc -p tsconfig.vite.json --noEmit && rm tsconfig.vite.tsbuildinfo", + "test": "npx vitest --run", + "test-node": "npm run test node/", + "test-browser": "npm run test browser/", + "check": "rm -rf dist/modules && tsc -p src && tsc -p packages/mp3-encoder/src --noEmit && tsc -p tsconfig.vitest.json --noEmit && tsc -p scripts --noEmit && tsc -p tsconfig.vite.json --noEmit", "check-docblocks": "tsx scripts/check-docblocks.ts dist/mediabunny.d.ts", "docs:dev": "vitepress dev docs", "docs:build": "npm run build && npm run docs:generate && vitepress build docs && npm run examples:build && cp dist/mediabunny.d.ts dist-docs/", diff --git a/scripts/check-docblocks.ts b/scripts/check-docblocks.ts index 5589a9c..1232750 100644 --- a/scripts/check-docblocks.ts +++ b/scripts/check-docblocks.ts @@ -207,11 +207,11 @@ const main = () => { console.log(`✅ All symbols in ${filePath} have meaningful docblocks.`); } else { console.log( - `❌ Found ${result.missingDocblocks.length} symbols with insufficient docblocks in ${filePath}:`, + `❌ Found ${result.missingDocblocks.length} symbols with insufficient docblocks:`, ); result.missingDocblocks.forEach((item) => { - console.log(` - ${item.kind} '${item.name}' at line ${item.line}: ${item.reason}`); + console.log(` - ${item.kind} '${item.name}' at ${filePath}:${item.line}: ${item.reason}`); }); process.exit(1); diff --git a/src/codec-data.ts b/src/codec-data.ts index f9673e5..377fdea 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -6,8 +6,7 @@ * file, You can obtain one at https://mozilla.org/MPL/2.0/. */ -import { VP9_LEVEL_TABLE } from './codec'; -import { InputVideoTrack } from './input-track'; +import { VideoCodec, VP9_LEVEL_TABLE } from './codec'; import { assert, assertNever, @@ -24,7 +23,7 @@ import { toDataView, toUint8Array, } from './misc'; -import { EncodedPacket, PacketType } from './packet'; +import { PacketType } from './packet'; import { MetadataTags } from './tags'; // References for AVC/HEVC code: @@ -1479,28 +1478,21 @@ export const parseModesFromVorbisSetupPacket = (setupHeader: Uint8Array) => { }; /** Determines a packet's type (key or delta) by digging into the packet bitstream. */ -export const determineVideoPacketType = async ( - videoTrack: InputVideoTrack, - packet: EncodedPacket, -): Promise => { - assert(videoTrack.codec); - - switch (videoTrack.codec) { +export const determineVideoPacketType = ( + codec: VideoCodec, + decoderConfig: VideoDecoderConfig, + packetData: Uint8Array, +): PacketType | null => { + switch (codec) { case 'avc': { - const decoderConfig = await videoTrack.getDecoderConfig(); - assert(decoderConfig); - - const nalUnits = extractAvcNalUnits(packet.data, decoderConfig); + const nalUnits = extractAvcNalUnits(packetData, decoderConfig); const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR); return isKeyframe ? 'key' : 'delta'; }; case 'hevc': { - const decoderConfig = await videoTrack.getDecoderConfig(); - assert(decoderConfig); - - const nalUnits = extractHevcNalUnits(packet.data, decoderConfig); + const nalUnits = extractHevcNalUnits(packetData, decoderConfig); const isKeyframe = nalUnits.some((x) => { const type = extractNalUnitTypeForHevc(x); return HevcNalUnitType.BLA_W_LP <= type && type <= HevcNalUnitType.RSV_IRAP_VCL23; @@ -1511,12 +1503,12 @@ export const determineVideoPacketType = async ( case 'vp8': { // VP8, once again, by far the easiest to deal with. - const frameType = packet.data[0]! & 0b1; + const frameType = packetData[0]! & 0b1; return frameType === 0 ? 'key' : 'delta'; }; case 'vp9': { - const bitstream = new Bitstream(packet.data); + const bitstream = new Bitstream(packetData); if (bitstream.readBits(2) !== 2) { return null; @@ -1543,7 +1535,7 @@ export const determineVideoPacketType = async ( case 'av1': { let reducedStillPictureHeader = false; - for (const { type, data } of iterateAv1PacketObus(packet.data)) { + for (const { type, data } of iterateAv1PacketObus(packetData)) { if (type === 1) { // OBU_SEQUENCE_HEADER const bitstream = new Bitstream(data); @@ -1573,7 +1565,7 @@ export const determineVideoPacketType = async ( }; default: { - assertNever(videoTrack.codec); + assertNever(codec); assert(false); }; } diff --git a/src/encode.ts b/src/encode.ts index b77bb52..bf7834b 100644 --- a/src/encode.ts +++ b/src/encode.ts @@ -107,6 +107,14 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => { * @public */ export type VideoEncodingAdditionalOptions = { + /** + * What to do with alpha data contained in the video samples. + * + * - `'discard'` (default): Only the samples' color data is kept; the video is opaque. + * - `'keep'`: The samples' alpha data is also encoded as side data. Make sure to pair this mode with a container + * format that supports transparency (such as WebM or Matroska). + */ + alpha?: 'discard' | 'keep'; /** Configures the bitrate mode. */ bitrateMode?: 'constant' | 'variable'; /** The latency mode used by the encoder; controls the performance-quality tradeoff. */ @@ -136,6 +144,9 @@ export const validateVideoEncodingAdditionalOptions = (codec: VideoCodec, option if (!options || typeof options !== 'object') { throw new TypeError('Encoding options must be an object.'); } + if (options.alpha !== undefined && !['discard', 'keep'].includes(options.alpha)) { + throw new TypeError('options.alpha, when provided, must be \'discard\' or \'keep\'.'); + } if (options.bitrateMode !== undefined && !['constant', 'variable'].includes(options.bitrateMode)) { throw new TypeError('bitrateMode, when provided, must be \'constant\' or \'variable\'.'); } @@ -189,7 +200,8 @@ export const buildVideoEncoderConfig = (options: { height: options.height, bitrate: resolvedBitrate, bitrateMode: options.bitrateMode, - framerate: options.framerate, // this.source._connectedTrack?.metadata.frameRate, + alpha: options.alpha ?? 'discard', + framerate: options.framerate, latencyMode: options.latencyMode, hardwareAcceleration: options.hardwareAcceleration, scalabilityMode: options.scalabilityMode, @@ -506,6 +518,7 @@ export const canEncodeVideo = async ( bitrate, framerate: undefined, ...restOptions, + alpha: 'discard', // Since we handle alpha ourselves }); const support = await VideoEncoder.isConfigSupported(encoderConfig); diff --git a/src/index.ts b/src/index.ts index 9ec8163..5925cf5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -157,6 +157,7 @@ export { } from './input-track'; export { EncodedPacket, + EncodedPacketSideData, PacketType, } from './packet'; export { diff --git a/src/input-track.ts b/src/input-track.ts index e0680cd..25b5100 100644 --- a/src/input-track.ts +++ b/src/input-track.ts @@ -192,6 +192,7 @@ export interface InputVideoTrackBacking extends InputTrackBacking { getCodedHeight(): number; getRotation(): Rotation; getColorSpace(): Promise; + canBeTransparent(): Promise; getDecoderConfig(): Promise; } @@ -260,6 +261,11 @@ export class InputVideoTrack extends InputTrack { || (colorSpace.matrix as string) === 'bt2020-ncl'; } + /** Checks if this track may contain transparent samples with alpha data. */ + canBeTransparent() { + return this._backing.canBeTransparent(); + } + /** * Returns the [decoder configuration](https://www.w3.org/TR/webcodecs/#video-decoder-config) for decoding the * track's packets using a [`VideoDecoder`](https://developer.mozilla.org/en-US/docs/Web/API/VideoDecoder). Returns @@ -312,7 +318,10 @@ export class InputVideoTrack extends InputTrack { return null; } - return determineVideoPacketType(this, packet); + const decoderConfig = await this.getDecoderConfig(); + assert(decoderConfig); + + return determineVideoPacketType(this.codec, decoderConfig, packet.data); } } diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index d0027f4..16aa9a5 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -2966,6 +2966,10 @@ class IsobmffVideoTrackBacking extends IsobmffTrackBacking implements InputVideo }; } + async canBeTransparent() { + return false; + } + async getDecoderConfig(): Promise { if (!this.internalTrack.info.codec) { return null; diff --git a/src/matroska/ebml.ts b/src/matroska/ebml.ts index 14dd625..4fc70bd 100644 --- a/src/matroska/ebml.ts +++ b/src/matroska/ebml.ts @@ -100,6 +100,7 @@ export enum EBMLId { Video = 0xe0, PixelWidth = 0xb0, PixelHeight = 0xba, + AlphaMode = 0x53c0, Audio = 0xe1, SamplingFrequency = 0xb5, Channels = 0x9f, diff --git a/src/matroska/matroska-demuxer.ts b/src/matroska/matroska-demuxer.ts index 7f1b030..e287652 100644 --- a/src/matroska/matroska-demuxer.ts +++ b/src/matroska/matroska-demuxer.ts @@ -50,7 +50,7 @@ import { TRANSFER_CHARACTERISTICS_MAP_INVERSE, UNDETERMINED_LANGUAGE, } from '../misc'; -import { EncodedPacket, PLACEHOLDER_DATA } from '../packet'; +import { EncodedPacket, EncodedPacketSideData, PLACEHOLDER_DATA } from '../packet'; import { assertDefinedSize, CODEC_STRING_MAP, @@ -143,6 +143,7 @@ type ClusterBlock = { data: Uint8Array; lacing: BlockLacing; decoded: boolean; + mainAdditional: Uint8Array | null; }; type CuePoint = { @@ -204,6 +205,7 @@ type InternalTrack = { codec: VideoCodec | null; codecDescription: Uint8Array | null; colorSpace: VideoColorSpaceInit | null; + alphaMode: boolean; } | { type: 'audio'; @@ -236,6 +238,11 @@ export class MatroskaDemuxer extends Demuxer { currentTrack: InternalTrack | null = null; currentCluster: Cluster | null = null; currentBlock: ClusterBlock | null = null; + currentBlockAdditional: { + addId: number; + data: Uint8Array | null; + } | null = null; + currentCueTime: number | null = null; currentDecodingInstruction: DecodingInstruction | null = null; currentTagTargetIsMovie: boolean = true; @@ -845,6 +852,7 @@ export class MatroskaDemuxer extends Demuxer { data: frameData, lacing: BlockLacing.None, decoded: true, + mainAdditional: originalBlock.mainAdditional, }); } @@ -1110,6 +1118,7 @@ export class MatroskaDemuxer extends Demuxer { codec: null, codecDescription: null, colorSpace: null, + alphaMode: false, }; } else if (type === 2) { this.currentTrack.info = { @@ -1214,6 +1223,12 @@ export class MatroskaDemuxer extends Demuxer { this.currentTrack.info.height = readUnsignedInt(slice, size); }; break; + case EBMLId.AlphaMode: { + if (this.currentTrack?.info?.type !== 'video') break; + + this.currentTrack.info.alphaMode = readUnsignedInt(slice, size) === 1; + }; break; + case EBMLId.Colour: { if (this.currentTrack?.info?.type !== 'video') break; @@ -1365,6 +1380,7 @@ export class MatroskaDemuxer extends Demuxer { data: blockData, lacing, decoded: !hasDecodingInstructions, + mainAdditional: null, }); }; break; @@ -1407,10 +1423,43 @@ export class MatroskaDemuxer extends Demuxer { data: blockData, lacing, decoded: !hasDecodingInstructions, + mainAdditional: null, }; trackData.blocks.push(this.currentBlock); }; break; + case EBMLId.BlockAdditions: { + this.readContiguousElements(slice.slice(dataStartPos, size)); + }; break; + + case EBMLId.BlockMore: { + if (!this.currentBlock) break; + + this.currentBlockAdditional = { + addId: 1, + data: null, + }; + + this.readContiguousElements(slice.slice(dataStartPos, size)); + + if (this.currentBlockAdditional.data && this.currentBlockAdditional.addId === 1) { + this.currentBlock.mainAdditional = this.currentBlockAdditional.data; + } + this.currentBlockAdditional = null; + }; break; + + case EBMLId.BlockAdditional: { + if (!this.currentBlockAdditional) break; + + this.currentBlockAdditional.data = readBytes(slice, size); + }; break; + + case EBMLId.BlockAddID: { + if (!this.currentBlockAdditional) break; + + this.currentBlockAdditional.addId = readUnsignedInt(slice, size); + }; break; + case EBMLId.BlockDuration: { if (!this.currentBlock) break; @@ -2004,6 +2053,13 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { const data = options.metadataOnly ? PLACEHOLDER_DATA : block.data; const timestamp = block.timestamp / this.internalTrack.segment.timestampFactor; const duration = block.duration / this.internalTrack.segment.timestampFactor; + + const sideData: EncodedPacketSideData = {}; + if (block.mainAdditional && this.internalTrack.info?.type === 'video' && this.internalTrack.info.alphaMode) { + sideData.alpha = options.metadataOnly ? PLACEHOLDER_DATA : block.mainAdditional; + sideData.alphaByteLength = block.mainAdditional.byteLength; + } + const packet = new EncodedPacket( data, block.isKeyFrame ? 'key' : 'delta', @@ -2011,6 +2067,7 @@ abstract class MatroskaTrackBacking implements InputTrackBacking { duration, cluster.dataStartPos + blockIndex, block.data.byteLength, + sideData, ); this.packetToClusterLocation.set(packet, { cluster, blockIndex }); @@ -2319,6 +2376,10 @@ class MatroskaVideoTrackBacking extends MatroskaTrackBacking implements InputVid }; } + async canBeTransparent() { + return this.internalTrack.info.alphaMode; + } + async getDecoderConfig(): Promise { if (!this.internalTrack.info.codec) { return null; diff --git a/src/matroska/matroska-muxer.ts b/src/matroska/matroska-muxer.ts index c839e63..b12b95a 100644 --- a/src/matroska/matroska-muxer.ts +++ b/src/matroska/matroska-muxer.ts @@ -89,6 +89,7 @@ type MatroskaTrackData = { width: number; height: number; decoderConfig: VideoDecoderConfig; + alphaMode: boolean; }; } | { track: OutputAudioTrack; @@ -343,6 +344,7 @@ export class MatroskaMuxer extends Muxer { const videoElement: EBMLElement = { id: EBMLId.Video, data: [ { id: EBMLId.PixelWidth, data: trackData.info.width }, { id: EBMLId.PixelHeight, data: trackData.info.height }, + trackData.info.alphaMode ? { id: EBMLId.AlphaMode, data: 1 } : null, (colorSpaceIsComplete(colorSpace) ? { id: EBMLId.Colour, @@ -695,7 +697,7 @@ export class MatroskaMuxer extends Muxer { }); } - private getVideoTrackData(track: OutputVideoTrack, meta?: EncodedVideoChunkMetadata) { + private getVideoTrackData(track: OutputVideoTrack, packet: EncodedPacket, meta?: EncodedVideoChunkMetadata) { const existingTrackData = this.trackDatas.find(x => x.track === track); if (existingTrackData) { return existingTrackData as MatroskaVideoTrackData; @@ -715,6 +717,7 @@ export class MatroskaMuxer extends Muxer { width: meta.decoderConfig.codedWidth, height: meta.decoderConfig.codedHeight, decoderConfig: meta.decoderConfig, + alphaMode: !!packet.sideData.alpha, // The first packet determines if this track has alpha or not }, chunkQueue: [], lastWrittenMsTimestamp: null, @@ -819,7 +822,7 @@ export class MatroskaMuxer extends Muxer { const release = await this.mutex.acquire(); try { - const trackData = this.getVideoTrackData(track, meta); + const trackData = this.getVideoTrackData(track, packet, meta); const isKeyFrame = packet.type === 'key'; let timestamp = this.validateAndNormalizeTimestamp(trackData.track, packet.timestamp, isKeyFrame); @@ -831,7 +834,11 @@ export class MatroskaMuxer extends Muxer { duration = roundToMultiple(duration, 1 / track.metadata.frameRate); } - const videoChunk = this.createInternalChunk(packet.data, timestamp, duration, packet.type); + const additions = trackData.info.alphaMode + ? packet.sideData.alpha ?? null + : null; + + const videoChunk = this.createInternalChunk(packet.data, timestamp, duration, packet.type, additions); if (track.source._codec === 'vp9') this.fixVP9ColorSpace(trackData, videoChunk); trackData.chunkQueue.push(videoChunk); @@ -1086,8 +1093,8 @@ export class MatroskaMuxer extends Muxer { chunk.additions ? { id: EBMLId.BlockAdditions, data: [ { id: EBMLId.BlockMore, data: [ + { id: EBMLId.BlockAddID, data: 1 }, // Some players expect BlockAddID to come first { id: EBMLId.BlockAdditional, data: chunk.additions }, - { id: EBMLId.BlockAddID, data: 1 }, ] }, ] } : null, diff --git a/src/media-sink.ts b/src/media-sink.ts index 1dc6dc0..793b0b7 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -7,7 +7,12 @@ */ import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec'; -import { extractHevcNalUnits, extractNalUnitTypeForHevc, HevcNalUnitType } from './codec-data'; +import { + determineVideoPacketType, + extractHevcNalUnits, + extractNalUnitTypeForHevc, + HevcNalUnitType, +} from './codec-data'; import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder'; import { InputDisposedError } from './input'; import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track'; @@ -365,7 +370,7 @@ abstract class DecoderWrapper< > { constructor( public onSample: (sample: MediaSample) => unknown, - public onError: (error: DOMException) => unknown, + public onError: (error: Error) => unknown, ) {} abstract getDecodeQueueSize(): number; @@ -388,7 +393,7 @@ export abstract class BaseMediaSampleSink< /** @internal */ abstract _createDecoder( onSample: (sample: MediaSample) => unknown, - onError: (error: DOMException) => unknown + onError: (error: Error) => unknown ): Promise>; /** @internal */ abstract _createPacketSink(): EncodedPacketSink; @@ -811,9 +816,23 @@ class VideoDecoderWrapper extends DecoderWrapper { currentPacketIndex = 0; raslSkipped = false; // For HEVC stuff + // Alpha stuff + alphaDecoder: VideoDecoder | null = null; + alphaHadKeyframe = false; + colorQueue: VideoFrame[] = []; + alphaQueue: (VideoFrame | null)[] = []; + merger: ColorAlphaMerger | null = null; + mergerCreationFailed = false; + decodedAlphaChunkCount = 0; + alphaDecoderQueueSize = 0; + /** Each value is the number of decoded alpha chunks at which a null alpha frame should be added. */ + nullAlphaFrameQueue: number[] = []; + currentAlphaPacketIndex = 0; + alphaRaslSkipped = false; // For HEVC stuff + constructor( onSample: (sample: VideoSample) => unknown, - onError: (error: DOMException) => unknown, + onError: (error: Error) => unknown, public codec: VideoCodec, public decoderConfig: VideoDecoderConfig, public rotation: Rotation, @@ -840,79 +859,48 @@ class VideoDecoderWrapper extends DecoderWrapper { void this.customDecoderCallSerializer.call(() => this.customDecoder!.init()); } else { - // Specific handler for the WebCodecs VideoDecoder to iron out browser differences - const sampleHandler = (sample: VideoSample) => { - if (isSafari()) { - // For correct B-frame handling, we don't just hand over the frames directly but instead add them to - // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue - // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we - // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are - // emitted in presentation order, but Safari doesn't always follow this rule. - if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) { - for (const sample of this.sampleQueue) { - this.finalizeAndEmitSample(sample); - } + const colorHandler = (frame: VideoFrame) => { + if (this.alphaQueue.length > 0) { + // Even when no alpha data is present (most of the time), there will be nulls in this queue + const alphaFrame = this.alphaQueue.shift(); + assert(alphaFrame !== undefined); - this.sampleQueue.length = 0; - } - - insertSorted(this.sampleQueue, sample, x => x.timestamp); + this.mergeAlpha(frame, alphaFrame); } else { - // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are - // required to emit decoded frames in presentation order *while* retaining the timestamp of their - // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a - // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and - // ensure they are sorted by doing this. - const timestamp = this.inputTimestamps.shift(); - - // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the - // correspondence should be 1:1. - assert(timestamp !== undefined); - - sample.setTimestamp(timestamp); - this.finalizeAndEmitSample(sample); + this.colorQueue.push(frame); } }; this.decoder = new VideoDecoder({ - output: frame => sampleHandler(new VideoSample(frame)), + output: (frame) => { + try { + colorHandler(frame); + } catch (error) { + this.onError(error as Error); + } + }, error: onError, }); this.decoder.configure(decoderConfig); } } - finalizeAndEmitSample(sample: VideoSample) { - // Round the timestamps to the time resolution - sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution); - sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution); - sample.setRotation(this.rotation); - - this.onSample(sample); - } - getDecodeQueueSize() { if (this.customDecoder) { return this.customDecoderQueueSize; } else { assert(this.decoder); - return this.decoder.decodeQueueSize; + + return Math.max( + this.decoder.decodeQueueSize, + this.alphaDecoder?.decodeQueueSize ?? 0, + ); } } decode(packet: EncodedPacket) { if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) { - // If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as - // CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave - // differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good - // and causes bugs upstream. So, let's take the dropping into our own hands. - const nalUnits = extractHevcNalUnits(packet.data, this.decoderConfig); - const hasRaslPicture = nalUnits.some((x) => { - const type = extractNalUnitTypeForHevc(x); - return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R; - }); - - if (hasRaslPicture) { + if (this.hasHevcRaslPicture(packet.data)) { return; // Drop } @@ -934,15 +922,218 @@ class VideoDecoderWrapper extends DecoderWrapper { } this.decoder.decode(packet.toEncodedVideoChunk()); + this.decodeAlphaData(packet); } } + decodeAlphaData(packet: EncodedPacket) { + if (!packet.sideData.alpha || this.mergerCreationFailed) { + // No alpha side data in the packet, most common case + this.pushNullAlphaFrame(); + return; + } + + if (!this.merger) { + try { + this.merger = new ColorAlphaMerger(); + } catch (error) { + console.error('Due to an error, only color data will be decoded.', error); + + this.mergerCreationFailed = true; + this.decodeAlphaData(packet); // Go again + + return; + } + } + + // Check if we need to set up the alpha decoder + if (!this.alphaDecoder) { + const alphaHandler = (frame: VideoFrame) => { + this.alphaDecoderQueueSize--; + + if (this.colorQueue.length > 0) { + const colorFrame = this.colorQueue.shift(); + assert(colorFrame !== undefined); + + this.mergeAlpha(colorFrame, frame); + } else { + this.alphaQueue.push(frame); + } + + // Check if any null frames have been queued for this point + this.decodedAlphaChunkCount++; + while ( + this.nullAlphaFrameQueue.length > 0 + && this.nullAlphaFrameQueue[0] === this.decodedAlphaChunkCount + ) { + this.nullAlphaFrameQueue.shift(); + + if (this.colorQueue.length > 0) { + const colorFrame = this.colorQueue.shift(); + assert(colorFrame !== undefined); + + this.mergeAlpha(colorFrame, null); + } else { + this.alphaQueue.push(null); + } + } + }; + + this.alphaDecoder = new VideoDecoder({ + output: (frame) => { + try { + alphaHandler(frame); + } catch (error) { + this.onError(error as Error); + } + }, + error: this.onError, + }); + this.alphaDecoder.configure(this.decoderConfig); + } + + const type = determineVideoPacketType(this.codec, this.decoderConfig, packet.sideData.alpha); + + // Alpha packets might follow a different key frame rhythm than the main packets. Therefore, before we start + // decoding, we must first find a packet that's actually a key frame. Until then, we treat the image as opaque. + if (!this.alphaHadKeyframe) { + this.alphaHadKeyframe = type === 'key'; + } + + if (this.alphaHadKeyframe) { + // Same RASL skipping logic as for color, unlikely to be hit (since who uses HEVC with separate alpha??) but + // here for symmetry. + if (this.codec === 'hevc' && this.currentAlphaPacketIndex > 0 && !this.alphaRaslSkipped) { + if (this.hasHevcRaslPicture(packet.sideData.alpha)) { + this.pushNullAlphaFrame(); + return; + } + + this.alphaRaslSkipped = true; + } + + this.currentAlphaPacketIndex++; + this.alphaDecoder.decode(packet.alphaToEncodedVideoChunk(type ?? packet.type)); + this.alphaDecoderQueueSize++; + } else { + this.pushNullAlphaFrame(); + } + } + + pushNullAlphaFrame() { + if (this.alphaDecoderQueueSize === 0) { + // Easy + this.alphaQueue.push(null); + } else { + // There are still alpha chunks being decoded, so pushing `null` immediately would result in out-of-order + // data and be incorrect. Instead, we need to enqueue a "null frame" for when the current decoder workload + // has finished. + this.nullAlphaFrameQueue.push(this.decodedAlphaChunkCount + this.alphaDecoderQueueSize); + } + } + + /** + * If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as + * CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave + * differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good + * and causes bugs upstream. So, let's take the dropping into our own hands. + */ + hasHevcRaslPicture(packetData: Uint8Array) { + const nalUnits = extractHevcNalUnits(packetData, this.decoderConfig); + return nalUnits.some((x) => { + const type = extractNalUnitTypeForHevc(x); + return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R; + }); + } + + /** Handler for the WebCodecs VideoDecoder for ironing out browser differences. */ + sampleHandler(sample: VideoSample) { + if (isSafari()) { + // For correct B-frame handling, we don't just hand over the frames directly but instead add them to + // a queue, because we want to ensure frames are emitted in presentation order. We flush the queue + // each time we receive a frame with a timestamp larger than the highest we've seen so far, as we + // can sure that is not a B-frame. Typically, WebCodecs automatically guarantees that frames are + // emitted in presentation order, but Safari doesn't always follow this rule. + if (this.sampleQueue.length > 0 && (sample.timestamp >= last(this.sampleQueue)!.timestamp)) { + for (const sample of this.sampleQueue) { + this.finalizeAndEmitSample(sample); + } + + this.sampleQueue.length = 0; + } + + insertSorted(this.sampleQueue, sample, x => x.timestamp); + } else { + // Assign it the next earliest timestamp from the input. We do this because browsers, by spec, are + // required to emit decoded frames in presentation order *while* retaining the timestamp of their + // originating EncodedVideoChunk. For files with B-frames but no out-of-order timestamps (like a + // missing ctts box, for example), this causes a mismatch. We therefore fix the timestamps and + // ensure they are sorted by doing this. + const timestamp = this.inputTimestamps.shift(); + + // There's no way we'd have more decoded frames than encoded packets we passed in. Actually, the + // correspondence should be 1:1. + assert(timestamp !== undefined); + + sample.setTimestamp(timestamp); + this.finalizeAndEmitSample(sample); + } + } + + finalizeAndEmitSample(sample: VideoSample) { + // Round the timestamps to the time resolution + sample.setTimestamp(Math.round(sample.timestamp * this.timeResolution) / this.timeResolution); + sample.setDuration(Math.round(sample.duration * this.timeResolution) / this.timeResolution); + sample.setRotation(this.rotation); + + this.onSample(sample); + } + + mergeAlpha(color: VideoFrame, alpha: VideoFrame | null) { + if (!alpha) { + // Nothing needs to be merged + const finalSample = new VideoSample(color); + this.sampleHandler(finalSample); + + return; + } + + assert(this.merger); + + this.merger.update(color, alpha); + color.close(); + alpha.close(); + + const finalFrame = new VideoFrame(this.merger.canvas, { + timestamp: color.timestamp, + duration: color.duration ?? undefined, + }); + + const finalSample = new VideoSample(finalFrame); + this.sampleHandler(finalSample); + } + async flush() { if (this.customDecoder) { await this.customDecoderCallSerializer.call(() => this.customDecoder!.flush()); } else { assert(this.decoder); - await this.decoder.flush(); + await Promise.all([ + this.decoder.flush(), + this.alphaDecoder?.flush(), + ]); + + this.colorQueue.forEach(x => x.close()); + this.colorQueue.length = 0; + this.alphaQueue.forEach(x => x?.close()); + this.alphaQueue.length = 0; + + this.alphaHadKeyframe = false; + this.decodedAlphaChunkCount = 0; + this.alphaDecoderQueueSize = 0; + this.nullAlphaFrameQueue.length = 0; + this.currentAlphaPacketIndex = 0; + this.alphaRaslSkipped = false; } if (isSafari()) { @@ -963,6 +1154,14 @@ class VideoDecoderWrapper extends DecoderWrapper { } else { assert(this.decoder); this.decoder.close(); + this.alphaDecoder?.close(); + + this.colorQueue.forEach(x => x.close()); + this.colorQueue.length = 0; + this.alphaQueue.forEach(x => x?.close()); + this.alphaQueue.length = 0; + + this.merger?.close(); } for (const sample of this.sampleQueue) { @@ -972,6 +1171,150 @@ class VideoDecoderWrapper extends DecoderWrapper { } } +/** Utility class that merges together color and alpha information using simple WebGL 2 shaders. */ +class ColorAlphaMerger { + canvas: OffscreenCanvas | HTMLCanvasElement; + private gl: WebGL2RenderingContext; + private program: WebGLProgram; + private vao: WebGLVertexArrayObject; + private colorTexture: WebGLTexture; + private alphaTexture: WebGLTexture; + + constructor() { + // Canvas will be resized later + if (typeof OffscreenCanvas !== 'undefined') { + // Prefer OffscreenCanvas for Worker environments + this.canvas = new OffscreenCanvas(300, 150); + } else { + this.canvas = document.createElement('canvas'); + } + + const gl = this.canvas.getContext('webgl2', { + premultipliedAlpha: false, + }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness + if (!gl) { + throw new Error('Couldn\'t acquire WebGL 2 context.'); + } + + this.gl = gl; + this.program = this.createProgram(); + this.vao = this.createVAO(); + this.colorTexture = this.createTexture(); + this.alphaTexture = this.createTexture(); + + this.gl.useProgram(this.program); + this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_colorTexture'), 0); + this.gl.uniform1i(this.gl.getUniformLocation(this.program, 'u_alphaTexture'), 1); + } + + private createProgram(): WebGLProgram { + const vertexShader = this.createShader(this.gl.VERTEX_SHADER, `#version 300 es + in vec2 a_position; + in vec2 a_texCoord; + out vec2 v_texCoord; + + void main() { + gl_Position = vec4(a_position, 0.0, 1.0); + v_texCoord = a_texCoord; + } + `); + + const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es + precision highp float; + + uniform sampler2D u_colorTexture; + uniform sampler2D u_alphaTexture; + in vec2 v_texCoord; + out vec4 fragColor; + + void main() { + vec3 color = texture(u_colorTexture, v_texCoord).rgb; + float alpha = texture(u_alphaTexture, v_texCoord).r; + fragColor = vec4(color, alpha); + } + `); + + const program = this.gl.createProgram(); + this.gl.attachShader(program, vertexShader); + this.gl.attachShader(program, fragmentShader); + this.gl.linkProgram(program); + + return program; + } + + private createShader(type: number, source: string): WebGLShader { + const shader = this.gl.createShader(type)!; + this.gl.shaderSource(shader, source); + this.gl.compileShader(shader); + return shader; + } + + private createVAO(): WebGLVertexArrayObject { + const vao = this.gl.createVertexArray(); + this.gl.bindVertexArray(vao); + + const vertices = new Float32Array([ + -1, -1, 0, 1, + 1, -1, 1, 1, + -1, 1, 0, 0, + 1, 1, 1, 0, + ]); + + const buffer = this.gl.createBuffer(); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer); + this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW); + + const positionLocation = this.gl.getAttribLocation(this.program, 'a_position'); + const texCoordLocation = this.gl.getAttribLocation(this.program, 'a_texCoord'); + + this.gl.enableVertexAttribArray(positionLocation); + this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0); + + this.gl.enableVertexAttribArray(texCoordLocation); + this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8); + + return vao; + } + + private createTexture(): WebGLTexture { + const texture = this.gl.createTexture(); + + this.gl.bindTexture(this.gl.TEXTURE_2D, texture); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR); + + return texture; + } + + update(color: VideoFrame, alpha: VideoFrame): void { + if (color.displayWidth !== this.canvas.width || color.displayHeight !== this.canvas.height) { + this.canvas.width = color.displayWidth; + this.canvas.height = color.displayHeight; + } + + this.gl.activeTexture(this.gl.TEXTURE0); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.colorTexture); + this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, color); + + this.gl.activeTexture(this.gl.TEXTURE1); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.alphaTexture); + this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, alpha); + + this.gl.viewport(0, 0, this.canvas.width, this.canvas.height); + this.gl.clear(this.gl.COLOR_BUFFER_BIT); + + this.gl.bindVertexArray(this.vao); + this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); + } + + close() { + this.gl.getExtension('WEBGL_lose_context')?.loseContext(); + this.gl = null as unknown as WebGL2RenderingContext; + } +} + /** * A sink that retrieves decoded video samples (video frames) from a video track. * @group Media sinks @@ -995,7 +1338,7 @@ export class VideoSampleSink extends BaseMediaSampleSink { /** @internal */ async _createDecoder( onSample: (sample: VideoSample) => unknown, - onError: (error: DOMException) => unknown, + onError: (error: Error) => unknown, ) { if (!(await this._track.canDecode())) { throw new Error( @@ -1078,6 +1421,11 @@ export type WrappedCanvas = { * @public */ export type CanvasSinkOptions = { + /** + * Whether the output canvases should have transparency instead of a black background. Defaults to `false`. Set + * this to `true` when using this sink to read transparent videos. + */ + alpha?: boolean; /** * The width of the output canvas in pixels, defaulting to the display width of the video track. If height is not * set, it will be deduced automatically based on aspect ratio. @@ -1130,6 +1478,8 @@ export class CanvasSink { /** @internal */ _videoTrack: InputVideoTrack; /** @internal */ + _alpha: boolean; + /** @internal */ _width: number; /** @internal */ _height: number; @@ -1154,6 +1504,9 @@ export class CanvasSink { if (options && typeof options !== 'object') { throw new TypeError('options must be an object.'); } + if (options.alpha !== undefined && typeof options.alpha !== 'boolean') { + throw new TypeError('options.alpha, when provided, must be a boolean.'); + } if (options.width !== undefined && (!Number.isInteger(options.width) || options.width <= 0)) { throw new TypeError('options.width, when defined, must be a positive integer.'); } @@ -1214,6 +1567,7 @@ export class CanvasSink { } this._videoTrack = videoTrack; + this._alpha = options.alpha ?? false; this._width = width; this._height = height; this._rotation = rotation; @@ -1250,14 +1604,14 @@ export class CanvasSink { } const context = canvas.getContext('2d', { - alpha: isFirefox(), // Firefox has VideoFrame glitches with opaque canvases + alpha: this._alpha || isFirefox(), // Firefox has VideoFrame glitches with opaque canvases }) as CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D; assert(context); context.resetTransform(); if (!canvasIsNew) { - if (isFirefox()) { + if (!this._alpha && isFirefox()) { context.fillStyle = 'black'; context.fillRect(0, 0, this._width, this._height); } else { @@ -1338,7 +1692,7 @@ class AudioDecoderWrapper extends DecoderWrapper { constructor( onSample: (sample: AudioSample) => unknown, - onError: (error: DOMException) => unknown, + onError: (error: Error) => unknown, codec: AudioCodec, decoderConfig: AudioDecoderConfig, ) { @@ -1390,7 +1744,13 @@ class AudioDecoderWrapper extends DecoderWrapper { void this.customDecoderCallSerializer.call(() => this.customDecoder!.init()); } else { this.decoder = new AudioDecoder({ - output: data => sampleHandler(new AudioSample(data)), + output: (data) => { + try { + sampleHandler(new AudioSample(data)); + } catch (error) { + this.onError(error as Error); + } + }, error: onError, }); this.decoder.configure(decoderConfig); @@ -1455,7 +1815,7 @@ class PcmAudioDecoderWrapper extends DecoderWrapper { constructor( onSample: (sample: AudioSample) => unknown, - onError: (error: DOMException) => unknown, + onError: (error: Error) => unknown, public decoderConfig: AudioDecoderConfig, ) { super(onSample, onError); @@ -1645,7 +2005,7 @@ export class AudioSampleSink extends BaseMediaSampleSink { /** @internal */ async _createDecoder( onSample: (sample: AudioSample) => unknown, - onError: (error: DOMException) => unknown, + onError: (error: Error) => unknown, ) { if (!(await this._track.canDecode())) { throw new Error( diff --git a/src/media-source.ts b/src/media-source.ts index 7df06fb..625348f 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -37,7 +37,7 @@ import { customVideoEncoders, customAudioEncoders, } from './custom-coder'; -import { EncodedPacket } from './packet'; +import { EncodedPacket, EncodedPacketSideData } from './packet'; import { AudioSample, VideoSample } from './sample'; import { AudioEncodingConfig, @@ -213,12 +213,19 @@ class VideoEncoderWrapper { private customEncoderCallSerializer = new CallSerializer(); private customEncoderQueueSize = 0; + // Alpha stuff + private alphaEncoder: VideoEncoder | null = null; + private splitter: ColorAlphaSplitter | null = null; + private splitterCreationFailed = false; + private alphaFrameQueue: (VideoFrame | null)[] = []; + /** * Encoders typically throw their errors "out of band", meaning asynchronously in some other execution context. * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. * So, we keep track of the encoder error and throw it as soon as we get the chance. */ - private encoderError: Error | null = null; + private error: Error | null = null; + private errorNeedsNewStack = true; constructor(private source: VideoSource, private encodingConfig: VideoEncodingConfig) {} @@ -329,7 +336,7 @@ class VideoEncoderWrapper { const promise = this.customEncoderCallSerializer .call(() => this.customEncoder!.encode(clonedSample, finalEncodeOptions)) .then(() => this.customEncoderQueueSize--) - .catch((error: Error) => this.encoderError ??= error) + .catch((error: Error) => this.error ??= error) .finally(() => { clonedSample.close(); // `videoSample` gets closed in the finally block at the end of the method @@ -340,9 +347,49 @@ class VideoEncoderWrapper { } } else { assert(this.encoder); + const videoFrame = videoSample.toVideoFrame(); - this.encoder.encode(videoFrame, finalEncodeOptions); - videoFrame.close(); + + if (!this.alphaEncoder) { + // No alpha encoder, simple case + this.encoder.encode(videoFrame, finalEncodeOptions); + videoFrame.close(); + } else { + // We're expected to encode alpha as well + const frameDefinitelyHasNoAlpha = !!videoFrame.format && !videoFrame.format.includes('A'); + + if (frameDefinitelyHasNoAlpha || this.splitterCreationFailed) { + this.alphaFrameQueue.push(null); + this.encoder.encode(videoFrame, finalEncodeOptions); + videoFrame.close(); + } else { + const width = videoFrame.displayWidth; + const height = videoFrame.displayHeight; + + if (!this.splitter) { + try { + this.splitter = new ColorAlphaSplitter(width, height); + } catch (error) { + console.error('Due to an error, only color data will be encoded.', error); + + this.splitterCreationFailed = true; + this.alphaFrameQueue.push(null); + this.encoder.encode(videoFrame, finalEncodeOptions); + videoFrame.close(); + } + } + + if (this.splitter) { + const alphaFrame = this.splitter.extractAlpha(videoFrame); + const colorFrame = this.splitter.extractColor(videoFrame); + + this.alphaFrameQueue.push(alphaFrame); + this.encoder.encode(colorFrame, finalEncodeOptions); + colorFrame.close(); + videoFrame.close(); + } + } + } if (shouldClose) { videoSample.close(); @@ -364,10 +411,6 @@ class VideoEncoderWrapper { } private ensureEncoder(videoSample: VideoSample) { - if (this.encoder) { - return; - } - const encoderError = new Error(); this.ensureEncoderPromise = (async () => { const encoderConfig = buildVideoEncoderConfig({ @@ -400,7 +443,11 @@ class VideoEncoderWrapper { } this.encodingConfig.onEncodedPacket?.(packet, meta); - void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta); + void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta) + .catch((error) => { + this.error ??= error; + this.errorNeedsNewStack = false; + }); }; await this.customEncoder.init(); @@ -409,6 +456,15 @@ class VideoEncoderWrapper { throw new Error('VideoEncoder is not supported by this browser.'); } + encoderConfig.alpha = 'discard'; // Since we handle alpha ourselves + + if (this.encodingConfig.alpha === 'keep') { + // Encoding alpha requires using two parallel encoders, so we need to make sure they stay in sync + // and that neither of them drops frames. Setting latencyMode to 'quality' achieves this, because + // "User Agents MUST not drop frames to achieve the target bitrate and/or framerate." + encoderConfig.latencyMode = 'quality'; + } + const hasOddDimension = encoderConfig.width % 2 === 1 || encoderConfig.height % 2 === 1; if ( hasOddDimension @@ -432,19 +488,116 @@ class VideoEncoderWrapper { ); } + /** Queue of color chunks waiting for their alpha counterpart. */ + const colorChunkQueue: { + chunk: EncodedVideoChunk; + meta: EncodedVideoChunkMetadata | undefined; + }[] = []; + /** Each value is the number of encoded alpha chunks at which a null alpha chunk should be added. */ + const nullAlphaChunkQueue: number[] = []; + let encodedAlphaChunkCount = 0; + let alphaEncoderQueue = 0; + + const addPacket = ( + colorChunk: EncodedVideoChunk, + alphaChunk: EncodedVideoChunk | null, + meta: EncodedVideoChunkMetadata | undefined, + ) => { + const sideData: EncodedPacketSideData = {}; + + if (alphaChunk) { + const alphaData = new Uint8Array(alphaChunk.byteLength); + alphaChunk.copyTo(alphaData); + + sideData.alpha = alphaData; + } + + const packet = EncodedPacket.fromEncodedChunk(colorChunk, sideData); + + this.encodingConfig.onEncodedPacket?.(packet, meta); + void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta) + .catch((error) => { + this.error ??= error; + this.errorNeedsNewStack = false; + }); + }; + this.encoder = new VideoEncoder({ output: (chunk, meta) => { - const packet = EncodedPacket.fromEncodedChunk(chunk); + if (!this.alphaEncoder) { + // We're done + addPacket(chunk, null, meta); + return; + } - this.encodingConfig.onEncodedPacket?.(packet, meta); - void this.muxer!.addEncodedVideoPacket(this.source._connectedTrack!, packet, meta); + const alphaFrame = this.alphaFrameQueue.shift(); + assert(alphaFrame !== undefined); + + if (alphaFrame) { + this.alphaEncoder.encode(alphaFrame, { + // Crucial: The alpha frame is forced to be a key frame whenever the color frame + // also is. Without this, playback can glitch and even crash in some browsers. + // This is the reason why the two encoders are wired in series and not in parallel. + keyFrame: chunk.type === 'key', + }); + alphaEncoderQueue++; + alphaFrame.close(); + colorChunkQueue.push({ chunk, meta }); + } else { + // There was no alpha component for this frame + if (alphaEncoderQueue === 0) { + // No pending alpha encodes either, so we're done + addPacket(chunk, null, meta); + } else { + // There are still alpha encodes pending, so we can't add the packet immediately since + // we'd end up with out-of-order packets. Instead, let's queue a null alpha chunk to be + // added in the future, after the current encoder workload has completed: + nullAlphaChunkQueue.push(encodedAlphaChunkCount + alphaEncoderQueue); + colorChunkQueue.push({ chunk, meta }); + } + } }, error: (error) => { error.stack = encoderError.stack; // Provide a more useful stack trace - this.encoderError ??= error; + this.error ??= error; }, }); this.encoder.configure(encoderConfig); + + if (this.encodingConfig.alpha === 'keep') { + // We need to encode alpha as well, which we do with a separate encoder + this.alphaEncoder = new VideoEncoder({ + // We ignore the alpha chunk's metadata + // eslint-disable-next-line @typescript-eslint/no-unused-vars + output: (chunk, meta) => { + alphaEncoderQueue--; + + // There has to be a color chunk because the encoders are wired in series + const colorChunk = colorChunkQueue.shift(); + assert(colorChunk !== undefined); + + addPacket(colorChunk.chunk, chunk, colorChunk.meta); + + // See if there are any null alpha chunks queued up + encodedAlphaChunkCount++; + while ( + nullAlphaChunkQueue.length > 0 + && nullAlphaChunkQueue[0] === encodedAlphaChunkCount + ) { + nullAlphaChunkQueue.shift(); + const colorChunk = colorChunkQueue.shift(); + assert(colorChunk !== undefined); + + addPacket(colorChunk.chunk, null, colorChunk.meta); + } + }, + error: (error) => { + error.stack = encoderError.stack; // Provide a more useful stack trace + this.error ??= error; + }, + }); + this.alphaEncoder.configure(encoderConfig); + } } assert(this.source._connectedTrack); @@ -465,12 +618,21 @@ class VideoEncoderWrapper { await this.customEncoderCallSerializer.call(() => this.customEncoder!.close()); } else if (this.encoder) { if (!forceClose) { + // These are wired in series, therefore they must also be flushed in series await this.encoder.flush(); + await this.alphaEncoder?.flush(); } if (this.encoder.state !== 'closed') { this.encoder.close(); } + if (this.alphaEncoder && this.alphaEncoder.state !== 'closed') { + this.alphaEncoder.close(); + } + + this.alphaFrameQueue.forEach(x => x?.close()); + + this.splitter?.close(); } if (!forceClose) this.checkForEncoderError(); @@ -480,18 +642,289 @@ class VideoEncoderWrapper { if (this.customEncoder) { return this.customEncoderQueueSize; } else { + // Because the color and alpha encoders are wired in series, there's no need to also include the alpha + // encoder's queue size here return this.encoder?.encodeQueueSize ?? 0; } } checkForEncoderError() { - if (this.encoderError) { - this.encoderError.stack = new Error().stack; // Provide an even more useful stack trace - throw this.encoderError; + if (this.error) { + if (this.errorNeedsNewStack) { + this.error.stack = new Error().stack; // Provide an even more useful stack trace + } + + throw this.error; } } } +/** Utility class for splitting a composite frame into separate color and alpha components. */ +class ColorAlphaSplitter { + canvas: OffscreenCanvas | HTMLCanvasElement; + + private gl: WebGL2RenderingContext; + private colorProgram: WebGLProgram; + private alphaProgram: WebGLProgram; + private vao: WebGLVertexArrayObject; + private sourceTexture: WebGLTexture; + private lastFrame: VideoFrame | null = null; + private alphaResolutionLocation: WebGLUniformLocation; + + constructor(initialWidth: number, initialHeight: number) { + if (typeof OffscreenCanvas !== 'undefined') { + this.canvas = new OffscreenCanvas(initialWidth, initialHeight); + } else { + this.canvas = document.createElement('canvas'); + this.canvas.width = initialWidth; + this.canvas.height = initialHeight; + } + + const gl = this.canvas.getContext('webgl2', { + alpha: true, // Needed due to the YUV thing we do for alpha + }) as unknown as WebGL2RenderingContext | null; // Casting because of some TypeScript weirdness + if (!gl) { + throw new Error('Couldn\'t acquire WebGL 2 context.'); + } + + this.gl = gl; + + this.colorProgram = this.createColorProgram(); + this.alphaProgram = this.createAlphaProgram(); + this.vao = this.createVAO(); + this.sourceTexture = this.createTexture(); + + this.alphaResolutionLocation = this.gl.getUniformLocation(this.alphaProgram, 'u_resolution')!; + + this.gl.useProgram(this.colorProgram); + this.gl.uniform1i(this.gl.getUniformLocation(this.colorProgram, 'u_sourceTexture'), 0); + + this.gl.useProgram(this.alphaProgram); + this.gl.uniform1i(this.gl.getUniformLocation(this.alphaProgram, 'u_sourceTexture'), 0); + } + + private createVertexShader(): WebGLShader { + return this.createShader(this.gl.VERTEX_SHADER, `#version 300 es + in vec2 a_position; + in vec2 a_texCoord; + out vec2 v_texCoord; + + void main() { + gl_Position = vec4(a_position, 0.0, 1.0); + v_texCoord = a_texCoord; + } + `); + } + + private createColorProgram(): WebGLProgram { + const vertexShader = this.createVertexShader(); + + // This shader is simple, simply copy the color information while setting alpha to 1 + const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es + precision highp float; + + uniform sampler2D u_sourceTexture; + in vec2 v_texCoord; + out vec4 fragColor; + + void main() { + vec4 source = texture(u_sourceTexture, v_texCoord); + fragColor = vec4(source.rgb, 1.0); + } + `); + + const program = this.gl.createProgram(); + this.gl.attachShader(program, vertexShader); + this.gl.attachShader(program, fragmentShader); + this.gl.linkProgram(program); + + return program; + } + + private createAlphaProgram(): WebGLProgram { + const vertexShader = this.createVertexShader(); + + // This shader's more complex. The main reason is that this shader writes data in I420 (yuv420) pixel format + // instead of regular RGBA. In other words, we use the shader to write out I420 data into an RGBA canvas, which + // we then later read out with JavaScript. The reason being that browsers weirdly encode canvases and mess up + // the color spaces, and the only way to have full control over the color space is by outputting YUV data + // directly (avoiding the RGB conversion). Doing this conversion in JS is painfully slow, so let's utlize the + // GPU since we're already calling it anyway. + const fragmentShader = this.createShader(this.gl.FRAGMENT_SHADER, `#version 300 es + precision highp float; + + uniform sampler2D u_sourceTexture; + uniform vec2 u_resolution; // The width and height of the canvas + in vec2 v_texCoord; + out vec4 fragColor; + + // This function determines the value for a single byte in the YUV stream + float getByteValue(float byteOffset) { + float width = u_resolution.x; + float height = u_resolution.y; + + float yPlaneSize = width * height; + + if (byteOffset < yPlaneSize) { + // This byte is in the luma plane. Find the corresponding pixel coordinates to sample from + float y = floor(byteOffset / width); + float x = mod(byteOffset, width); + + // Add 0.5 to sample the center of the texel + vec2 sampleCoord = (vec2(x, y) + 0.5) / u_resolution; + + // The luma value is the alpha from the source texture + return texture(u_sourceTexture, sampleCoord).a; + } else { + // Write a fixed value for chroma and beyond + return 128.0 / 255.0; + } + } + + void main() { + // Each fragment writes 4 bytes (R, G, B, A) + float pixelIndex = floor(gl_FragCoord.y) * u_resolution.x + floor(gl_FragCoord.x); + float baseByteOffset = pixelIndex * 4.0; + + vec4 result; + for (int i = 0; i < 4; i++) { + float currentByteOffset = baseByteOffset + float(i); + result[i] = getByteValue(currentByteOffset); + } + + fragColor = result; + } + `); + + const program = this.gl.createProgram(); + this.gl.attachShader(program, vertexShader); + this.gl.attachShader(program, fragmentShader); + this.gl.linkProgram(program); + + return program; + } + + private createShader(type: number, source: string): WebGLShader { + const shader = this.gl.createShader(type)!; + this.gl.shaderSource(shader, source); + this.gl.compileShader(shader); + if (!this.gl.getShaderParameter(shader, this.gl.COMPILE_STATUS)) { + console.error('Shader compile error:', this.gl.getShaderInfoLog(shader)); + } + return shader; + } + + private createVAO(): WebGLVertexArrayObject { + const vao = this.gl.createVertexArray(); + this.gl.bindVertexArray(vao); + + const vertices = new Float32Array([ + -1, -1, 0, 1, + 1, -1, 1, 1, + -1, 1, 0, 0, + 1, 1, 1, 0, + ]); + + const buffer = this.gl.createBuffer(); + this.gl.bindBuffer(this.gl.ARRAY_BUFFER, buffer); + this.gl.bufferData(this.gl.ARRAY_BUFFER, vertices, this.gl.STATIC_DRAW); + + const positionLocation = this.gl.getAttribLocation(this.colorProgram, 'a_position'); + const texCoordLocation = this.gl.getAttribLocation(this.colorProgram, 'a_texCoord'); + + this.gl.enableVertexAttribArray(positionLocation); + this.gl.vertexAttribPointer(positionLocation, 2, this.gl.FLOAT, false, 16, 0); + + this.gl.enableVertexAttribArray(texCoordLocation); + this.gl.vertexAttribPointer(texCoordLocation, 2, this.gl.FLOAT, false, 16, 8); + + return vao; + } + + private createTexture(): WebGLTexture { + const texture = this.gl.createTexture(); + + this.gl.bindTexture(this.gl.TEXTURE_2D, texture); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_S, this.gl.CLAMP_TO_EDGE); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_WRAP_T, this.gl.CLAMP_TO_EDGE); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MIN_FILTER, this.gl.LINEAR); + this.gl.texParameteri(this.gl.TEXTURE_2D, this.gl.TEXTURE_MAG_FILTER, this.gl.LINEAR); + + return texture; + } + + private updateTexture(sourceFrame: VideoFrame): void { + if (this.lastFrame === sourceFrame) { + return; + } + + if (sourceFrame.displayWidth !== this.canvas.width || sourceFrame.displayHeight !== this.canvas.height) { + this.canvas.width = sourceFrame.displayWidth; + this.canvas.height = sourceFrame.displayHeight; + } + + this.gl.activeTexture(this.gl.TEXTURE0); + this.gl.bindTexture(this.gl.TEXTURE_2D, this.sourceTexture); + this.gl.texImage2D(this.gl.TEXTURE_2D, 0, this.gl.RGBA, this.gl.RGBA, this.gl.UNSIGNED_BYTE, sourceFrame); + + this.lastFrame = sourceFrame; + } + + extractColor(sourceFrame: VideoFrame) { + this.updateTexture(sourceFrame); + + this.gl.useProgram(this.colorProgram); + this.gl.viewport(0, 0, this.canvas.width, this.canvas.height); + this.gl.clear(this.gl.COLOR_BUFFER_BIT); + this.gl.bindVertexArray(this.vao); + this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); + + return new VideoFrame(this.canvas, { + timestamp: sourceFrame.timestamp, + duration: sourceFrame.duration ?? undefined, + alpha: 'discard', + }); + } + + extractAlpha(sourceFrame: VideoFrame) { + this.updateTexture(sourceFrame); + + this.gl.useProgram(this.alphaProgram); + this.gl.uniform2f(this.alphaResolutionLocation, this.canvas.width, this.canvas.height); + + this.gl.viewport(0, 0, this.canvas.width, this.canvas.height); + this.gl.clear(this.gl.COLOR_BUFFER_BIT); + this.gl.bindVertexArray(this.vao); + this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 4); + + const { width, height } = this.canvas; + + const chromaSamples = Math.ceil(width / 2) * Math.ceil(height / 2); + const yuvSize = width * height + chromaSamples * 2; + const requiredHeight = Math.ceil(yuvSize / (width * 4)); + + let yuv = new Uint8Array(4 * width * requiredHeight); + this.gl.readPixels(0, 0, width, requiredHeight, this.gl.RGBA, this.gl.UNSIGNED_BYTE, yuv); + yuv = yuv.subarray(0, yuvSize); + + assert(yuv[width * height] === 128); // Where chroma data starts + assert(yuv[yuv.length - 1] === 128); // Assert the YUV data has been fully written + + return new VideoFrame(yuv, { + format: 'I420', + codedWidth: width, + codedHeight: height, + timestamp: sourceFrame.timestamp, + duration: sourceFrame.duration ?? undefined, + }); + } + + close() { + this.gl.getExtension('WEBGL_lose_context')?.loseContext(); + this.gl = null as unknown as WebGL2RenderingContext; + } +} + /** * This source can be used to add raw, unencoded video samples (frames) to an output video track. These frames will * automatically be encoded and then piped into the output. @@ -858,7 +1291,8 @@ class AudioEncoderWrapper { * However, we want to surface these errors to the user within the normal control flow, so they don't go uncaught. * So, we keep track of the encoder error and throw it as soon as we get the chance. */ - private encoderError: Error | null = null; + private error: Error | null = null; + private errorNeedsNewStack = true; constructor(private source: AudioSource, private encodingConfig: AudioEncodingConfig) {} @@ -909,7 +1343,7 @@ class AudioEncoderWrapper { const promise = this.customEncoderCallSerializer .call(() => this.customEncoder!.encode(clonedSample)) .then(() => this.customEncoderQueueSize--) - .catch((error: Error) => this.encoderError ??= error) + .catch((error: Error) => this.error ??= error) .finally(() => { clonedSample.close(); // `audioSample` gets closed in the finally block at the end of the method @@ -1018,10 +1452,6 @@ class AudioEncoderWrapper { } private ensureEncoder(audioSample: AudioSample) { - if (this.encoderInitialized) { - return; - } - const encoderError = new Error(); this.ensureEncoderPromise = (async () => { const { numberOfChannels, sampleRate } = audioSample; @@ -1055,7 +1485,11 @@ class AudioEncoderWrapper { } this.encodingConfig.onEncodedPacket?.(packet, meta); - void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta); + void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta) + .catch((error) => { + this.error ??= error; + this.errorNeedsNewStack = false; + }); }; await this.customEncoder.init(); @@ -1080,11 +1514,15 @@ class AudioEncoderWrapper { const packet = EncodedPacket.fromEncodedChunk(chunk); this.encodingConfig.onEncodedPacket?.(packet, meta); - void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta); + void this.muxer!.addEncodedAudioPacket(this.source._connectedTrack!, packet, meta) + .catch((error) => { + this.error ??= error; + this.errorNeedsNewStack = false; + }); }, error: (error) => { error.stack = encoderError.stack; // Provide a more useful stack trace - this.encoderError ??= error; + this.error ??= error; }, }); this.encoder.configure(encoderConfig); @@ -1223,9 +1661,12 @@ class AudioEncoderWrapper { } checkForEncoderError() { - if (this.encoderError) { - this.encoderError.stack = new Error().stack; // Provide an even more useful stack trace - throw this.encoderError; + if (this.error) { + if (this.errorNeedsNewStack) { + this.error.stack = new Error().stack; // Provide an even more useful stack trace + } + + throw this.error; } } } @@ -1679,6 +2120,8 @@ export abstract class SubtitleSource extends MediaSource { export class TextSubtitleSource extends SubtitleSource { /** @internal */ private _parser: SubtitleParser; + /** @internal */ + private _error: Error | null = null; /** Creates a new {@link TextSubtitleSource} where added text chunks are in the specified `codec`. */ constructor(codec: SubtitleCodec) { @@ -1686,8 +2129,12 @@ export class TextSubtitleSource extends SubtitleSource { this._parser = new SubtitleParser({ codec, - output: (cue, metadata) => - this._connectedTrack?.output._muxer.addSubtitleCue(this._connectedTrack, cue, metadata), + output: (cue, metadata) => { + void this._connectedTrack?.output._muxer.addSubtitleCue(this._connectedTrack, cue, metadata) + .catch((error) => { + this._error ??= error; + }); + }, }); } @@ -1703,9 +2150,25 @@ export class TextSubtitleSource extends SubtitleSource { throw new TypeError('text must be a string.'); } + this._checkForError(); + this._ensureValidAdd(); this._parser.parse(text); return this._connectedTrack!.output._muxer.mutex.currentPromise; } + + /** @internal */ + _checkForError() { + if (this._error) { + throw this._error; + } + } + + /** @internal */ + override async _flushAndClose(forceClose: boolean) { + if (!forceClose) { + this._checkForError(); + } + } } diff --git a/src/output-format.ts b/src/output-format.ts index 8a5d4ed..de50fe3 100644 --- a/src/output-format.ts +++ b/src/output-format.ts @@ -403,6 +403,10 @@ export type MkvOutputFormatOptions = { /** * Matroska file format. + * + * Supports writing transparent video. For a video track to be marked as transparent, the first packet added must + * contain alpha side data. + * * @group Output formats * @public */ @@ -490,6 +494,10 @@ export type WebMOutputFormatOptions = MkvOutputFormatOptions; /** * WebM file format, based on Matroska. + * + * Supports writing transparent video. For a video track to be marked as transparent, the first packet added must + * contain alpha side data. + * * @group Output formats * @public */ diff --git a/src/packet.ts b/src/packet.ts index 5221267..c71890c 100644 --- a/src/packet.ts +++ b/src/packet.ts @@ -18,6 +18,24 @@ export const PLACEHOLDER_DATA = new Uint8Array(0); */ export type PacketType = 'key' | 'delta'; +/** + * Holds additional data accompanying an {@link EncodedPacket}. + * @group Packets + * @public + */ +export type EncodedPacketSideData = { + /** + * An encoded alpha frame, encoded with the same codec as the packet. Typically used for transparent videos, where + * the alpha information is stored separately from the color information. + */ + alpha?: Uint8Array; + /** + * The actual byte length of the alpha data. This field is useful for metadata-only packets where the + * `alpha` field contains no bytes. + */ + alphaByteLength?: number; +}; + /** * Represents an encoded chunk of media. Mainly used as an expressive wrapper around WebCodecs API's * [`EncodedVideoChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedVideoChunk) and @@ -33,6 +51,9 @@ export class EncodedPacket { */ readonly byteLength: number; + /** Additional data carried with this packet. */ + readonly sideData: EncodedPacketSideData; + /** Creates a new {@link EncodedPacket} from raw bytes and timing information. */ constructor( /** The encoded data of this packet. */ @@ -54,6 +75,7 @@ export class EncodedPacket { */ public readonly sequenceNumber = -1, byteLength?: number, + sideData?: EncodedPacketSideData, ) { if (data === PLACEHOLDER_DATA && byteLength === undefined) { throw new Error( @@ -83,8 +105,25 @@ export class EncodedPacket { if (!Number.isInteger(byteLength) || byteLength < 0) { throw new TypeError('byteLength must be a non-negative integer.'); } + if (sideData !== undefined && (typeof sideData !== 'object' || !sideData)) { + throw new TypeError('sideData, when provided, must be an object.'); + } + if (sideData?.alpha !== undefined && !(sideData.alpha instanceof Uint8Array)) { + throw new TypeError('sideData.alpha, when provided, must be a Uint8Array.'); + } + if ( + sideData?.alphaByteLength !== undefined + && (!Number.isInteger(sideData.alphaByteLength) || sideData.alphaByteLength < 0) + ) { + throw new TypeError('sideData.alphaByteLength, when provided, must be a non-negative integer.'); + } this.byteLength = byteLength; + this.sideData = sideData ?? {}; + + if (this.sideData.alpha && this.sideData.alphaByteLength === undefined) { + this.sideData.alphaByteLength = this.sideData.alpha.byteLength; + } } /** If this packet is a metadata-only packet. Metadata-only packets don't contain their packet data. */ @@ -102,7 +141,9 @@ export class EncodedPacket { return Math.trunc(SECOND_TO_MICROSECOND_FACTOR * this.duration); } - /** Converts this packet to an EncodedVideoChunk for use with the WebCodecs API. */ + /** Converts this packet to an + * [`EncodedVideoChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedVideoChunk) for use with the + * WebCodecs API. */ toEncodedVideoChunk() { if (this.isMetadataOnly) { throw new TypeError('Metadata-only packets cannot be converted to a video chunk.'); @@ -119,7 +160,33 @@ export class EncodedPacket { }); } - /** Converts this packet to an EncodedAudioChunk for use with the WebCodecs API. */ + /** + * Converts this packet to an + * [`EncodedVideoChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedVideoChunk) for use with the + * WebCodecs API, using the alpha side data instead of the color data. Throws if no alpha side data is defined. + */ + alphaToEncodedVideoChunk(type = this.type) { + if (!this.sideData.alpha) { + throw new TypeError('This packet does not contain alpha side data.'); + } + if (this.isMetadataOnly) { + throw new TypeError('Metadata-only packets cannot be converted to a video chunk.'); + } + if (typeof EncodedVideoChunk === 'undefined') { + throw new Error('Your browser does not support EncodedVideoChunk.'); + } + + return new EncodedVideoChunk({ + data: this.sideData.alpha, + type, + timestamp: this.microsecondTimestamp, + duration: this.microsecondDuration, + }); + } + + /** Converts this packet to an + * [`EncodedAudioChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedAudioChunk) for use with the + * WebCodecs API. */ toEncodedAudioChunk() { if (this.isMetadataOnly) { throw new TypeError('Metadata-only packets cannot be converted to an audio chunk.'); @@ -137,10 +204,15 @@ export class EncodedPacket { } /** - * Creates an EncodedPacket from an EncodedVideoChunk or EncodedAudioChunk. This method is useful for converting - * chunks from the WebCodecs API to EncodedPackets. + * Creates an {@link EncodedPacket} from an + * [`EncodedVideoChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedVideoChunk) or + * [`EncodedAudioChunk`](https://developer.mozilla.org/en-US/docs/Web/API/EncodedAudioChunk). This method is useful + * for converting chunks from the WebCodecs API to `EncodedPacket` instances. */ - static fromEncodedChunk(chunk: EncodedVideoChunk | EncodedAudioChunk): EncodedPacket { + static fromEncodedChunk( + chunk: EncodedVideoChunk | EncodedAudioChunk, + sideData?: EncodedPacketSideData, + ): EncodedPacket { if (!(chunk instanceof EncodedVideoChunk || chunk instanceof EncodedAudioChunk)) { throw new TypeError('chunk must be an EncodedVideoChunk or EncodedAudioChunk.'); } @@ -153,6 +225,9 @@ export class EncodedPacket { chunk.type as PacketType, chunk.timestamp / 1e6, (chunk.duration ?? 0) / 1e6, + undefined, + undefined, + sideData, ); } diff --git a/src/sample.ts b/src/sample.ts index c713aa8..924d016 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -96,6 +96,14 @@ export class VideoSample { return Math.trunc(SECOND_TO_MICROSECOND_FACTOR * this.duration); } + /** + * Whether this sample uses a pixel format that can hold transparency data. Note that this doesn't necessarily mean + * that the sample is transparent. + */ + get hasAlpha() { + return this.format && this.format.includes('A'); + } + /** * Creates a new {@link VideoSample} from a * [`VideoFrame`](https://developer.mozilla.org/en-US/docs/Web/API/VideoFrame). This is essentially a near zero-cost diff --git a/src/tags.ts b/src/tags.ts index cc58e97..9055efc 100644 --- a/src/tags.ts +++ b/src/tags.ts @@ -12,7 +12,7 @@ * directly read or write the underlying metadata tags (which differ by format). * * - For MP4/QuickTime files, the metadata refers to the data in `'moov'`-level `'udta'` and `'meta'` atoms. - * - For Matroska files, the metadata refers to the Tags and Attachments elements whose target is 50 (MOVIE). + * - For WebM/Matroska files, the metadata refers to the Tags and Attachments elements whose target is 50 (MOVIE). * - For MP3 files, the metadata refers to the ID3v2 or ID3v1 tags. * - For Ogg files, there is no global metadata so instead, the metadata refers to the combined metadata of all tracks, * in Vorbis-style comment headers. @@ -65,9 +65,9 @@ export type MetadataTags = { * is also used, then the keys reflect the keys specified there (such as `'com.apple.quicktime.version'`). * Additionally, any atoms within the `'udta'` atom are dumped into here, however with unknown internal format * (`Uint8Array`). - * - Matroska: `SimpleTag` elements whose target is 50 (MOVIE), either containing string or `Uint8Array` values. - * Additionally, all attached files (such as font files) are included here, where the key corresponds to the FileUID - * and the value is an {@link AttachedFile}. + * - WebM/Matroska: `SimpleTag` elements whose target is 50 (MOVIE), either containing string or `Uint8Array` + * values. Additionally, all attached files (such as font files) are included here, where the key corresponds to + * the FileUID and the value is an {@link AttachedFile}. * - MP3: The ID3v2 tags, or a single `'TAG'` key with the contents of the ID3v1 tag. * - Ogg: The key-value string pairs from the Vorbis-style comment header (see RFC 7845, Section 5.2). * Additionally, the `'vendor'` key refers to the vendor string within this header. diff --git a/test/browser/transparency.test.ts b/test/browser/transparency.test.ts new file mode 100644 index 0000000..f7cb489 --- /dev/null +++ b/test/browser/transparency.test.ts @@ -0,0 +1,271 @@ +import { expect, test } from 'vitest'; +import { Input } from '../../src/input.js'; +import { BufferSource, UrlSource } from '../../src/source.js'; +import { ALL_FORMATS } from '../../src/input-format.js'; +import { CanvasSink, EncodedPacketSink, VideoSampleSink } from '../../src/media-sink.js'; +import { Output } from '../../src/output.js'; +import { WebMOutputFormat } from '../../src/output-format.js'; +import { BufferTarget } from '../../src/target.js'; +import { CanvasSource, VideoSampleSource } from '../../src/media-source.js'; +import { canEncodeVideo, QUALITY_HIGH } from '../../src/encode.js'; +import { VideoSample } from '../../src/sample.js'; + +test('Can decode transparent video', async () => { + using input = new Input({ + source: new UrlSource('/transparency.webm'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + expect(await videoTrack.canBeTransparent()).toBe(true); + + const sink = new VideoSampleSink(videoTrack); + const sample = (await sink.getSample(0.5))!; + + expect(sample.format).toContain('A'); // Probably RGBA + expect(sample.hasAlpha).toBe(true); + + const canvas = new OffscreenCanvas(sample.displayWidth, sample.displayHeight); + const context = canvas.getContext('2d')!; + + sample.draw(context, 0, 0); + + const imageData = context.getImageData(0, 0, canvas.width, canvas.height); + expect(imageData.data[3]).toBeLessThan(255); // Check that there's actually transparent pixels +}); + +test('Can decode faulty transparent video and behaves gracefully', async () => { + using input = new Input({ + source: new UrlSource('/transparency-faulty.webm'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const packetSink = new EncodedPacketSink(videoTrack); + const secondKeyPacket = (await packetSink.getNextKeyPacket((await packetSink.getFirstPacket())!))!; + + const sink = new VideoSampleSink(videoTrack); + + const startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + expect(startSample.format).toContain('A'); + + const secondSample = (await sink.getSample(secondKeyPacket.timestamp))!; + expect(secondSample.format).not.toContain('A'); // There was no alpha key frame for this one + expect(secondSample.hasAlpha).toBe(false); +}); + +test('Can extract transparent frames via CanvasSink', async () => { + using input = new Input({ + source: new UrlSource('/transparency.webm'), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const sink = new CanvasSink(videoTrack, { alpha: true }); + const wrappedCanvas = (await sink.getCanvas(await videoTrack.getFirstTimestamp()))!; + + const canvas = new OffscreenCanvas(wrappedCanvas.canvas.width, wrappedCanvas.canvas.height); + const context = canvas.getContext('2d')!; + context.drawImage(wrappedCanvas.canvas, 0, 0); + + let imageData = context.getImageData(0, 0, canvas.width, canvas.height); + expect(imageData.data[3]).toBeLessThan(255); // Check that there's actually transparent pixels + + const opaqueSink = new CanvasSink(videoTrack); // Default is alpha: false + const opaqueWrappedCanvas = (await opaqueSink.getCanvas(await videoTrack.getFirstTimestamp()))!; + + context.drawImage(opaqueWrappedCanvas.canvas, 0, 0); + + imageData = context.getImageData(0, 0, canvas.width, canvas.height); + expect(imageData.data[3]).toBe(255); +}); + +test('Can encode transparent video', async () => { + const output = new Output({ + format: new WebMOutputFormat(), + target: new BufferTarget(), + }); + + const canvas = new OffscreenCanvas(1280, 720); + const context = canvas.getContext('2d')!; + + const source = new CanvasSource(canvas, { + codec: 'vp9', + bitrate: QUALITY_HIGH, + alpha: 'keep', + }); + output.addVideoTrack(source); + + await output.start(); + + context.fillStyle = '#ff0000'; + context.fillRect(200, 200, 200, 200); + await source.add(0, 1); + + context.fillStyle = '#00ff00'; + context.fillRect(300, 300, 200, 200); + await source.add(1, 1); + + context.fillStyle = '#0000ff'; + context.fillRect(400, 400, 200, 200); + await source.add(2, 1); + + await output.finalize(); + + const blob = new Blob([output.target.buffer!], { + type: output.format.mimeType, + }); + const url = URL.createObjectURL(blob); + + const video = document.createElement('video'); + video.src = url; + video.muted = true; + void video.play(); + + await new Promise(resolve => video.addEventListener('loadeddata', resolve)); + + // Let the video play for a little bit to prevent flake + while (video.currentTime < 0.1) { + await new Promise(resolve => setTimeout(resolve, 0)); + } + + expect(video.videoWidth).toBe(1280); + expect(video.videoHeight).toBe(720); + + const probeCanvas = new OffscreenCanvas(1280, 720); + const probeContext = probeCanvas.getContext('2d')!; + + probeContext.drawImage(video, 0, 0); + + let imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height); + expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error) + + const pos = { x: 300, y: 300 }; // Dead center in the red square + const index = (pos.x + pos.y * probeCanvas.width) * 4; + + // Red (within error) + expect(imageData.data[index + 0]).greaterThanOrEqual(253); + expect(imageData.data[index + 1]).lessThanOrEqual(2); + expect(imageData.data[index + 2]).lessThanOrEqual(2); + + expect(imageData.data[index + 3]).greaterThanOrEqual(253); // Opaque (within error) + + // Let's also check it's read correctly by Mediabunny + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + expect(await videoTrack.canBeTransparent()).toBe(true); + + const sink = new VideoSampleSink(videoTrack); + + const firstSample = (await sink.getSample(0))!; + expect(firstSample.format).toContain('A'); + + probeContext.clearRect(0, 0, probeCanvas.width, probeCanvas.height); + firstSample.draw(probeContext, 0, 0); + + imageData = probeContext.getImageData(0, 0, probeCanvas.width, probeCanvas.height); + expect(imageData.data[3]).lessThanOrEqual(2); // Transparent (within error) +}); + +test('Can encode video with alternating transparency', async () => { + const output = new Output({ + format: new WebMOutputFormat(), + target: new BufferTarget(), + }); + + const canvas1 = new OffscreenCanvas(640, 480); + const context1 = canvas1.getContext('2d', { alpha: true })!; + context1.fillStyle = '#ff000080'; + context1.fillRect(0, 0, canvas1.width, canvas1.height); + + const canvas2 = new OffscreenCanvas(640, 480); + const context2 = canvas2.getContext('2d', { alpha: false })!; + context2.fillStyle = '#0000ff'; + context2.fillRect(0, 0, canvas2.width, canvas2.height); + + const source = new VideoSampleSource({ + codec: 'vp9', + bitrate: QUALITY_HIGH, + alpha: 'keep', + }); + output.addVideoTrack(source); + + await output.start(); + + for (let i = 0; i < 64; i++) { + const sample = new VideoSample(new Uint8Array(640 * 480 * 4), { + format: i % 2 ? 'RGBX' : 'RGBA', + codedWidth: 640, + codedHeight: 480, + timestamp: i, + duration: 1, + }); + await source.add(sample); + } + + await output.finalize(); + + using input = new Input({ + source: new BufferSource(output.target.buffer!), + formats: ALL_FORMATS, + }); + + const videoTrack = (await input.getPrimaryVideoTrack())!; + const packetSink = new EncodedPacketSink(videoTrack); + + let i = 0; + for await (const packet of packetSink.packets()) { + if (i % 2) { + expect(packet.sideData.alpha).toBeUndefined(); + } else { + expect(packet.sideData.alpha).toBeDefined(); + } + + i++; + } + + const sampleSink = new VideoSampleSink(videoTrack); + + i = 0; + for await (const sample of sampleSink.samples()) { + if (i % 2) { + expect(sample.format).not.toContain('A'); + } else { + expect(sample.format).toContain('A'); + } + + i++; + } +}); + +test('Can encode transparent video with odd dimensions', async () => { + const output = new Output({ + format: new WebMOutputFormat(), + target: new BufferTarget(), + }); + + const canvas = new OffscreenCanvas(641, 479); + const context = canvas.getContext('2d', { alpha: true })!; + context.fillStyle = '#ff000080'; + context.fillRect(0, 0, canvas.width, canvas.height); + + const source = new CanvasSource(canvas, { + codec: 'vp9', + bitrate: QUALITY_HIGH, + alpha: 'keep', + }); + output.addVideoTrack(source); + + await output.start(); + await source.add(0, 1); + await output.finalize(); +}); + +test('Positive encodability check with alpha', async () => { + const result = await canEncodeVideo('vp9', { alpha: 'keep' }); + expect(result).toBe(true); +}); diff --git a/test/public/transparency-faulty.webm b/test/public/transparency-faulty.webm new file mode 100644 index 0000000..7538ad2 Binary files /dev/null and b/test/public/transparency-faulty.webm differ diff --git a/test/public/transparency.webm b/test/public/transparency.webm new file mode 100644 index 0000000..5c13062 Binary files /dev/null and b/test/public/transparency.webm differ diff --git a/test/tsconfig.json b/test/tsconfig.json deleted file mode 100644 index 461d735..0000000 --- a/test/tsconfig.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "extends": "../tsconfig.json", - "compilerOptions": { - "noEmit": true, - "moduleResolution": "nodenext", - "module": "NodeNext", - "declaration": true, - "declarationMap": true, - "stripInternal": true - }, - "include": ["**/*"], - "references": [{ "path": "../src" }] -} diff --git a/test/vitest.config.ts b/test/vitest.config.ts deleted file mode 100644 index 4fda357..0000000 --- a/test/vitest.config.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { defineConfig } from 'vitest/config'; - -export default defineConfig({ - test: { - browser: { - provider: 'webdriverio', - instances: [{ browser: 'chrome' }], - headless: true, - }, - }, -}); diff --git a/tsconfig.json b/tsconfig.json index 9f6de94..2542ebc 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -11,5 +11,8 @@ "allowJs": true, "noEmit": true, }, - "references": [{ "path": "./tsconfig.vite.json" }] + "references": [ + { "path": "./tsconfig.vite.json" }, + { "path": "./tsconfig.vitest.json" } + ] } diff --git a/tsconfig.vitest.json b/tsconfig.vitest.json new file mode 100644 index 0000000..f9a4276 --- /dev/null +++ b/tsconfig.vitest.json @@ -0,0 +1,11 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "moduleResolution": "nodenext", + "module": "NodeNext", + "composite": true, + "noEmit": false + }, + "include": ["vitest.config.ts", "./test/**/*"], + "references": [{ "path": "./src" }] +} diff --git a/vitest.config.ts b/vitest.config.ts new file mode 100644 index 0000000..33a53df --- /dev/null +++ b/vitest.config.ts @@ -0,0 +1,34 @@ +/// + +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + projects: [ + { + test: { + name: 'node', + root: 'test', + include: ['node/**/*.test.ts'], + environment: 'node', + }, + }, + { + test: { + name: 'browser', + root: 'test', + include: ['browser/**/*.test.ts'], + browser: { + enabled: true, + provider: 'webdriverio', + instances: [{ + browser: 'chrome', + }], + headless: false, // A bunch of features need the head + screenshotFailures: false, + }, + }, + }, + ], + }, +});