diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bc40201..b2b2155 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,6 +5,7 @@ on: types: [published] permissions: + id-token: write # Required for OIDC contents: read jobs: @@ -14,7 +15,7 @@ jobs: permissions: write-all steps: - name: Checkout repository - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + uses: actions/checkout@v4 with: fetch-depth: 0 @@ -27,12 +28,15 @@ jobs: git push origin release - name: Set up Node.js - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + uses: actions/setup-node@v4 with: node-version: 22 cache: 'npm' registry-url: https://registry.npmjs.org + - name: Update npm + run: npm install -g npm@latest + - name: Get package.json version id: package-json-version run: echo "version=v$(cat package.json | jq '.version' --raw-output)" >> $GITHUB_OUTPUT @@ -58,11 +62,7 @@ jobs: run: gh release upload ${{ github.event.release.tag_name }} dist/bundles/mediabunny.cjs dist/bundles/mediabunny.min.cjs dist/bundles/mediabunny.mjs dist/bundles/mediabunny.min.mjs dist/mediabunny.d.ts packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.js packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.min.js packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.mjs packages/mp3-encoder/dist/bundles/mediabunny-mp3-encoder.min.mjs packages/mp3-encoder/dist/mediabunny-mp3-encoder.d.ts - name: Publish Mediabunny to npm - run: npm publish --provenance --access public - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public - name: Publish workspace packages to npm - run: npm publish --provenance --access public --workspaces - env: - NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm publish --access public --workspaces diff --git a/README.md b/README.md index 6d108d4..738f4fb 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,10 @@ Mediabunny is a JavaScript library for reading, writing, and converting media fi
- Remotion + + + Remotion +      diff --git a/dev/convert.html b/dev/convert.html index e548520..36a2645 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -100,9 +100,10 @@ }, */ video: () => ({ - width: 720, - frameRate: 30, - bitrate: Mediabunny.QUALITY_VERY_LOW, + allowRotationMetadata: false, + //width: 720, + //frameRate: 30, + //bitrate: Mediabunny.QUALITY_VERY_LOW, //discard: true, /* process: (sample) => { @@ -180,7 +181,7 @@ }, trim: { start: 0, - end: 20 + end: 4 }, }); console.log(conversion); diff --git a/dev/live.html b/dev/live.html index afcb991..8eef45b 100644 --- a/dev/live.html +++ b/dev/live.html @@ -1,6 +1,7 @@ + diff --git a/docs/public/sponsors/remotion-dark.png b/docs/public/sponsors/remotion-dark.png new file mode 100644 index 0000000..04e6ea1 Binary files /dev/null and b/docs/public/sponsors/remotion-dark.png differ diff --git a/docs/public/sponsors/remotion-light.png b/docs/public/sponsors/remotion-light.png new file mode 100644 index 0000000..74b3592 Binary files /dev/null and b/docs/public/sponsors/remotion-light.png differ diff --git a/docs/public/sponsors/remotion.png b/docs/public/sponsors/remotion.png deleted file mode 100644 index f76b061..0000000 Binary files a/docs/public/sponsors/remotion.png and /dev/null differ diff --git a/package-lock.json b/package-lock.json index 81249a8..c35e2b0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.25.2", + "version": "1.26.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.25.2", + "version": "1.26.0", "license": "MPL-2.0", "workspaces": [ "packages/*" @@ -7739,9 +7739,9 @@ } }, "node_modules/mediabunny": { - "version": "1.25.1", - "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.25.1.tgz", - "integrity": "sha512-0Rrd47PMCVJbTPA7IJaXPCupV5/RZ/icgr+a0qExRJAr0n5vB4fsGSo+fdHIehG0CrddXtVRvNZwFtJz709yfA==", + "version": "1.25.8", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.25.8.tgz", + "integrity": "sha512-2WCa9WtEbHOvg5rAWXQjVE+O/r01fpj65ymZdA2XhIeaWmdDQ40p32F0WY6tdBi+aeY+4ldwhAOpo0eyVrYTGg==", "license": "MPL-2.0", "peer": true, "workspaces": [ @@ -12065,7 +12065,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.25.2", + "version": "1.26.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" diff --git a/package.json b/package.json index 9a6a387..178d998 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.25.2", + "version": "1.26.0", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index 172e319..fe80545 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.25.2", + "version": "1.26.0", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", diff --git a/packages/mp3-encoder/src/index.ts b/packages/mp3-encoder/src/index.ts index 92aedbc..5c99658 100644 --- a/packages/mp3-encoder/src/index.ts +++ b/packages/mp3-encoder/src/index.ts @@ -22,7 +22,7 @@ class Mp3Encoder extends CustomAudioEncoder { private buffer = new Uint8Array(2 ** 16); private currentBufferOffset = 0; - private currentTimestamp = 0; + private currentTimestamp: number | null = null; private chunkMetadata: EncodedAudioChunkMetadata = {}; static override supports(codec: AudioCodec, config: AudioDecoderConfig): boolean { @@ -79,6 +79,11 @@ class Mp3Encoder extends CustomAudioEncoder { } async encode(audioSample: AudioSample) { + if (this.currentTimestamp === null) { + // The first sample's timestamp determines where we start + this.currentTimestamp = audioSample.timestamp; + } + const sizePerChannel = audioSample.allocationSize({ format: 's16-planar', planeIndex: 0, @@ -123,6 +128,8 @@ class Mp3Encoder extends CustomAudioEncoder { * these chunks and extract the MP3 frames only when they're complete. */ private digestOutput(bytes: Uint8Array) { + assert(this.currentTimestamp !== null); + const requiredBufferSize = this.currentBufferOffset + bytes.length; if (requiredBufferSize > this.buffer.length) { // Grow the buffer to the required size diff --git a/src/codec-data.ts b/src/codec-data.ts index eef04ec..2fd8ca7 100644 --- a/src/codec-data.ts +++ b/src/codec-data.ts @@ -22,6 +22,8 @@ import { textEncoder, toDataView, toUint8Array, + getChromiumVersion, + isChromium, } from './misc'; import { PacketType } from './packet'; import { MetadataTags } from './metadata'; @@ -34,6 +36,7 @@ import { MetadataTags } from './metadata'; export enum AvcNalUnitType { IDR = 5, + SEI = 6, SPS = 7, PPS = 8, SPS_EXT = 13, @@ -590,9 +593,7 @@ export const extractNalUnitTypeForHevc = (data: Uint8Array) => { }; /** Builds a HevcDecoderConfigurationRecord from an HEVC packet in Annex B format. */ -export const extractHevcDecoderConfigurationRecord = ( - packetData: Uint8Array, -) => { +export const extractHevcDecoderConfigurationRecord = (packetData: Uint8Array) => { try { const nalUnits = findNalUnitsInAnnexB(packetData); @@ -1456,6 +1457,69 @@ export const extractAv1CodecInfoFromPacket = ( } } + // Frame size + const frameWidthBitsMinus1 = bitstream.readBits(4); + const frameHeightBitsMinus1 = bitstream.readBits(4); + const n1 = frameWidthBitsMinus1 + 1; + bitstream.skipBits(n1); // max_frame_width_minus_1 + const n2 = frameHeightBitsMinus1 + 1; + bitstream.skipBits(n2); // max_frame_height_minus_1 + + // Frame IDs + let frameIdNumbersPresentFlag = 0; + if (reducedStillPictureHeader) { + frameIdNumbersPresentFlag = 0; + } else { + frameIdNumbersPresentFlag = bitstream.readBits(1); + } + + if (frameIdNumbersPresentFlag) { + bitstream.skipBits(4); // delta_frame_id_length_minus_2 + bitstream.skipBits(3); // additional_frame_id_length_minus_1 + } + + bitstream.skipBits(1); // use_128x128_superblock + bitstream.skipBits(1); // enable_filter_intra + bitstream.skipBits(1); // enable_intra_edge_filter + + if (!reducedStillPictureHeader) { + bitstream.skipBits(1); // enable_interintra_compound + bitstream.skipBits(1); // enable_masked_compound + bitstream.skipBits(1); // enable_warped_motion + bitstream.skipBits(1); // enable_dual_filter + const enableOrderHint = bitstream.readBits(1); + + if (enableOrderHint) { + bitstream.skipBits(1); // enable_jnt_comp + bitstream.skipBits(1); // enable_ref_frame_mvs + } + + const seqChooseScreenContentTools = bitstream.readBits(1); + let seqForceScreenContentTools = 0; + + if (seqChooseScreenContentTools) { + seqForceScreenContentTools = 2; // SELECT_SCREEN_CONTENT_TOOLS + } else { + seqForceScreenContentTools = bitstream.readBits(1); + } + + if (seqForceScreenContentTools > 0) { + const seqChooseIntegerMv = bitstream.readBits(1); + if (!seqChooseIntegerMv) { + bitstream.skipBits(1); // seq_force_integer_mv + } + } + + if (enableOrderHint) { + bitstream.skipBits(3); // order_hint_bits_minus_1 + } + } + + bitstream.skipBits(1); // enable_superres + bitstream.skipBits(1); // enable_cdef + bitstream.skipBits(1); // enable_restoration + + // color_config() const highBitdepth = bitstream.readBits(1); let bitDepth = 8; @@ -1653,7 +1717,68 @@ export const determineVideoPacketType = ( switch (codec) { case 'avc': { const nalUnits = extractAvcNalUnits(packetData, decoderConfig); - const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR); + let isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR); + + if (!isKeyframe && (!isChromium() || getChromiumVersion()! >= 144)) { + // In addition to IDR, Recovery Point SEI also counts as a valid H.264 keyframe by current consensus. + // See https://github.com/w3c/webcodecs/issues/650 for the relevant discussion. WebKit and Firefox have + // always supported them, but Chromium hasn't, therefore the (admittedly dirty) version check. + + for (const nalUnit of nalUnits) { + const type = extractNalUnitTypeForAvc(nalUnit); + if (type !== AvcNalUnitType.SEI) { + continue; + } + + const bytes = removeEmulationPreventionBytes(nalUnit); + let pos = 1; // Skip NALU header + + // sei_rbsp() + do { + // sei_message() + let payloadType = 0; + while (true) { + const nextByte = bytes[pos++]; + if (nextByte === undefined) break; + payloadType += nextByte; + + if (nextByte < 255) { + break; + } + } + + let payloadSize = 0; + while (true) { + const nextByte = bytes[pos++]; + if (nextByte === undefined) break; + payloadSize += nextByte; + + if (nextByte < 255) { + break; + } + } + + // sei_payload() + const PAYLOAD_TYPE_RECOVERY_POINT = 6; + if (payloadType === PAYLOAD_TYPE_RECOVERY_POINT) { + const bitstream = new Bitstream(bytes); + bitstream.pos = 8 * pos; + + const recoveryFrameCount = readExpGolomb(bitstream); + const exactMatchFlag = bitstream.readBits(1); + + if (recoveryFrameCount === 0 && exactMatchFlag === 1) { + // https://github.com/w3c/webcodecs/pull/910 + // "recovery_frame_cnt == 0 and exact_match_flag=1 in the SEI recovery payload" + isKeyframe = true; + break; + } + } + + pos += payloadSize; + } while (pos < bytes.length - 1); + } + } return isKeyframe ? 'key' : 'delta'; }; diff --git a/src/conversion.ts b/src/conversion.ts index 632b300..433f125 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -82,10 +82,16 @@ export type ConversionOptions = { /** Options to trim the input file. */ trim?: { - /** The time in the input file in seconds at which the output file should start. Must be less than `end`. */ - start: number; - /** The time in the input file in seconds at which the output file should end. Must be greater than `start`. */ - end: number; + /** + * The time in the input file in seconds at which the output file should start. Must be less than `end`. + * Defaults to 0 when omitted. + */ + start?: number; + /** + * The time in the input file in seconds at which the output file should end. Must be greater than `start`. + * Defaults to the duration of the input when omitted. + */ + end?: number; }; /** @@ -138,6 +144,12 @@ export type ConversionVideoOptions = { * metadata. */ rotate?: Rotation; + /** + * Defaults to `true`. When enabaled, Mediabunny will use the rotation metadata in the output file to perform video + * rotation whenever possible. Set this field to `false` if you want to ensure the output file does not make use of + * rotation metadata and that any rotation is baked into the video frames directly. + */ + allowRotationMetadata?: boolean; /** * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to * the dimensions of the input video track. Cropping is performed after rotation but before resizing. @@ -299,6 +311,9 @@ const validateVideoOptions = (videoOptions: ConversionVideoOptions | undefined) if (videoOptions?.rotate !== undefined && ![0, 90, 180, 270].includes(videoOptions.rotate)) { throw new TypeError('options.video.rotate, when provided, must be 0, 90, 180 or 270.'); } + if (videoOptions?.allowRotationMetadata !== undefined && typeof videoOptions.allowRotationMetadata !== 'boolean') { + throw new TypeError('options.video.allowRotationMetadata, when provided, must be a boolean.'); + } if (videoOptions?.crop !== undefined) { validateCropRectangle(videoOptions.crop, 'options.video.'); } @@ -833,7 +848,8 @@ export class Conversion { let videoSource: VideoSource; const totalRotation = normalizeRotation(track.rotation + (trackOptions.rotate ?? 0)); - const outputSupportsRotation = this.output.format.supportsVideoRotationMetadata; + const canUseRotationMetadata = this.output.format.supportsVideoRotationMetadata + && (trackOptions.allowRotationMetadata ?? true); const [rotatedWidth, rotatedHeight] = totalRotation % 180 === 0 ? [track.codedWidth, track.codedHeight] @@ -878,7 +894,7 @@ export class Conversion { // TODO This is suboptimal: Forcing a rerender when both rotation and process are set is not // performance-optimal, but right now there's no other way because we can't change the track rotation // metadata after the output has already started. Should be possible with API changes in v2, though! - || (totalRotation !== 0 && (!outputSupportsRotation || trackOptions.process !== undefined)) + || (totalRotation !== 0 && (!canUseRotationMetadata || trackOptions.process !== undefined)) || !!crop; const alpha = trackOptions.alpha ?? 'discard'; @@ -1510,7 +1526,10 @@ export class Conversion { targetSampleRate, startTime: this._startTimestamp, endTime: this._endTimestamp, - onSample: sample => this._registerAudioSample(track, trackOptions, source, sample), + onSample: async (sample) => { + await this._registerAudioSample(track, trackOptions, source, sample); + sample.close(); + }, }); const sink = new AudioSampleSink(track); @@ -1522,6 +1541,7 @@ export class Conversion { } await resampler.add(sample); + sample.close(); } await resampler.finalize(); diff --git a/src/isobmff/isobmff-demuxer.ts b/src/isobmff/isobmff-demuxer.ts index be4e1db..9d42aae 100644 --- a/src/isobmff/isobmff-demuxer.ts +++ b/src/isobmff/isobmff-demuxer.ts @@ -2524,12 +2524,8 @@ abstract class IsobmffTrackBacking implements InputTrackBacking { const timestampInTimescale = this.mapTimestampIntoTimescale(timestamp); const sampleTable = this.internalTrack.demuxer.getSampleTableForTrack(this.internalTrack); - const sampleIndex = getSampleIndexForTimestamp(sampleTable, timestampInTimescale); - const keyFrameSampleIndex = sampleIndex === -1 - ? -1 - : getRelevantKeyframeIndexForSample(sampleTable, sampleIndex); - - const promise = this.fetchPacketForSampleIndex(res, keyFrameSampleIndex, options); + const sampleIndex = getKeyframeSampleIndexForTimestamp(sampleTable, timestampInTimescale); + const promise = this.fetchPacketForSampleIndex(res, sampleIndex, options); if (res.pending) await promise; if (!sampleTableIsEmpty(sampleTable) || !this.internalTrack.demuxer.isFragmented) { @@ -2968,7 +2964,45 @@ const getSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: nu const entry = sampleTable.sampleTimingEntries[index]!; return entry.startIndex - + Math.min(Math.floor((timescaleUnits - entry.startDecodeTimestamp) / entry.delta), entry.count - 1); + + Math.min( + Math.floor((timescaleUnits - entry.startDecodeTimestamp) / entry.delta), + entry.count - 1, + ); + } +}; + +const getKeyframeSampleIndexForTimestamp = (sampleTable: SampleTable, timescaleUnits: number) => { + if (!sampleTable.keySampleIndices) { + // Every sample is a keyframe + return getSampleIndexForTimestamp(sampleTable, timescaleUnits); + } + + if (sampleTable.presentationTimestamps) { + const index = binarySearchLessOrEqual( + sampleTable.presentationTimestamps, + timescaleUnits, + x => x.presentationTimestamp, + ); + if (index === -1) { + return -1; + } + + // Walk the samples in presentation order until we find one that's a keyframe + for (let i = index; i >= 0; i--) { + const sampleIndex = sampleTable.presentationTimestamps[i]!.sampleIndex; + const isKeyFrame = binarySearchExact(sampleTable.keySampleIndices, sampleIndex, x => x) !== -1; + + if (isKeyFrame) { + return sampleIndex; + } + } + + return -1; + } else { + const sampleIndex = getSampleIndexForTimestamp(sampleTable, timescaleUnits); + + const index = binarySearchLessOrEqual(sampleTable.keySampleIndices, sampleIndex, x => x); + return sampleTable.keySampleIndices[index] ?? -1; } }; @@ -3046,15 +3080,6 @@ const getSampleInfo = (sampleTable: SampleTable, sampleIndex: number): SampleInf }; }; -const getRelevantKeyframeIndexForSample = (sampleTable: SampleTable, sampleIndex: number) => { - if (!sampleTable.keySampleIndices) { - return sampleIndex; - } - - const index = binarySearchLessOrEqual(sampleTable.keySampleIndices, sampleIndex, x => x); - return sampleTable.keySampleIndices[index] ?? -1; -}; - const getNextKeyframeIndexForSample = (sampleTable: SampleTable, sampleIndex: number) => { if (!sampleTable.keySampleIndices) { return sampleIndex + 1; diff --git a/src/media-sink.ts b/src/media-sink.ts index c4bf404..8d3ab8a 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -216,9 +216,10 @@ export class EncodedPacketSink { } const packet = await this._track._backing.getKeyPacket(timestamp, options); - if (!packet || packet.type === 'delta') { + if (!packet) { return packet; } + assert(packet.type === 'key'); const determinedType = await this._track.determinePacketType(packet); if (determinedType === 'delta') { @@ -250,9 +251,10 @@ export class EncodedPacketSink { } const nextPacket = await this._track._backing.getNextKeyPacket(packet, options); - if (!nextPacket || nextPacket.type === 'delta') { + if (!nextPacket) { return nextPacket; } + assert(nextPacket.type === 'key'); const determinedType = await this._track.determinePacketType(nextPacket); if (determinedType === 'delta') { @@ -489,9 +491,6 @@ export abstract class BaseMediaSampleSink< const packetSink = this._createPacketSink(); const keyPacket = await packetSink.getKeyPacket(startTimestamp, { verifyKeyPackets: true }) ?? await packetSink.getFirstPacket(); - if (!keyPacket) { - return; - } let currentPacket: EncodedPacket | null = keyPacket; @@ -513,7 +512,7 @@ export abstract class BaseMediaSampleSink< } } - const packets = packetSink.packets(keyPacket, endPacket); + const packets = packetSink.packets(keyPacket ?? undefined, endPacket); await packets.next(); // Skip the start packet as we already have it while (currentPacket && !ended && !this._track.input._disposed) { @@ -2177,11 +2176,14 @@ export class AudioBufferSink { /** @internal */ _audioSampleToWrappedArrayBuffer(sample: AudioSample): WrappedAudioBuffer { - return { + const result: WrappedAudioBuffer = { buffer: sample.toAudioBuffer(), timestamp: sample.timestamp, duration: sample.duration, }; + + sample.close(); + return result; } /** diff --git a/src/media-source.ts b/src/media-source.ts index c2b4a6b..7b5cae6 100644 --- a/src/media-source.ts +++ b/src/media-source.ts @@ -1055,6 +1055,12 @@ export class MediaStreamVideoTrackSource extends VideoSource { private _promiseWithResolvers = promiseWithResolvers(); /** @internal */ private _errorPromiseAccessed = false; + /** @internal */ + private _paused = false; + /** @internal */ + private _lastSampleTimestamp: number | null = null; + /** @internal */ + private _pauseOffset = 0; /** A promise that rejects upon any error within this source. This promise never resolves. */ get errorPromise() { @@ -1062,6 +1068,11 @@ export class MediaStreamVideoTrackSource extends VideoSource { return this._promiseWithResolvers.promise; } + /** Whether this source is currently paused as a result of calling `.pause()`. */ + get paused() { + return this._paused; + } + /** * Creates a new {@link MediaStreamVideoTrackSource} from a * [`MediaStreamVideoTrack`](https://developer.mozilla.org/en-US/docs/Web/API/MediaStreamTrack), which will pull @@ -1103,8 +1114,29 @@ export class MediaStreamVideoTrackSource extends VideoSource { return; } + const currentTimestamp = videoFrame.timestamp / 1e6; + + if (this._paused) { + const frameSeen = firstVideoFrameTimestamp !== null; + if (frameSeen) { + if (this._lastSampleTimestamp !== null) { + // In addition to dropping this frame, let's also keep track of the time we have lost due to the + // pause. Doing it like this instead of simply keeping track of the paused time is better since + // it retains the frame rate of the underlying source. + const timeDelta = currentTimestamp - this._lastSampleTimestamp; + // We modify this field instead of _timestampOffset since we still might have data in flight + // in the encoder, with which we don't want to mess. + this._pauseOffset -= timeDelta; + } + this._lastSampleTimestamp = currentTimestamp; + } + + videoFrame.close(); + return; + } + if (firstVideoFrameTimestamp === null) { - firstVideoFrameTimestamp = videoFrame.timestamp / 1e6; + firstVideoFrameTimestamp = currentTimestamp; const muxer = this._connectedTrack!.output._muxer; if (muxer.firstMediaStreamTimestamp === null) { @@ -1116,13 +1148,19 @@ export class MediaStreamVideoTrackSource extends VideoSource { } } + this._lastSampleTimestamp = currentTimestamp; + if (this._encoder.getQueueSize() >= 4) { // Drop frames if the encoder is overloaded videoFrame.close(); return; } - void this._encoder.add(new VideoSample(videoFrame), true) + const sample = new VideoSample(videoFrame, { + timestamp: currentTimestamp + this._pauseOffset, + }); + + void this._encoder.add(sample, true) .catch((error) => { errored = true; @@ -1182,6 +1220,19 @@ export class MediaStreamVideoTrackSource extends VideoSource { } } + /** + * Pauses the capture of video frames - any video frames emitted by the underlying media stream will be ignored + * while paused. This does *not* close the underlying `MediaStreamVideoTrack`, it just ignores its output. + */ + pause() { + this._paused = true; + } + + /** Resumes the capture of video frames after being paused. */ + resume() { + this._paused = false; + } + /** @internal */ override async _flushAndClose(forceClose: boolean) { if (this._abortController) { @@ -1356,21 +1407,27 @@ class AudioEncoderWrapper { (audioSample.timestamp + audioSample.duration) * audioSample.sampleRate, ); - if (this.lastEndSampleIndex !== null && startSampleIndex > this.lastEndSampleIndex) { - const sampleCount = startSampleIndex - this.lastEndSampleIndex; - const fillSample = new AudioSample({ - data: new Float32Array(sampleCount * audioSample.numberOfChannels), - format: 'f32-planar', - sampleRate: audioSample.sampleRate, - numberOfChannels: audioSample.numberOfChannels, - numberOfFrames: sampleCount, - timestamp: this.lastEndSampleIndex / audioSample.sampleRate, - }); + if (this.lastEndSampleIndex === null) { + this.lastEndSampleIndex = endSampleIndex; + } else { + const sampleDiff = startSampleIndex - this.lastEndSampleIndex; - await this.add(fillSample, true); // Recursive call + if (sampleDiff >= 64) { + // The gap is big enough, let's add a correction sample + const fillSample = new AudioSample({ + data: new Float32Array(sampleDiff * audioSample.numberOfChannels), + format: 'f32-planar', + sampleRate: audioSample.sampleRate, + numberOfChannels: audioSample.numberOfChannels, + numberOfFrames: sampleDiff, + timestamp: this.lastEndSampleIndex / audioSample.sampleRate, + }); + + await this.add(fillSample, true); // Recursive call + } + + this.lastEndSampleIndex += audioSample.numberOfFrames; } - - this.lastEndSampleIndex = endSampleIndex; } if (this.customEncoder) { @@ -1851,6 +1908,12 @@ export class MediaStreamAudioTrackSource extends AudioSource { private _promiseWithResolvers = promiseWithResolvers(); /** @internal */ private _errorPromiseAccessed = false; + /** @internal */ + private _paused = false; + /** @internal */ + private _lastSampleTimestamp: number | null = null; + /** @internal */ + private _pauseOffset = 0; /** A promise that rejects upon any error within this source. This promise never resolves. */ get errorPromise() { @@ -1858,6 +1921,11 @@ export class MediaStreamAudioTrackSource extends AudioSource { return this._promiseWithResolvers.promise; } + /** Whether this source is currently paused as a result of calling `.pause()`. */ + get paused() { + return this._paused; + } + /** * Creates a new {@link MediaStreamAudioTrackSource} from a `MediaStreamAudioTrack`, which will pull audio samples * from the stream in real time and encode them according to {@link AudioEncodingConfig}. @@ -1884,38 +1952,74 @@ export class MediaStreamAudioTrackSource extends AudioSource { this._abortController = new AbortController(); + let firstAudioDataTimestamp: number | null = null; + let errored = false; + + const onAudioSample = (audioSample: AudioSample) => { + if (errored) { + audioSample.close(); + return; + } + + const currentTimestamp = audioSample.timestamp; + + if (this._paused) { + const dataSeen = firstAudioDataTimestamp !== null; + if (dataSeen) { + if (this._lastSampleTimestamp !== null) { + // In addition to dropping this sample, let's also keep track of the time we have lost due to + // the pause. Doing it like this instead of simply keeping track of the paused time is better + // since it retains the sample rate of the underlying source. + const timeDelta = currentTimestamp - this._lastSampleTimestamp; + // We modify this field instead of _timestampOffset since we still might have data in flight + // in the encoder, with which we don't want to mess. + this._pauseOffset -= timeDelta; + } + this._lastSampleTimestamp = currentTimestamp; + } + + audioSample.close(); + return; + } + + if (firstAudioDataTimestamp === null) { + firstAudioDataTimestamp = audioSample.timestamp; + + const muxer = this._connectedTrack!.output._muxer; + if (muxer.firstMediaStreamTimestamp === null) { + muxer.firstMediaStreamTimestamp = performance.now() / 1000; + this._timestampOffset = -firstAudioDataTimestamp; + } else { + this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp) + - firstAudioDataTimestamp; + } + } + + this._lastSampleTimestamp = currentTimestamp; + + if (this._encoder.getQueueSize() >= 4) { + // Drop data if the encoder is overloaded + audioSample.close(); + return; + } + + audioSample.setTimestamp(currentTimestamp + this._pauseOffset); + + void this._encoder.add(audioSample, true) + .catch((error) => { + errored = true; + + this._abortController?.abort(); + this._promiseWithResolvers.reject(error); + void this._audioContext?.suspend(); + }); + }; + if (typeof MediaStreamTrackProcessor !== 'undefined') { // Great, MediaStreamTrackProcessor is supported, this is the preferred way of doing things - let firstAudioDataTimestamp: number | null = null; - const processor = new MediaStreamTrackProcessor({ track: this._track }); const consumer = new WritableStream({ - write: (audioData) => { - if (firstAudioDataTimestamp === null) { - firstAudioDataTimestamp = audioData.timestamp / 1e6; - - const muxer = this._connectedTrack!.output._muxer; - if (muxer.firstMediaStreamTimestamp === null) { - muxer.firstMediaStreamTimestamp = performance.now() / 1000; - this._timestampOffset = -firstAudioDataTimestamp; - } else { - this._timestampOffset = (performance.now() / 1000 - muxer.firstMediaStreamTimestamp) - - firstAudioDataTimestamp; - } - } - - if (this._encoder.getQueueSize() >= 4) { - // Drop data if the encoder is overloaded - audioData.close(); - return; - } - - void this._encoder.add(new AudioSample(audioData), true) - .catch((error) => { - this._abortController?.abort(); - this._promiseWithResolvers.reject(error); - }); - }, + write: audioData => onAudioSample(new AudioSample(audioData)), }); processor.readable.pipeTo(consumer, { @@ -1943,7 +2047,6 @@ export class MediaStreamAudioTrackSource extends AudioSource { sourceNode.connect(this._scriptProcessorNode); this._scriptProcessorNode.connect(this._audioContext.destination); - let audioReceived = false; let totalDuration = 0; this._scriptProcessorNode.onaudioprocess = (event) => { @@ -1951,33 +2054,25 @@ export class MediaStreamAudioTrackSource extends AudioSource { totalDuration += event.inputBuffer.duration; for (const audioSample of iterator) { - if (!audioReceived) { - audioReceived = true; - - const muxer = this._connectedTrack!.output._muxer; - if (muxer.firstMediaStreamTimestamp === null) { - muxer.firstMediaStreamTimestamp = performance.now() / 1000; - } else { - this._timestampOffset = performance.now() / 1000 - muxer.firstMediaStreamTimestamp; - } - } - - if (this._encoder.getQueueSize() >= 4) { - // Drop data if the encoder is overloaded - audioSample.close(); - continue; - } - - void this._encoder.add(audioSample, true) - .catch((error) => { - void this._audioContext!.suspend(); - this._promiseWithResolvers.reject(error); - }); + onAudioSample(audioSample); } }; } } + /** + * Pauses the capture of audio data - any audio data emitted by the underlying media stream will be ignored + * while paused. This does *not* close the underlying `MediaStreamAudioTrack`, it just ignores its output. + */ + pause() { + this._paused = true; + } + + /** Resumes the capture of audio data after being paused. */ + resume() { + this._paused = false; + } + /** @internal */ override async _flushAndClose(forceClose: boolean) { if (this._abortController) { diff --git a/src/misc.ts b/src/misc.ts index 29c04c2..2677f8b 100644 --- a/src/misc.ts +++ b/src/misc.ts @@ -685,7 +685,15 @@ export const isWebKit = () => { } // This even returns true for WebKit-wrapping browsers such as Chrome on iOS - return isWebKitCache = !!(typeof navigator !== 'undefined' && navigator.vendor?.match(/apple/i)); + return isWebKitCache = !!( + typeof navigator !== 'undefined' + && ( + navigator.vendor?.match(/apple/i) + // Or, in workers: + || (/AppleWebKit/.test(navigator.userAgent) && !/Chrome/.test(navigator.userAgent)) + || /\b(iPad|iPhone|iPod)\b/.test(navigator.userAgent) + ) + ); }; let isFirefoxCache: boolean | null = null; @@ -703,7 +711,28 @@ export const isChromium = () => { return isChromiumCache; } - return isChromiumCache = !!(typeof navigator !== 'undefined' && navigator.vendor?.includes('Google Inc')); + return isChromiumCache = !!( + typeof navigator !== 'undefined' + && (navigator.vendor?.includes('Google Inc') || /Chrome/.test(navigator.userAgent)) + ); +}; + +let chromiumVersionCache: number | null = null; +export const getChromiumVersion = () => { + if (chromiumVersionCache !== null) { + return chromiumVersionCache; + } + + if (typeof navigator === 'undefined') { + return null; + } + + const match = /\bChrome\/(\d+)/.exec(navigator.userAgent); + if (!match) { + return null; + } + + return chromiumVersionCache = Number(match[1]!); }; /** diff --git a/src/sample.ts b/src/sample.ts index b8b3be0..9d61724 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -21,6 +21,54 @@ import { polyfillSymbolDispose(); +type FinalizationRegistryValue = { + type: 'video'; + data: VideoFrame | OffscreenCanvas | Uint8Array; +} | { + type: 'audio'; + data: AudioData | Uint8Array; +}; + +// Let's manually handle logging the garbage collection errors that are typically logged by the browser. This way, they +// also kick for audio samples (which is normally not the case), making sure any incorrect code is quickly caught. +let lastVideoGcErrorLog = -Infinity; +let lastAudioGcErrorLog = -Infinity; +let finalizationRegistry: FinalizationRegistry | null = null; +if (typeof FinalizationRegistry !== 'undefined') { + finalizationRegistry = new FinalizationRegistry((value) => { + const now = Date.now(); + + if (value.type === 'video') { + if (now - lastVideoGcErrorLog >= 1000) { + // This error is annoying but oh so important + console.error( + `A VideoSample was garbage collected without first being closed. For proper resource management,` + + ` make sure to call close() on all your VideoSamples as soon as you're done using them.`, + ); + + lastVideoGcErrorLog = now; + } + + if (typeof VideoFrame !== 'undefined' && value.data instanceof VideoFrame) { + value.data.close(); // Prevent the browser error since we're logging our own + } + } else { + if (now - lastAudioGcErrorLog >= 1000) { + console.error( + `An AudioSample was garbage collected without first being closed. For proper resource management,` + + ` make sure to call close() on all your AudioSamples as soon as you're done using them.`, + ); + + lastAudioGcErrorLog = now; + } + + if (typeof AudioData !== 'undefined' && value.data instanceof AudioData) { + value.data.close(); + } + } + }); +} + /** * Metadata used for VideoSample initialization. * @group Samples @@ -144,7 +192,11 @@ export class VideoSample implements Disposable { data: VideoFrame | CanvasImageSource | AllowSharedBufferSource, init?: VideoSampleInit, ) { - if (data instanceof ArrayBuffer || ArrayBuffer.isView(data)) { + if ( + data instanceof ArrayBuffer + || (typeof SharedArrayBuffer !== 'undefined' && data instanceof SharedArrayBuffer) + || ArrayBuffer.isView(data) + ) { if (!init || typeof init !== 'object') { throw new TypeError('init must be an object.'); } @@ -278,6 +330,7 @@ export class VideoSample implements Disposable { } VideoSample._openSampleCount++; + finalizationRegistry?.register(this, { type: 'video', data: this._data }, this); } /** Clones this video sample. */ @@ -326,6 +379,8 @@ export class VideoSample implements Disposable { return; } + finalizationRegistry?.unregister(this); + if (isVideoFrame(this._data)) { this._data.close(); } else { @@ -977,6 +1032,8 @@ export class AudioSample implements Disposable { this._data = dataBuffer; } + + finalizationRegistry?.register(this, { type: 'audio', data: this._data }, this); } /** Returns the number of bytes required to hold the audio sample's data as specified by the given options. */ @@ -1138,9 +1195,8 @@ export class AudioSample implements Disposable { } } } else { - // Branch for Uint8Array data (non-AudioData) const uint8Data = this._data; - const srcView = new DataView(uint8Data.buffer, uint8Data.byteOffset, uint8Data.byteLength); + const srcView = toDataView(uint8Data); const srcFormat = this.format; const readFn = getReadFunction(srcFormat); @@ -1210,6 +1266,8 @@ export class AudioSample implements Disposable { return; } + finalizationRegistry?.unregister(this); + if (isAudioData(this._data)) { this._data.close(); } else { diff --git a/test/browser/flac.test.ts b/test/browser/flac.test.ts index 2aed0e3..ddc065b 100644 --- a/test/browser/flac.test.ts +++ b/test/browser/flac.test.ts @@ -19,7 +19,7 @@ test('can decode samples from a FLAC file', async () => { const sink = new AudioSampleSink(track); - const sample = await sink.getSample(1); + using sample = await sink.getSample(1); assert(sample); expect(sample.timestamp).toBe(0.9287981859410431); }); diff --git a/test/browser/transparency.test.ts b/test/browser/transparency.test.ts index 35527ae..394c827 100644 --- a/test/browser/transparency.test.ts +++ b/test/browser/transparency.test.ts @@ -21,7 +21,7 @@ test('Can decode transparent video', async () => { expect(await videoTrack.canBeTransparent()).toBe(true); const sink = new VideoSampleSink(videoTrack); - const sample = (await sink.getSample(0.5))!; + using sample = (await sink.getSample(0.5))!; expect(sample.format).toContain('A'); // Probably RGBA expect(sample.hasAlpha).toBe(true); @@ -47,10 +47,10 @@ test('Can decode faulty transparent video and behaves gracefully', async () => { const sink = new VideoSampleSink(videoTrack); - const startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + using startSample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; expect(startSample.format).toContain('A'); - const secondSample = (await sink.getSample(secondKeyPacket.timestamp))!; + using secondSample = (await sink.getSample(secondKeyPacket.timestamp))!; expect(secondSample.format).not.toContain('A'); // There was no alpha key frame for this one expect(secondSample.hasAlpha).toBe(false); }); @@ -164,7 +164,7 @@ test('Can encode transparent video', async () => { const sink = new VideoSampleSink(videoTrack); - const firstSample = (await sink.getSample(0))!; + using firstSample = (await sink.getSample(0))!; expect(firstSample.format).toContain('A'); probeContext.clearRect(0, 0, probeCanvas.width, probeCanvas.height); @@ -201,7 +201,7 @@ test('Can encode video with alternating transparency', async () => { await output.start(); for (let i = 0; i < 64; i++) { - const sample = new VideoSample(new Uint8Array(640 * 480 * 4), { + using sample = new VideoSample(new Uint8Array(640 * 480 * 4), { format: i % 2 ? 'RGBX' : 'RGBA', codedWidth: 640, codedHeight: 480, @@ -235,7 +235,7 @@ test('Can encode video with alternating transparency', async () => { const sampleSink = new VideoSampleSink(videoTrack); i = 0; - for await (const sample of sampleSink.samples()) { + for await (using sample of sampleSink.samples()) { if (i % 2) { expect(sample.format).not.toContain('A'); } else { @@ -299,7 +299,7 @@ test('Can transmux transparent video, discards alpha by default', async () => { expect(await videoTrack.canBeTransparent()).toBe(false); const sink = new VideoSampleSink(videoTrack); - const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; expect(sample.hasAlpha).toBe(false); }); @@ -331,7 +331,7 @@ test('Can transmux transparent video, can keep alpha', async () => { expect(await videoTrack.canBeTransparent()).toBe(true); const sink = new VideoSampleSink(videoTrack); - const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; expect(sample.format).toContain('A'); expect(sample.hasAlpha).toBe(true); }); @@ -370,7 +370,7 @@ test('Can reencode transparent video, keeping alpha', async () => { expect(videoTrack.displayWidth).toBe(320); const sink = new VideoSampleSink(videoTrack); - const sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; + using sample = (await sink.getSample(await videoTrack.getFirstTimestamp()))!; expect(sample.format).toContain('A'); expect(sample.hasAlpha).toBe(true); });