diff --git a/dev/convert.html b/dev/convert.html index a50cbd5..7824830 100644 --- a/dev/convert.html +++ b/dev/convert.html @@ -24,7 +24,7 @@ chunked: true, chunkSize: 2**20 }); - const outputFormat = new Mediabunny.WavOutputFormat({}); + const outputFormat = new Mediabunny.Mp4OutputFormat({}); const button = document.createElement('button'); button.textContent = 'Cancel'; @@ -72,7 +72,7 @@ }), output, audio: { - codec: 'pcm-s16', + //codec: 'pcm-s16', //sampleRate: 16000, //numberOfChannels: 1, //discard: true, @@ -109,6 +109,16 @@ */ video: () => ({ //discard: true, + crop: { + left: 0, + top: 0, + width: 500, + height: 500, + }, + rotate: 90, + width: 200, + height: 500, + fit: 'contain', //forceTranscode: true, //codec: 'avc', //fit: 'contain', @@ -132,8 +142,8 @@ //height: 100, }), trim: { - start: 0, - end: 10 + start: 10, + end: 20 }, }); console.log(conversion); diff --git a/docs/guide/converting-media-files.md b/docs/guide/converting-media-files.md index 03fc298..76f4296 100644 --- a/docs/guide/converting-media-files.md +++ b/docs/guide/converting-media-files.md @@ -109,6 +109,7 @@ type ConversionVideoOptions = { height?: number; fit?: 'fill' | 'contain' | 'cover'; rotate?: 0 | 90 | 180 | 270; + crop?: { left: number; top: number; width: number; height: number }; frameRate?: number; codec?: VideoCodec; bitrate?: number | Quality; @@ -137,21 +138,27 @@ The provided configuration will apply equally to all video tracks of the input. If you want to get rid of the video track, use `discard: true`. -### Resizing/rotating video +### Resizing video The `width`, `height` and `fit` properties control how the video is resized. If only `width` or `height` is provided, the other value is deduced automatically to preserve the video's original aspect ratio. If both are used, `fit` must be set to control the fitting algorithm: - `'fill'` will stretch the image to fill the entire box, potentially altering aspect ratio. - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to letterboxing. - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio. -`rotation` rotates the video by the specified number of degrees clockwise. This rotation is applied on top of any rotation metadata in the original input file. - -If `width` or `height` is used in conjunction with `rotation`, they control the post-rotation dimensions. +If `width` or `height` is used in conjunction with `rotation` or `crop`, they control the post-rotation, post-crop dimensions. If you want to apply max/min constraints to a video's dimensions, check out [track-specific options](#track-specific-options). In the rare case that the input video changes size over time, the `fit` field can be used to control the size change behavior (see [`VideoEncodingConfig`](./media-sources#video-encoding-config)). When unset, the behavior is `'passThrough'`. +### Rotating video + +`rotation` rotates the video by the specified number of degrees clockwise. This rotation is applied on top of any rotation metadata in the original input file and happens before cropping and resizing. + +### Cropping video + +`crop` can be used to extract a rectangular region from the original video. The rectangle is specified using `left`, `top`, `width` and `height` and is clamped to the dimensions of the video. Cropping is applied after rotation but before resizing. + ### Adjusting frame rate The `frameRate` property can be used to set the frame rate of the output video in Hz. If not specified, the original input frame rate will be used (which may be variable). diff --git a/docs/guide/media-sinks.md b/docs/guide/media-sinks.md index d42e166..ac76161 100644 --- a/docs/guide/media-sinks.md +++ b/docs/guide/media-sinks.md @@ -314,7 +314,7 @@ for await (const sample of keyFrameSamples) { ### `CanvasSink` -While `VideoSampleSink` extracts raw decoded video samples, you can use `CanvasSink` to extract these samples as canvases instead. In doing so, certain operations such as scaling and rotating can also be handled by the sink. The downside is the additional VRAM requirements for the canvases' framebuffers. +While `VideoSampleSink` extracts raw decoded video samples, you can use `CanvasSink` to extract these samples as canvases instead. In doing so, certain operations such as scaling, rotating, and cropping can also be handled by the sink. The downside is the additional VRAM requirements for the canvases' framebuffers. ::: info This sink yields `HTMLCanvasElement` whenever possible, and falls back to `OffscreenCanvas` otherwise (in Worker contexts, for example). @@ -334,6 +334,7 @@ type CanvasSinkOptions = { height?: number; fit?: 'fill' | 'contain' | 'cover'; rotation?: 0 | 90 | 180 | 270; + crop?: { left: number; top: number; width: number; height: number }; poolSize?: number; }; ``` @@ -347,7 +348,9 @@ type CanvasSinkOptions = { - `'contain'` will contain the entire image within the box while preserving aspect ratio. This may lead to letterboxing. - `'cover'` will scale the image until the entire box is filled, while preserving aspect ratio. - `rotation`\ - The clockwise rotation by which to rotate the raw video frame. Defaults to the rotation set in the file metadata. Rotation is applied before resizing. + The clockwise rotation by which to rotate the raw video frame. Defaults to the rotation set in the file metadata. Rotation is applied before cropping and resizing. +- `crop`\ + Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to the dimensions of the input video track. Cropping is performed after rotation but before resizing. - `poolSize`\ See [Canvas pool](#canvas-pool). diff --git a/docs/guide/packets-and-samples.md b/docs/guide/packets-and-samples.md index b9d8a82..18baa04 100644 --- a/docs/guide/packets-and-samples.md +++ b/docs/guide/packets-and-samples.md @@ -350,6 +350,7 @@ drawWithFit( options: { fit: 'fill' | 'contain' | 'cover'; rotation?: Rotation; // Overrides the sample's rotation + crop?: CropRectangle; }, ): void; ``` diff --git a/package-lock.json b/package-lock.json index 72f8bfe..87d97d8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "mediabunny", - "version": "1.14.4", + "version": "1.15.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "mediabunny", - "version": "1.14.4", + "version": "1.15.0", "license": "MPL-2.0", "workspaces": [ "packages/*" @@ -7749,9 +7749,9 @@ } }, "node_modules/mediabunny": { - "version": "1.14.3", - "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.14.3.tgz", - "integrity": "sha512-kCvieRo6X1QDcdWLjn7o2BY/VCDeyU9nNGBVjOIOiWPoTtIekHR+viKAYaZafLEm0poBv+O2PwttL37PaSo/kA==", + "version": "1.14.4", + "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.14.4.tgz", + "integrity": "sha512-WXY384sVkUOGKF2OuQgLqo8+dwX1MLYnpkSDoaPa+oNed4tP4L5u0/Vq0yZQEmcaBO7rRFusMDSBcbrp66R++A==", "license": "MPL-2.0", "peer": true, "workspaces": [ @@ -12242,7 +12242,7 @@ }, "packages/mp3-encoder": { "name": "@mediabunny/mp3-encoder", - "version": "1.14.4", + "version": "1.15.0", "license": "MPL-2.0", "devDependencies": { "@types/emscripten": "^1.40.1" diff --git a/package.json b/package.json index 0ffae19..3ecca8c 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "mediabunny", "author": "Vanilagy", - "version": "1.14.4", + "version": "1.15.0", "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.", "type": "module", "workspaces": [ diff --git a/packages/mp3-encoder/package.json b/packages/mp3-encoder/package.json index cc977a9..47290fe 100644 --- a/packages/mp3-encoder/package.json +++ b/packages/mp3-encoder/package.json @@ -1,7 +1,7 @@ { "name": "@mediabunny/mp3-encoder", "author": "Vanilagy", - "version": "1.14.4", + "version": "1.15.0", "description": "MP3 encoder extension for Mediabunny, based on LAME.", "main": "./dist/bundles/mediabunny-mp3-encoder.mjs", "module": "./dist/bundles/mediabunny-mp3-encoder.mjs", diff --git a/src/conversion.ts b/src/conversion.ts index 0192590..10eb8f8 100644 --- a/src/conversion.ts +++ b/src/conversion.ts @@ -47,7 +47,7 @@ import { } from './misc'; import { Output, TrackType } from './output'; import { Mp4OutputFormat } from './output-format'; -import { AudioSample, VideoSample } from './sample'; +import { AudioSample, clampCropRectangle, validateCropRectangle, VideoSample } from './sample'; import { MetadataTags, validateMetadataTags } from './tags'; import { NullTarget } from './target'; @@ -126,10 +126,24 @@ export type ConversionVideoOptions = { */ fit?: 'fill' | 'contain' | 'cover'; /** - * The angle in degrees to rotate the input video by, clockwise. Rotation is applied before resizing. This - * rotation is _in addition to_ the natural rotation of the input video as specified in input file's metadata. + * The angle in degrees to rotate the input video by, clockwise. Rotation is applied before cropping and resizing. + * This rotation is _in addition to_ the natural rotation of the input video as specified in input file's metadata. */ rotate?: Rotation; + /** + * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to + * the dimensions of the input video track. Cropping is performed after rotation but before resizing. + */ + crop?: { + /** The distance in pixels from the left edge of the source frame to the left edge of the crop rectangle. */ + left: number; + /** The distance in pixels from the top edge of the source frame to the top edge of the crop rectangle. */ + top: number; + /** The width in pixels of the crop rectangle. */ + width: number; + /** The height in pixels of the crop rectangle. */ + height: number; + }; /** * The desired frame rate of the output video, in hertz. If not specified, the original input frame rate will * be used (which may be variable). @@ -213,6 +227,9 @@ const validateVideoOptions = (videoOptions: ConversionVideoOptions | undefined) if (videoOptions?.rotate !== undefined && ![0, 90, 180, 270].includes(videoOptions.rotate)) { throw new TypeError('options.video.rotate, when provided, must be 0, 90, 180 or 270.'); } + if (videoOptions?.crop !== undefined) { + validateCropRectangle(videoOptions.crop, 'options.video.'); + } if ( videoOptions?.frameRate !== undefined && (!Number.isFinite(videoOptions.frameRate) || videoOptions.frameRate <= 0) @@ -601,10 +618,19 @@ export class Conversion { const totalRotation = normalizeRotation(track.rotation + (trackOptions.rotate ?? 0)); const outputSupportsRotation = this.output.format.supportsVideoRotationMetadata; - const [originalWidth, originalHeight] = totalRotation % 180 === 0 + const [rotatedWidth, rotatedHeight] = totalRotation % 180 === 0 ? [track.codedWidth, track.codedHeight] : [track.codedHeight, track.codedWidth]; + const crop = trackOptions.crop; + if (crop) { + clampCropRectangle(crop, rotatedWidth, rotatedHeight); + } + + const [originalWidth, originalHeight] = crop + ? [crop.width, crop.height] + : [rotatedWidth, rotatedHeight]; + let width = originalWidth; let height = originalHeight; const aspectRatio = width / height; @@ -630,7 +656,8 @@ export class Conversion { || !!trackOptions.frameRate; let needsRerender = width !== originalWidth || height !== originalHeight - || (totalRotation !== 0 && !outputSupportsRotation); + || (totalRotation !== 0 && !outputSupportsRotation) + || !!crop; let videoCodecs = this.output.format.getSupportedVideoCodecs(); if ( @@ -754,6 +781,7 @@ export class Conversion { height, fit: trackOptions.fit ?? 'fill', rotation: totalRotation, // Bake the rotation into the output + crop: trackOptions.crop, poolSize: 1, }); const iterator = sink.canvases(this._startTimestamp, this._endTimestamp); diff --git a/src/index.ts b/src/index.ts index c7bbedd..4662fc6 100644 --- a/src/index.ts +++ b/src/index.ts @@ -160,6 +160,7 @@ export { AudioSampleCopyToOptions, VideoSample, VideoSampleInit, + CropRectangle, } from './sample'; export { AudioBufferSink, diff --git a/src/media-sink.ts b/src/media-sink.ts index e11b4b3..989d9b2 100644 --- a/src/media-sink.ts +++ b/src/media-sink.ts @@ -29,7 +29,7 @@ import { } from './misc'; import { EncodedPacket } from './packet'; import { fromAlaw, fromUlaw } from './pcm'; -import { AudioSample, VideoSample } from './sample'; +import { AudioSample, clampCropRectangle, CropRectangle, validateCropRectangle, VideoSample } from './sample'; /** * Additional options for controlling packet retrieval. @@ -1052,6 +1052,11 @@ export type CanvasSinkOptions = { * Rotation is applied before resizing. */ rotation?: Rotation; + /** + * Specifies the rectangular region of the input video to crop to. The crop region will automatically be clamped to + * the dimensions of the input video track. Cropping is performed after rotation but before resizing. + */ + crop?: CropRectangle; /** * When set, specifies the number of canvases in the pool. These canvases will be reused in a ring buffer / * round-robin type fashion. This keeps the amount of allocated VRAM constant and relieves the browser from @@ -1083,6 +1088,8 @@ export class CanvasSink { /** @internal */ _rotation: Rotation; /** @internal */ + _crop?: { left: number; top: number; width: number; height: number }; + /** @internal */ _videoSampleSink: VideoSampleSink; /** @internal */ _canvasPool: (HTMLCanvasElement | OffscreenCanvas | null)[]; @@ -1118,6 +1125,9 @@ export class CanvasSink { if (options.rotation !== undefined && ![0, 90, 180, 270].includes(options.rotation)) { throw new TypeError('options.rotation, when provided, must be 0, 90, 180 or 270.'); } + if (options.crop !== undefined) { + validateCropRectangle(options.crop, 'options.'); + } if ( options.poolSize !== undefined && (typeof options.poolSize !== 'number' || !Number.isInteger(options.poolSize) || options.poolSize < 0) @@ -1126,9 +1136,19 @@ export class CanvasSink { } const rotation = options.rotation ?? videoTrack.rotation; - let [width, height] = rotation % 180 === 0 + + const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 ? [videoTrack.codedWidth, videoTrack.codedHeight] : [videoTrack.codedHeight, videoTrack.codedWidth]; + + const crop = options.crop; + if (crop) { + clampCropRectangle(crop, rotatedWidth, rotatedHeight); + } + + let [width, height] = crop + ? [crop.width, crop.height] + : [rotatedWidth, rotatedHeight]; const originalAspectRatio = width / height; // If width and height aren't defined together, deduce the missing value using the aspect ratio @@ -1147,6 +1167,7 @@ export class CanvasSink { this._width = width; this._height = height; this._rotation = rotation; + this._crop = crop; this._fit = options.fit ?? 'fill'; this._videoSampleSink = new VideoSampleSink(videoTrack); this._canvasPool = Array.from({ length: options.poolSize ?? 0 }, () => null); @@ -1191,6 +1212,7 @@ export class CanvasSink { sample.drawWithFit(context, { fit: this._fit, rotation: this._rotation, + crop: this._crop, }); const result = { diff --git a/src/sample.ts b/src/sample.ts index 1e2947a..a9fd911 100644 --- a/src/sample.ts +++ b/src/sample.ts @@ -507,28 +507,7 @@ export class VideoSample { throw new Error('VideoSample is closed.'); } - // The provided sx,sy,sWidth,sHeight refer to the final rotated image, but that's not actually how the image is - // stored. Therefore, we must map these back onto the original, pre-rotation image. - if (this.rotation === 90) { - [sx, sy, sWidth, sHeight] = [ - sy, - this.codedHeight - sx - sWidth, - sHeight, - sWidth, - ]; - } else if (this.rotation === 180) { - [sx, sy] = [ - this.codedWidth - sx - sWidth, - this.codedHeight - sy - sHeight, - ]; - } else if (this.rotation === 270) { - [sx, sy, sWidth, sHeight] = [ - this.codedWidth - sy - sHeight, - sx, - sHeight, - sWidth, - ]; - } + ({ sx, sy, sWidth, sHeight } = this._rotateSourceRegion(sx, sy, sWidth, sHeight, this.rotation)); const source = this.toCanvasImageSource(); @@ -576,26 +555,69 @@ export class VideoSample { fit: 'fill' | 'contain' | 'cover'; /** A way to override rotation. Defaults to the rotation of the sample. */ rotation?: Rotation; + /** + * Specifies the rectangular region of the video sample to crop to. The crop region will automatically be + * clamped to the dimensions of the video sample. Cropping is performed after rotation but before resizing. + */ + crop?: CropRectangle; }) { + if (!( + (typeof CanvasRenderingContext2D !== 'undefined' && context instanceof CanvasRenderingContext2D) + || ( + typeof OffscreenCanvasRenderingContext2D !== 'undefined' + && context instanceof OffscreenCanvasRenderingContext2D + ) + )) { + throw new TypeError('context must be a CanvasRenderingContext2D or OffscreenCanvasRenderingContext2D.'); + } + if (!options || typeof options !== 'object') { + throw new TypeError('options must be an object.'); + } + if (!['fill', 'contain', 'cover'].includes(options.fit)) { + throw new TypeError('options.fit must be \'fill\', \'contain\', or \'cover\'.'); + } + if (options.rotation !== undefined && ![0, 90, 180, 270].includes(options.rotation)) { + throw new TypeError('options.rotation, when provided, must be 0, 90, 180, or 270.'); + } + if (options.crop !== undefined) { + validateCropRectangle(options.crop, 'options.'); + } + const canvasWidth = context.canvas.width; const canvasHeight = context.canvas.height; const rotation = options.rotation ?? this.rotation; + const [rotatedWidth, rotatedHeight] = rotation % 180 === 0 + ? [this.codedWidth, this.codedHeight] + : [this.codedHeight, this.codedWidth]; + + if (options.crop) { + clampCropRectangle(options.crop, rotatedWidth, rotatedHeight); + } + // These variables specify where the final sample will be drawn on the canvas let dx: number; let dy: number; let newWidth: number; let newHeight: number; + const { sx, sy, sWidth, sHeight } = this._rotateSourceRegion( + options.crop?.left ?? 0, + options.crop?.top ?? 0, + options.crop?.width ?? this.codedWidth, + options.crop?.height ?? this.codedHeight, + rotation, + ); + if (options.fit === 'fill') { dx = 0; dy = 0; newWidth = canvasWidth; newHeight = canvasHeight; } else { - const [sampleWidth, sampleHeight] = rotation % 180 === 0 - ? [this.codedWidth, this.codedHeight] - : [this.codedHeight, this.codedWidth]; + const [sampleWidth, sampleHeight] = options.crop + ? [options.crop.width, options.crop.height] + : [rotatedWidth, rotatedHeight]; const scale = options.fit === 'contain' ? Math.min(canvasWidth / sampleWidth, canvasHeight / sampleHeight) @@ -616,7 +638,35 @@ export class VideoSample { // Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it // ourselves here - context.drawImage(this.toCanvasImageSource(), dx, dy, newWidth, newHeight); + context.drawImage(this.toCanvasImageSource(), sx, sy, sWidth, sHeight, dx, dy, newWidth, newHeight); + } + + /** @internal */ + _rotateSourceRegion(sx: number, sy: number, sWidth: number, sHeight: number, rotation: number) { + // The provided sx,sy,sWidth,sHeight refer to the final rotated image, but that's not actually how the image is + // stored. Therefore, we must map these back onto the original, pre-rotation image. + if (rotation === 90) { + [sx, sy, sWidth, sHeight] = [ + sy, + this.codedHeight - sx - sWidth, + sHeight, + sWidth, + ]; + } else if (rotation === 180) { + [sx, sy] = [ + this.codedWidth - sx - sWidth, + this.codedHeight - sy - sHeight, + ]; + } else if (rotation === 270) { + [sx, sy, sWidth, sHeight] = [ + this.codedWidth - sy - sHeight, + sx, + sHeight, + sWidth, + ]; + } + + return { sx, sy, sWidth, sHeight }; } /** @@ -679,6 +729,49 @@ const isVideoFrame = (x: unknown): x is VideoFrame => { return typeof VideoFrame !== 'undefined' && x instanceof VideoFrame; }; +/** + * Specifies the rectangular cropping region. + * @public + */ +export type CropRectangle = { + /** The distance in pixels from the left edge of the source frame to the left edge of the crop rectangle. */ + left: number; + /** The distance in pixels from the top edge of the source frame to the top edge of the crop rectangle. */ + top: number; + /** The width in pixels of the crop rectangle. */ + width: number; + /** The height in pixels of the crop rectangle. */ + height: number; +}; + +export const clampCropRectangle = (crop: CropRectangle, outerWidth: number, outerHeight: number) => { + crop.left = Math.min(crop.left, outerWidth); + crop.top = Math.min(crop.top, outerHeight); + crop.width = Math.min(crop.width, outerWidth - crop.left); + crop.height = Math.min(crop.height, outerHeight - crop.top); + + assert(crop.width >= 0); + assert(crop.height >= 0); +}; + +export const validateCropRectangle = (crop: CropRectangle, prefix: string) => { + if (!crop || typeof crop !== 'object') { + throw new TypeError(prefix + 'crop, when provided, must be an object.'); + } + if (!Number.isInteger(crop.left) || crop.left < 0) { + throw new TypeError(prefix + 'crop.left must be a non-negative integer.'); + } + if (!Number.isInteger(crop.top) || crop.top < 0) { + throw new TypeError(prefix + 'crop.top must be a non-negative integer.'); + } + if (!Number.isInteger(crop.width) || crop.width < 0) { + throw new TypeError(prefix + 'crop.width must be a non-negative integer.'); + } + if (!Number.isInteger(crop.height) || crop.height < 0) { + throw new TypeError(prefix + 'crop.height must be a non-negative integer.'); + } +}; + const AUDIO_SAMPLE_FORMATS = new Set( ['f32', 'f32-planar', 's16', 's16-planar', 's32', 's32-planar', 'u8', 'u8-planar'], );