Fix regular Language element in Matroska not having an effect anymore

Instruct VSCode to use ESLint as default formatter

.gitignore

@@ -1,4 +1,3 @@
-.vscode
 node_modules
 /dist
 /dist-docs

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+	"editor.defaultFormatter": "dbaeumer.vscode-eslint"
+}

dev/convert.html

@@ -74,6 +74,8 @@
 			},
 			*/
 			video: () => ({
+				codec: 'avc',
+				//fit: 'contain',
 				//frameRate: 27.123,
 				//width: 320,
 				//forceTranscode: true,

docs/guide/converting-media-files.md

@@ -141,6 +141,8 @@ If `width` or `height` is used in conjunction with `rotation`, they control the
 
 If you want to apply max/min constraints to a video's dimensions, check out [track-specific options](#track-specific-options).
 
+In the rare case that the input video changes size over time, the `fit` field can be used to control the size change behavior (see [`VideoEncodingConfig`](./media-sources#video-encoding-config)). When unset, the behavior is `'passThrough'`.
+
 ### Adjusting frame rate
 
 The `frameRate` property can be used to set the frame rate of the output video in Hz. If not specified, the original input frame rate will be used (which may be variable).

docs/guide/media-sources.md

@@ -54,6 +54,7 @@ type VideoEncodingConfig = {
 	hardwareAcceleration?: 'no-preference' | 'prefer-hardware' | 'prefer-software';
 	scalabilityMode?: string;
 	contentHint?: string;
+	sizeChangeBehavior?: 'deny' | 'passThrough' | 'fill' | 'contain' | 'cover';
 
 	onEncodedPacket?: (
 		packet: EncodedPacket,
@@ -73,6 +74,7 @@ type VideoEncodingConfig = {
 - `hardwareAcceleration`: A hint that configures the hardware acceleration method of this codec. This is best left on `'no-preference'`.
 - `scalabilityMode`: An encoding scalability mode identifier as defined by [WebRTC-SVC](https://w3c.github.io/webrtc-svc/#scalabilitymodes*).
 - `contentHint`: An encoding video content hint as defined by [mst-content-hint](https://w3c.github.io/mst-content-hint/#video-content-hints).
+- `sizeChangeBehavior`: Video frames may change size overtime. This field controls the behavior in case this happens. Defaults to `'deny'`. 
 - `onEncodedPacket`: Called for each successfully encoded packet. Useful for determining encoding progress.
 - `onEncoderConfig`: Called when the internal encoder config, as used by the WebCodecs API, is created. You can use this to introspect the full codec string.
 

docs/guide/packets-and-samples.md

@@ -343,6 +343,17 @@ draw(
 ```
 These methods behave like [drawImage](https://developer.mozilla.org/en-US/docs/Web/API/CanvasRenderingContext2D/drawImage) and paint the video frame at the given position with the given dimensions. This method will automatically draw the frame with the correct rotation based on its `rotation` property.
 
+The `drawWithFit` method can be used to draw the video sample to fill an entire canvas with a specified fitting algorithm:
+```ts
+drawWithFit(
+	context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D,
+	options: {
+		fit: 'fill' | 'contain' | 'cover';
+		rotation?: Rotation; // Overrides the sample's rotation
+	},
+): void;
+```
+
 If you want to draw the raw underlying image to a canvas directly (without respecting the rotation metadata), then you can use the following method:
 ```ts
 videoSample.toCanvasImageSource(); // => VideoFrame | OffscreenCanvas;

examples/media-player/media-player.ts

@@ -144,7 +144,10 @@ const initMediaPlayer = async (file: File) => {
 
 		// For video, let's use a CanvasSink as it handles rotation and closing video samples for us.
 		// Pool size of 2: We'll only ever have the current and the next frame around, so we only need two canvases.
-		videoSink = videoTrack && new CanvasSink(videoTrack, { poolSize: 2 });
+		videoSink = videoTrack && new CanvasSink(videoTrack, {
+			poolSize: 2,
+			fit: 'contain', // In case the video changes dimensions over time
+		});
 		// For audio, we'll use an AudioBufferSink to directly retrieve AudioBuffers compatible with the Web Audio API
 		audioSink = audioTrack && new AudioBufferSink(audioTrack);
 

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.10.0",
+  "version": "1.11.2",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.10.0",
+      "version": "1.11.2",
       "license": "MPL-2.0",
       "workspaces": [
         "packages/*"
@@ -5900,9 +5900,9 @@
       }
     },
     "node_modules/mediabunny": {
-      "version": "1.9.3",
-      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.9.3.tgz",
-      "integrity": "sha512-uVspdzrpwvW8NxtGspHPOJvdNSY0dNFEmOQngSheX9PYX9oXijZENBjfYWOgm9c9IgSSlN2bdJF9iyPyY3f+fQ==",
+      "version": "1.11.1",
+      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.11.1.tgz",
+      "integrity": "sha512-UsoLZi9aHcQWMHsQF0Y4xSREy2eByFvHRjK3kmvfuaYDHPiqsj0tN6HE+daSzr+WHUh98pFlu2XKXwzBp2AHcQ==",
       "license": "MPL-2.0",
       "peer": true,
       "workspaces": [
@@ -9017,7 +9017,7 @@
     },
     "packages/mp3-encoder": {
       "name": "@mediabunny/mp3-encoder",
-      "version": "1.10.0",
+      "version": "1.11.2",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.10.0",
+  "version": "1.11.2",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "workspaces": [

packages/mp3-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/mp3-encoder",
   "author": "Vanilagy",
-  "version": "1.10.0",
+  "version": "1.11.2",
   "description": "MP3 encoder extension for Mediabunny, based on LAME.",
   "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
   "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",

src/conversion.ts

@@ -37,10 +37,55 @@ import {
 	VideoSampleSource,
 	AudioSampleSource,
 } from './media-source';
-import { assert, clamp, MaybePromise, normalizeRotation, promiseWithResolvers, Rotation } from './misc';
+import {
+	assert,
+	clamp,
+	isIso639Dash2LanguageCode,
+	MaybePromise,
+	normalizeRotation,
+	promiseWithResolvers,
+	Rotation,
+} from './misc';
 import { Output, TrackType } from './output';
 import { AudioSample, VideoSample } from './sample';
 
+/**
+ * The options for media file conversion.
+ * @public
+ */
+export type ConversionOptions = {
+	/** The input file. */
+	input: Input;
+	/** The output file. */
+	output: Output;
+
+	/**
+	 * Video-specific options. When passing an object, the same options are applied to all video tracks. When passing a
+	 * function, it will be invoked for each video track and is expected to return or resolve to the options
+	 * for that specific track. The function is passed an instance of `InputVideoTrack` as well as a number `n`, which
+	 * is the 1-based index of the track in the list of all video tracks.
+	 */
+	video?: ConversionVideoOptions
+		| ((track: InputVideoTrack, n: number) => MaybePromise<ConversionVideoOptions | undefined>);
+
+	/**
+	 * Audio-specific options. When passing an object, the same options are applied to all audio tracks. When passing a
+	 * function, it will be invoked for each audio track and is expected to return or resolve to the options
+	 * for that specific track. The function is passed an instance of `InputAudioTrack` as well as a number `n`, which
+	 * is the 1-based index of the track in the list of all audio tracks.
+	 */
+	audio?: ConversionAudioOptions
+		| ((track: InputAudioTrack, n: number) => MaybePromise<ConversionAudioOptions | undefined>);
+
+	/** Options to trim the input file. */
+	trim?: {
+		/** The time in the input file in seconds at which the output file should start. Must be less than `end`.  */
+		start: number;
+		/** The time in the input file in seconds at which the output file should end. Must be greater than `start`. */
+		end: number;
+	};
+};
+
 /**
  * Video-specific options.
  * @public
@@ -59,7 +104,7 @@ export type ConversionVideoOptions = {
 	 */
 	height?: number;
 	/**
-	 * The fitting algorithm in case both width and height are set.
+	 * The fitting algorithm in case both width and height are set, or if the input video changes its size over time.
 	 *
 	 * - 'fill' will stretch the image to fill the entire box, potentially altering aspect ratio.
 	 * - 'contain' will contain the entire image within the box while preserving aspect ratio. This may lead to
@@ -162,43 +207,6 @@ const validateVideoOptions = (videoOptions: ConversionVideoOptions | undefined)
 	}
 };
 
-/**
- * The options for media file conversion.
- * @public
- */
-export type ConversionOptions = {
-	/** The input file. */
-	input: Input;
-	/** The output file. */
-	output: Output;
-
-	/**
-	 * Video-specific options. When passing an object, the same options are applied to all video tracks. When passing a
-	 * function, it will be invoked for each video track and is expected to return or resolve to the options
-	 * for that specific track. The function is passed an instance of `InputVideoTrack` as well as a number `n`, which
-	 * is the 1-based index of the track in the list of all video tracks.
-	 */
-	video?: ConversionVideoOptions
-		| ((track: InputVideoTrack, n: number) => MaybePromise<ConversionVideoOptions | undefined>);
-
-	/**
-	 * Audio-specific options. When passing an object, the same options are applied to all audio tracks. When passing a
-	 * function, it will be invoked for each audio track and is expected to return or resolve to the options
-	 * for that specific track. The function is passed an instance of `InputAudioTrack` as well as a number `n`, which
-	 * is the 1-based index of the track in the list of all audio tracks.
-	 */
-	audio?: ConversionAudioOptions
-		| ((track: InputAudioTrack, n: number) => MaybePromise<ConversionAudioOptions | undefined>);
-
-	/** Options to trim the input file. */
-	trim?: {
-		/** The time in the input file in seconds at which the output file should start. Must be less than `end`.  */
-		start: number;
-		/** The time in the input file in seconds at which the output file should end. Must be greater than `start`. */
-		end: number;
-	};
-};
-
 const validateAudioOptions = (audioOptions: ConversionAudioOptions | undefined) => {
 	if (audioOptions !== undefined && (!audioOptions || typeof audioOptions !== 'object')) {
 		throw new TypeError('options.audio, when provided, must be an object.');
@@ -623,6 +631,7 @@ export class Conversion {
 			const encodingConfig: VideoEncodingConfig = {
 				codec: encodableCodec,
 				bitrate,
+				sizeChangeBehavior: trackOptions.fit ?? 'passThrough',
 				onEncodedPacket: sample => this._reportProgress(track.id, sample.timestamp + sample.duration),
 			};
 
@@ -810,7 +819,9 @@ export class Conversion {
 
 		this.output.addVideoTrack(videoSource, {
 			frameRate: trackOptions.frameRate,
-			languageCode: track.languageCode,
+			// TEMP: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
+			languageCode: isIso639Dash2LanguageCode(track.languageCode) ? track.languageCode : undefined,
+			name: track.name ?? undefined,
 			rotation: needsRerender ? 0 : totalRotation, // Rerendering will bake the rotation into the output
 		});
 		this._addedCounts.video++;
@@ -979,7 +990,9 @@ export class Conversion {
 		}
 
 		this.output.addAudioTrack(audioSource, {
-			languageCode: track.languageCode,
+			// TEMP: This condition can be removed when all demuxers properly homogenize to BCP47 in v2
+			languageCode: isIso639Dash2LanguageCode(track.languageCode) ? track.languageCode : undefined,
+			name: track.name ?? undefined,
 		});
 		this._addedCounts.audio++;
 		this._totalTrackCount++;

src/encode.ts

@@ -43,6 +43,19 @@ export type VideoEncodingConfig = {
 	 * all the same key frame interval.
 	 */
 	keyFrameInterval?: number;
+	/**
+	 * Video frames may change size overtime. This field controls the behavior in case this happens.
+	 *
+	 * - 'deny' (default) will throw an error, requiring all frames to have the exact same dimensions.
+	 * - 'passThrough' will allow the change and directly pass the frame to the encoder.
+	 * - 'fill' will stretch the image to fill the entire original box, potentially altering aspect ratio.
+	 * - 'contain' will contain the entire image within the  originalbox while preserving aspect ratio. This may lead to
+	 * letterboxing.
+	 * - 'cover' will scale the image until the entire original box is filled, while preserving aspect ratio.
+	 *
+	 * The "original box" refers to the dimensions of the first encoded frame.
+	 */
+	sizeChangeBehavior?: 'deny' | 'passThrough' | 'fill' | 'contain' | 'cover';
 
 	/** Called for each successfully encoded packet. Both the packet and the encoding metadata are passed. */
 	onEncodedPacket?: (packet: EncodedPacket, meta: EncodedVideoChunkMetadata | undefined) => unknown;
@@ -66,6 +79,7 @@ export const validateVideoEncodingConfig = (config: VideoEncodingConfig) => {
 	) {
 		throw new TypeError('config.keyFrameInterval, when provided, must be a non-negative number.');
 	}
+	// todo here
 	if (config.onEncodedPacket !== undefined && typeof config.onEncodedPacket !== 'function') {
 		throw new TypeError('config.onEncodedChunk, when provided, must be a function.');
 	}

src/isobmff/isobmff-demuxer.ts

@@ -2755,7 +2755,14 @@ const extractRotationFromMatrix = (matrix: TransformationMatrix) => {
 	const sinTheta = m21 / scaleX;
 
 	// Invert the rotation because matrices are post-multiplied in ISOBMFF
-	return -Math.atan2(sinTheta, cosTheta) * (180 / Math.PI);
+	const result = -Math.atan2(sinTheta, cosTheta) * (180 / Math.PI);
+
+	if (!Number.isFinite(result)) {
+		// Can happen if the entire matrix is 0, for example
+		return 0;
+	}
+
+	return result;
 };
 
 const sampleTableIsEmpty = (sampleTable: SampleTable) => {

src/matroska/matroska-demuxer.ts

@@ -1014,7 +1014,10 @@ export class MatroskaDemuxer extends Demuxer {
 
 			case EBMLId.Language: {
 				if (!this.currentTrack) break;
-				if (this.currentTrack.languageCode) break; // LanguageBCP47 was present, which takes precedence
+				if (this.currentTrack.languageCode !== UNDETERMINED_LANGUAGE) {
+					// LanguageBCP47 was present, which takes precedence
+					break;
+				}
 
 				this.currentTrack.languageCode = reader.readAsciiString(size);
 

src/media-sink.ts

@@ -1118,6 +1118,8 @@ export class CanvasSink {
 	/** @internal */
 	_videoSampleToWrappedCanvas(sample: VideoSample): WrappedCanvas {
 		let canvas = this._canvasPool[this._nextCanvasIndex];
+		let canvasIsNew = false;
+
 		if (!canvas) {
 			if (typeof document !== 'undefined') {
 				// Prefer an HTMLCanvasElement
@@ -1131,6 +1133,8 @@ export class CanvasSink {
 			if (this._canvasPool.length > 0) {
 				this._canvasPool[this._nextCanvasIndex] = canvas;
 			}
+
+			canvasIsNew = true;
 		}
 
 		if (this._canvasPool.length > 0) {
@@ -1143,40 +1147,14 @@ export class CanvasSink {
 
 		context.resetTransform();
 
-		// These variables specify where the final sample will be drawn on the canvas
-		let dx: number;
-		let dy: number;
-		let newWidth: number;
-		let newHeight: number;
-
-		if (this._fit === 'fill') {
-			dx = 0;
-			dy = 0;
-			newWidth = this._width;
-			newHeight = this._height;
-		} else {
-			const [sampleWidth, sampleHeight] = this._rotation % 180 === 0
-				? [sample.codedWidth, sample.codedHeight]
-				: [sample.codedHeight, sample.codedWidth];
-
-			const scale = this._fit === 'contain'
-				? Math.min(this._width / sampleWidth, this._height / sampleHeight)
-				: Math.max(this._width / sampleWidth, this._height / sampleHeight);
-			newWidth = sampleWidth * scale;
-			newHeight = sampleHeight * scale;
-			dx = (this._width - newWidth) / 2;
-			dy = (this._height - newHeight) / 2;
+		if (!canvasIsNew) {
+			context.clearRect(0, 0, this._width, this._height);
 		}
 
-		const aspectRatioChange = this._rotation % 180 === 0 ? 1 : newWidth / newHeight;
-		context.translate(this._width / 2, this._height / 2);
-		context.rotate(this._rotation * Math.PI / 180);
-		// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
-		// don't need to think about how those dimensions change after the rotation
-		context.scale(1 / aspectRatioChange, aspectRatioChange);
-		context.translate(-this._width / 2, -this._height / 2);
-
-		context.drawImage(sample.toCanvasImageSource(), dx, dy, newWidth, newHeight);
+		sample.drawWithFit(context, {
+			fit: this._fit,
+			rotation: this._rotation,
+		});
 
 		const result = {
 			canvas,

src/media-source.ts

@@ -191,8 +191,9 @@ class VideoEncoderWrapper {
 	private encoder: VideoEncoder | null = null;
 	private muxer: Muxer | null = null;
 	private lastMultipleOfKeyFrameInterval = -1;
-	private lastWidth: number | null = null;
-	private lastHeight: number | null = null;
+	private codedWidth: number | null = null;
+	private codedHeight: number | null = null;
+	private resizeCanvas: HTMLCanvasElement | OffscreenCanvas | null = null;
 
 	private customEncoder: CustomVideoEncoder | null = null;
 	private customEncoderCallSerializer = new CallSerializer();
@@ -213,16 +214,60 @@ class VideoEncoderWrapper {
 			this.source._ensureValidAdd();
 
 			// Ensure video sample size remains constant
-			if (this.lastWidth !== null && this.lastHeight !== null) {
-				if (videoSample.codedWidth !== this.lastWidth || videoSample.codedHeight !== this.lastHeight) {
-					throw new Error(
-						`Video sample size must remain constant. Expected ${this.lastWidth}x${this.lastHeight},`
-						+ ` got ${videoSample.codedWidth}x${videoSample.codedHeight}.`,
-					);
+			if (this.codedWidth !== null && this.codedHeight !== null) {
+				if (videoSample.codedWidth !== this.codedWidth || videoSample.codedHeight !== this.codedHeight) {
+					const sizeChangeBehavior = this.encodingConfig.sizeChangeBehavior ?? 'deny';
+
+					if (sizeChangeBehavior === 'passThrough') {
+						// Do nada
+					} else if (sizeChangeBehavior === 'deny') {
+						throw new Error(
+							`Video sample size must remain constant. Expected ${this.codedWidth}x${this.codedHeight},`
+							+ ` got ${videoSample.codedWidth}x${videoSample.codedHeight}. To allow the sample size to`
+							+ ` change over time, set \`sizeChangeBehavior\` to a value other than 'strict' in the`
+							+ ` encoding options.`,
+						);
+					} else {
+						let canvasIsNew = false;
+
+						if (!this.resizeCanvas) {
+							if (typeof document !== 'undefined') {
+								// Prefer an HTMLCanvasElement
+								this.resizeCanvas = document.createElement('canvas');
+								this.resizeCanvas.width = this.codedWidth;
+								this.resizeCanvas.height = this.codedHeight;
+							} else {
+								this.resizeCanvas = new OffscreenCanvas(this.codedWidth, this.codedHeight);
+							}
+
+							canvasIsNew = true;
+						}
+
+						const context = this.resizeCanvas.getContext('2d', { alpha: false }) as
+							CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D;
+						assert(context);
+
+						if (!canvasIsNew) {
+							context.clearRect(0, 0, this.codedWidth, this.codedHeight);
+						}
+
+						videoSample.drawWithFit(context, { fit: sizeChangeBehavior });
+
+						if (shouldClose) {
+							videoSample.close();
+						}
+
+						videoSample = new VideoSample(this.resizeCanvas, {
+							timestamp: videoSample.timestamp,
+							duration: videoSample.duration,
+							rotation: videoSample.rotation,
+						});
+						shouldClose = true;
+					}
 				}
 			} else {
-				this.lastWidth = videoSample.codedWidth;
-				this.lastHeight = videoSample.codedHeight;
+				this.codedWidth = videoSample.codedWidth;
+				this.codedHeight = videoSample.codedHeight;
 			}
 
 			if (!this.encoderInitialized) {

src/sample.ts

@@ -534,6 +534,64 @@ export class VideoSample {
 		context.restore();
 	}
 
+	/**
+	 * Draws the sample in the middle of the canvas corresponding to the context with the specified fit behavior.
+	 */
+	drawWithFit(context: CanvasRenderingContext2D | OffscreenCanvasRenderingContext2D, options: {
+		/**
+		 * Controls the fitting algorithm.
+		 *
+		 * - 'fill' will stretch the image to fill the entire box, potentially altering aspect ratio.
+		 * - 'contain' will contain the entire image within the box while preserving aspect ratio. This may lead to
+		 * letterboxing.
+		 * - 'cover' will scale the image until the entire box is filled, while preserving aspect ratio.
+		 */
+		fit: 'fill' | 'contain' | 'cover';
+		/** A way to override rotation. Defaults to the rotation of the sample. */
+		rotation?: Rotation;
+	}) {
+		const canvasWidth = context.canvas.width;
+		const canvasHeight = context.canvas.height;
+		const rotation = options.rotation ?? this.rotation;
+
+		// These variables specify where the final sample will be drawn on the canvas
+		let dx: number;
+		let dy: number;
+		let newWidth: number;
+		let newHeight: number;
+
+		if (options.fit === 'fill') {
+			dx = 0;
+			dy = 0;
+			newWidth = canvasWidth;
+			newHeight = canvasHeight;
+		} else {
+			const [sampleWidth, sampleHeight] = rotation % 180 === 0
+				? [this.codedWidth, this.codedHeight]
+				: [this.codedHeight, this.codedWidth];
+
+			const scale = options.fit === 'contain'
+				? Math.min(canvasWidth / sampleWidth, canvasHeight / sampleHeight)
+				: Math.max(canvasWidth / sampleWidth, canvasHeight / sampleHeight);
+			newWidth = sampleWidth * scale;
+			newHeight = sampleHeight * scale;
+			dx = (canvasWidth - newWidth) / 2;
+			dy = (canvasHeight - newHeight) / 2;
+		}
+
+		const aspectRatioChange = rotation % 180 === 0 ? 1 : newWidth / newHeight;
+		context.translate(canvasWidth / 2, canvasHeight / 2);
+		context.rotate(rotation * Math.PI / 180);
+		// This aspect ratio compensation is done so that we can draw the sample with the intended dimensions and
+		// don't need to think about how those dimensions change after the rotation
+		context.scale(1 / aspectRatioChange, aspectRatioChange);
+		context.translate(-canvasWidth / 2, -canvasHeight / 2);
+
+		// Important that we don't use .draw() here since that would take rotation into account, but we wanna handle it
+		// ourselves here
+		context.drawImage(this.toCanvasImageSource(), dx, dy, newWidth, newHeight);
+	}
+
 	/**
 	 * Converts this video sample to a CanvasImageSource for drawing to a canvas.
 	 *