Skip HEVC RASL slices following a decoder reset

Release

dev/convert.html

@@ -24,7 +24,7 @@
 			chunked: true,
 			chunkSize: 2**20
 		});
-		const outputFormat = new Mediabunny.WavOutputFormat({});
+		const outputFormat = new Mediabunny.Mp4OutputFormat({});
 
 		const button = document.createElement('button');
 		button.textContent = 'Cancel';
@@ -47,7 +47,7 @@
 			format: outputFormat,
 			target
 		});
-		output.setMetadata({
+		output.setMetadataTags({
 			title: 'Bigggy',
 			artist: 'Buck Bunny',
 			images: [{
@@ -105,7 +105,8 @@
 			},
 			*/
 			video: () => ({
-				discard: true,
+				//discard: true,
+				forceTranscode: true,
 				//codec: 'avc',
 				//fit: 'contain',
 				//frameRate: 27.123,
@@ -128,7 +129,7 @@
 				//height: 100,
 			}),
 			trim: {
-				start: 0,
+				start: 1,
 				end: 10
 			},
 		});

dev/demux.html

@@ -14,12 +14,30 @@
 			source: new Mediabunny.BlobSource(file),
 		});
 
-		const audioTrack = await input.getPrimaryAudioTrack();
-		const sink = new Mediabunny.EncodedPacketSink(audioTrack);
+		const videoTrack = await input.getPrimaryVideoTrack();
+		const sink = new Mediabunny.VideoSampleSink(videoTrack);
 
-		console.log(await audioTrack.getDecoderConfig())
-		console.log(await sink.getFirstPacket());
+		console.log(await sink.getSample(2.131875));
 
+		/*
+		const sink = new Mediabunny.EncodedPacketSink(videoTrack);
+
+		for await (const packet of sink.packets()) {
+			console.log(packet)
+		}
+		*/
+
+		/*
+		for await (const sample of sink.samples(0.99)) {
+			console.log(sample);
+
+			if (sample.timestamp >= 2) {
+				break;
+			}
+			
+		}
+		*/
+		//console.log(await sink.getSample(1))
 
 		/*
 		const screenStream = await navigator.mediaDevices.getDisplayMedia({ video: true });

docs/guide/reading-media-files.md

@@ -60,6 +60,12 @@ await input.computeDuration(); // => 1905.4615
 ```
 More specifically, the duration is defined as the maximum end timestamp across all tracks.
 
+Mediabunny also lets you read descriptive metadata tags from media files, such as title, artist, or cover art:
+```ts
+await input.getMetadataTags(); // => MetadataTags
+```
+For more info, see [`MetadataTags`](../api/MetadataTags).
+
 ## Reading track metadata
 
 You can extract the list of all media tracks in the file like so:
@@ -76,12 +82,6 @@ await input.getPrimaryVideoTrack(); // => InputVideoTrack | null
 await input.getPrimaryAudioTrack(); // => InputAudioTrack | null
 ```
 
-Mediabunny also lets you read descriptive metadata tags from media files, such as title, artist, or cover art:
-```ts
-await input.getMetadataTags(); // => MetadataTags
-```
-For more info, see [`MetadataTags`](../api/MetadataTags).
-
 ::: info
 Subtitle tracks are currently not supported for reading.
 :::

examples/metadata-extraction/index.html

@@ -57,7 +57,7 @@
 		@reference "../base.css";
 
 		ul {
-			@apply py-1 px-4 bg-gray-500/10 rounded-lg;
+			@apply py-1 px-4 bg-gray-500/10 rounded-lg max-w-lg;
 		}
 
 		b {

examples/metadata-extraction/metadata-extraction.ts

@@ -80,6 +80,37 @@ const extractMetadata = (resource: File | string) => {
 				: {}
 			),
 		}))),
+		'Metadata tags': input.getMetadataTags().then((tags) => {
+			const result = {
+				'Title': tags.title,
+				'Description': tags.description,
+				'Artist': tags.artist,
+				'Album': tags.album,
+				'Album artist': tags.albumArtist,
+				'Track number': tags.trackNumber,
+				'Tracks total': tags.tracksTotal,
+				'Disc number': tags.discNumber,
+				'Discs total': tags.discsTotal,
+				'Genre': tags.genre,
+				'Date': tags.date?.toISOString().slice(0, 10),
+				'Lyrics': tags.lyrics,
+				'Comment': tags.comment,
+				'Images': tags.images?.map((image) => {
+					const blob = new Blob([image.data], { type: image.mimeType });
+					const element = new Image();
+					element.src = URL.createObjectURL(blob);
+
+					return element;
+				}),
+				'Raw tag count': tags.raw && Object.keys(tags.raw).length,
+			};
+
+			if (Object.values(result).some(x => x !== undefined)) {
+				return result;
+			} else {
+				return undefined;
+			}
+		}),
 	};
 
 	fileNameElement.textContent = resource instanceof File ? resource.name : resource;
@@ -93,7 +124,9 @@ const extractMetadata = (resource: File | string) => {
 
 // Creates an HTML element to display any given value
 const renderValue = (value: unknown) => {
-	if (Array.isArray(value)) {
+	if (value instanceof HTMLElement) {
+		return value;
+	} else if (Array.isArray(value)) {
 		const arrayAsObject: Record<string, unknown> = Object.fromEntries(
 			value.map((item, index) => [(index + 1).toString(), item]),
 		);
@@ -116,6 +149,7 @@ const renderObject = (object: Record<string, unknown>) => {
 	for (const key of keys) {
 		const value = object[key];
 		const listItem = document.createElement('li');
+		listItem.style.wordBreak = 'break-word';
 		const keySpan = document.createElement('b');
 		keySpan.textContent = `${key}: `;
 
@@ -132,6 +166,10 @@ const renderObject = (object: Record<string, unknown>) => {
 				// Replace the loading text with the resolved value
 				listItem.removeChild(loadingSpan);
 				listItem.appendChild(renderValue(resolvedValue));
+
+				if (resolvedValue === undefined) {
+					listElement.removeChild(listItem);
+				}
 			}).catch((error) => {
 				console.error(error);
 
@@ -146,7 +184,9 @@ const renderObject = (object: Record<string, unknown>) => {
 			listItem.appendChild(renderValue(value));
 		}
 
-		listElement.appendChild(listItem);
+		if (value !== undefined) {
+			listElement.appendChild(listItem);
+		}
 	}
 
 	return listElement;

examples/thumbnail-generation/thumbnail-generation.ts

@@ -96,6 +96,13 @@ const generateThumbnails = async (resource: File | string) => {
 				timestampElement.className
                     = 'absolute bottom-0 right-0 bg-black/30 text-white px-1 py-0.5 text-[11px] rounded-tl-lg';
 				container.append(timestampElement);
+			} else {
+				// Add something to indicate that the thumbnail is missing
+				const p = document.createElement('p');
+				p.textContent = '?';
+				p.className = 'absolute inset-0 flex items-center justify-center text-3xl opacity-50';
+
+				container.append(p);
 			}
 
 			i++;

package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "mediabunny",
-  "version": "1.14.0",
+  "version": "1.14.3",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "mediabunny",
-      "version": "1.14.0",
+      "version": "1.14.3",
       "license": "MPL-2.0",
       "workspaces": [
         "packages/*"
@@ -7749,9 +7749,9 @@
       }
     },
     "node_modules/mediabunny": {
-      "version": "1.13.3",
-      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.13.3.tgz",
-      "integrity": "sha512-kV12CVwMdhMQILGrVcyDWOgnZHwAytStQYKZeEaUj32XamYhlDX0Od4SkXVr0Yms8iapJsiCoEQdSK4UKUfooA==",
+      "version": "1.14.2",
+      "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.14.2.tgz",
+      "integrity": "sha512-JCup3rpsJGIw3I7E7bZSmVEANCr/OwfwkH3g/Wq1TcHEJ7ymLLw+jC3+/Nnn/JLXzFnw8/I6fV3xECLnFedfHg==",
       "license": "MPL-2.0",
       "peer": true,
       "workspaces": [
@@ -12242,7 +12242,7 @@
     },
     "packages/mp3-encoder": {
       "name": "@mediabunny/mp3-encoder",
-      "version": "1.14.0",
+      "version": "1.14.3",
       "license": "MPL-2.0",
       "devDependencies": {
         "@types/emscripten": "^1.40.1"

package.json

@@ -1,7 +1,7 @@
 {
   "name": "mediabunny",
   "author": "Vanilagy",
-  "version": "1.14.0",
+  "version": "1.14.3",
   "description": "Pure TypeScript media toolkit for reading, writing, and converting media files, directly in the browser.",
   "type": "module",
   "workspaces": [

packages/mp3-encoder/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@mediabunny/mp3-encoder",
   "author": "Vanilagy",
-  "version": "1.14.0",
+  "version": "1.14.3",
   "description": "MP3 encoder extension for Mediabunny, based on LAME.",
   "main": "./dist/bundles/mediabunny-mp3-encoder.mjs",
   "module": "./dist/bundles/mediabunny-mp3-encoder.mjs",

scripts/generate-api-docs.ts

@@ -1,6 +1,6 @@
-// This script has been 100% vibe-coded with Claude, meaning I literally haven't looked at any of the code. It's
-// probably a mess, but it solves a one-off problem where only the output matters, and the output is indeed good, which
-// is the point of a custom script for this: full, precise control.
+// This script has been 100% vibe-coded with Claude (and Gemini!), meaning I literally haven't looked at any of the
+// code. It's probably a mess, but it solves a one-off problem where only the output matters, and the output is indeed
+// good, which is the point of a custom script for this: full, precise control.
 
 /* eslint-disable @typescript-eslint/restrict-template-expressions */
 /* eslint-disable @typescript-eslint/no-unused-vars */
@@ -386,7 +386,34 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 		}
 	});
 
-	// Phase 2: Generate documentation for each symbol
+	// Data structures for "Used by" feature
+	const usedByReferences = new Map<string, Set<{ user: string; context: string; type: 'constructor' | 'method' | 'property' | 'extends' | 'type_param' | 'type_alias' | 'variable' | 'function' }>>();
+	const generatedDocs = new Map<string, string>();
+
+	const addUsage = (
+		used: string,
+		user: string,
+		context: string,
+		type: 'constructor' | 'method' | 'property' | 'extends' | 'type_param' | 'type_alias' | 'variable' | 'function',
+	) => {
+		// No self-references
+		if (used === user) return;
+
+		if (!usedByReferences.has(used)) {
+			usedByReferences.set(used, new Set());
+		}
+		const usageSet = usedByReferences.get(used)!;
+
+		// Check for duplicates before adding
+		for (const item of usageSet) {
+			if (item.user === user && item.context === context && item.type === type) {
+				return;
+			}
+		}
+		usageSet.add({ user, context, type });
+	};
+
+	// Phase 2: Generate documentation for each symbol (and collect usage data)
 	allSymbols.forEach((exportSymbol) => {
 		const declaration = exportSymbol.valueDeclaration || exportSymbol.declarations?.[0];
 		if (!declaration) return;
@@ -492,9 +519,10 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 					const allReferences = filterToExportedTypes([...new Set(allTypeStrings.flatMap(findAllTypeReferences))], variableName);
 					markdown += formatReferences(allReferences);
 
-					const outputPath = path.join(outputDir, `${variableName}.md`);
-					fs.writeFileSync(outputPath, markdown);
-					console.log(`Generated: ${outputPath}`);
+					// In tandem: update usage map
+					allReferences.forEach(ref => addUsage(ref, variableName, variableName, 'function'));
+
+					generatedDocs.set(variableName, markdown);
 				}
 			} else {
 				// Handle regular variables
@@ -507,9 +535,10 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 				const references = filterToExportedTypes(findAllTypeReferences(variableValue), variableName);
 				markdown += formatReferences(references);
 
-				const outputPath = path.join(outputDir, `${variableName}.md`);
-				fs.writeFileSync(outputPath, markdown);
-				console.log(`Generated: ${outputPath}`);
+				// In tandem: update usage map
+				references.forEach(ref => addUsage(ref, variableName, variableName, 'variable'));
+
+				generatedDocs.set(variableName, markdown);
 			}
 
 			return;
@@ -643,6 +672,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 				if (typeParamReferencesText) {
 					typeParameters += typeParamReferencesText;
 				}
+
+				// In tandem: update usage map
+				typeParamReferences.forEach(ref => addUsage(ref, className, className, 'type_param'));
 			}
 
 			// Check for extends clause (only for classes/interfaces)
@@ -653,6 +685,11 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 				if (extendsClauseNode && extendsClauseNode.types[0]) {
 					const superClassName = extendsClauseNode.types[0].expression.getText();
 					extendsClause = `\n\n**Extends:** [\`${superClassName}\`](./${superClassName}.md)\n`;
+
+					// In tandem: update usage map
+					if (exportedTypes.has(superClassName)) {
+						addUsage(superClassName, className, className, 'extends');
+					}
 				}
 			}
 
@@ -835,6 +872,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 										const references = filterToExportedTypes(findAllTypeReferences(paramType), className);
 										const referencesText = formatReferences(references);
 
+										// In tandem: update usage map
+										references.forEach(ref => addUsage(ref, className, paramName, 'property'));
+
 										const propertyContent = `### \`${paramName}\`\n\n\`\`\`ts\n${propertyDef}\n\`\`\`${desc ? `\n\n${desc}` : ''}${referencesText}`;
 										properties.push(propertyContent);
 									});
@@ -905,6 +945,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 							const overloadReferences = filterToExportedTypes([...new Set(ctorTypeStrings.flatMap(findAllTypeReferences))], className);
 							constructorBlock += formatReferences(overloadReferences, linkedTypes);
 
+							// In tandem: update usage map
+							overloadReferences.forEach(ref => addUsage(ref, className, className, 'constructor'));
+
 							constructorBlocks.push(constructorBlock);
 						});
 
@@ -950,6 +993,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 					const references = filterToExportedTypes(findAllTypeReferences(type), className);
 					const referencesText = formatReferences(references, linkedTypes);
 
+					// In tandem: update usage map
+					references.forEach(ref => addUsage(ref, className, name, 'property'));
+
 					// Check if this is an event handler (starts with "on" and can be a function)
 					const isEventHandler = name.startsWith('on') && (
 						type.includes('=>')
@@ -993,6 +1039,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 					const references = filterToExportedTypes(findAllTypeReferences(type), className);
 					const referencesText = formatReferences(references, linkedTypes);
 
+					// In tandem: update usage map
+					references.forEach(ref => addUsage(ref, className, name, 'property'));
+
 					const inheritedBadge = '';
 					properties.push(`### \`${name}\`${inheritedBadge}\n\n\`\`\`ts\n${accessorDef}\n\`\`\`${desc ? `\n\n${desc}` : ''}${referencesText}`);
 				} else if (ts.isSetAccessorDeclaration(member) && member.name) {
@@ -1013,6 +1062,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 					const references = filterToExportedTypes(findAllTypeReferences(paramType), className);
 					const referencesText = formatReferences(references, linkedTypes);
 
+					// In tandem: update usage map
+					references.forEach(ref => addUsage(ref, className, name, 'property'));
+
 					const inheritedBadge = '';
 					properties.push(`### \`${name}\`${inheritedBadge}\n\n\`\`\`ts\n${accessorDef}\n\`\`\`${desc ? `\n\n${desc}` : ''}${referencesText}`);
 				} else if ((ts.isMethodDeclaration(member) || ts.isMethodSignature(member)) && member.name) {
@@ -1106,6 +1158,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 					const allReferences = filterToExportedTypes([...new Set(allTypeStrings.flatMap(findAllTypeReferences))], className);
 					methodContent += formatReferences(allReferences, linkedTypes);
 
+					// In tandem: update usage map
+					allReferences.forEach(ref => addUsage(ref, className, name, 'method'));
+
 					if (isStatic) {
 						staticMethods.push(methodContent);
 					} else {
@@ -1174,6 +1229,9 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 						const references = filterToExportedTypes(findAllTypeReferences(cleanedType), className);
 						const referencesText = formatReferences(references);
 
+						// In tandem: update usage map
+						references.forEach(ref => addUsage(ref, className, propName, 'property'));
+
 						// Check if this is an event handler (starts with "on" and can be a function)
 						const isEventHandler = propName.startsWith('on') && (
 							cleanedType.includes('=>')
@@ -1252,19 +1310,39 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 
 			// Add subclasses section for classes that have subclasses
 			if (ts.isClassDeclaration(declaration) && classHierarchy.has(className)) {
-				const subclasses = classHierarchy.get(className)!;
-				// Sort by definition order instead of alphabetically
-				subclasses.sort((a, b) => {
-					const orderA = symbolOrderMap.get(a);
-					const orderB = symbolOrderMap.get(b);
-					if (orderA === undefined) throw new Error(`Symbol '${a}' not found in entry files export order`);
-					if (orderB === undefined) throw new Error(`Symbol '${b}' not found in entry files export order`);
-					return orderA - orderB;
-				});
-				markdown += `\n## Subclasses\n\n`;
-				subclasses.forEach((sub) => {
-					markdown += `- [\`${sub}\`](./${sub}.md)\n`;
-				});
+				// Recursively build hierarchical list
+				const buildHierarchicalList = (parentClass: string, depth = 0, visited = new Set<string>()): string => {
+					if (visited.has(parentClass)) return ''; // Prevent infinite loops
+					visited.add(parentClass);
+
+					const directChildren = classHierarchy.get(parentClass) || [];
+					if (directChildren.length === 0) return '';
+
+					// Sort children by definition order
+					const sortedChildren = [...directChildren].sort((a, b) => {
+						const orderA = symbolOrderMap.get(a);
+						const orderB = symbolOrderMap.get(b);
+						if (orderA === undefined) throw new Error(`Symbol '${a}' not found in entry files export order`);
+						if (orderB === undefined) throw new Error(`Symbol '${b}' not found in entry files export order`);
+						return orderA - orderB;
+					});
+
+					let result = '';
+					const indent = '  '.repeat(depth);
+
+					for (const child of sortedChildren) {
+						result += `${indent}- [\`${child}\`](./${child}.md)\n`;
+						// Recursively add children of this child
+						result += buildHierarchicalList(child, depth + 1, visited);
+					}
+
+					return result;
+				};
+
+				const hierarchicalList = buildHierarchicalList(className);
+				if (hierarchicalList) {
+					markdown += `\n## Subclasses\n\n${hierarchicalList}`;
+				}
 			}
 
 			// Add instances section for classes that have instances
@@ -1284,6 +1362,11 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 				});
 			}
 
+			// Add placeholder for "Used by" section for classes and interfaces
+			if (ts.isClassDeclaration(declaration) || ts.isInterfaceDeclaration(declaration)) {
+				markdown += '\n<!-- USED_BY_SECTION -->\n';
+			}
+
 			// Add type definition for type aliases
 			if (ts.isTypeAliasDeclaration(declaration) && declaration.type) {
 				const resolvedType = typeChecker.getTypeAtLocation(declaration);
@@ -1346,7 +1429,13 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 				const typeReferences = filterToExportedTypes([...new Set(allTypeRefs)], className);
 				const typeReferencesText = formatReferences(typeReferences);
 
+				// In tandem: update usage map
+				typeReferences.forEach(ref => addUsage(ref, className, className, 'type_alias'));
+
 				markdown += `\n\`\`\`ts\n${typeDefinition}\n\`\`\`${typeReferencesText}`;
+
+				// Add placeholder for "Used by" section for type aliases
+				markdown += '\n<!-- USED_BY_SECTION -->\n';
 			}
 
 			if (typeParameters) {
@@ -1372,12 +1461,89 @@ const generateDocs = (entryFiles: string[], apiConfigFile: string) => {
 			if (methods.length > 0) {
 				markdown += `\n## Methods\n\n${methods.join('\n\n')}\n`;
 			}
-			const outputPath = path.join(outputDir, `${className}.md`);
-			fs.writeFileSync(outputPath, markdown);
-			console.log(`Generated: ${outputPath}`);
+
+			generatedDocs.set(className, markdown);
 		}
 	});
 
+	// Phase 3: Assemble final docs with "Used by" sections and write files
+	generatedDocs.forEach((markdown, symbolName) => {
+		const usages = usedByReferences.get(symbolName);
+		let usedByMarkdown = '';
+
+		if (usages && usages.size > 0) {
+			const symbolSubclasses = classHierarchy.get(symbolName) || [];
+
+			const usedByLines = [...usages]
+				.filter((usage) => {
+					// Filter out subclasses from "Used by" since they already appear in "Subclasses" section
+					if (usage.type === 'extends' && symbolSubclasses.includes(usage.user)) {
+						return false;
+					}
+
+					// Filter out top-level type references when there are more specific contexts available
+					// This prevents redundancy where both "TypeName" and "TypeName.property" appear
+					if (usage.type === 'type_alias' || usage.type === 'type_param') {
+						// Check if there are more specific usages from the same user (property, method, constructor, etc.)
+						const hasMoreSpecificUsage = [...usages].some(otherUsage =>
+							otherUsage.user === usage.user
+							&& otherUsage.type !== 'type_alias'
+							&& otherUsage.type !== 'type_param'
+							&& otherUsage.type !== 'extends',
+						);
+						if (hasMoreSpecificUsage) {
+							return false;
+						}
+					}
+
+					return true;
+				})
+				.map((usage) => {
+					let displayText = '';
+					let link = '';
+
+					switch (usage.type) {
+						case 'constructor':
+							displayText = `new ${usage.user}()`;
+							link = `./${usage.user}.md#constructor`;
+							break;
+						case 'method':
+							displayText = `${usage.user}.${usage.context}()`;
+							link = `./${usage.user}.md#${usage.context.toLowerCase()}`;
+							break;
+						case 'property':
+						case 'variable':
+							displayText = `${usage.user}.${usage.context}`;
+							link = `./${usage.user}.md#${usage.context.toLowerCase()}`;
+							break;
+						case 'function':
+							displayText = `${usage.user}()`;
+							link = `./${usage.user}.md`;
+							break;
+						case 'extends':
+						case 'type_param':
+						case 'type_alias':
+							displayText = usage.user;
+							link = `./${usage.user}.md`;
+							break;
+					}
+					return { text: `[\`${displayText}\`](${link})`, sortKey: displayText.toLowerCase() };
+				});
+
+			if (usedByLines.length > 0) {
+				// Sort alphabetically by display text
+				usedByLines.sort((a, b) => a.sortKey.localeCompare(b.sortKey));
+				const listItems = usedByLines.map(item => `- ${item.text}`).join('\n');
+				usedByMarkdown = `\n## Used by\n\n${listItems}\n`;
+			}
+		}
+
+		const finalMarkdown = markdown.replace('<!-- USED_BY_SECTION -->', usedByMarkdown);
+		const outputPath = path.join(outputDir, `${symbolName}.md`);
+		fs.writeFileSync(outputPath, finalMarkdown);
+		console.log(`Generated: ${outputPath}`);
+	});
+
 	// Generate index.md with all exported symbols grouped by group
 	const entriesByGroup = new Map<string, Array<{ name: string; type: string; order: number }>>();
 

src/codec-data.ts

@@ -26,8 +26,27 @@ import { EncodedPacket, PacketType } from './packet';
 // Rec. ITU-T H.265
 // https://stackoverflow.com/questions/24884827
 
+export enum AvcNalUnitType {
+	IDR = 5,
+	SPS = 7,
+	PPS = 8,
+	SPS_EXT = 13,
+}
+
+export enum HevcNalUnitType {
+	RASL_N = 8,
+	RASL_R = 9,
+	BLA_W_LP = 16,
+	RSV_IRAP_VCL23 = 23,
+	VPS_NUT = 32,
+	SPS_NUT = 33,
+	PPS_NUT = 34,
+	PREFIX_SEI_NUT = 39,
+	SUFFIX_SEI_NUT = 40,
+}
+
 /** Finds all NAL units in an AVC packet in Annex B format. */
-const findNalUnitsInAnnexB = (packetData: Uint8Array) => {
+export const findNalUnitsInAnnexB = (packetData: Uint8Array) => {
 	const nalUnits: Uint8Array[] = [];
 	let i = 0;
 
@@ -184,6 +203,21 @@ export type AvcDecoderConfigurationRecord = {
 	sequenceParameterSetExt: Uint8Array[] | null;
 };
 
+export const extractAvcNalUnits = (packetData: Uint8Array, decoderConfig: VideoDecoderConfig) => {
+	if (decoderConfig.description) {
+		// Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
+
+		const bytes = toUint8Array(decoderConfig.description);
+		const lengthSizeMinusOne = bytes[4]! & 0b11;
+		const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
+
+		return findNalUnitsInLengthPrefixed(packetData, lengthSize);
+	} else {
+		// Stream is in Annex B format
+		return findNalUnitsInAnnexB(packetData);
+	}
+};
+
 const extractNalUnitTypeForAvc = (data: Uint8Array) => {
 	return data[0]! & 0x1F;
 };
@@ -193,9 +227,9 @@ export const extractAvcDecoderConfigurationRecord = (packetData: Uint8Array) =>
 	try {
 		const nalUnits = findNalUnitsInAnnexB(packetData);
 
-		const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 7);
-		const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 8);
-		const spsExtUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === 13);
+		const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.SPS);
+		const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.PPS);
+		const spsExtUnits = nalUnits.filter(unit => extractNalUnitTypeForAvc(unit) === AvcNalUnitType.SPS_EXT);
 
 		if (spsUnits.length === 0) {
 			return null;
@@ -337,12 +371,6 @@ export const serializeAvcDecoderConfigurationRecord = (record: AvcDecoderConfigu
 	return new Uint8Array(bytes);
 };
 
-const NALU_TYPE_VPS = 32;
-const NALU_TYPE_SPS = 33;
-const NALU_TYPE_PPS = 34;
-const NALU_TYPE_SEI_PREFIX = 39;
-const NALU_TYPE_SEI_SUFFIX = 40;
-
 // Data specified in ISO 14496-15
 export type HevcDecoderConfigurationRecord = {
 	configurationVersion: number;
@@ -369,7 +397,22 @@ export type HevcDecoderConfigurationRecord = {
 	}[];
 };
 
-const extractNalUnitTypeForHevc = (data: Uint8Array) => {
+export const extractHevcNalUnits = (packetData: Uint8Array, decoderConfig: VideoDecoderConfig) => {
+	if (decoderConfig.description) {
+		// Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
+
+		const bytes = toUint8Array(decoderConfig.description);
+		const lengthSizeMinusOne = bytes[21]! & 0b11;
+		const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
+
+		return findNalUnitsInLengthPrefixed(packetData, lengthSize);
+	} else {
+		// Stream is in Annex B format
+		return findNalUnitsInAnnexB(packetData);
+	}
+};
+
+export const extractNalUnitTypeForHevc = (data: Uint8Array) => {
 	return (data[0]! >> 1) & 0x3F;
 };
 
@@ -380,12 +423,12 @@ export const extractHevcDecoderConfigurationRecord = (
 	try {
 		const nalUnits = findNalUnitsInAnnexB(packetData);
 
-		const vpsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_VPS);
-		const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_SPS);
-		const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_PPS);
+		const vpsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.VPS_NUT);
+		const spsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.SPS_NUT);
+		const ppsUnits = nalUnits.filter(unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.PPS_NUT);
 		const seiUnits = nalUnits.filter(
-			unit => extractNalUnitTypeForHevc(unit) === NALU_TYPE_SEI_PREFIX
-				|| extractNalUnitTypeForHevc(unit) === NALU_TYPE_SEI_SUFFIX,
+			unit => extractNalUnitTypeForHevc(unit) === HevcNalUnitType.PREFIX_SEI_NUT
+				|| extractNalUnitTypeForHevc(unit) === HevcNalUnitType.SUFFIX_SEI_NUT,
 		);
 
 		if (spsUnits.length === 0 || ppsUnits.length === 0) return null;
@@ -521,7 +564,7 @@ export const extractHevcDecoderConfigurationRecord = (
 				? [
 						{
 							arrayCompleteness: 1,
-							nalUnitType: NALU_TYPE_VPS,
+							nalUnitType: HevcNalUnitType.VPS_NUT,
 							nalUnits: vpsUnits,
 						},
 					]
@@ -530,7 +573,7 @@ export const extractHevcDecoderConfigurationRecord = (
 				? [
 						{
 							arrayCompleteness: 1,
-							nalUnitType: NALU_TYPE_SPS,
+							nalUnitType: HevcNalUnitType.SPS_NUT,
 							nalUnits: spsUnits,
 						},
 					]
@@ -539,7 +582,7 @@ export const extractHevcDecoderConfigurationRecord = (
 				? [
 						{
 							arrayCompleteness: 1,
-							nalUnitType: NALU_TYPE_PPS,
+							nalUnitType: HevcNalUnitType.PPS_NUT,
 							nalUnits: ppsUnits,
 						},
 					]
@@ -1440,22 +1483,9 @@ export const determineVideoPacketType = async (
 			const decoderConfig = await videoTrack.getDecoderConfig();
 			assert(decoderConfig);
 
-			let nalUnits: Uint8Array[];
+			const nalUnits = extractAvcNalUnits(packet.data, decoderConfig);
+			const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === AvcNalUnitType.IDR);
 
-			if (decoderConfig.description) {
-				// Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
-
-				const bytes = toUint8Array(decoderConfig.description);
-				const lengthSizeMinusOne = bytes[4]! & 0b11;
-				const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
-
-				nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize);
-			} else {
-				// Stream is in Annex B format
-				nalUnits = findNalUnitsInAnnexB(packet.data);
-			}
-
-			const isKeyframe = nalUnits.some(x => extractNalUnitTypeForAvc(x) === 5);
 			return isKeyframe ? 'key' : 'delta';
 		};
 
@@ -1463,25 +1493,12 @@ export const determineVideoPacketType = async (
 			const decoderConfig = await videoTrack.getDecoderConfig();
 			assert(decoderConfig);
 
-			let nalUnits: Uint8Array[];
-
-			if (decoderConfig.description) {
-				// Stream is length-prefixed. Let's extract the size of the length prefix from the decoder config
-
-				const bytes = toUint8Array(decoderConfig.description);
-				const lengthSizeMinusOne = bytes[21]! & 0b11;
-				const lengthSize = (lengthSizeMinusOne + 1) as 1 | 2 | 3 | 4;
-
-				nalUnits = findNalUnitsInLengthPrefixed(packet.data, lengthSize);
-			} else {
-				// Stream is in Annex B format
-				nalUnits = findNalUnitsInAnnexB(packet.data);
-			}
-
+			const nalUnits = extractHevcNalUnits(packet.data, decoderConfig);
 			const isKeyframe = nalUnits.some((x) => {
 				const type = extractNalUnitTypeForHevc(x);
-				return 16 <= type && type <= 23;
+				return HevcNalUnitType.BLA_W_LP <= type && type <= HevcNalUnitType.RSV_IRAP_VCL23;
 			});
+
 			return isKeyframe ? 'key' : 'delta';
 		};
 

src/conversion.ts

@@ -534,9 +534,14 @@ export class Conversion {
 		if (this.onProgress) {
 			this._computeProgress = true;
 			this._totalDuration = Math.min(
-				await this.input.computeDuration() - this._startTimestamp,
+				(await this.input.computeDuration()) - this._startTimestamp,
 				this._endTimestamp - this._startTimestamp,
 			);
+
+			for (const track of this.utilizedTracks) {
+				this._maxTimestamps.set(track.id, 0);
+			}
+
 			this.onProgress?.(0);
 		}
 
@@ -723,7 +728,7 @@ export class Conversion {
 				await tempOutput.start();
 
 				const sink = new VideoSampleSink(track);
-				const firstSample = await sink.getSample(this._startTimestamp);
+				const firstSample = await sink.getSample(firstTimestamp); // Let's just use the first sample
 
 				if (firstSample) {
 					try {
@@ -1160,7 +1165,10 @@ export class Conversion {
 		}
 		assert(this._totalDuration !== null);
 
-		this._maxTimestamps.set(trackId, Math.max(endTimestamp, this._maxTimestamps.get(trackId) ?? -Infinity));
+		this._maxTimestamps.set(
+			trackId,
+			Math.max(endTimestamp, this._maxTimestamps.get(trackId)!),
+		);
 
 		const minTimestamp = Math.min(...this._maxTimestamps.values());
 		const newProgress = clamp(minTimestamp / this._totalDuration, 0, 1);

src/media-sink.ts

@@ -7,6 +7,7 @@
  */
 
 import { parsePcmCodec, PCM_AUDIO_CODECS, PcmAudioCodec, VideoCodec, AudioCodec } from './codec';
+import { extractHevcNalUnits, extractNalUnitTypeForHevc, HevcNalUnitType } from './codec-data';
 import { CustomVideoDecoder, customVideoDecoders, CustomAudioDecoder, customAudioDecoders } from './custom-coder';
 import { InputAudioTrack, InputTrack, InputVideoTrack } from './input-track';
 import {
@@ -624,8 +625,8 @@ export abstract class BaseMediaSampleSink<
 					const nextPacket = await packetSink.getNextPacket(currentPacket);
 					assert(nextPacket);
 
-					currentPacket = nextPacket;
 					decoder.decode(nextPacket);
+					currentPacket = nextPacket;
 				}
 
 				maxSequenceNumber = -1;
@@ -757,12 +758,14 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 
 	inputTimestamps: number[] = []; // Timestamps input into the decoder, sorted.
 	sampleQueue: VideoSample[] = []; // Safari-specific thing, check usage.
+	currentPacketIndex = 0;
+	raslSkipped = false; // For HEVC stuff
 
 	constructor(
 		onSample: (sample: VideoSample) => unknown,
 		onError: (error: DOMException) => unknown,
-		codec: VideoCodec,
-		decoderConfig: VideoDecoderConfig,
+		public codec: VideoCodec,
+		public decoderConfig: VideoDecoderConfig,
 		public rotation: Rotation,
 		public timeResolution: number,
 	) {
@@ -848,6 +851,26 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 	}
 
 	decode(packet: EncodedPacket) {
+		if (this.codec === 'hevc' && this.currentPacketIndex > 0 && !this.raslSkipped) {
+			// If we're using HEVC, we need to make sure to skip any RASL slices that follow a non-IDR key frame such as
+			// CRA_NUT. This is because RASL slices cannot be decoded without data before the CRA_NUT. Browsers behave
+			// differently here: Chromium drops the packets, Safari throws a decoder error. Either way, it's not good
+			// and causes bugs upstream. So, let's take the dropping into our own hands.
+			const nalUnits = extractHevcNalUnits(packet.data, this.decoderConfig);
+			const hasRaslPicture = nalUnits.some((x) => {
+				const type = extractNalUnitTypeForHevc(x);
+				return type === HevcNalUnitType.RASL_N || type === HevcNalUnitType.RASL_R;
+			});
+
+			if (hasRaslPicture) {
+				return; // Drop
+			}
+
+			this.raslSkipped = true;
+		}
+
+		this.currentPacketIndex++;
+
 		if (this.customDecoder) {
 			this.customDecoderQueueSize++;
 			void this.customDecoderCallSerializer
@@ -879,6 +902,9 @@ class VideoDecoderWrapper extends DecoderWrapper<VideoSample> {
 
 			this.sampleQueue.length = 0;
 		}
+
+		this.currentPacketIndex = 0;
+		this.raslSkipped = false;
 	}
 
 	close() {